direct-io.hg

view linux-2.6-xen-sparse/drivers/xen/netback/netback.c @ 11730:fd80b7e1e333

[NET] back: Refcount fixes to vif rate-limiting code.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kfraser@localhost.localdomain
date Mon Oct 09 17:08:18 2006 +0100 (2006-10-09)
parents 7efaaae37415
children cb337b2f817b
line source
1 /******************************************************************************
2 * drivers/xen/netback/netback.c
3 *
4 * Back-end of the driver for virtual network devices. This portion of the
5 * driver exports a 'unified' network-device interface that can be accessed
6 * by any operating system that implements a compatible front end. A
7 * reference front-end implementation can be found in:
8 * drivers/xen/netfront/netfront.c
9 *
10 * Copyright (c) 2002-2005, K A Fraser
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
37 #include "common.h"
38 #include <xen/balloon.h>
39 #include <xen/interface/memory.h>
41 /*#define NETBE_DEBUG_INTERRUPT*/
43 struct netbk_rx_meta {
44 skb_frag_t frag;
45 int id;
46 int copy:1;
47 };
49 static void netif_idx_release(u16 pending_idx);
50 static void netif_page_release(struct page *page);
51 static void make_tx_response(netif_t *netif,
52 netif_tx_request_t *txp,
53 s8 st);
54 static netif_rx_response_t *make_rx_response(netif_t *netif,
55 u16 id,
56 s8 st,
57 u16 offset,
58 u16 size,
59 u16 flags);
61 static void net_tx_action(unsigned long unused);
62 static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0);
64 static void net_rx_action(unsigned long unused);
65 static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
67 static struct timer_list net_timer;
69 #define MAX_PENDING_REQS 256
71 static struct sk_buff_head rx_queue;
73 static struct page **mmap_pages;
74 static inline unsigned long idx_to_kaddr(unsigned int idx)
75 {
76 return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx]));
77 }
79 #define PKT_PROT_LEN 64
81 static struct {
82 netif_tx_request_t req;
83 netif_t *netif;
84 } pending_tx_info[MAX_PENDING_REQS];
85 static u16 pending_ring[MAX_PENDING_REQS];
86 typedef unsigned int PEND_RING_IDX;
87 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
88 static PEND_RING_IDX pending_prod, pending_cons;
89 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
91 /* Freed TX SKBs get batched on this ring before return to pending_ring. */
92 static u16 dealloc_ring[MAX_PENDING_REQS];
93 static PEND_RING_IDX dealloc_prod, dealloc_cons;
95 static struct sk_buff_head tx_queue;
97 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
98 static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS];
99 static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS];
101 static struct list_head net_schedule_list;
102 static spinlock_t net_schedule_list_lock;
104 #define MAX_MFN_ALLOC 64
105 static unsigned long mfn_list[MAX_MFN_ALLOC];
106 static unsigned int alloc_index = 0;
108 static inline unsigned long alloc_mfn(void)
109 {
110 return mfn_list[--alloc_index];
111 }
113 static int check_mfn(int nr)
114 {
115 struct xen_memory_reservation reservation = {
116 .extent_order = 0,
117 .domid = DOMID_SELF
118 };
120 if (likely(alloc_index >= nr))
121 return 0;
123 set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index);
124 reservation.nr_extents = MAX_MFN_ALLOC - alloc_index;
125 alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation,
126 &reservation);
128 return alloc_index >= nr ? 0 : -ENOMEM;
129 }
131 static inline void maybe_schedule_tx_action(void)
132 {
133 smp_mb();
134 if ((NR_PENDING_REQS < (MAX_PENDING_REQS/2)) &&
135 !list_empty(&net_schedule_list))
136 tasklet_schedule(&net_tx_tasklet);
137 }
139 /*
140 * A gross way of confirming the origin of an skb data page. The slab
141 * allocator abuses a field in the page struct to cache the kmem_cache_t ptr.
142 */
143 static inline int is_xen_skb(struct sk_buff *skb)
144 {
145 extern kmem_cache_t *skbuff_cachep;
146 kmem_cache_t *cp = (kmem_cache_t *)virt_to_page(skb->head)->lru.next;
147 return (cp == skbuff_cachep);
148 }
150 /*
151 * We can flip without copying the packet unless:
152 * 1. The data is not allocated from our special cache; or
153 * 2. The main data area is shared; or
154 * 3. One or more fragments are shared; or
155 * 4. There are chained fragments.
156 */
157 static inline int is_flippable_skb(struct sk_buff *skb)
158 {
159 int frag;
161 if (!is_xen_skb(skb) || skb_cloned(skb))
162 return 0;
164 for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
165 if (page_count(skb_shinfo(skb)->frags[frag].page) > 1)
166 return 0;
167 }
169 if (skb_shinfo(skb)->frag_list != NULL)
170 return 0;
172 return 1;
173 }
175 static struct sk_buff *netbk_copy_skb(struct sk_buff *skb)
176 {
177 struct skb_shared_info *ninfo;
178 struct sk_buff *nskb;
179 unsigned long offset;
180 int ret;
181 int len;
182 int headlen;
184 BUG_ON(skb_shinfo(skb)->frag_list != NULL);
186 nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC);
187 if (unlikely(!nskb))
188 goto err;
190 skb_reserve(nskb, 16);
191 headlen = nskb->end - nskb->data;
192 if (headlen > skb_headlen(skb))
193 headlen = skb_headlen(skb);
194 ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen);
195 BUG_ON(ret);
197 ninfo = skb_shinfo(nskb);
198 ninfo->gso_size = skb_shinfo(skb)->gso_size;
199 ninfo->gso_type = skb_shinfo(skb)->gso_type;
201 offset = headlen;
202 len = skb->len - headlen;
204 nskb->len = skb->len;
205 nskb->data_len = len;
206 nskb->truesize += len;
208 while (len) {
209 struct page *page;
210 int copy;
211 int zero;
213 if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) {
214 dump_stack();
215 goto err_free;
216 }
218 copy = len >= PAGE_SIZE ? PAGE_SIZE : len;
219 zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO;
221 page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | zero);
222 if (unlikely(!page))
223 goto err_free;
225 ret = skb_copy_bits(skb, offset, page_address(page), copy);
226 BUG_ON(ret);
228 ninfo->frags[ninfo->nr_frags].page = page;
229 ninfo->frags[ninfo->nr_frags].page_offset = 0;
230 ninfo->frags[ninfo->nr_frags].size = copy;
231 ninfo->nr_frags++;
233 offset += copy;
234 len -= copy;
235 }
237 offset = nskb->data - skb->data;
239 nskb->h.raw = skb->h.raw + offset;
240 nskb->nh.raw = skb->nh.raw + offset;
241 nskb->mac.raw = skb->mac.raw + offset;
243 return nskb;
245 err_free:
246 kfree_skb(nskb);
247 err:
248 return NULL;
249 }
251 static inline int netbk_max_required_rx_slots(netif_t *netif)
252 {
253 if (netif->features & (NETIF_F_SG|NETIF_F_TSO))
254 return MAX_SKB_FRAGS + 2; /* header + extra_info + frags */
255 return 1; /* all in one */
256 }
258 static inline int netbk_queue_full(netif_t *netif)
259 {
260 RING_IDX peek = netif->rx_req_cons_peek;
261 RING_IDX needed = netbk_max_required_rx_slots(netif);
263 return ((netif->rx.sring->req_prod - peek) < needed) ||
264 ((netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek) < needed);
265 }
267 int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
268 {
269 netif_t *netif = netdev_priv(dev);
271 BUG_ON(skb->dev != dev);
273 /* Drop the packet if the target domain has no receive buffers. */
274 if (unlikely(!netif_running(dev) || !netif_carrier_ok(dev)))
275 goto drop;
277 if (unlikely(netbk_queue_full(netif))) {
278 /* Not a BUG_ON() -- misbehaving netfront can trigger this. */
279 if (netbk_can_queue(dev))
280 DPRINTK("Queue full but not stopped!\n");
281 goto drop;
282 }
284 /* Copy the packet here if it's destined for a flipping
285 interface but isn't flippable (e.g. extra references to
286 data)
287 */
288 if (!netif->copying_receiver && !is_flippable_skb(skb)) {
289 struct sk_buff *nskb = netbk_copy_skb(skb);
290 if ( unlikely(nskb == NULL) )
291 goto drop;
292 /* Copy only the header fields we use in this driver. */
293 nskb->dev = skb->dev;
294 nskb->ip_summed = skb->ip_summed;
295 nskb->proto_data_valid = skb->proto_data_valid;
296 dev_kfree_skb(skb);
297 skb = nskb;
298 }
300 netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1 +
301 !!skb_shinfo(skb)->gso_size;
302 netif_get(netif);
304 if (netbk_can_queue(dev) && netbk_queue_full(netif)) {
305 netif->rx.sring->req_event = netif->rx_req_cons_peek +
306 netbk_max_required_rx_slots(netif);
307 mb(); /* request notification /then/ check & stop the queue */
308 if (netbk_queue_full(netif))
309 netif_stop_queue(dev);
310 }
312 skb_queue_tail(&rx_queue, skb);
313 tasklet_schedule(&net_rx_tasklet);
315 return 0;
317 drop:
318 netif->stats.tx_dropped++;
319 dev_kfree_skb(skb);
320 return 0;
321 }
323 #if 0
324 static void xen_network_done_notify(void)
325 {
326 static struct net_device *eth0_dev = NULL;
327 if (unlikely(eth0_dev == NULL))
328 eth0_dev = __dev_get_by_name("eth0");
329 netif_rx_schedule(eth0_dev);
330 }
331 /*
332 * Add following to poll() function in NAPI driver (Tigon3 is example):
333 * if ( xen_network_done() )
334 * tg3_enable_ints(tp);
335 */
336 int xen_network_done(void)
337 {
338 return skb_queue_empty(&rx_queue);
339 }
340 #endif
342 struct netrx_pending_operations {
343 unsigned trans_prod, trans_cons;
344 unsigned mmu_prod, mmu_cons;
345 unsigned mcl_prod, mcl_cons;
346 unsigned copy_prod, copy_cons;
347 unsigned meta_prod, meta_cons;
348 mmu_update_t *mmu;
349 gnttab_transfer_t *trans;
350 gnttab_copy_t *copy;
351 multicall_entry_t *mcl;
352 struct netbk_rx_meta *meta;
353 };
355 /* Set up the grant operations for this fragment. If it's a flipping
356 interface, we also set up the unmap request from here. */
357 static u16 netbk_gop_frag(netif_t *netif, struct netbk_rx_meta *meta,
358 int i, struct netrx_pending_operations *npo,
359 struct page *page, unsigned long size,
360 unsigned long offset)
361 {
362 mmu_update_t *mmu;
363 gnttab_transfer_t *gop;
364 gnttab_copy_t *copy_gop;
365 multicall_entry_t *mcl;
366 netif_rx_request_t *req;
367 unsigned long old_mfn, new_mfn;
369 old_mfn = virt_to_mfn(page_address(page));
371 req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i);
372 if (netif->copying_receiver) {
373 /* The fragment needs to be copied rather than
374 flipped. */
375 meta->copy = 1;
376 copy_gop = npo->copy + npo->copy_prod++;
377 copy_gop->source.domid = DOMID_SELF;
378 copy_gop->source.offset = offset;
379 copy_gop->source.u.gmfn = old_mfn;
380 copy_gop->dest.domid = netif->domid;
381 copy_gop->dest.offset = 0;
382 copy_gop->dest.u.ref = req->gref;
383 copy_gop->len = size;
384 copy_gop->flags = GNTCOPY_dest_gref;
385 } else {
386 meta->copy = 0;
387 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
388 new_mfn = alloc_mfn();
390 /*
391 * Set the new P2M table entry before
392 * reassigning the old data page. Heed the
393 * comment in pgtable-2level.h:pte_page(). :-)
394 */
395 set_phys_to_machine(page_to_pfn(page), new_mfn);
397 mcl = npo->mcl + npo->mcl_prod++;
398 MULTI_update_va_mapping(mcl,
399 (unsigned long)page_address(page),
400 pfn_pte_ma(new_mfn, PAGE_KERNEL),
401 0);
403 mmu = npo->mmu + npo->mmu_prod++;
404 mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) |
405 MMU_MACHPHYS_UPDATE;
406 mmu->val = page_to_pfn(page);
407 }
409 gop = npo->trans + npo->trans_prod++;
410 gop->mfn = old_mfn;
411 gop->domid = netif->domid;
412 gop->ref = req->gref;
413 }
414 return req->id;
415 }
417 static void netbk_gop_skb(struct sk_buff *skb,
418 struct netrx_pending_operations *npo)
419 {
420 netif_t *netif = netdev_priv(skb->dev);
421 int nr_frags = skb_shinfo(skb)->nr_frags;
422 int i;
423 int extra;
424 struct netbk_rx_meta *head_meta, *meta;
426 head_meta = npo->meta + npo->meta_prod++;
427 head_meta->frag.page_offset = skb_shinfo(skb)->gso_type;
428 head_meta->frag.size = skb_shinfo(skb)->gso_size;
429 extra = !!head_meta->frag.size + 1;
431 for (i = 0; i < nr_frags; i++) {
432 meta = npo->meta + npo->meta_prod++;
433 meta->frag = skb_shinfo(skb)->frags[i];
434 meta->id = netbk_gop_frag(netif, meta, i + extra, npo,
435 meta->frag.page,
436 meta->frag.size,
437 meta->frag.page_offset);
438 }
440 /*
441 * This must occur at the end to ensure that we don't trash
442 * skb_shinfo until we're done.
443 */
444 head_meta->id = netbk_gop_frag(netif, head_meta, 0, npo,
445 virt_to_page(skb->data),
446 skb_headlen(skb),
447 offset_in_page(skb->data));
449 netif->rx.req_cons += nr_frags + extra;
450 }
452 static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta)
453 {
454 int i;
456 for (i = 0; i < nr_frags; i++)
457 put_page(meta[i].frag.page);
458 }
460 /* This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
461 used to set up the operations on the top of
462 netrx_pending_operations, which have since been done. Check that
463 they didn't give any errors and advance over them. */
464 static int netbk_check_gop(int nr_frags, domid_t domid,
465 struct netrx_pending_operations *npo)
466 {
467 multicall_entry_t *mcl;
468 gnttab_transfer_t *gop;
469 gnttab_copy_t *copy_op;
470 int status = NETIF_RSP_OKAY;
471 int i;
473 for (i = 0; i <= nr_frags; i++) {
474 if (npo->meta[npo->meta_cons + i].copy) {
475 copy_op = npo->copy + npo->copy_cons++;
476 if (copy_op->status != GNTST_okay) {
477 DPRINTK("Bad status %d from copy to DOM%d.\n",
478 gop->status, domid);
479 status = NETIF_RSP_ERROR;
480 }
481 } else {
482 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
483 mcl = npo->mcl + npo->mcl_cons++;
484 /* The update_va_mapping() must not fail. */
485 BUG_ON(mcl->result != 0);
486 }
488 gop = npo->trans + npo->trans_cons++;
489 /* Check the reassignment error code. */
490 if (gop->status != 0) {
491 DPRINTK("Bad status %d from grant transfer to DOM%u\n",
492 gop->status, domid);
493 /*
494 * Page no longer belongs to us unless
495 * GNTST_bad_page, but that should be
496 * a fatal error anyway.
497 */
498 BUG_ON(gop->status == GNTST_bad_page);
499 status = NETIF_RSP_ERROR;
500 }
501 }
502 }
504 return status;
505 }
507 static void netbk_add_frag_responses(netif_t *netif, int status,
508 struct netbk_rx_meta *meta, int nr_frags)
509 {
510 int i;
511 unsigned long offset;
513 for (i = 0; i < nr_frags; i++) {
514 int id = meta[i].id;
515 int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data;
517 if (meta[i].copy)
518 offset = 0;
519 else
520 offset = meta[i].frag.page_offset;
521 make_rx_response(netif, id, status, offset,
522 meta[i].frag.size, flags);
523 }
524 }
526 static void net_rx_action(unsigned long unused)
527 {
528 netif_t *netif = NULL;
529 s8 status;
530 u16 id, irq, flags;
531 netif_rx_response_t *resp;
532 multicall_entry_t *mcl;
533 struct sk_buff_head rxq;
534 struct sk_buff *skb;
535 int notify_nr = 0;
536 int ret;
537 int nr_frags;
538 int count;
539 unsigned long offset;
541 /*
542 * Putting hundreds of bytes on the stack is considered rude.
543 * Static works because a tasklet can only be on one CPU at any time.
544 */
545 static multicall_entry_t rx_mcl[NET_RX_RING_SIZE+3];
546 static mmu_update_t rx_mmu[NET_RX_RING_SIZE];
547 static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE];
548 static gnttab_copy_t grant_copy_op[NET_RX_RING_SIZE];
549 static unsigned char rx_notify[NR_IRQS];
550 static u16 notify_list[NET_RX_RING_SIZE];
551 static struct netbk_rx_meta meta[NET_RX_RING_SIZE];
553 struct netrx_pending_operations npo = {
554 mmu: rx_mmu,
555 trans: grant_trans_op,
556 copy: grant_copy_op,
557 mcl: rx_mcl,
558 meta: meta};
560 skb_queue_head_init(&rxq);
562 count = 0;
564 while ((skb = skb_dequeue(&rx_queue)) != NULL) {
565 nr_frags = skb_shinfo(skb)->nr_frags;
566 *(int *)skb->cb = nr_frags;
568 if (!xen_feature(XENFEAT_auto_translated_physmap) &&
569 check_mfn(nr_frags + 1)) {
570 /* Memory squeeze? Back off for an arbitrary while. */
571 if ( net_ratelimit() )
572 WPRINTK("Memory squeeze in netback "
573 "driver.\n");
574 mod_timer(&net_timer, jiffies + HZ);
575 skb_queue_head(&rx_queue, skb);
576 break;
577 }
579 netbk_gop_skb(skb, &npo);
581 count += nr_frags + 1;
583 __skb_queue_tail(&rxq, skb);
585 /* Filled the batch queue? */
586 if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE)
587 break;
588 }
590 if (npo.mcl_prod &&
591 !xen_feature(XENFEAT_auto_translated_physmap)) {
592 mcl = npo.mcl + npo.mcl_prod++;
594 BUG_ON(mcl[-1].op != __HYPERVISOR_update_va_mapping);
595 mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL;
597 mcl->op = __HYPERVISOR_mmu_update;
598 mcl->args[0] = (unsigned long)rx_mmu;
599 mcl->args[1] = npo.mmu_prod;
600 mcl->args[2] = 0;
601 mcl->args[3] = DOMID_SELF;
602 }
604 if (npo.trans_prod) {
605 mcl = npo.mcl + npo.mcl_prod++;
606 mcl->op = __HYPERVISOR_grant_table_op;
607 mcl->args[0] = GNTTABOP_transfer;
608 mcl->args[1] = (unsigned long)grant_trans_op;
609 mcl->args[2] = npo.trans_prod;
610 }
612 if (npo.copy_prod) {
613 mcl = npo.mcl + npo.mcl_prod++;
614 mcl->op = __HYPERVISOR_grant_table_op;
615 mcl->args[0] = GNTTABOP_copy;
616 mcl->args[1] = (unsigned long)grant_copy_op;
617 mcl->args[2] = npo.copy_prod;
618 }
620 /* Nothing to do? */
621 if (!npo.mcl_prod)
622 return;
624 BUG_ON(npo.copy_prod > NET_RX_RING_SIZE);
625 BUG_ON(npo.mmu_prod > NET_RX_RING_SIZE);
626 BUG_ON(npo.trans_prod > NET_RX_RING_SIZE);
627 BUG_ON(npo.mcl_prod > NET_RX_RING_SIZE+3);
628 BUG_ON(npo.meta_prod > NET_RX_RING_SIZE);
630 ret = HYPERVISOR_multicall(npo.mcl, npo.mcl_prod);
631 BUG_ON(ret != 0);
633 while ((skb = __skb_dequeue(&rxq)) != NULL) {
634 nr_frags = *(int *)skb->cb;
636 netif = netdev_priv(skb->dev);
637 /* We can't rely on skb_release_data to release the
638 pages used by fragments for us, since it tries to
639 touch the pages in the fraglist. If we're in
640 flipping mode, that doesn't work. In copying mode,
641 we still have access to all of the pages, and so
642 it's safe to let release_data deal with it. */
643 /* (Freeing the fragments is safe since we copy
644 non-linear skbs destined for flipping interfaces) */
645 if (!netif->copying_receiver) {
646 atomic_set(&(skb_shinfo(skb)->dataref), 1);
647 skb_shinfo(skb)->frag_list = NULL;
648 skb_shinfo(skb)->nr_frags = 0;
649 netbk_free_pages(nr_frags, meta + npo.meta_cons + 1);
650 }
652 netif->stats.tx_bytes += skb->len;
653 netif->stats.tx_packets++;
655 status = netbk_check_gop(nr_frags, netif->domid, &npo);
657 id = meta[npo.meta_cons].id;
658 flags = nr_frags ? NETRXF_more_data : 0;
660 if (skb->ip_summed == CHECKSUM_HW) /* local packet? */
661 flags |= NETRXF_csum_blank | NETRXF_data_validated;
662 else if (skb->proto_data_valid) /* remote but checksummed? */
663 flags |= NETRXF_data_validated;
665 if (meta[npo.meta_cons].copy)
666 offset = 0;
667 else
668 offset = offset_in_page(skb->data);
669 resp = make_rx_response(netif, id, status, offset,
670 skb_headlen(skb), flags);
672 if (meta[npo.meta_cons].frag.size) {
673 struct netif_extra_info *gso =
674 (struct netif_extra_info *)
675 RING_GET_RESPONSE(&netif->rx,
676 netif->rx.rsp_prod_pvt++);
678 resp->flags |= NETRXF_extra_info;
680 gso->u.gso.size = meta[npo.meta_cons].frag.size;
681 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
682 gso->u.gso.pad = 0;
683 gso->u.gso.features = 0;
685 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
686 gso->flags = 0;
687 }
689 netbk_add_frag_responses(netif, status,
690 meta + npo.meta_cons + 1,
691 nr_frags);
693 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret);
694 irq = netif->irq;
695 if (ret && !rx_notify[irq]) {
696 rx_notify[irq] = 1;
697 notify_list[notify_nr++] = irq;
698 }
700 if (netif_queue_stopped(netif->dev) &&
701 !netbk_queue_full(netif))
702 netif_wake_queue(netif->dev);
704 netif_put(netif);
705 dev_kfree_skb(skb);
706 npo.meta_cons += nr_frags + 1;
707 }
709 while (notify_nr != 0) {
710 irq = notify_list[--notify_nr];
711 rx_notify[irq] = 0;
712 notify_remote_via_irq(irq);
713 }
715 /* More work to do? */
716 if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer))
717 tasklet_schedule(&net_rx_tasklet);
718 #if 0
719 else
720 xen_network_done_notify();
721 #endif
722 }
724 static void net_alarm(unsigned long unused)
725 {
726 tasklet_schedule(&net_rx_tasklet);
727 }
729 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
730 {
731 netif_t *netif = netdev_priv(dev);
732 return &netif->stats;
733 }
735 static int __on_net_schedule_list(netif_t *netif)
736 {
737 return netif->list.next != NULL;
738 }
740 static void remove_from_net_schedule_list(netif_t *netif)
741 {
742 spin_lock_irq(&net_schedule_list_lock);
743 if (likely(__on_net_schedule_list(netif))) {
744 list_del(&netif->list);
745 netif->list.next = NULL;
746 netif_put(netif);
747 }
748 spin_unlock_irq(&net_schedule_list_lock);
749 }
751 static void add_to_net_schedule_list_tail(netif_t *netif)
752 {
753 if (__on_net_schedule_list(netif))
754 return;
756 spin_lock_irq(&net_schedule_list_lock);
757 if (!__on_net_schedule_list(netif) &&
758 likely(netif_running(netif->dev) &&
759 netif_carrier_ok(netif->dev))) {
760 list_add_tail(&netif->list, &net_schedule_list);
761 netif_get(netif);
762 }
763 spin_unlock_irq(&net_schedule_list_lock);
764 }
766 /*
767 * Note on CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER:
768 * If this driver is pipelining transmit requests then we can be very
769 * aggressive in avoiding new-packet notifications -- frontend only needs to
770 * send a notification if there are no outstanding unreceived responses.
771 * If we may be buffer transmit buffers for any reason then we must be rather
772 * more conservative and treat this as the final check for pending work.
773 */
774 void netif_schedule_work(netif_t *netif)
775 {
776 int more_to_do;
778 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
779 more_to_do = RING_HAS_UNCONSUMED_REQUESTS(&netif->tx);
780 #else
781 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
782 #endif
784 if (more_to_do) {
785 add_to_net_schedule_list_tail(netif);
786 maybe_schedule_tx_action();
787 }
788 }
790 void netif_deschedule_work(netif_t *netif)
791 {
792 remove_from_net_schedule_list(netif);
793 }
796 static void tx_credit_callback(unsigned long data)
797 {
798 netif_t *netif = (netif_t *)data;
799 netif->remaining_credit = netif->credit_bytes;
800 netif_schedule_work(netif);
801 }
803 inline static void net_tx_action_dealloc(void)
804 {
805 gnttab_unmap_grant_ref_t *gop;
806 u16 pending_idx;
807 PEND_RING_IDX dc, dp;
808 netif_t *netif;
809 int ret;
811 dc = dealloc_cons;
812 dp = dealloc_prod;
814 /* Ensure we see all indexes enqueued by netif_idx_release(). */
815 smp_rmb();
817 /*
818 * Free up any grants we have finished using
819 */
820 gop = tx_unmap_ops;
821 while (dc != dp) {
822 pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
823 gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
824 GNTMAP_host_map,
825 grant_tx_handle[pending_idx]);
826 gop++;
827 }
828 ret = HYPERVISOR_grant_table_op(
829 GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops);
830 BUG_ON(ret);
832 while (dealloc_cons != dp) {
833 pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
835 netif = pending_tx_info[pending_idx].netif;
837 make_tx_response(netif, &pending_tx_info[pending_idx].req,
838 NETIF_RSP_OKAY);
840 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
842 netif_put(netif);
843 }
844 }
846 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
847 {
848 RING_IDX cons = netif->tx.req_cons;
850 do {
851 make_tx_response(netif, txp, NETIF_RSP_ERROR);
852 if (cons >= end)
853 break;
854 txp = RING_GET_REQUEST(&netif->tx, cons++);
855 } while (1);
856 netif->tx.req_cons = cons;
857 netif_schedule_work(netif);
858 netif_put(netif);
859 }
861 static int netbk_count_requests(netif_t *netif, netif_tx_request_t *txp,
862 int work_to_do)
863 {
864 netif_tx_request_t *first = txp;
865 RING_IDX cons = netif->tx.req_cons;
866 int frags = 0;
868 while (txp->flags & NETTXF_more_data) {
869 if (frags >= work_to_do) {
870 DPRINTK("Need more frags\n");
871 return -frags;
872 }
874 txp = RING_GET_REQUEST(&netif->tx, cons + frags);
875 if (txp->size > first->size) {
876 DPRINTK("Frags galore\n");
877 return -frags;
878 }
880 first->size -= txp->size;
881 frags++;
883 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
884 DPRINTK("txp->offset: %x, size: %u\n",
885 txp->offset, txp->size);
886 return -frags;
887 }
888 }
890 return frags;
891 }
893 static gnttab_map_grant_ref_t *netbk_get_requests(netif_t *netif,
894 struct sk_buff *skb,
895 gnttab_map_grant_ref_t *mop)
896 {
897 struct skb_shared_info *shinfo = skb_shinfo(skb);
898 skb_frag_t *frags = shinfo->frags;
899 netif_tx_request_t *txp;
900 unsigned long pending_idx = *((u16 *)skb->data);
901 RING_IDX cons = netif->tx.req_cons;
902 int i, start;
904 /* Skip first skb fragment if it is on same page as header fragment. */
905 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
907 for (i = start; i < shinfo->nr_frags; i++) {
908 txp = RING_GET_REQUEST(&netif->tx, cons++);
909 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons++)];
911 gnttab_set_map_op(mop++, idx_to_kaddr(pending_idx),
912 GNTMAP_host_map | GNTMAP_readonly,
913 txp->gref, netif->domid);
915 memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp));
916 netif_get(netif);
917 pending_tx_info[pending_idx].netif = netif;
918 frags[i].page = (void *)pending_idx;
919 }
921 return mop;
922 }
924 static int netbk_tx_check_mop(struct sk_buff *skb,
925 gnttab_map_grant_ref_t **mopp)
926 {
927 gnttab_map_grant_ref_t *mop = *mopp;
928 int pending_idx = *((u16 *)skb->data);
929 netif_t *netif = pending_tx_info[pending_idx].netif;
930 netif_tx_request_t *txp;
931 struct skb_shared_info *shinfo = skb_shinfo(skb);
932 int nr_frags = shinfo->nr_frags;
933 int i, err, start;
935 /* Check status of header. */
936 err = mop->status;
937 if (unlikely(err)) {
938 txp = &pending_tx_info[pending_idx].req;
939 make_tx_response(netif, txp, NETIF_RSP_ERROR);
940 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
941 netif_put(netif);
942 } else {
943 set_phys_to_machine(
944 __pa(idx_to_kaddr(pending_idx)) >> PAGE_SHIFT,
945 FOREIGN_FRAME(mop->dev_bus_addr >> PAGE_SHIFT));
946 grant_tx_handle[pending_idx] = mop->handle;
947 }
949 /* Skip first skb fragment if it is on same page as header fragment. */
950 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
952 for (i = start; i < nr_frags; i++) {
953 int j, newerr;
955 pending_idx = (unsigned long)shinfo->frags[i].page;
957 /* Check error status: if okay then remember grant handle. */
958 newerr = (++mop)->status;
959 if (likely(!newerr)) {
960 set_phys_to_machine(
961 __pa(idx_to_kaddr(pending_idx))>>PAGE_SHIFT,
962 FOREIGN_FRAME(mop->dev_bus_addr>>PAGE_SHIFT));
963 grant_tx_handle[pending_idx] = mop->handle;
964 /* Had a previous error? Invalidate this fragment. */
965 if (unlikely(err))
966 netif_idx_release(pending_idx);
967 continue;
968 }
970 /* Error on this fragment: respond to client with an error. */
971 txp = &pending_tx_info[pending_idx].req;
972 make_tx_response(netif, txp, NETIF_RSP_ERROR);
973 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
974 netif_put(netif);
976 /* Not the first error? Preceding frags already invalidated. */
977 if (err)
978 continue;
980 /* First error: invalidate header and preceding fragments. */
981 pending_idx = *((u16 *)skb->data);
982 netif_idx_release(pending_idx);
983 for (j = start; j < i; j++) {
984 pending_idx = (unsigned long)shinfo->frags[i].page;
985 netif_idx_release(pending_idx);
986 }
988 /* Remember the error: invalidate all subsequent fragments. */
989 err = newerr;
990 }
992 *mopp = mop + 1;
993 return err;
994 }
996 static void netbk_fill_frags(struct sk_buff *skb)
997 {
998 struct skb_shared_info *shinfo = skb_shinfo(skb);
999 int nr_frags = shinfo->nr_frags;
1000 int i;
1002 for (i = 0; i < nr_frags; i++) {
1003 skb_frag_t *frag = shinfo->frags + i;
1004 netif_tx_request_t *txp;
1005 unsigned long pending_idx;
1007 pending_idx = (unsigned long)frag->page;
1008 txp = &pending_tx_info[pending_idx].req;
1009 frag->page = virt_to_page(idx_to_kaddr(pending_idx));
1010 frag->size = txp->size;
1011 frag->page_offset = txp->offset;
1013 skb->len += txp->size;
1014 skb->data_len += txp->size;
1015 skb->truesize += txp->size;
1019 int netbk_get_extras(netif_t *netif, struct netif_extra_info *extras,
1020 int work_to_do)
1022 struct netif_extra_info *extra;
1023 RING_IDX cons = netif->tx.req_cons;
1025 do {
1026 if (unlikely(work_to_do-- <= 0)) {
1027 DPRINTK("Missing extra info\n");
1028 return -EBADR;
1031 extra = (struct netif_extra_info *)
1032 RING_GET_REQUEST(&netif->tx, cons);
1033 if (unlikely(!extra->type ||
1034 extra->type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1035 netif->tx.req_cons = ++cons;
1036 DPRINTK("Invalid extra type: %d\n", extra->type);
1037 return -EINVAL;
1040 memcpy(&extras[extra->type - 1], extra, sizeof(*extra));
1041 netif->tx.req_cons = ++cons;
1042 } while (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE);
1044 return work_to_do;
1047 static int netbk_set_skb_gso(struct sk_buff *skb, struct netif_extra_info *gso)
1049 if (!gso->u.gso.size) {
1050 DPRINTK("GSO size must not be zero.\n");
1051 return -EINVAL;
1054 /* Currently only TCPv4 S.O. is supported. */
1055 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) {
1056 DPRINTK("Bad GSO type %d.\n", gso->u.gso.type);
1057 return -EINVAL;
1060 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1061 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1063 /* Header must be checked, and gso_segs computed. */
1064 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1065 skb_shinfo(skb)->gso_segs = 0;
1067 return 0;
1070 /* Called after netfront has transmitted */
1071 static void net_tx_action(unsigned long unused)
1073 struct list_head *ent;
1074 struct sk_buff *skb;
1075 netif_t *netif;
1076 netif_tx_request_t txreq;
1077 struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
1078 u16 pending_idx;
1079 RING_IDX i;
1080 gnttab_map_grant_ref_t *mop;
1081 unsigned int data_len;
1082 int ret, work_to_do;
1084 if (dealloc_cons != dealloc_prod)
1085 net_tx_action_dealloc();
1087 mop = tx_map_ops;
1088 while (((NR_PENDING_REQS + MAX_SKB_FRAGS) < MAX_PENDING_REQS) &&
1089 !list_empty(&net_schedule_list)) {
1090 /* Get a netif from the list with work to do. */
1091 ent = net_schedule_list.next;
1092 netif = list_entry(ent, netif_t, list);
1093 netif_get(netif);
1094 remove_from_net_schedule_list(netif);
1096 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do);
1097 if (!work_to_do) {
1098 netif_put(netif);
1099 continue;
1102 i = netif->tx.req_cons;
1103 rmb(); /* Ensure that we see the request before we copy it. */
1104 memcpy(&txreq, RING_GET_REQUEST(&netif->tx, i), sizeof(txreq));
1106 /* Credit-based scheduling. */
1107 if (txreq.size > netif->remaining_credit) {
1108 unsigned long now = jiffies;
1109 unsigned long next_credit =
1110 netif->credit_timeout.expires +
1111 msecs_to_jiffies(netif->credit_usec / 1000);
1113 /* Timer could already be pending in rare cases. */
1114 if (timer_pending(&netif->credit_timeout)) {
1115 netif_put(netif);
1116 continue;
1119 /* Passed the point where we can replenish credit? */
1120 if (time_after_eq(now, next_credit)) {
1121 netif->credit_timeout.expires = now;
1122 netif->remaining_credit = netif->credit_bytes;
1125 /* Still too big to send right now? Set a callback. */
1126 if (txreq.size > netif->remaining_credit) {
1127 netif->remaining_credit = 0;
1128 netif->credit_timeout.data =
1129 (unsigned long)netif;
1130 netif->credit_timeout.function =
1131 tx_credit_callback;
1132 __mod_timer(&netif->credit_timeout,
1133 next_credit);
1134 netif_put(netif);
1135 continue;
1138 netif->remaining_credit -= txreq.size;
1140 work_to_do--;
1141 netif->tx.req_cons = ++i;
1143 memset(extras, 0, sizeof(extras));
1144 if (txreq.flags & NETTXF_extra_info) {
1145 work_to_do = netbk_get_extras(netif, extras,
1146 work_to_do);
1147 i = netif->tx.req_cons;
1148 if (unlikely(work_to_do < 0)) {
1149 netbk_tx_err(netif, &txreq, i);
1150 continue;
1154 ret = netbk_count_requests(netif, &txreq, work_to_do);
1155 if (unlikely(ret < 0)) {
1156 netbk_tx_err(netif, &txreq, i - ret);
1157 continue;
1159 i += ret;
1161 if (unlikely(ret > MAX_SKB_FRAGS)) {
1162 DPRINTK("Too many frags\n");
1163 netbk_tx_err(netif, &txreq, i);
1164 continue;
1167 if (unlikely(txreq.size < ETH_HLEN)) {
1168 DPRINTK("Bad packet size: %d\n", txreq.size);
1169 netbk_tx_err(netif, &txreq, i);
1170 continue;
1173 /* No crossing a page as the payload mustn't fragment. */
1174 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1175 DPRINTK("txreq.offset: %x, size: %u, end: %lu\n",
1176 txreq.offset, txreq.size,
1177 (txreq.offset &~PAGE_MASK) + txreq.size);
1178 netbk_tx_err(netif, &txreq, i);
1179 continue;
1182 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
1184 data_len = (txreq.size > PKT_PROT_LEN &&
1185 ret < MAX_SKB_FRAGS) ?
1186 PKT_PROT_LEN : txreq.size;
1188 skb = alloc_skb(data_len+16, GFP_ATOMIC);
1189 if (unlikely(skb == NULL)) {
1190 DPRINTK("Can't allocate a skb in start_xmit.\n");
1191 netbk_tx_err(netif, &txreq, i);
1192 break;
1195 /* Packets passed to netif_rx() must have some headroom. */
1196 skb_reserve(skb, 16);
1198 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1199 struct netif_extra_info *gso;
1200 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1202 if (netbk_set_skb_gso(skb, gso)) {
1203 kfree_skb(skb);
1204 netbk_tx_err(netif, &txreq, i);
1205 continue;
1209 gnttab_set_map_op(mop, idx_to_kaddr(pending_idx),
1210 GNTMAP_host_map | GNTMAP_readonly,
1211 txreq.gref, netif->domid);
1212 mop++;
1214 memcpy(&pending_tx_info[pending_idx].req,
1215 &txreq, sizeof(txreq));
1216 pending_tx_info[pending_idx].netif = netif;
1217 *((u16 *)skb->data) = pending_idx;
1219 __skb_put(skb, data_len);
1221 skb_shinfo(skb)->nr_frags = ret;
1222 if (data_len < txreq.size) {
1223 skb_shinfo(skb)->nr_frags++;
1224 skb_shinfo(skb)->frags[0].page =
1225 (void *)(unsigned long)pending_idx;
1226 } else {
1227 /* Discriminate from any valid pending_idx value. */
1228 skb_shinfo(skb)->frags[0].page = (void *)~0UL;
1231 __skb_queue_tail(&tx_queue, skb);
1233 pending_cons++;
1235 mop = netbk_get_requests(netif, skb, mop);
1237 netif->tx.req_cons = i;
1238 netif_schedule_work(netif);
1240 if ((mop - tx_map_ops) >= ARRAY_SIZE(tx_map_ops))
1241 break;
1244 if (mop == tx_map_ops)
1245 return;
1247 ret = HYPERVISOR_grant_table_op(
1248 GNTTABOP_map_grant_ref, tx_map_ops, mop - tx_map_ops);
1249 BUG_ON(ret);
1251 mop = tx_map_ops;
1252 while ((skb = __skb_dequeue(&tx_queue)) != NULL) {
1253 netif_tx_request_t *txp;
1255 pending_idx = *((u16 *)skb->data);
1256 netif = pending_tx_info[pending_idx].netif;
1257 txp = &pending_tx_info[pending_idx].req;
1259 /* Check the remap error code. */
1260 if (unlikely(netbk_tx_check_mop(skb, &mop))) {
1261 printk(KERN_ALERT "#### netback grant fails\n");
1262 skb_shinfo(skb)->nr_frags = 0;
1263 kfree_skb(skb);
1264 continue;
1267 data_len = skb->len;
1268 memcpy(skb->data,
1269 (void *)(idx_to_kaddr(pending_idx)|txp->offset),
1270 data_len);
1271 if (data_len < txp->size) {
1272 /* Append the packet payload as a fragment. */
1273 txp->offset += data_len;
1274 txp->size -= data_len;
1275 } else {
1276 /* Schedule a response immediately. */
1277 netif_idx_release(pending_idx);
1280 /*
1281 * Old frontends do not assert data_validated but we
1282 * can infer it from csum_blank so test both flags.
1283 */
1284 if (txp->flags & (NETTXF_data_validated|NETTXF_csum_blank)) {
1285 skb->ip_summed = CHECKSUM_UNNECESSARY;
1286 skb->proto_data_valid = 1;
1287 } else {
1288 skb->ip_summed = CHECKSUM_NONE;
1289 skb->proto_data_valid = 0;
1291 skb->proto_csum_blank = !!(txp->flags & NETTXF_csum_blank);
1293 netbk_fill_frags(skb);
1295 skb->dev = netif->dev;
1296 skb->protocol = eth_type_trans(skb, skb->dev);
1298 netif->stats.rx_bytes += skb->len;
1299 netif->stats.rx_packets++;
1301 netif_rx(skb);
1302 netif->dev->last_rx = jiffies;
1306 static void netif_idx_release(u16 pending_idx)
1308 static DEFINE_SPINLOCK(_lock);
1309 unsigned long flags;
1311 spin_lock_irqsave(&_lock, flags);
1312 dealloc_ring[MASK_PEND_IDX(dealloc_prod)] = pending_idx;
1313 /* Sync with net_tx_action_dealloc: insert idx /then/ incr producer. */
1314 smp_wmb();
1315 dealloc_prod++;
1316 spin_unlock_irqrestore(&_lock, flags);
1318 tasklet_schedule(&net_tx_tasklet);
1321 static void netif_page_release(struct page *page)
1323 /* Ready for next use. */
1324 set_page_count(page, 1);
1326 netif_idx_release(page->index);
1329 irqreturn_t netif_be_int(int irq, void *dev_id, struct pt_regs *regs)
1331 netif_t *netif = dev_id;
1333 add_to_net_schedule_list_tail(netif);
1334 maybe_schedule_tx_action();
1336 if (netif_queue_stopped(netif->dev) && !netbk_queue_full(netif))
1337 netif_wake_queue(netif->dev);
1339 return IRQ_HANDLED;
1342 static void make_tx_response(netif_t *netif,
1343 netif_tx_request_t *txp,
1344 s8 st)
1346 RING_IDX i = netif->tx.rsp_prod_pvt;
1347 netif_tx_response_t *resp;
1348 int notify;
1350 resp = RING_GET_RESPONSE(&netif->tx, i);
1351 resp->id = txp->id;
1352 resp->status = st;
1354 if (txp->flags & NETTXF_extra_info)
1355 RING_GET_RESPONSE(&netif->tx, ++i)->status = NETIF_RSP_NULL;
1357 netif->tx.rsp_prod_pvt = ++i;
1358 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify);
1359 if (notify)
1360 notify_remote_via_irq(netif->irq);
1362 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
1363 if (i == netif->tx.req_cons) {
1364 int more_to_do;
1365 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
1366 if (more_to_do)
1367 add_to_net_schedule_list_tail(netif);
1369 #endif
1372 static netif_rx_response_t *make_rx_response(netif_t *netif,
1373 u16 id,
1374 s8 st,
1375 u16 offset,
1376 u16 size,
1377 u16 flags)
1379 RING_IDX i = netif->rx.rsp_prod_pvt;
1380 netif_rx_response_t *resp;
1382 resp = RING_GET_RESPONSE(&netif->rx, i);
1383 resp->offset = offset;
1384 resp->flags = flags;
1385 resp->id = id;
1386 resp->status = (s16)size;
1387 if (st < 0)
1388 resp->status = (s16)st;
1390 netif->rx.rsp_prod_pvt = ++i;
1392 return resp;
1395 #ifdef NETBE_DEBUG_INTERRUPT
1396 static irqreturn_t netif_be_dbg(int irq, void *dev_id, struct pt_regs *regs)
1398 struct list_head *ent;
1399 netif_t *netif;
1400 int i = 0;
1402 printk(KERN_ALERT "netif_schedule_list:\n");
1403 spin_lock_irq(&net_schedule_list_lock);
1405 list_for_each (ent, &net_schedule_list) {
1406 netif = list_entry(ent, netif_t, list);
1407 printk(KERN_ALERT " %d: private(rx_req_cons=%08x "
1408 "rx_resp_prod=%08x\n",
1409 i, netif->rx.req_cons, netif->rx.rsp_prod_pvt);
1410 printk(KERN_ALERT " tx_req_cons=%08x tx_resp_prod=%08x)\n",
1411 netif->tx.req_cons, netif->tx.rsp_prod_pvt);
1412 printk(KERN_ALERT " shared(rx_req_prod=%08x "
1413 "rx_resp_prod=%08x\n",
1414 netif->rx.sring->req_prod, netif->rx.sring->rsp_prod);
1415 printk(KERN_ALERT " rx_event=%08x tx_req_prod=%08x\n",
1416 netif->rx.sring->rsp_event, netif->tx.sring->req_prod);
1417 printk(KERN_ALERT " tx_resp_prod=%08x, tx_event=%08x)\n",
1418 netif->tx.sring->rsp_prod, netif->tx.sring->rsp_event);
1419 i++;
1422 spin_unlock_irq(&net_schedule_list_lock);
1423 printk(KERN_ALERT " ** End of netif_schedule_list **\n");
1425 return IRQ_HANDLED;
1427 #endif
1429 static int __init netback_init(void)
1431 int i;
1432 struct page *page;
1434 if (!is_running_on_xen())
1435 return -ENODEV;
1437 /* We can increase reservation by this much in net_rx_action(). */
1438 balloon_update_driver_allowance(NET_RX_RING_SIZE);
1440 skb_queue_head_init(&rx_queue);
1441 skb_queue_head_init(&tx_queue);
1443 init_timer(&net_timer);
1444 net_timer.data = 0;
1445 net_timer.function = net_alarm;
1447 mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS);
1448 if (mmap_pages == NULL) {
1449 printk("%s: out of memory\n", __FUNCTION__);
1450 return -ENOMEM;
1453 for (i = 0; i < MAX_PENDING_REQS; i++) {
1454 page = mmap_pages[i];
1455 SetPageForeign(page, netif_page_release);
1456 page->index = i;
1459 pending_cons = 0;
1460 pending_prod = MAX_PENDING_REQS;
1461 for (i = 0; i < MAX_PENDING_REQS; i++)
1462 pending_ring[i] = i;
1464 spin_lock_init(&net_schedule_list_lock);
1465 INIT_LIST_HEAD(&net_schedule_list);
1467 netif_xenbus_init();
1469 #ifdef NETBE_DEBUG_INTERRUPT
1470 (void)bind_virq_to_irqhandler(
1471 VIRQ_DEBUG,
1472 0,
1473 netif_be_dbg,
1474 SA_SHIRQ,
1475 "net-be-dbg",
1476 &netif_be_dbg);
1477 #endif
1479 return 0;
1482 module_init(netback_init);
1484 MODULE_LICENSE("Dual BSD/GPL");