ia64/xen-unstable

view linux-2.6-xen-sparse/drivers/xen/netback/netback.c @ 11796:dcee7eca71af

[NET] back: Copy tx_ring data before verification

We need to make a copy of data from tx_ring before
verifying them as otherwise what we end up using may
be different from what was verified.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
author kfraser@localhost.localdomain
date Thu Oct 12 16:09:15 2006 +0100 (2006-10-12)
parents bd2be8a8fc72
children 797430d25f1b
line source
1 /******************************************************************************
2 * drivers/xen/netback/netback.c
3 *
4 * Back-end of the driver for virtual network devices. This portion of the
5 * driver exports a 'unified' network-device interface that can be accessed
6 * by any operating system that implements a compatible front end. A
7 * reference front-end implementation can be found in:
8 * drivers/xen/netfront/netfront.c
9 *
10 * Copyright (c) 2002-2005, K A Fraser
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
37 #include "common.h"
38 #include <xen/balloon.h>
39 #include <xen/interface/memory.h>
41 /*#define NETBE_DEBUG_INTERRUPT*/
43 struct netbk_rx_meta {
44 skb_frag_t frag;
45 int id;
46 int copy:1;
47 };
49 static void netif_idx_release(u16 pending_idx);
50 static void netif_page_release(struct page *page);
51 static void make_tx_response(netif_t *netif,
52 netif_tx_request_t *txp,
53 s8 st);
54 static netif_rx_response_t *make_rx_response(netif_t *netif,
55 u16 id,
56 s8 st,
57 u16 offset,
58 u16 size,
59 u16 flags);
61 static void net_tx_action(unsigned long unused);
62 static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0);
64 static void net_rx_action(unsigned long unused);
65 static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
67 static struct timer_list net_timer;
69 #define MAX_PENDING_REQS 256
71 static struct sk_buff_head rx_queue;
73 static struct page **mmap_pages;
74 static inline unsigned long idx_to_kaddr(unsigned int idx)
75 {
76 return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx]));
77 }
79 #define PKT_PROT_LEN 64
81 static struct pending_tx_info {
82 netif_tx_request_t req;
83 netif_t *netif;
84 } pending_tx_info[MAX_PENDING_REQS];
85 static u16 pending_ring[MAX_PENDING_REQS];
86 typedef unsigned int PEND_RING_IDX;
87 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
88 static PEND_RING_IDX pending_prod, pending_cons;
89 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
91 /* Freed TX SKBs get batched on this ring before return to pending_ring. */
92 static u16 dealloc_ring[MAX_PENDING_REQS];
93 static PEND_RING_IDX dealloc_prod, dealloc_cons;
95 static struct sk_buff_head tx_queue;
97 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
98 static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS];
99 static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS];
101 static struct list_head net_schedule_list;
102 static spinlock_t net_schedule_list_lock;
104 #define MAX_MFN_ALLOC 64
105 static unsigned long mfn_list[MAX_MFN_ALLOC];
106 static unsigned int alloc_index = 0;
108 static inline unsigned long alloc_mfn(void)
109 {
110 return mfn_list[--alloc_index];
111 }
113 static int check_mfn(int nr)
114 {
115 struct xen_memory_reservation reservation = {
116 .extent_order = 0,
117 .domid = DOMID_SELF
118 };
120 if (likely(alloc_index >= nr))
121 return 0;
123 set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index);
124 reservation.nr_extents = MAX_MFN_ALLOC - alloc_index;
125 alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation,
126 &reservation);
128 return alloc_index >= nr ? 0 : -ENOMEM;
129 }
131 static inline void maybe_schedule_tx_action(void)
132 {
133 smp_mb();
134 if ((NR_PENDING_REQS < (MAX_PENDING_REQS/2)) &&
135 !list_empty(&net_schedule_list))
136 tasklet_schedule(&net_tx_tasklet);
137 }
139 /*
140 * A gross way of confirming the origin of an skb data page. The slab
141 * allocator abuses a field in the page struct to cache the kmem_cache_t ptr.
142 */
143 static inline int is_xen_skb(struct sk_buff *skb)
144 {
145 extern kmem_cache_t *skbuff_cachep;
146 kmem_cache_t *cp = (kmem_cache_t *)virt_to_page(skb->head)->lru.next;
147 return (cp == skbuff_cachep);
148 }
150 /*
151 * We can flip without copying the packet unless:
152 * 1. The data is not allocated from our special cache; or
153 * 2. The main data area is shared; or
154 * 3. One or more fragments are shared; or
155 * 4. There are chained fragments.
156 */
157 static inline int is_flippable_skb(struct sk_buff *skb)
158 {
159 int frag;
161 if (!is_xen_skb(skb) || skb_cloned(skb))
162 return 0;
164 for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
165 if (page_count(skb_shinfo(skb)->frags[frag].page) > 1)
166 return 0;
167 }
169 if (skb_shinfo(skb)->frag_list != NULL)
170 return 0;
172 return 1;
173 }
175 static struct sk_buff *netbk_copy_skb(struct sk_buff *skb)
176 {
177 struct skb_shared_info *ninfo;
178 struct sk_buff *nskb;
179 unsigned long offset;
180 int ret;
181 int len;
182 int headlen;
184 BUG_ON(skb_shinfo(skb)->frag_list != NULL);
186 nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC);
187 if (unlikely(!nskb))
188 goto err;
190 skb_reserve(nskb, 16);
191 headlen = nskb->end - nskb->data;
192 if (headlen > skb_headlen(skb))
193 headlen = skb_headlen(skb);
194 ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen);
195 BUG_ON(ret);
197 ninfo = skb_shinfo(nskb);
198 ninfo->gso_size = skb_shinfo(skb)->gso_size;
199 ninfo->gso_type = skb_shinfo(skb)->gso_type;
201 offset = headlen;
202 len = skb->len - headlen;
204 nskb->len = skb->len;
205 nskb->data_len = len;
206 nskb->truesize += len;
208 while (len) {
209 struct page *page;
210 int copy;
211 int zero;
213 if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) {
214 dump_stack();
215 goto err_free;
216 }
218 copy = len >= PAGE_SIZE ? PAGE_SIZE : len;
219 zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO;
221 page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | zero);
222 if (unlikely(!page))
223 goto err_free;
225 ret = skb_copy_bits(skb, offset, page_address(page), copy);
226 BUG_ON(ret);
228 ninfo->frags[ninfo->nr_frags].page = page;
229 ninfo->frags[ninfo->nr_frags].page_offset = 0;
230 ninfo->frags[ninfo->nr_frags].size = copy;
231 ninfo->nr_frags++;
233 offset += copy;
234 len -= copy;
235 }
237 offset = nskb->data - skb->data;
239 nskb->h.raw = skb->h.raw + offset;
240 nskb->nh.raw = skb->nh.raw + offset;
241 nskb->mac.raw = skb->mac.raw + offset;
243 return nskb;
245 err_free:
246 kfree_skb(nskb);
247 err:
248 return NULL;
249 }
251 static inline int netbk_max_required_rx_slots(netif_t *netif)
252 {
253 if (netif->features & (NETIF_F_SG|NETIF_F_TSO))
254 return MAX_SKB_FRAGS + 2; /* header + extra_info + frags */
255 return 1; /* all in one */
256 }
258 static inline int netbk_queue_full(netif_t *netif)
259 {
260 RING_IDX peek = netif->rx_req_cons_peek;
261 RING_IDX needed = netbk_max_required_rx_slots(netif);
263 return ((netif->rx.sring->req_prod - peek) < needed) ||
264 ((netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek) < needed);
265 }
267 int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
268 {
269 netif_t *netif = netdev_priv(dev);
271 BUG_ON(skb->dev != dev);
273 /* Drop the packet if the target domain has no receive buffers. */
274 if (unlikely(!netif_running(dev) || !netif_carrier_ok(dev)))
275 goto drop;
277 if (unlikely(netbk_queue_full(netif))) {
278 /* Not a BUG_ON() -- misbehaving netfront can trigger this. */
279 if (netbk_can_queue(dev))
280 DPRINTK("Queue full but not stopped!\n");
281 goto drop;
282 }
284 /* Copy the packet here if it's destined for a flipping
285 interface but isn't flippable (e.g. extra references to
286 data)
287 */
288 if (!netif->copying_receiver && !is_flippable_skb(skb)) {
289 struct sk_buff *nskb = netbk_copy_skb(skb);
290 if ( unlikely(nskb == NULL) )
291 goto drop;
292 /* Copy only the header fields we use in this driver. */
293 nskb->dev = skb->dev;
294 nskb->ip_summed = skb->ip_summed;
295 nskb->proto_data_valid = skb->proto_data_valid;
296 dev_kfree_skb(skb);
297 skb = nskb;
298 }
300 netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1 +
301 !!skb_shinfo(skb)->gso_size;
302 netif_get(netif);
304 if (netbk_can_queue(dev) && netbk_queue_full(netif)) {
305 netif->rx.sring->req_event = netif->rx_req_cons_peek +
306 netbk_max_required_rx_slots(netif);
307 mb(); /* request notification /then/ check & stop the queue */
308 if (netbk_queue_full(netif))
309 netif_stop_queue(dev);
310 }
312 skb_queue_tail(&rx_queue, skb);
313 tasklet_schedule(&net_rx_tasklet);
315 return 0;
317 drop:
318 netif->stats.tx_dropped++;
319 dev_kfree_skb(skb);
320 return 0;
321 }
323 #if 0
324 static void xen_network_done_notify(void)
325 {
326 static struct net_device *eth0_dev = NULL;
327 if (unlikely(eth0_dev == NULL))
328 eth0_dev = __dev_get_by_name("eth0");
329 netif_rx_schedule(eth0_dev);
330 }
331 /*
332 * Add following to poll() function in NAPI driver (Tigon3 is example):
333 * if ( xen_network_done() )
334 * tg3_enable_ints(tp);
335 */
336 int xen_network_done(void)
337 {
338 return skb_queue_empty(&rx_queue);
339 }
340 #endif
342 struct netrx_pending_operations {
343 unsigned trans_prod, trans_cons;
344 unsigned mmu_prod, mmu_cons;
345 unsigned mcl_prod, mcl_cons;
346 unsigned copy_prod, copy_cons;
347 unsigned meta_prod, meta_cons;
348 mmu_update_t *mmu;
349 gnttab_transfer_t *trans;
350 gnttab_copy_t *copy;
351 multicall_entry_t *mcl;
352 struct netbk_rx_meta *meta;
353 };
355 /* Set up the grant operations for this fragment. If it's a flipping
356 interface, we also set up the unmap request from here. */
357 static u16 netbk_gop_frag(netif_t *netif, struct netbk_rx_meta *meta,
358 int i, struct netrx_pending_operations *npo,
359 struct page *page, unsigned long size,
360 unsigned long offset)
361 {
362 mmu_update_t *mmu;
363 gnttab_transfer_t *gop;
364 gnttab_copy_t *copy_gop;
365 multicall_entry_t *mcl;
366 netif_rx_request_t *req;
367 unsigned long old_mfn, new_mfn;
369 old_mfn = virt_to_mfn(page_address(page));
371 req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i);
372 if (netif->copying_receiver) {
373 /* The fragment needs to be copied rather than
374 flipped. */
375 meta->copy = 1;
376 copy_gop = npo->copy + npo->copy_prod++;
377 copy_gop->flags = GNTCOPY_dest_gref;
378 if (PageForeign(page)) {
379 struct pending_tx_info *src_pend =
380 &pending_tx_info[page->index];
381 copy_gop->source.domid = src_pend->netif->domid;
382 copy_gop->source.u.ref = src_pend->req.gref;
383 copy_gop->flags |= GNTCOPY_source_gref;
384 } else {
385 copy_gop->source.domid = DOMID_SELF;
386 copy_gop->source.u.gmfn = old_mfn;
387 }
388 copy_gop->source.offset = offset;
389 copy_gop->dest.domid = netif->domid;
390 copy_gop->dest.offset = 0;
391 copy_gop->dest.u.ref = req->gref;
392 copy_gop->len = size;
393 } else {
394 meta->copy = 0;
395 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
396 new_mfn = alloc_mfn();
398 /*
399 * Set the new P2M table entry before
400 * reassigning the old data page. Heed the
401 * comment in pgtable-2level.h:pte_page(). :-)
402 */
403 set_phys_to_machine(page_to_pfn(page), new_mfn);
405 mcl = npo->mcl + npo->mcl_prod++;
406 MULTI_update_va_mapping(mcl,
407 (unsigned long)page_address(page),
408 pfn_pte_ma(new_mfn, PAGE_KERNEL),
409 0);
411 mmu = npo->mmu + npo->mmu_prod++;
412 mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) |
413 MMU_MACHPHYS_UPDATE;
414 mmu->val = page_to_pfn(page);
415 }
417 gop = npo->trans + npo->trans_prod++;
418 gop->mfn = old_mfn;
419 gop->domid = netif->domid;
420 gop->ref = req->gref;
421 }
422 return req->id;
423 }
425 static void netbk_gop_skb(struct sk_buff *skb,
426 struct netrx_pending_operations *npo)
427 {
428 netif_t *netif = netdev_priv(skb->dev);
429 int nr_frags = skb_shinfo(skb)->nr_frags;
430 int i;
431 int extra;
432 struct netbk_rx_meta *head_meta, *meta;
434 head_meta = npo->meta + npo->meta_prod++;
435 head_meta->frag.page_offset = skb_shinfo(skb)->gso_type;
436 head_meta->frag.size = skb_shinfo(skb)->gso_size;
437 extra = !!head_meta->frag.size + 1;
439 for (i = 0; i < nr_frags; i++) {
440 meta = npo->meta + npo->meta_prod++;
441 meta->frag = skb_shinfo(skb)->frags[i];
442 meta->id = netbk_gop_frag(netif, meta, i + extra, npo,
443 meta->frag.page,
444 meta->frag.size,
445 meta->frag.page_offset);
446 }
448 /*
449 * This must occur at the end to ensure that we don't trash
450 * skb_shinfo until we're done.
451 */
452 head_meta->id = netbk_gop_frag(netif, head_meta, 0, npo,
453 virt_to_page(skb->data),
454 skb_headlen(skb),
455 offset_in_page(skb->data));
457 netif->rx.req_cons += nr_frags + extra;
458 }
460 static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta)
461 {
462 int i;
464 for (i = 0; i < nr_frags; i++)
465 put_page(meta[i].frag.page);
466 }
468 /* This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
469 used to set up the operations on the top of
470 netrx_pending_operations, which have since been done. Check that
471 they didn't give any errors and advance over them. */
472 static int netbk_check_gop(int nr_frags, domid_t domid,
473 struct netrx_pending_operations *npo)
474 {
475 multicall_entry_t *mcl;
476 gnttab_transfer_t *gop;
477 gnttab_copy_t *copy_op;
478 int status = NETIF_RSP_OKAY;
479 int i;
481 for (i = 0; i <= nr_frags; i++) {
482 if (npo->meta[npo->meta_cons + i].copy) {
483 copy_op = npo->copy + npo->copy_cons++;
484 if (copy_op->status != GNTST_okay) {
485 DPRINTK("Bad status %d from copy to DOM%d.\n",
486 gop->status, domid);
487 status = NETIF_RSP_ERROR;
488 }
489 } else {
490 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
491 mcl = npo->mcl + npo->mcl_cons++;
492 /* The update_va_mapping() must not fail. */
493 BUG_ON(mcl->result != 0);
494 }
496 gop = npo->trans + npo->trans_cons++;
497 /* Check the reassignment error code. */
498 if (gop->status != 0) {
499 DPRINTK("Bad status %d from grant transfer to DOM%u\n",
500 gop->status, domid);
501 /*
502 * Page no longer belongs to us unless
503 * GNTST_bad_page, but that should be
504 * a fatal error anyway.
505 */
506 BUG_ON(gop->status == GNTST_bad_page);
507 status = NETIF_RSP_ERROR;
508 }
509 }
510 }
512 return status;
513 }
515 static void netbk_add_frag_responses(netif_t *netif, int status,
516 struct netbk_rx_meta *meta, int nr_frags)
517 {
518 int i;
519 unsigned long offset;
521 for (i = 0; i < nr_frags; i++) {
522 int id = meta[i].id;
523 int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data;
525 if (meta[i].copy)
526 offset = 0;
527 else
528 offset = meta[i].frag.page_offset;
529 make_rx_response(netif, id, status, offset,
530 meta[i].frag.size, flags);
531 }
532 }
534 static void net_rx_action(unsigned long unused)
535 {
536 netif_t *netif = NULL;
537 s8 status;
538 u16 id, irq, flags;
539 netif_rx_response_t *resp;
540 multicall_entry_t *mcl;
541 struct sk_buff_head rxq;
542 struct sk_buff *skb;
543 int notify_nr = 0;
544 int ret;
545 int nr_frags;
546 int count;
547 unsigned long offset;
549 /*
550 * Putting hundreds of bytes on the stack is considered rude.
551 * Static works because a tasklet can only be on one CPU at any time.
552 */
553 static multicall_entry_t rx_mcl[NET_RX_RING_SIZE+3];
554 static mmu_update_t rx_mmu[NET_RX_RING_SIZE];
555 static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE];
556 static gnttab_copy_t grant_copy_op[NET_RX_RING_SIZE];
557 static unsigned char rx_notify[NR_IRQS];
558 static u16 notify_list[NET_RX_RING_SIZE];
559 static struct netbk_rx_meta meta[NET_RX_RING_SIZE];
561 struct netrx_pending_operations npo = {
562 mmu: rx_mmu,
563 trans: grant_trans_op,
564 copy: grant_copy_op,
565 mcl: rx_mcl,
566 meta: meta};
568 skb_queue_head_init(&rxq);
570 count = 0;
572 while ((skb = skb_dequeue(&rx_queue)) != NULL) {
573 nr_frags = skb_shinfo(skb)->nr_frags;
574 *(int *)skb->cb = nr_frags;
576 if (!xen_feature(XENFEAT_auto_translated_physmap) &&
577 check_mfn(nr_frags + 1)) {
578 /* Memory squeeze? Back off for an arbitrary while. */
579 if ( net_ratelimit() )
580 WPRINTK("Memory squeeze in netback "
581 "driver.\n");
582 mod_timer(&net_timer, jiffies + HZ);
583 skb_queue_head(&rx_queue, skb);
584 break;
585 }
587 netbk_gop_skb(skb, &npo);
589 count += nr_frags + 1;
591 __skb_queue_tail(&rxq, skb);
593 /* Filled the batch queue? */
594 if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE)
595 break;
596 }
598 if (npo.mcl_prod &&
599 !xen_feature(XENFEAT_auto_translated_physmap)) {
600 mcl = npo.mcl + npo.mcl_prod++;
602 BUG_ON(mcl[-1].op != __HYPERVISOR_update_va_mapping);
603 mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL;
605 mcl->op = __HYPERVISOR_mmu_update;
606 mcl->args[0] = (unsigned long)rx_mmu;
607 mcl->args[1] = npo.mmu_prod;
608 mcl->args[2] = 0;
609 mcl->args[3] = DOMID_SELF;
610 }
612 if (npo.trans_prod) {
613 mcl = npo.mcl + npo.mcl_prod++;
614 mcl->op = __HYPERVISOR_grant_table_op;
615 mcl->args[0] = GNTTABOP_transfer;
616 mcl->args[1] = (unsigned long)grant_trans_op;
617 mcl->args[2] = npo.trans_prod;
618 }
620 if (npo.copy_prod) {
621 mcl = npo.mcl + npo.mcl_prod++;
622 mcl->op = __HYPERVISOR_grant_table_op;
623 mcl->args[0] = GNTTABOP_copy;
624 mcl->args[1] = (unsigned long)grant_copy_op;
625 mcl->args[2] = npo.copy_prod;
626 }
628 /* Nothing to do? */
629 if (!npo.mcl_prod)
630 return;
632 BUG_ON(npo.copy_prod > NET_RX_RING_SIZE);
633 BUG_ON(npo.mmu_prod > NET_RX_RING_SIZE);
634 BUG_ON(npo.trans_prod > NET_RX_RING_SIZE);
635 BUG_ON(npo.mcl_prod > NET_RX_RING_SIZE+3);
636 BUG_ON(npo.meta_prod > NET_RX_RING_SIZE);
638 ret = HYPERVISOR_multicall(npo.mcl, npo.mcl_prod);
639 BUG_ON(ret != 0);
641 while ((skb = __skb_dequeue(&rxq)) != NULL) {
642 nr_frags = *(int *)skb->cb;
644 netif = netdev_priv(skb->dev);
645 /* We can't rely on skb_release_data to release the
646 pages used by fragments for us, since it tries to
647 touch the pages in the fraglist. If we're in
648 flipping mode, that doesn't work. In copying mode,
649 we still have access to all of the pages, and so
650 it's safe to let release_data deal with it. */
651 /* (Freeing the fragments is safe since we copy
652 non-linear skbs destined for flipping interfaces) */
653 if (!netif->copying_receiver) {
654 atomic_set(&(skb_shinfo(skb)->dataref), 1);
655 skb_shinfo(skb)->frag_list = NULL;
656 skb_shinfo(skb)->nr_frags = 0;
657 netbk_free_pages(nr_frags, meta + npo.meta_cons + 1);
658 }
660 netif->stats.tx_bytes += skb->len;
661 netif->stats.tx_packets++;
663 status = netbk_check_gop(nr_frags, netif->domid, &npo);
665 id = meta[npo.meta_cons].id;
666 flags = nr_frags ? NETRXF_more_data : 0;
668 if (skb->ip_summed == CHECKSUM_HW) /* local packet? */
669 flags |= NETRXF_csum_blank | NETRXF_data_validated;
670 else if (skb->proto_data_valid) /* remote but checksummed? */
671 flags |= NETRXF_data_validated;
673 if (meta[npo.meta_cons].copy)
674 offset = 0;
675 else
676 offset = offset_in_page(skb->data);
677 resp = make_rx_response(netif, id, status, offset,
678 skb_headlen(skb), flags);
680 if (meta[npo.meta_cons].frag.size) {
681 struct netif_extra_info *gso =
682 (struct netif_extra_info *)
683 RING_GET_RESPONSE(&netif->rx,
684 netif->rx.rsp_prod_pvt++);
686 resp->flags |= NETRXF_extra_info;
688 gso->u.gso.size = meta[npo.meta_cons].frag.size;
689 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
690 gso->u.gso.pad = 0;
691 gso->u.gso.features = 0;
693 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
694 gso->flags = 0;
695 }
697 netbk_add_frag_responses(netif, status,
698 meta + npo.meta_cons + 1,
699 nr_frags);
701 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret);
702 irq = netif->irq;
703 if (ret && !rx_notify[irq]) {
704 rx_notify[irq] = 1;
705 notify_list[notify_nr++] = irq;
706 }
708 if (netif_queue_stopped(netif->dev) &&
709 !netbk_queue_full(netif))
710 netif_wake_queue(netif->dev);
712 netif_put(netif);
713 dev_kfree_skb(skb);
714 npo.meta_cons += nr_frags + 1;
715 }
717 while (notify_nr != 0) {
718 irq = notify_list[--notify_nr];
719 rx_notify[irq] = 0;
720 notify_remote_via_irq(irq);
721 }
723 /* More work to do? */
724 if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer))
725 tasklet_schedule(&net_rx_tasklet);
726 #if 0
727 else
728 xen_network_done_notify();
729 #endif
730 }
732 static void net_alarm(unsigned long unused)
733 {
734 tasklet_schedule(&net_rx_tasklet);
735 }
737 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
738 {
739 netif_t *netif = netdev_priv(dev);
740 return &netif->stats;
741 }
743 static int __on_net_schedule_list(netif_t *netif)
744 {
745 return netif->list.next != NULL;
746 }
748 static void remove_from_net_schedule_list(netif_t *netif)
749 {
750 spin_lock_irq(&net_schedule_list_lock);
751 if (likely(__on_net_schedule_list(netif))) {
752 list_del(&netif->list);
753 netif->list.next = NULL;
754 netif_put(netif);
755 }
756 spin_unlock_irq(&net_schedule_list_lock);
757 }
759 static void add_to_net_schedule_list_tail(netif_t *netif)
760 {
761 if (__on_net_schedule_list(netif))
762 return;
764 spin_lock_irq(&net_schedule_list_lock);
765 if (!__on_net_schedule_list(netif) &&
766 likely(netif_running(netif->dev) &&
767 netif_carrier_ok(netif->dev))) {
768 list_add_tail(&netif->list, &net_schedule_list);
769 netif_get(netif);
770 }
771 spin_unlock_irq(&net_schedule_list_lock);
772 }
774 /*
775 * Note on CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER:
776 * If this driver is pipelining transmit requests then we can be very
777 * aggressive in avoiding new-packet notifications -- frontend only needs to
778 * send a notification if there are no outstanding unreceived responses.
779 * If we may be buffer transmit buffers for any reason then we must be rather
780 * more conservative and treat this as the final check for pending work.
781 */
782 void netif_schedule_work(netif_t *netif)
783 {
784 int more_to_do;
786 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
787 more_to_do = RING_HAS_UNCONSUMED_REQUESTS(&netif->tx);
788 #else
789 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
790 #endif
792 if (more_to_do) {
793 add_to_net_schedule_list_tail(netif);
794 maybe_schedule_tx_action();
795 }
796 }
798 void netif_deschedule_work(netif_t *netif)
799 {
800 remove_from_net_schedule_list(netif);
801 }
804 static void tx_add_credit(netif_t *netif)
805 {
806 unsigned long max_burst;
808 /*
809 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
810 * Otherwise the interface can seize up due to insufficient credit.
811 */
812 max_burst = RING_GET_REQUEST(&netif->tx, netif->tx.req_cons)->size;
813 max_burst = min(max_burst, 131072UL);
814 max_burst = max(max_burst, netif->credit_bytes);
816 netif->remaining_credit = min(netif->remaining_credit +
817 netif->credit_bytes,
818 max_burst);
819 }
821 static void tx_credit_callback(unsigned long data)
822 {
823 netif_t *netif = (netif_t *)data;
824 tx_add_credit(netif);
825 netif_schedule_work(netif);
826 }
828 inline static void net_tx_action_dealloc(void)
829 {
830 gnttab_unmap_grant_ref_t *gop;
831 u16 pending_idx;
832 PEND_RING_IDX dc, dp;
833 netif_t *netif;
834 int ret;
836 dc = dealloc_cons;
837 dp = dealloc_prod;
839 /* Ensure we see all indexes enqueued by netif_idx_release(). */
840 smp_rmb();
842 /*
843 * Free up any grants we have finished using
844 */
845 gop = tx_unmap_ops;
846 while (dc != dp) {
847 pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
848 gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
849 GNTMAP_host_map,
850 grant_tx_handle[pending_idx]);
851 gop++;
852 }
853 ret = HYPERVISOR_grant_table_op(
854 GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops);
855 BUG_ON(ret);
857 while (dealloc_cons != dp) {
858 pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
860 netif = pending_tx_info[pending_idx].netif;
862 make_tx_response(netif, &pending_tx_info[pending_idx].req,
863 NETIF_RSP_OKAY);
865 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
867 netif_put(netif);
868 }
869 }
871 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
872 {
873 RING_IDX cons = netif->tx.req_cons;
875 do {
876 make_tx_response(netif, txp, NETIF_RSP_ERROR);
877 if (cons >= end)
878 break;
879 txp = RING_GET_REQUEST(&netif->tx, cons++);
880 } while (1);
881 netif->tx.req_cons = cons;
882 netif_schedule_work(netif);
883 netif_put(netif);
884 }
886 static int netbk_count_requests(netif_t *netif, netif_tx_request_t *first,
887 netif_tx_request_t *txp, int work_to_do)
888 {
889 RING_IDX cons = netif->tx.req_cons;
890 int frags = 0;
892 if (!(first->flags & NETTXF_more_data))
893 return 0;
895 do {
896 if (frags >= work_to_do) {
897 DPRINTK("Need more frags\n");
898 return -frags;
899 }
901 if (unlikely(frags >= MAX_SKB_FRAGS)) {
902 DPRINTK("Too many frags\n");
903 return -frags;
904 }
906 memcpy(txp, RING_GET_REQUEST(&netif->tx, cons + frags),
907 sizeof(*txp));
908 if (txp->size > first->size) {
909 DPRINTK("Frags galore\n");
910 return -frags;
911 }
913 first->size -= txp->size;
914 frags++;
916 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
917 DPRINTK("txp->offset: %x, size: %u\n",
918 txp->offset, txp->size);
919 return -frags;
920 }
921 } while ((txp++)->flags & NETTXF_more_data);
923 return frags;
924 }
926 static gnttab_map_grant_ref_t *netbk_get_requests(netif_t *netif,
927 struct sk_buff *skb,
928 netif_tx_request_t *txp,
929 gnttab_map_grant_ref_t *mop)
930 {
931 struct skb_shared_info *shinfo = skb_shinfo(skb);
932 skb_frag_t *frags = shinfo->frags;
933 unsigned long pending_idx = *((u16 *)skb->data);
934 int i, start;
936 /* Skip first skb fragment if it is on same page as header fragment. */
937 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
939 for (i = start; i < shinfo->nr_frags; i++, txp++) {
940 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons++)];
942 gnttab_set_map_op(mop++, idx_to_kaddr(pending_idx),
943 GNTMAP_host_map | GNTMAP_readonly,
944 txp->gref, netif->domid);
946 memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp));
947 netif_get(netif);
948 pending_tx_info[pending_idx].netif = netif;
949 frags[i].page = (void *)pending_idx;
950 }
952 return mop;
953 }
955 static int netbk_tx_check_mop(struct sk_buff *skb,
956 gnttab_map_grant_ref_t **mopp)
957 {
958 gnttab_map_grant_ref_t *mop = *mopp;
959 int pending_idx = *((u16 *)skb->data);
960 netif_t *netif = pending_tx_info[pending_idx].netif;
961 netif_tx_request_t *txp;
962 struct skb_shared_info *shinfo = skb_shinfo(skb);
963 int nr_frags = shinfo->nr_frags;
964 int i, err, start;
966 /* Check status of header. */
967 err = mop->status;
968 if (unlikely(err)) {
969 txp = &pending_tx_info[pending_idx].req;
970 make_tx_response(netif, txp, NETIF_RSP_ERROR);
971 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
972 netif_put(netif);
973 } else {
974 set_phys_to_machine(
975 __pa(idx_to_kaddr(pending_idx)) >> PAGE_SHIFT,
976 FOREIGN_FRAME(mop->dev_bus_addr >> PAGE_SHIFT));
977 grant_tx_handle[pending_idx] = mop->handle;
978 }
980 /* Skip first skb fragment if it is on same page as header fragment. */
981 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
983 for (i = start; i < nr_frags; i++) {
984 int j, newerr;
986 pending_idx = (unsigned long)shinfo->frags[i].page;
988 /* Check error status: if okay then remember grant handle. */
989 newerr = (++mop)->status;
990 if (likely(!newerr)) {
991 set_phys_to_machine(
992 __pa(idx_to_kaddr(pending_idx))>>PAGE_SHIFT,
993 FOREIGN_FRAME(mop->dev_bus_addr>>PAGE_SHIFT));
994 grant_tx_handle[pending_idx] = mop->handle;
995 /* Had a previous error? Invalidate this fragment. */
996 if (unlikely(err))
997 netif_idx_release(pending_idx);
998 continue;
999 }
1001 /* Error on this fragment: respond to client with an error. */
1002 txp = &pending_tx_info[pending_idx].req;
1003 make_tx_response(netif, txp, NETIF_RSP_ERROR);
1004 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
1005 netif_put(netif);
1007 /* Not the first error? Preceding frags already invalidated. */
1008 if (err)
1009 continue;
1011 /* First error: invalidate header and preceding fragments. */
1012 pending_idx = *((u16 *)skb->data);
1013 netif_idx_release(pending_idx);
1014 for (j = start; j < i; j++) {
1015 pending_idx = (unsigned long)shinfo->frags[i].page;
1016 netif_idx_release(pending_idx);
1019 /* Remember the error: invalidate all subsequent fragments. */
1020 err = newerr;
1023 *mopp = mop + 1;
1024 return err;
1027 static void netbk_fill_frags(struct sk_buff *skb)
1029 struct skb_shared_info *shinfo = skb_shinfo(skb);
1030 int nr_frags = shinfo->nr_frags;
1031 int i;
1033 for (i = 0; i < nr_frags; i++) {
1034 skb_frag_t *frag = shinfo->frags + i;
1035 netif_tx_request_t *txp;
1036 unsigned long pending_idx;
1038 pending_idx = (unsigned long)frag->page;
1039 txp = &pending_tx_info[pending_idx].req;
1040 frag->page = virt_to_page(idx_to_kaddr(pending_idx));
1041 frag->size = txp->size;
1042 frag->page_offset = txp->offset;
1044 skb->len += txp->size;
1045 skb->data_len += txp->size;
1046 skb->truesize += txp->size;
1050 int netbk_get_extras(netif_t *netif, struct netif_extra_info *extras,
1051 int work_to_do)
1053 struct netif_extra_info extra;
1054 RING_IDX cons = netif->tx.req_cons;
1056 do {
1057 if (unlikely(work_to_do-- <= 0)) {
1058 DPRINTK("Missing extra info\n");
1059 return -EBADR;
1062 memcpy(&extra, RING_GET_REQUEST(&netif->tx, cons),
1063 sizeof(extra));
1064 if (unlikely(!extra.type ||
1065 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1066 netif->tx.req_cons = ++cons;
1067 DPRINTK("Invalid extra type: %d\n", extra.type);
1068 return -EINVAL;
1071 memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
1072 netif->tx.req_cons = ++cons;
1073 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
1075 return work_to_do;
1078 static int netbk_set_skb_gso(struct sk_buff *skb, struct netif_extra_info *gso)
1080 if (!gso->u.gso.size) {
1081 DPRINTK("GSO size must not be zero.\n");
1082 return -EINVAL;
1085 /* Currently only TCPv4 S.O. is supported. */
1086 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) {
1087 DPRINTK("Bad GSO type %d.\n", gso->u.gso.type);
1088 return -EINVAL;
1091 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1092 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1094 /* Header must be checked, and gso_segs computed. */
1095 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1096 skb_shinfo(skb)->gso_segs = 0;
1098 return 0;
1101 /* Called after netfront has transmitted */
1102 static void net_tx_action(unsigned long unused)
1104 struct list_head *ent;
1105 struct sk_buff *skb;
1106 netif_t *netif;
1107 netif_tx_request_t txreq;
1108 netif_tx_request_t txfrags[MAX_SKB_FRAGS];
1109 struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
1110 u16 pending_idx;
1111 RING_IDX i;
1112 gnttab_map_grant_ref_t *mop;
1113 unsigned int data_len;
1114 int ret, work_to_do;
1116 if (dealloc_cons != dealloc_prod)
1117 net_tx_action_dealloc();
1119 mop = tx_map_ops;
1120 while (((NR_PENDING_REQS + MAX_SKB_FRAGS) < MAX_PENDING_REQS) &&
1121 !list_empty(&net_schedule_list)) {
1122 /* Get a netif from the list with work to do. */
1123 ent = net_schedule_list.next;
1124 netif = list_entry(ent, netif_t, list);
1125 netif_get(netif);
1126 remove_from_net_schedule_list(netif);
1128 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do);
1129 if (!work_to_do) {
1130 netif_put(netif);
1131 continue;
1134 i = netif->tx.req_cons;
1135 rmb(); /* Ensure that we see the request before we copy it. */
1136 memcpy(&txreq, RING_GET_REQUEST(&netif->tx, i), sizeof(txreq));
1138 /* Credit-based scheduling. */
1139 if (txreq.size > netif->remaining_credit) {
1140 unsigned long now = jiffies;
1141 unsigned long next_credit =
1142 netif->credit_timeout.expires +
1143 msecs_to_jiffies(netif->credit_usec / 1000);
1145 /* Timer could already be pending in rare cases. */
1146 if (timer_pending(&netif->credit_timeout)) {
1147 netif_put(netif);
1148 continue;
1151 /* Passed the point where we can replenish credit? */
1152 if (time_after_eq(now, next_credit)) {
1153 netif->credit_timeout.expires = now;
1154 tx_add_credit(netif);
1157 /* Still too big to send right now? Set a callback. */
1158 if (txreq.size > netif->remaining_credit) {
1159 netif->credit_timeout.data =
1160 (unsigned long)netif;
1161 netif->credit_timeout.function =
1162 tx_credit_callback;
1163 __mod_timer(&netif->credit_timeout,
1164 next_credit);
1165 netif_put(netif);
1166 continue;
1169 netif->remaining_credit -= txreq.size;
1171 work_to_do--;
1172 netif->tx.req_cons = ++i;
1174 memset(extras, 0, sizeof(extras));
1175 if (txreq.flags & NETTXF_extra_info) {
1176 work_to_do = netbk_get_extras(netif, extras,
1177 work_to_do);
1178 i = netif->tx.req_cons;
1179 if (unlikely(work_to_do < 0)) {
1180 netbk_tx_err(netif, &txreq, i);
1181 continue;
1185 ret = netbk_count_requests(netif, &txreq, txfrags, work_to_do);
1186 if (unlikely(ret < 0)) {
1187 netbk_tx_err(netif, &txreq, i - ret);
1188 continue;
1190 i += ret;
1192 if (unlikely(txreq.size < ETH_HLEN)) {
1193 DPRINTK("Bad packet size: %d\n", txreq.size);
1194 netbk_tx_err(netif, &txreq, i);
1195 continue;
1198 /* No crossing a page as the payload mustn't fragment. */
1199 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1200 DPRINTK("txreq.offset: %x, size: %u, end: %lu\n",
1201 txreq.offset, txreq.size,
1202 (txreq.offset &~PAGE_MASK) + txreq.size);
1203 netbk_tx_err(netif, &txreq, i);
1204 continue;
1207 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
1209 data_len = (txreq.size > PKT_PROT_LEN &&
1210 ret < MAX_SKB_FRAGS) ?
1211 PKT_PROT_LEN : txreq.size;
1213 skb = alloc_skb(data_len+16, GFP_ATOMIC);
1214 if (unlikely(skb == NULL)) {
1215 DPRINTK("Can't allocate a skb in start_xmit.\n");
1216 netbk_tx_err(netif, &txreq, i);
1217 break;
1220 /* Packets passed to netif_rx() must have some headroom. */
1221 skb_reserve(skb, 16);
1223 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1224 struct netif_extra_info *gso;
1225 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1227 if (netbk_set_skb_gso(skb, gso)) {
1228 kfree_skb(skb);
1229 netbk_tx_err(netif, &txreq, i);
1230 continue;
1234 gnttab_set_map_op(mop, idx_to_kaddr(pending_idx),
1235 GNTMAP_host_map | GNTMAP_readonly,
1236 txreq.gref, netif->domid);
1237 mop++;
1239 memcpy(&pending_tx_info[pending_idx].req,
1240 &txreq, sizeof(txreq));
1241 pending_tx_info[pending_idx].netif = netif;
1242 *((u16 *)skb->data) = pending_idx;
1244 __skb_put(skb, data_len);
1246 skb_shinfo(skb)->nr_frags = ret;
1247 if (data_len < txreq.size) {
1248 skb_shinfo(skb)->nr_frags++;
1249 skb_shinfo(skb)->frags[0].page =
1250 (void *)(unsigned long)pending_idx;
1251 } else {
1252 /* Discriminate from any valid pending_idx value. */
1253 skb_shinfo(skb)->frags[0].page = (void *)~0UL;
1256 __skb_queue_tail(&tx_queue, skb);
1258 pending_cons++;
1260 mop = netbk_get_requests(netif, skb, txfrags, mop);
1262 netif->tx.req_cons = i;
1263 netif_schedule_work(netif);
1265 if ((mop - tx_map_ops) >= ARRAY_SIZE(tx_map_ops))
1266 break;
1269 if (mop == tx_map_ops)
1270 return;
1272 ret = HYPERVISOR_grant_table_op(
1273 GNTTABOP_map_grant_ref, tx_map_ops, mop - tx_map_ops);
1274 BUG_ON(ret);
1276 mop = tx_map_ops;
1277 while ((skb = __skb_dequeue(&tx_queue)) != NULL) {
1278 netif_tx_request_t *txp;
1280 pending_idx = *((u16 *)skb->data);
1281 netif = pending_tx_info[pending_idx].netif;
1282 txp = &pending_tx_info[pending_idx].req;
1284 /* Check the remap error code. */
1285 if (unlikely(netbk_tx_check_mop(skb, &mop))) {
1286 printk(KERN_ALERT "#### netback grant fails\n");
1287 skb_shinfo(skb)->nr_frags = 0;
1288 kfree_skb(skb);
1289 continue;
1292 data_len = skb->len;
1293 memcpy(skb->data,
1294 (void *)(idx_to_kaddr(pending_idx)|txp->offset),
1295 data_len);
1296 if (data_len < txp->size) {
1297 /* Append the packet payload as a fragment. */
1298 txp->offset += data_len;
1299 txp->size -= data_len;
1300 } else {
1301 /* Schedule a response immediately. */
1302 netif_idx_release(pending_idx);
1305 /*
1306 * Old frontends do not assert data_validated but we
1307 * can infer it from csum_blank so test both flags.
1308 */
1309 if (txp->flags & (NETTXF_data_validated|NETTXF_csum_blank)) {
1310 skb->ip_summed = CHECKSUM_UNNECESSARY;
1311 skb->proto_data_valid = 1;
1312 } else {
1313 skb->ip_summed = CHECKSUM_NONE;
1314 skb->proto_data_valid = 0;
1316 skb->proto_csum_blank = !!(txp->flags & NETTXF_csum_blank);
1318 netbk_fill_frags(skb);
1320 skb->dev = netif->dev;
1321 skb->protocol = eth_type_trans(skb, skb->dev);
1323 netif->stats.rx_bytes += skb->len;
1324 netif->stats.rx_packets++;
1326 netif_rx(skb);
1327 netif->dev->last_rx = jiffies;
1331 static void netif_idx_release(u16 pending_idx)
1333 static DEFINE_SPINLOCK(_lock);
1334 unsigned long flags;
1336 spin_lock_irqsave(&_lock, flags);
1337 dealloc_ring[MASK_PEND_IDX(dealloc_prod)] = pending_idx;
1338 /* Sync with net_tx_action_dealloc: insert idx /then/ incr producer. */
1339 smp_wmb();
1340 dealloc_prod++;
1341 spin_unlock_irqrestore(&_lock, flags);
1343 tasklet_schedule(&net_tx_tasklet);
1346 static void netif_page_release(struct page *page)
1348 /* Ready for next use. */
1349 set_page_count(page, 1);
1351 netif_idx_release(page->index);
1354 irqreturn_t netif_be_int(int irq, void *dev_id, struct pt_regs *regs)
1356 netif_t *netif = dev_id;
1358 add_to_net_schedule_list_tail(netif);
1359 maybe_schedule_tx_action();
1361 if (netif_queue_stopped(netif->dev) && !netbk_queue_full(netif))
1362 netif_wake_queue(netif->dev);
1364 return IRQ_HANDLED;
1367 static void make_tx_response(netif_t *netif,
1368 netif_tx_request_t *txp,
1369 s8 st)
1371 RING_IDX i = netif->tx.rsp_prod_pvt;
1372 netif_tx_response_t *resp;
1373 int notify;
1375 resp = RING_GET_RESPONSE(&netif->tx, i);
1376 resp->id = txp->id;
1377 resp->status = st;
1379 if (txp->flags & NETTXF_extra_info)
1380 RING_GET_RESPONSE(&netif->tx, ++i)->status = NETIF_RSP_NULL;
1382 netif->tx.rsp_prod_pvt = ++i;
1383 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify);
1384 if (notify)
1385 notify_remote_via_irq(netif->irq);
1387 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
1388 if (i == netif->tx.req_cons) {
1389 int more_to_do;
1390 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
1391 if (more_to_do)
1392 add_to_net_schedule_list_tail(netif);
1394 #endif
1397 static netif_rx_response_t *make_rx_response(netif_t *netif,
1398 u16 id,
1399 s8 st,
1400 u16 offset,
1401 u16 size,
1402 u16 flags)
1404 RING_IDX i = netif->rx.rsp_prod_pvt;
1405 netif_rx_response_t *resp;
1407 resp = RING_GET_RESPONSE(&netif->rx, i);
1408 resp->offset = offset;
1409 resp->flags = flags;
1410 resp->id = id;
1411 resp->status = (s16)size;
1412 if (st < 0)
1413 resp->status = (s16)st;
1415 netif->rx.rsp_prod_pvt = ++i;
1417 return resp;
1420 #ifdef NETBE_DEBUG_INTERRUPT
1421 static irqreturn_t netif_be_dbg(int irq, void *dev_id, struct pt_regs *regs)
1423 struct list_head *ent;
1424 netif_t *netif;
1425 int i = 0;
1427 printk(KERN_ALERT "netif_schedule_list:\n");
1428 spin_lock_irq(&net_schedule_list_lock);
1430 list_for_each (ent, &net_schedule_list) {
1431 netif = list_entry(ent, netif_t, list);
1432 printk(KERN_ALERT " %d: private(rx_req_cons=%08x "
1433 "rx_resp_prod=%08x\n",
1434 i, netif->rx.req_cons, netif->rx.rsp_prod_pvt);
1435 printk(KERN_ALERT " tx_req_cons=%08x tx_resp_prod=%08x)\n",
1436 netif->tx.req_cons, netif->tx.rsp_prod_pvt);
1437 printk(KERN_ALERT " shared(rx_req_prod=%08x "
1438 "rx_resp_prod=%08x\n",
1439 netif->rx.sring->req_prod, netif->rx.sring->rsp_prod);
1440 printk(KERN_ALERT " rx_event=%08x tx_req_prod=%08x\n",
1441 netif->rx.sring->rsp_event, netif->tx.sring->req_prod);
1442 printk(KERN_ALERT " tx_resp_prod=%08x, tx_event=%08x)\n",
1443 netif->tx.sring->rsp_prod, netif->tx.sring->rsp_event);
1444 i++;
1447 spin_unlock_irq(&net_schedule_list_lock);
1448 printk(KERN_ALERT " ** End of netif_schedule_list **\n");
1450 return IRQ_HANDLED;
1452 #endif
1454 static int __init netback_init(void)
1456 int i;
1457 struct page *page;
1459 if (!is_running_on_xen())
1460 return -ENODEV;
1462 /* We can increase reservation by this much in net_rx_action(). */
1463 balloon_update_driver_allowance(NET_RX_RING_SIZE);
1465 skb_queue_head_init(&rx_queue);
1466 skb_queue_head_init(&tx_queue);
1468 init_timer(&net_timer);
1469 net_timer.data = 0;
1470 net_timer.function = net_alarm;
1472 mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS);
1473 if (mmap_pages == NULL) {
1474 printk("%s: out of memory\n", __FUNCTION__);
1475 return -ENOMEM;
1478 for (i = 0; i < MAX_PENDING_REQS; i++) {
1479 page = mmap_pages[i];
1480 SetPageForeign(page, netif_page_release);
1481 page->index = i;
1484 pending_cons = 0;
1485 pending_prod = MAX_PENDING_REQS;
1486 for (i = 0; i < MAX_PENDING_REQS; i++)
1487 pending_ring[i] = i;
1489 spin_lock_init(&net_schedule_list_lock);
1490 INIT_LIST_HEAD(&net_schedule_list);
1492 netif_xenbus_init();
1494 #ifdef NETBE_DEBUG_INTERRUPT
1495 (void)bind_virq_to_irqhandler(
1496 VIRQ_DEBUG,
1497 0,
1498 netif_be_dbg,
1499 SA_SHIRQ,
1500 "net-be-dbg",
1501 &netif_be_dbg);
1502 #endif
1504 return 0;
1507 module_init(netback_init);
1509 MODULE_LICENSE("Dual BSD/GPL");