ia64/xen-unstable

view linux-2.6-xen-sparse/drivers/xen/netback/netback.c @ 12483:ba84d697b1ea

[NET] back: Fix bug in 12262:ac2097d71e06dbbf77279af10d6ae7359d921ab0.
Pointed out by Jan Beulich.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Thu Nov 16 08:56:58 2006 +0000 (2006-11-16)
parents ac2097d71e06
children 277bdeee88f8
line source
1 /******************************************************************************
2 * drivers/xen/netback/netback.c
3 *
4 * Back-end of the driver for virtual network devices. This portion of the
5 * driver exports a 'unified' network-device interface that can be accessed
6 * by any operating system that implements a compatible front end. A
7 * reference front-end implementation can be found in:
8 * drivers/xen/netfront/netfront.c
9 *
10 * Copyright (c) 2002-2005, K A Fraser
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
37 #include "common.h"
38 #include <xen/balloon.h>
39 #include <xen/interface/memory.h>
41 /*#define NETBE_DEBUG_INTERRUPT*/
43 struct netbk_rx_meta {
44 skb_frag_t frag;
45 int id;
46 int copy:1;
47 };
49 static void netif_idx_release(u16 pending_idx);
50 static void netif_page_release(struct page *page);
51 static void make_tx_response(netif_t *netif,
52 netif_tx_request_t *txp,
53 s8 st);
54 static netif_rx_response_t *make_rx_response(netif_t *netif,
55 u16 id,
56 s8 st,
57 u16 offset,
58 u16 size,
59 u16 flags);
61 static void net_tx_action(unsigned long unused);
62 static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0);
64 static void net_rx_action(unsigned long unused);
65 static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
67 static struct timer_list net_timer;
69 #define MAX_PENDING_REQS 256
71 static struct sk_buff_head rx_queue;
73 static struct page **mmap_pages;
74 static inline unsigned long idx_to_kaddr(unsigned int idx)
75 {
76 return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx]));
77 }
79 #define PKT_PROT_LEN 64
81 static struct pending_tx_info {
82 netif_tx_request_t req;
83 netif_t *netif;
84 } pending_tx_info[MAX_PENDING_REQS];
85 static u16 pending_ring[MAX_PENDING_REQS];
86 typedef unsigned int PEND_RING_IDX;
87 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
88 static PEND_RING_IDX pending_prod, pending_cons;
89 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
91 /* Freed TX SKBs get batched on this ring before return to pending_ring. */
92 static u16 dealloc_ring[MAX_PENDING_REQS];
93 static PEND_RING_IDX dealloc_prod, dealloc_cons;
95 static struct sk_buff_head tx_queue;
97 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
98 static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS];
99 static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS];
101 static struct list_head net_schedule_list;
102 static spinlock_t net_schedule_list_lock;
104 #define MAX_MFN_ALLOC 64
105 static unsigned long mfn_list[MAX_MFN_ALLOC];
106 static unsigned int alloc_index = 0;
108 static inline unsigned long alloc_mfn(void)
109 {
110 return mfn_list[--alloc_index];
111 }
113 static int check_mfn(int nr)
114 {
115 struct xen_memory_reservation reservation = {
116 .extent_order = 0,
117 .domid = DOMID_SELF
118 };
120 if (likely(alloc_index >= nr))
121 return 0;
123 set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index);
124 reservation.nr_extents = MAX_MFN_ALLOC - alloc_index;
125 alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation,
126 &reservation);
128 return alloc_index >= nr ? 0 : -ENOMEM;
129 }
131 static inline void maybe_schedule_tx_action(void)
132 {
133 smp_mb();
134 if ((NR_PENDING_REQS < (MAX_PENDING_REQS/2)) &&
135 !list_empty(&net_schedule_list))
136 tasklet_schedule(&net_tx_tasklet);
137 }
139 /*
140 * A gross way of confirming the origin of an skb data page. The slab
141 * allocator abuses a field in the page struct to cache the kmem_cache_t ptr.
142 */
143 static inline int is_xen_skb(struct sk_buff *skb)
144 {
145 extern kmem_cache_t *skbuff_cachep;
146 kmem_cache_t *cp = (kmem_cache_t *)virt_to_page(skb->head)->lru.next;
147 return (cp == skbuff_cachep);
148 }
150 /*
151 * We can flip without copying the packet unless:
152 * 1. The data is not allocated from our special cache; or
153 * 2. The main data area is shared; or
154 * 3. One or more fragments are shared; or
155 * 4. There are chained fragments.
156 */
157 static inline int is_flippable_skb(struct sk_buff *skb)
158 {
159 int frag;
161 if (!is_xen_skb(skb) || skb_cloned(skb))
162 return 0;
164 for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
165 if (page_count(skb_shinfo(skb)->frags[frag].page) > 1)
166 return 0;
167 }
169 if (skb_shinfo(skb)->frag_list != NULL)
170 return 0;
172 return 1;
173 }
175 static struct sk_buff *netbk_copy_skb(struct sk_buff *skb)
176 {
177 struct skb_shared_info *ninfo;
178 struct sk_buff *nskb;
179 unsigned long offset;
180 int ret;
181 int len;
182 int headlen;
184 BUG_ON(skb_shinfo(skb)->frag_list != NULL);
186 nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC);
187 if (unlikely(!nskb))
188 goto err;
190 skb_reserve(nskb, 16 + NET_IP_ALIGN);
191 headlen = nskb->end - nskb->data;
192 if (headlen > skb_headlen(skb))
193 headlen = skb_headlen(skb);
194 ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen);
195 BUG_ON(ret);
197 ninfo = skb_shinfo(nskb);
198 ninfo->gso_size = skb_shinfo(skb)->gso_size;
199 ninfo->gso_type = skb_shinfo(skb)->gso_type;
201 offset = headlen;
202 len = skb->len - headlen;
204 nskb->len = skb->len;
205 nskb->data_len = len;
206 nskb->truesize += len;
208 while (len) {
209 struct page *page;
210 int copy;
211 int zero;
213 if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) {
214 dump_stack();
215 goto err_free;
216 }
218 copy = len >= PAGE_SIZE ? PAGE_SIZE : len;
219 zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO;
221 page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | zero);
222 if (unlikely(!page))
223 goto err_free;
225 ret = skb_copy_bits(skb, offset, page_address(page), copy);
226 BUG_ON(ret);
228 ninfo->frags[ninfo->nr_frags].page = page;
229 ninfo->frags[ninfo->nr_frags].page_offset = 0;
230 ninfo->frags[ninfo->nr_frags].size = copy;
231 ninfo->nr_frags++;
233 offset += copy;
234 len -= copy;
235 }
237 offset = nskb->data - skb->data;
239 nskb->h.raw = skb->h.raw + offset;
240 nskb->nh.raw = skb->nh.raw + offset;
241 nskb->mac.raw = skb->mac.raw + offset;
243 return nskb;
245 err_free:
246 kfree_skb(nskb);
247 err:
248 return NULL;
249 }
251 static inline int netbk_max_required_rx_slots(netif_t *netif)
252 {
253 if (netif->features & (NETIF_F_SG|NETIF_F_TSO))
254 return MAX_SKB_FRAGS + 2; /* header + extra_info + frags */
255 return 1; /* all in one */
256 }
258 static inline int netbk_queue_full(netif_t *netif)
259 {
260 RING_IDX peek = netif->rx_req_cons_peek;
261 RING_IDX needed = netbk_max_required_rx_slots(netif);
263 return ((netif->rx.sring->req_prod - peek) < needed) ||
264 ((netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek) < needed);
265 }
267 static void tx_queue_callback(unsigned long data)
268 {
269 netif_t *netif = (netif_t *)data;
270 if (netif_schedulable(netif->dev))
271 netif_wake_queue(netif->dev);
272 }
274 int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
275 {
276 netif_t *netif = netdev_priv(dev);
278 BUG_ON(skb->dev != dev);
280 /* Drop the packet if the target domain has no receive buffers. */
281 if (unlikely(!netif_schedulable(dev) || netbk_queue_full(netif)))
282 goto drop;
284 /*
285 * Copy the packet here if it's destined for a flipping interface
286 * but isn't flippable (e.g. extra references to data).
287 */
288 if (!netif->copying_receiver && !is_flippable_skb(skb)) {
289 struct sk_buff *nskb = netbk_copy_skb(skb);
290 if ( unlikely(nskb == NULL) )
291 goto drop;
292 /* Copy only the header fields we use in this driver. */
293 nskb->dev = skb->dev;
294 nskb->ip_summed = skb->ip_summed;
295 nskb->proto_data_valid = skb->proto_data_valid;
296 dev_kfree_skb(skb);
297 skb = nskb;
298 }
300 netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1 +
301 !!skb_shinfo(skb)->gso_size;
302 netif_get(netif);
304 if (netbk_can_queue(dev) && netbk_queue_full(netif)) {
305 netif->rx.sring->req_event = netif->rx_req_cons_peek +
306 netbk_max_required_rx_slots(netif);
307 mb(); /* request notification /then/ check & stop the queue */
308 if (netbk_queue_full(netif)) {
309 netif_stop_queue(dev);
310 /*
311 * Schedule 500ms timeout to restart the queue, thus
312 * ensuring that an inactive queue will be drained.
313 * Packets will be immediately be dropped until more
314 * receive buffers become available (see
315 * netbk_queue_full() check above).
316 */
317 netif->tx_queue_timeout.data = (unsigned long)netif;
318 netif->tx_queue_timeout.function = tx_queue_callback;
319 __mod_timer(&netif->tx_queue_timeout, jiffies + HZ/2);
320 }
321 }
323 skb_queue_tail(&rx_queue, skb);
324 tasklet_schedule(&net_rx_tasklet);
326 return 0;
328 drop:
329 netif->stats.tx_dropped++;
330 dev_kfree_skb(skb);
331 return 0;
332 }
334 #if 0
335 static void xen_network_done_notify(void)
336 {
337 static struct net_device *eth0_dev = NULL;
338 if (unlikely(eth0_dev == NULL))
339 eth0_dev = __dev_get_by_name("eth0");
340 netif_rx_schedule(eth0_dev);
341 }
342 /*
343 * Add following to poll() function in NAPI driver (Tigon3 is example):
344 * if ( xen_network_done() )
345 * tg3_enable_ints(tp);
346 */
347 int xen_network_done(void)
348 {
349 return skb_queue_empty(&rx_queue);
350 }
351 #endif
353 struct netrx_pending_operations {
354 unsigned trans_prod, trans_cons;
355 unsigned mmu_prod, mmu_cons;
356 unsigned mcl_prod, mcl_cons;
357 unsigned copy_prod, copy_cons;
358 unsigned meta_prod, meta_cons;
359 mmu_update_t *mmu;
360 gnttab_transfer_t *trans;
361 gnttab_copy_t *copy;
362 multicall_entry_t *mcl;
363 struct netbk_rx_meta *meta;
364 };
366 /* Set up the grant operations for this fragment. If it's a flipping
367 interface, we also set up the unmap request from here. */
368 static u16 netbk_gop_frag(netif_t *netif, struct netbk_rx_meta *meta,
369 int i, struct netrx_pending_operations *npo,
370 struct page *page, unsigned long size,
371 unsigned long offset)
372 {
373 mmu_update_t *mmu;
374 gnttab_transfer_t *gop;
375 gnttab_copy_t *copy_gop;
376 multicall_entry_t *mcl;
377 netif_rx_request_t *req;
378 unsigned long old_mfn, new_mfn;
380 old_mfn = virt_to_mfn(page_address(page));
382 req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i);
383 if (netif->copying_receiver) {
384 /* The fragment needs to be copied rather than
385 flipped. */
386 meta->copy = 1;
387 copy_gop = npo->copy + npo->copy_prod++;
388 copy_gop->flags = GNTCOPY_dest_gref;
389 if (PageForeign(page)) {
390 struct pending_tx_info *src_pend =
391 &pending_tx_info[page->index];
392 copy_gop->source.domid = src_pend->netif->domid;
393 copy_gop->source.u.ref = src_pend->req.gref;
394 copy_gop->flags |= GNTCOPY_source_gref;
395 } else {
396 copy_gop->source.domid = DOMID_SELF;
397 copy_gop->source.u.gmfn = old_mfn;
398 }
399 copy_gop->source.offset = offset;
400 copy_gop->dest.domid = netif->domid;
401 copy_gop->dest.offset = 0;
402 copy_gop->dest.u.ref = req->gref;
403 copy_gop->len = size;
404 } else {
405 meta->copy = 0;
406 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
407 new_mfn = alloc_mfn();
409 /*
410 * Set the new P2M table entry before
411 * reassigning the old data page. Heed the
412 * comment in pgtable-2level.h:pte_page(). :-)
413 */
414 set_phys_to_machine(page_to_pfn(page), new_mfn);
416 mcl = npo->mcl + npo->mcl_prod++;
417 MULTI_update_va_mapping(mcl,
418 (unsigned long)page_address(page),
419 pfn_pte_ma(new_mfn, PAGE_KERNEL),
420 0);
422 mmu = npo->mmu + npo->mmu_prod++;
423 mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) |
424 MMU_MACHPHYS_UPDATE;
425 mmu->val = page_to_pfn(page);
426 }
428 gop = npo->trans + npo->trans_prod++;
429 gop->mfn = old_mfn;
430 gop->domid = netif->domid;
431 gop->ref = req->gref;
432 }
433 return req->id;
434 }
436 static void netbk_gop_skb(struct sk_buff *skb,
437 struct netrx_pending_operations *npo)
438 {
439 netif_t *netif = netdev_priv(skb->dev);
440 int nr_frags = skb_shinfo(skb)->nr_frags;
441 int i;
442 int extra;
443 struct netbk_rx_meta *head_meta, *meta;
445 head_meta = npo->meta + npo->meta_prod++;
446 head_meta->frag.page_offset = skb_shinfo(skb)->gso_type;
447 head_meta->frag.size = skb_shinfo(skb)->gso_size;
448 extra = !!head_meta->frag.size + 1;
450 for (i = 0; i < nr_frags; i++) {
451 meta = npo->meta + npo->meta_prod++;
452 meta->frag = skb_shinfo(skb)->frags[i];
453 meta->id = netbk_gop_frag(netif, meta, i + extra, npo,
454 meta->frag.page,
455 meta->frag.size,
456 meta->frag.page_offset);
457 }
459 /*
460 * This must occur at the end to ensure that we don't trash
461 * skb_shinfo until we're done.
462 */
463 head_meta->id = netbk_gop_frag(netif, head_meta, 0, npo,
464 virt_to_page(skb->data),
465 skb_headlen(skb),
466 offset_in_page(skb->data));
468 netif->rx.req_cons += nr_frags + extra;
469 }
471 static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta)
472 {
473 int i;
475 for (i = 0; i < nr_frags; i++)
476 put_page(meta[i].frag.page);
477 }
479 /* This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
480 used to set up the operations on the top of
481 netrx_pending_operations, which have since been done. Check that
482 they didn't give any errors and advance over them. */
483 static int netbk_check_gop(int nr_frags, domid_t domid,
484 struct netrx_pending_operations *npo)
485 {
486 multicall_entry_t *mcl;
487 gnttab_transfer_t *gop;
488 gnttab_copy_t *copy_op;
489 int status = NETIF_RSP_OKAY;
490 int i;
492 for (i = 0; i <= nr_frags; i++) {
493 if (npo->meta[npo->meta_cons + i].copy) {
494 copy_op = npo->copy + npo->copy_cons++;
495 if (copy_op->status != GNTST_okay) {
496 DPRINTK("Bad status %d from copy to DOM%d.\n",
497 gop->status, domid);
498 status = NETIF_RSP_ERROR;
499 }
500 } else {
501 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
502 mcl = npo->mcl + npo->mcl_cons++;
503 /* The update_va_mapping() must not fail. */
504 BUG_ON(mcl->result != 0);
505 }
507 gop = npo->trans + npo->trans_cons++;
508 /* Check the reassignment error code. */
509 if (gop->status != 0) {
510 DPRINTK("Bad status %d from grant transfer to DOM%u\n",
511 gop->status, domid);
512 /*
513 * Page no longer belongs to us unless
514 * GNTST_bad_page, but that should be
515 * a fatal error anyway.
516 */
517 BUG_ON(gop->status == GNTST_bad_page);
518 status = NETIF_RSP_ERROR;
519 }
520 }
521 }
523 return status;
524 }
526 static void netbk_add_frag_responses(netif_t *netif, int status,
527 struct netbk_rx_meta *meta, int nr_frags)
528 {
529 int i;
530 unsigned long offset;
532 for (i = 0; i < nr_frags; i++) {
533 int id = meta[i].id;
534 int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data;
536 if (meta[i].copy)
537 offset = 0;
538 else
539 offset = meta[i].frag.page_offset;
540 make_rx_response(netif, id, status, offset,
541 meta[i].frag.size, flags);
542 }
543 }
545 static void net_rx_action(unsigned long unused)
546 {
547 netif_t *netif = NULL;
548 s8 status;
549 u16 id, irq, flags;
550 netif_rx_response_t *resp;
551 multicall_entry_t *mcl;
552 struct sk_buff_head rxq;
553 struct sk_buff *skb;
554 int notify_nr = 0;
555 int ret;
556 int nr_frags;
557 int count;
558 unsigned long offset;
560 /*
561 * Putting hundreds of bytes on the stack is considered rude.
562 * Static works because a tasklet can only be on one CPU at any time.
563 */
564 static multicall_entry_t rx_mcl[NET_RX_RING_SIZE+3];
565 static mmu_update_t rx_mmu[NET_RX_RING_SIZE];
566 static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE];
567 static gnttab_copy_t grant_copy_op[NET_RX_RING_SIZE];
568 static unsigned char rx_notify[NR_IRQS];
569 static u16 notify_list[NET_RX_RING_SIZE];
570 static struct netbk_rx_meta meta[NET_RX_RING_SIZE];
572 struct netrx_pending_operations npo = {
573 mmu: rx_mmu,
574 trans: grant_trans_op,
575 copy: grant_copy_op,
576 mcl: rx_mcl,
577 meta: meta};
579 skb_queue_head_init(&rxq);
581 count = 0;
583 while ((skb = skb_dequeue(&rx_queue)) != NULL) {
584 nr_frags = skb_shinfo(skb)->nr_frags;
585 *(int *)skb->cb = nr_frags;
587 if (!xen_feature(XENFEAT_auto_translated_physmap) &&
588 check_mfn(nr_frags + 1)) {
589 /* Memory squeeze? Back off for an arbitrary while. */
590 if ( net_ratelimit() )
591 WPRINTK("Memory squeeze in netback "
592 "driver.\n");
593 mod_timer(&net_timer, jiffies + HZ);
594 skb_queue_head(&rx_queue, skb);
595 break;
596 }
598 netbk_gop_skb(skb, &npo);
600 count += nr_frags + 1;
602 __skb_queue_tail(&rxq, skb);
604 /* Filled the batch queue? */
605 if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE)
606 break;
607 }
609 if (npo.mcl_prod &&
610 !xen_feature(XENFEAT_auto_translated_physmap)) {
611 mcl = npo.mcl + npo.mcl_prod++;
613 BUG_ON(mcl[-1].op != __HYPERVISOR_update_va_mapping);
614 mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL;
616 mcl->op = __HYPERVISOR_mmu_update;
617 mcl->args[0] = (unsigned long)rx_mmu;
618 mcl->args[1] = npo.mmu_prod;
619 mcl->args[2] = 0;
620 mcl->args[3] = DOMID_SELF;
621 }
623 if (npo.trans_prod) {
624 mcl = npo.mcl + npo.mcl_prod++;
625 mcl->op = __HYPERVISOR_grant_table_op;
626 mcl->args[0] = GNTTABOP_transfer;
627 mcl->args[1] = (unsigned long)grant_trans_op;
628 mcl->args[2] = npo.trans_prod;
629 }
631 if (npo.copy_prod) {
632 mcl = npo.mcl + npo.mcl_prod++;
633 mcl->op = __HYPERVISOR_grant_table_op;
634 mcl->args[0] = GNTTABOP_copy;
635 mcl->args[1] = (unsigned long)grant_copy_op;
636 mcl->args[2] = npo.copy_prod;
637 }
639 /* Nothing to do? */
640 if (!npo.mcl_prod)
641 return;
643 BUG_ON(npo.copy_prod > NET_RX_RING_SIZE);
644 BUG_ON(npo.mmu_prod > NET_RX_RING_SIZE);
645 BUG_ON(npo.trans_prod > NET_RX_RING_SIZE);
646 BUG_ON(npo.mcl_prod > NET_RX_RING_SIZE+3);
647 BUG_ON(npo.meta_prod > NET_RX_RING_SIZE);
649 ret = HYPERVISOR_multicall(npo.mcl, npo.mcl_prod);
650 BUG_ON(ret != 0);
652 while ((skb = __skb_dequeue(&rxq)) != NULL) {
653 nr_frags = *(int *)skb->cb;
655 netif = netdev_priv(skb->dev);
656 /* We can't rely on skb_release_data to release the
657 pages used by fragments for us, since it tries to
658 touch the pages in the fraglist. If we're in
659 flipping mode, that doesn't work. In copying mode,
660 we still have access to all of the pages, and so
661 it's safe to let release_data deal with it. */
662 /* (Freeing the fragments is safe since we copy
663 non-linear skbs destined for flipping interfaces) */
664 if (!netif->copying_receiver) {
665 atomic_set(&(skb_shinfo(skb)->dataref), 1);
666 skb_shinfo(skb)->frag_list = NULL;
667 skb_shinfo(skb)->nr_frags = 0;
668 netbk_free_pages(nr_frags, meta + npo.meta_cons + 1);
669 }
671 netif->stats.tx_bytes += skb->len;
672 netif->stats.tx_packets++;
674 status = netbk_check_gop(nr_frags, netif->domid, &npo);
676 id = meta[npo.meta_cons].id;
677 flags = nr_frags ? NETRXF_more_data : 0;
679 if (skb->ip_summed == CHECKSUM_HW) /* local packet? */
680 flags |= NETRXF_csum_blank | NETRXF_data_validated;
681 else if (skb->proto_data_valid) /* remote but checksummed? */
682 flags |= NETRXF_data_validated;
684 if (meta[npo.meta_cons].copy)
685 offset = 0;
686 else
687 offset = offset_in_page(skb->data);
688 resp = make_rx_response(netif, id, status, offset,
689 skb_headlen(skb), flags);
691 if (meta[npo.meta_cons].frag.size) {
692 struct netif_extra_info *gso =
693 (struct netif_extra_info *)
694 RING_GET_RESPONSE(&netif->rx,
695 netif->rx.rsp_prod_pvt++);
697 resp->flags |= NETRXF_extra_info;
699 gso->u.gso.size = meta[npo.meta_cons].frag.size;
700 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
701 gso->u.gso.pad = 0;
702 gso->u.gso.features = 0;
704 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
705 gso->flags = 0;
706 }
708 netbk_add_frag_responses(netif, status,
709 meta + npo.meta_cons + 1,
710 nr_frags);
712 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret);
713 irq = netif->irq;
714 if (ret && !rx_notify[irq]) {
715 rx_notify[irq] = 1;
716 notify_list[notify_nr++] = irq;
717 }
719 if (netif_queue_stopped(netif->dev) &&
720 netif_schedulable(netif->dev) &&
721 !netbk_queue_full(netif))
722 netif_wake_queue(netif->dev);
724 netif_put(netif);
725 dev_kfree_skb(skb);
726 npo.meta_cons += nr_frags + 1;
727 }
729 while (notify_nr != 0) {
730 irq = notify_list[--notify_nr];
731 rx_notify[irq] = 0;
732 notify_remote_via_irq(irq);
733 }
735 /* More work to do? */
736 if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer))
737 tasklet_schedule(&net_rx_tasklet);
738 #if 0
739 else
740 xen_network_done_notify();
741 #endif
742 }
744 static void net_alarm(unsigned long unused)
745 {
746 tasklet_schedule(&net_rx_tasklet);
747 }
749 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
750 {
751 netif_t *netif = netdev_priv(dev);
752 return &netif->stats;
753 }
755 static int __on_net_schedule_list(netif_t *netif)
756 {
757 return netif->list.next != NULL;
758 }
760 static void remove_from_net_schedule_list(netif_t *netif)
761 {
762 spin_lock_irq(&net_schedule_list_lock);
763 if (likely(__on_net_schedule_list(netif))) {
764 list_del(&netif->list);
765 netif->list.next = NULL;
766 netif_put(netif);
767 }
768 spin_unlock_irq(&net_schedule_list_lock);
769 }
771 static void add_to_net_schedule_list_tail(netif_t *netif)
772 {
773 if (__on_net_schedule_list(netif))
774 return;
776 spin_lock_irq(&net_schedule_list_lock);
777 if (!__on_net_schedule_list(netif) &&
778 likely(netif_schedulable(netif->dev))) {
779 list_add_tail(&netif->list, &net_schedule_list);
780 netif_get(netif);
781 }
782 spin_unlock_irq(&net_schedule_list_lock);
783 }
785 /*
786 * Note on CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER:
787 * If this driver is pipelining transmit requests then we can be very
788 * aggressive in avoiding new-packet notifications -- frontend only needs to
789 * send a notification if there are no outstanding unreceived responses.
790 * If we may be buffer transmit buffers for any reason then we must be rather
791 * more conservative and treat this as the final check for pending work.
792 */
793 void netif_schedule_work(netif_t *netif)
794 {
795 int more_to_do;
797 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
798 more_to_do = RING_HAS_UNCONSUMED_REQUESTS(&netif->tx);
799 #else
800 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
801 #endif
803 if (more_to_do) {
804 add_to_net_schedule_list_tail(netif);
805 maybe_schedule_tx_action();
806 }
807 }
809 void netif_deschedule_work(netif_t *netif)
810 {
811 remove_from_net_schedule_list(netif);
812 }
815 static void tx_add_credit(netif_t *netif)
816 {
817 unsigned long max_burst, max_credit;
819 /*
820 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
821 * Otherwise the interface can seize up due to insufficient credit.
822 */
823 max_burst = RING_GET_REQUEST(&netif->tx, netif->tx.req_cons)->size;
824 max_burst = min(max_burst, 131072UL);
825 max_burst = max(max_burst, netif->credit_bytes);
827 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
828 max_credit = netif->remaining_credit + netif->credit_bytes;
829 if (max_credit < netif->remaining_credit)
830 max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */
832 netif->remaining_credit = min(max_credit, max_burst);
833 }
835 static void tx_credit_callback(unsigned long data)
836 {
837 netif_t *netif = (netif_t *)data;
838 tx_add_credit(netif);
839 netif_schedule_work(netif);
840 }
842 inline static void net_tx_action_dealloc(void)
843 {
844 gnttab_unmap_grant_ref_t *gop;
845 u16 pending_idx;
846 PEND_RING_IDX dc, dp;
847 netif_t *netif;
848 int ret;
850 dc = dealloc_cons;
851 dp = dealloc_prod;
853 /* Ensure we see all indexes enqueued by netif_idx_release(). */
854 smp_rmb();
856 /*
857 * Free up any grants we have finished using
858 */
859 gop = tx_unmap_ops;
860 while (dc != dp) {
861 pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
862 gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
863 GNTMAP_host_map,
864 grant_tx_handle[pending_idx]);
865 gop++;
866 }
867 ret = HYPERVISOR_grant_table_op(
868 GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops);
869 BUG_ON(ret);
871 while (dealloc_cons != dp) {
872 pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
874 netif = pending_tx_info[pending_idx].netif;
876 make_tx_response(netif, &pending_tx_info[pending_idx].req,
877 NETIF_RSP_OKAY);
879 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
881 netif_put(netif);
882 }
883 }
885 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
886 {
887 RING_IDX cons = netif->tx.req_cons;
889 do {
890 make_tx_response(netif, txp, NETIF_RSP_ERROR);
891 if (cons >= end)
892 break;
893 txp = RING_GET_REQUEST(&netif->tx, cons++);
894 } while (1);
895 netif->tx.req_cons = cons;
896 netif_schedule_work(netif);
897 netif_put(netif);
898 }
900 static int netbk_count_requests(netif_t *netif, netif_tx_request_t *first,
901 netif_tx_request_t *txp, int work_to_do)
902 {
903 RING_IDX cons = netif->tx.req_cons;
904 int frags = 0;
906 if (!(first->flags & NETTXF_more_data))
907 return 0;
909 do {
910 if (frags >= work_to_do) {
911 DPRINTK("Need more frags\n");
912 return -frags;
913 }
915 if (unlikely(frags >= MAX_SKB_FRAGS)) {
916 DPRINTK("Too many frags\n");
917 return -frags;
918 }
920 memcpy(txp, RING_GET_REQUEST(&netif->tx, cons + frags),
921 sizeof(*txp));
922 if (txp->size > first->size) {
923 DPRINTK("Frags galore\n");
924 return -frags;
925 }
927 first->size -= txp->size;
928 frags++;
930 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
931 DPRINTK("txp->offset: %x, size: %u\n",
932 txp->offset, txp->size);
933 return -frags;
934 }
935 } while ((txp++)->flags & NETTXF_more_data);
937 return frags;
938 }
940 static gnttab_map_grant_ref_t *netbk_get_requests(netif_t *netif,
941 struct sk_buff *skb,
942 netif_tx_request_t *txp,
943 gnttab_map_grant_ref_t *mop)
944 {
945 struct skb_shared_info *shinfo = skb_shinfo(skb);
946 skb_frag_t *frags = shinfo->frags;
947 unsigned long pending_idx = *((u16 *)skb->data);
948 int i, start;
950 /* Skip first skb fragment if it is on same page as header fragment. */
951 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
953 for (i = start; i < shinfo->nr_frags; i++, txp++) {
954 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons++)];
956 gnttab_set_map_op(mop++, idx_to_kaddr(pending_idx),
957 GNTMAP_host_map | GNTMAP_readonly,
958 txp->gref, netif->domid);
960 memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp));
961 netif_get(netif);
962 pending_tx_info[pending_idx].netif = netif;
963 frags[i].page = (void *)pending_idx;
964 }
966 return mop;
967 }
969 static int netbk_tx_check_mop(struct sk_buff *skb,
970 gnttab_map_grant_ref_t **mopp)
971 {
972 gnttab_map_grant_ref_t *mop = *mopp;
973 int pending_idx = *((u16 *)skb->data);
974 netif_t *netif = pending_tx_info[pending_idx].netif;
975 netif_tx_request_t *txp;
976 struct skb_shared_info *shinfo = skb_shinfo(skb);
977 int nr_frags = shinfo->nr_frags;
978 int i, err, start;
980 /* Check status of header. */
981 err = mop->status;
982 if (unlikely(err)) {
983 txp = &pending_tx_info[pending_idx].req;
984 make_tx_response(netif, txp, NETIF_RSP_ERROR);
985 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
986 netif_put(netif);
987 } else {
988 set_phys_to_machine(
989 __pa(idx_to_kaddr(pending_idx)) >> PAGE_SHIFT,
990 FOREIGN_FRAME(mop->dev_bus_addr >> PAGE_SHIFT));
991 grant_tx_handle[pending_idx] = mop->handle;
992 }
994 /* Skip first skb fragment if it is on same page as header fragment. */
995 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
997 for (i = start; i < nr_frags; i++) {
998 int j, newerr;
1000 pending_idx = (unsigned long)shinfo->frags[i].page;
1002 /* Check error status: if okay then remember grant handle. */
1003 newerr = (++mop)->status;
1004 if (likely(!newerr)) {
1005 set_phys_to_machine(
1006 __pa(idx_to_kaddr(pending_idx))>>PAGE_SHIFT,
1007 FOREIGN_FRAME(mop->dev_bus_addr>>PAGE_SHIFT));
1008 grant_tx_handle[pending_idx] = mop->handle;
1009 /* Had a previous error? Invalidate this fragment. */
1010 if (unlikely(err))
1011 netif_idx_release(pending_idx);
1012 continue;
1015 /* Error on this fragment: respond to client with an error. */
1016 txp = &pending_tx_info[pending_idx].req;
1017 make_tx_response(netif, txp, NETIF_RSP_ERROR);
1018 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
1019 netif_put(netif);
1021 /* Not the first error? Preceding frags already invalidated. */
1022 if (err)
1023 continue;
1025 /* First error: invalidate header and preceding fragments. */
1026 pending_idx = *((u16 *)skb->data);
1027 netif_idx_release(pending_idx);
1028 for (j = start; j < i; j++) {
1029 pending_idx = (unsigned long)shinfo->frags[i].page;
1030 netif_idx_release(pending_idx);
1033 /* Remember the error: invalidate all subsequent fragments. */
1034 err = newerr;
1037 *mopp = mop + 1;
1038 return err;
1041 static void netbk_fill_frags(struct sk_buff *skb)
1043 struct skb_shared_info *shinfo = skb_shinfo(skb);
1044 int nr_frags = shinfo->nr_frags;
1045 int i;
1047 for (i = 0; i < nr_frags; i++) {
1048 skb_frag_t *frag = shinfo->frags + i;
1049 netif_tx_request_t *txp;
1050 unsigned long pending_idx;
1052 pending_idx = (unsigned long)frag->page;
1053 txp = &pending_tx_info[pending_idx].req;
1054 frag->page = virt_to_page(idx_to_kaddr(pending_idx));
1055 frag->size = txp->size;
1056 frag->page_offset = txp->offset;
1058 skb->len += txp->size;
1059 skb->data_len += txp->size;
1060 skb->truesize += txp->size;
1064 int netbk_get_extras(netif_t *netif, struct netif_extra_info *extras,
1065 int work_to_do)
1067 struct netif_extra_info extra;
1068 RING_IDX cons = netif->tx.req_cons;
1070 do {
1071 if (unlikely(work_to_do-- <= 0)) {
1072 DPRINTK("Missing extra info\n");
1073 return -EBADR;
1076 memcpy(&extra, RING_GET_REQUEST(&netif->tx, cons),
1077 sizeof(extra));
1078 if (unlikely(!extra.type ||
1079 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1080 netif->tx.req_cons = ++cons;
1081 DPRINTK("Invalid extra type: %d\n", extra.type);
1082 return -EINVAL;
1085 memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
1086 netif->tx.req_cons = ++cons;
1087 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
1089 return work_to_do;
1092 static int netbk_set_skb_gso(struct sk_buff *skb, struct netif_extra_info *gso)
1094 if (!gso->u.gso.size) {
1095 DPRINTK("GSO size must not be zero.\n");
1096 return -EINVAL;
1099 /* Currently only TCPv4 S.O. is supported. */
1100 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) {
1101 DPRINTK("Bad GSO type %d.\n", gso->u.gso.type);
1102 return -EINVAL;
1105 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1106 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1108 /* Header must be checked, and gso_segs computed. */
1109 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1110 skb_shinfo(skb)->gso_segs = 0;
1112 return 0;
1115 /* Called after netfront has transmitted */
1116 static void net_tx_action(unsigned long unused)
1118 struct list_head *ent;
1119 struct sk_buff *skb;
1120 netif_t *netif;
1121 netif_tx_request_t txreq;
1122 netif_tx_request_t txfrags[MAX_SKB_FRAGS];
1123 struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
1124 u16 pending_idx;
1125 RING_IDX i;
1126 gnttab_map_grant_ref_t *mop;
1127 unsigned int data_len;
1128 int ret, work_to_do;
1130 if (dealloc_cons != dealloc_prod)
1131 net_tx_action_dealloc();
1133 mop = tx_map_ops;
1134 while (((NR_PENDING_REQS + MAX_SKB_FRAGS) < MAX_PENDING_REQS) &&
1135 !list_empty(&net_schedule_list)) {
1136 /* Get a netif from the list with work to do. */
1137 ent = net_schedule_list.next;
1138 netif = list_entry(ent, netif_t, list);
1139 netif_get(netif);
1140 remove_from_net_schedule_list(netif);
1142 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do);
1143 if (!work_to_do) {
1144 netif_put(netif);
1145 continue;
1148 i = netif->tx.req_cons;
1149 rmb(); /* Ensure that we see the request before we copy it. */
1150 memcpy(&txreq, RING_GET_REQUEST(&netif->tx, i), sizeof(txreq));
1152 /* Credit-based scheduling. */
1153 if (txreq.size > netif->remaining_credit) {
1154 unsigned long now = jiffies;
1155 unsigned long next_credit =
1156 netif->credit_timeout.expires +
1157 msecs_to_jiffies(netif->credit_usec / 1000);
1159 /* Timer could already be pending in rare cases. */
1160 if (timer_pending(&netif->credit_timeout)) {
1161 netif_put(netif);
1162 continue;
1165 /* Passed the point where we can replenish credit? */
1166 if (time_after_eq(now, next_credit)) {
1167 netif->credit_timeout.expires = now;
1168 tx_add_credit(netif);
1171 /* Still too big to send right now? Set a callback. */
1172 if (txreq.size > netif->remaining_credit) {
1173 netif->credit_timeout.data =
1174 (unsigned long)netif;
1175 netif->credit_timeout.function =
1176 tx_credit_callback;
1177 __mod_timer(&netif->credit_timeout,
1178 next_credit);
1179 netif_put(netif);
1180 continue;
1183 netif->remaining_credit -= txreq.size;
1185 work_to_do--;
1186 netif->tx.req_cons = ++i;
1188 memset(extras, 0, sizeof(extras));
1189 if (txreq.flags & NETTXF_extra_info) {
1190 work_to_do = netbk_get_extras(netif, extras,
1191 work_to_do);
1192 i = netif->tx.req_cons;
1193 if (unlikely(work_to_do < 0)) {
1194 netbk_tx_err(netif, &txreq, i);
1195 continue;
1199 ret = netbk_count_requests(netif, &txreq, txfrags, work_to_do);
1200 if (unlikely(ret < 0)) {
1201 netbk_tx_err(netif, &txreq, i - ret);
1202 continue;
1204 i += ret;
1206 if (unlikely(txreq.size < ETH_HLEN)) {
1207 DPRINTK("Bad packet size: %d\n", txreq.size);
1208 netbk_tx_err(netif, &txreq, i);
1209 continue;
1212 /* No crossing a page as the payload mustn't fragment. */
1213 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1214 DPRINTK("txreq.offset: %x, size: %u, end: %lu\n",
1215 txreq.offset, txreq.size,
1216 (txreq.offset &~PAGE_MASK) + txreq.size);
1217 netbk_tx_err(netif, &txreq, i);
1218 continue;
1221 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
1223 data_len = (txreq.size > PKT_PROT_LEN &&
1224 ret < MAX_SKB_FRAGS) ?
1225 PKT_PROT_LEN : txreq.size;
1227 skb = alloc_skb(data_len + 16 + NET_IP_ALIGN, GFP_ATOMIC);
1228 if (unlikely(skb == NULL)) {
1229 DPRINTK("Can't allocate a skb in start_xmit.\n");
1230 netbk_tx_err(netif, &txreq, i);
1231 break;
1234 /* Packets passed to netif_rx() must have some headroom. */
1235 skb_reserve(skb, 16 + NET_IP_ALIGN);
1237 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1238 struct netif_extra_info *gso;
1239 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1241 if (netbk_set_skb_gso(skb, gso)) {
1242 kfree_skb(skb);
1243 netbk_tx_err(netif, &txreq, i);
1244 continue;
1248 gnttab_set_map_op(mop, idx_to_kaddr(pending_idx),
1249 GNTMAP_host_map | GNTMAP_readonly,
1250 txreq.gref, netif->domid);
1251 mop++;
1253 memcpy(&pending_tx_info[pending_idx].req,
1254 &txreq, sizeof(txreq));
1255 pending_tx_info[pending_idx].netif = netif;
1256 *((u16 *)skb->data) = pending_idx;
1258 __skb_put(skb, data_len);
1260 skb_shinfo(skb)->nr_frags = ret;
1261 if (data_len < txreq.size) {
1262 skb_shinfo(skb)->nr_frags++;
1263 skb_shinfo(skb)->frags[0].page =
1264 (void *)(unsigned long)pending_idx;
1265 } else {
1266 /* Discriminate from any valid pending_idx value. */
1267 skb_shinfo(skb)->frags[0].page = (void *)~0UL;
1270 __skb_queue_tail(&tx_queue, skb);
1272 pending_cons++;
1274 mop = netbk_get_requests(netif, skb, txfrags, mop);
1276 netif->tx.req_cons = i;
1277 netif_schedule_work(netif);
1279 if ((mop - tx_map_ops) >= ARRAY_SIZE(tx_map_ops))
1280 break;
1283 if (mop == tx_map_ops)
1284 return;
1286 ret = HYPERVISOR_grant_table_op(
1287 GNTTABOP_map_grant_ref, tx_map_ops, mop - tx_map_ops);
1288 BUG_ON(ret);
1290 mop = tx_map_ops;
1291 while ((skb = __skb_dequeue(&tx_queue)) != NULL) {
1292 netif_tx_request_t *txp;
1294 pending_idx = *((u16 *)skb->data);
1295 netif = pending_tx_info[pending_idx].netif;
1296 txp = &pending_tx_info[pending_idx].req;
1298 /* Check the remap error code. */
1299 if (unlikely(netbk_tx_check_mop(skb, &mop))) {
1300 printk(KERN_ALERT "#### netback grant fails\n");
1301 skb_shinfo(skb)->nr_frags = 0;
1302 kfree_skb(skb);
1303 continue;
1306 data_len = skb->len;
1307 memcpy(skb->data,
1308 (void *)(idx_to_kaddr(pending_idx)|txp->offset),
1309 data_len);
1310 if (data_len < txp->size) {
1311 /* Append the packet payload as a fragment. */
1312 txp->offset += data_len;
1313 txp->size -= data_len;
1314 } else {
1315 /* Schedule a response immediately. */
1316 netif_idx_release(pending_idx);
1319 /*
1320 * Old frontends do not assert data_validated but we
1321 * can infer it from csum_blank so test both flags.
1322 */
1323 if (txp->flags & (NETTXF_data_validated|NETTXF_csum_blank)) {
1324 skb->ip_summed = CHECKSUM_UNNECESSARY;
1325 skb->proto_data_valid = 1;
1326 } else {
1327 skb->ip_summed = CHECKSUM_NONE;
1328 skb->proto_data_valid = 0;
1330 skb->proto_csum_blank = !!(txp->flags & NETTXF_csum_blank);
1332 netbk_fill_frags(skb);
1334 skb->dev = netif->dev;
1335 skb->protocol = eth_type_trans(skb, skb->dev);
1337 netif->stats.rx_bytes += skb->len;
1338 netif->stats.rx_packets++;
1340 netif_rx(skb);
1341 netif->dev->last_rx = jiffies;
1345 static void netif_idx_release(u16 pending_idx)
1347 static DEFINE_SPINLOCK(_lock);
1348 unsigned long flags;
1350 spin_lock_irqsave(&_lock, flags);
1351 dealloc_ring[MASK_PEND_IDX(dealloc_prod)] = pending_idx;
1352 /* Sync with net_tx_action_dealloc: insert idx /then/ incr producer. */
1353 smp_wmb();
1354 dealloc_prod++;
1355 spin_unlock_irqrestore(&_lock, flags);
1357 tasklet_schedule(&net_tx_tasklet);
1360 static void netif_page_release(struct page *page)
1362 /* Ready for next use. */
1363 set_page_count(page, 1);
1365 netif_idx_release(page->index);
1368 irqreturn_t netif_be_int(int irq, void *dev_id, struct pt_regs *regs)
1370 netif_t *netif = dev_id;
1372 add_to_net_schedule_list_tail(netif);
1373 maybe_schedule_tx_action();
1375 if (netif_schedulable(netif->dev) && !netbk_queue_full(netif))
1376 netif_wake_queue(netif->dev);
1378 return IRQ_HANDLED;
1381 static void make_tx_response(netif_t *netif,
1382 netif_tx_request_t *txp,
1383 s8 st)
1385 RING_IDX i = netif->tx.rsp_prod_pvt;
1386 netif_tx_response_t *resp;
1387 int notify;
1389 resp = RING_GET_RESPONSE(&netif->tx, i);
1390 resp->id = txp->id;
1391 resp->status = st;
1393 if (txp->flags & NETTXF_extra_info)
1394 RING_GET_RESPONSE(&netif->tx, ++i)->status = NETIF_RSP_NULL;
1396 netif->tx.rsp_prod_pvt = ++i;
1397 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify);
1398 if (notify)
1399 notify_remote_via_irq(netif->irq);
1401 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
1402 if (i == netif->tx.req_cons) {
1403 int more_to_do;
1404 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
1405 if (more_to_do)
1406 add_to_net_schedule_list_tail(netif);
1408 #endif
1411 static netif_rx_response_t *make_rx_response(netif_t *netif,
1412 u16 id,
1413 s8 st,
1414 u16 offset,
1415 u16 size,
1416 u16 flags)
1418 RING_IDX i = netif->rx.rsp_prod_pvt;
1419 netif_rx_response_t *resp;
1421 resp = RING_GET_RESPONSE(&netif->rx, i);
1422 resp->offset = offset;
1423 resp->flags = flags;
1424 resp->id = id;
1425 resp->status = (s16)size;
1426 if (st < 0)
1427 resp->status = (s16)st;
1429 netif->rx.rsp_prod_pvt = ++i;
1431 return resp;
1434 #ifdef NETBE_DEBUG_INTERRUPT
1435 static irqreturn_t netif_be_dbg(int irq, void *dev_id, struct pt_regs *regs)
1437 struct list_head *ent;
1438 netif_t *netif;
1439 int i = 0;
1441 printk(KERN_ALERT "netif_schedule_list:\n");
1442 spin_lock_irq(&net_schedule_list_lock);
1444 list_for_each (ent, &net_schedule_list) {
1445 netif = list_entry(ent, netif_t, list);
1446 printk(KERN_ALERT " %d: private(rx_req_cons=%08x "
1447 "rx_resp_prod=%08x\n",
1448 i, netif->rx.req_cons, netif->rx.rsp_prod_pvt);
1449 printk(KERN_ALERT " tx_req_cons=%08x tx_resp_prod=%08x)\n",
1450 netif->tx.req_cons, netif->tx.rsp_prod_pvt);
1451 printk(KERN_ALERT " shared(rx_req_prod=%08x "
1452 "rx_resp_prod=%08x\n",
1453 netif->rx.sring->req_prod, netif->rx.sring->rsp_prod);
1454 printk(KERN_ALERT " rx_event=%08x tx_req_prod=%08x\n",
1455 netif->rx.sring->rsp_event, netif->tx.sring->req_prod);
1456 printk(KERN_ALERT " tx_resp_prod=%08x, tx_event=%08x)\n",
1457 netif->tx.sring->rsp_prod, netif->tx.sring->rsp_event);
1458 i++;
1461 spin_unlock_irq(&net_schedule_list_lock);
1462 printk(KERN_ALERT " ** End of netif_schedule_list **\n");
1464 return IRQ_HANDLED;
1466 #endif
1468 static int __init netback_init(void)
1470 int i;
1471 struct page *page;
1473 if (!is_running_on_xen())
1474 return -ENODEV;
1476 /* We can increase reservation by this much in net_rx_action(). */
1477 balloon_update_driver_allowance(NET_RX_RING_SIZE);
1479 skb_queue_head_init(&rx_queue);
1480 skb_queue_head_init(&tx_queue);
1482 init_timer(&net_timer);
1483 net_timer.data = 0;
1484 net_timer.function = net_alarm;
1486 mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS);
1487 if (mmap_pages == NULL) {
1488 printk("%s: out of memory\n", __FUNCTION__);
1489 return -ENOMEM;
1492 for (i = 0; i < MAX_PENDING_REQS; i++) {
1493 page = mmap_pages[i];
1494 SetPageForeign(page, netif_page_release);
1495 page->index = i;
1498 pending_cons = 0;
1499 pending_prod = MAX_PENDING_REQS;
1500 for (i = 0; i < MAX_PENDING_REQS; i++)
1501 pending_ring[i] = i;
1503 spin_lock_init(&net_schedule_list_lock);
1504 INIT_LIST_HEAD(&net_schedule_list);
1506 netif_xenbus_init();
1508 #ifdef NETBE_DEBUG_INTERRUPT
1509 (void)bind_virq_to_irqhandler(
1510 VIRQ_DEBUG,
1511 0,
1512 netif_be_dbg,
1513 SA_SHIRQ,
1514 "net-be-dbg",
1515 &netif_be_dbg);
1516 #endif
1518 return 0;
1521 module_init(netback_init);
1523 MODULE_LICENSE("Dual BSD/GPL");