ia64/xen-unstable

view linux-2.6-xen-sparse/drivers/xen/netback/netback.c @ 12276:ac2097d71e06

[NET] back: Fix wrap to zero in transmit credit scheduler.
This could happen when credit_bytes == ~0UL (i.e., scheduling is 'disabled').
Signed-off-by: Kirk Allan <kallan@novell.com>
author kfraser@localhost.localdomain
date Tue Nov 07 09:48:19 2006 +0000 (2006-11-07)
parents 637eace6d5c6
children ba84d697b1ea
line source
1 /******************************************************************************
2 * drivers/xen/netback/netback.c
3 *
4 * Back-end of the driver for virtual network devices. This portion of the
5 * driver exports a 'unified' network-device interface that can be accessed
6 * by any operating system that implements a compatible front end. A
7 * reference front-end implementation can be found in:
8 * drivers/xen/netfront/netfront.c
9 *
10 * Copyright (c) 2002-2005, K A Fraser
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
17 *
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 * IN THE SOFTWARE.
35 */
37 #include "common.h"
38 #include <xen/balloon.h>
39 #include <xen/interface/memory.h>
41 /*#define NETBE_DEBUG_INTERRUPT*/
43 struct netbk_rx_meta {
44 skb_frag_t frag;
45 int id;
46 int copy:1;
47 };
49 static void netif_idx_release(u16 pending_idx);
50 static void netif_page_release(struct page *page);
51 static void make_tx_response(netif_t *netif,
52 netif_tx_request_t *txp,
53 s8 st);
54 static netif_rx_response_t *make_rx_response(netif_t *netif,
55 u16 id,
56 s8 st,
57 u16 offset,
58 u16 size,
59 u16 flags);
61 static void net_tx_action(unsigned long unused);
62 static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0);
64 static void net_rx_action(unsigned long unused);
65 static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0);
67 static struct timer_list net_timer;
69 #define MAX_PENDING_REQS 256
71 static struct sk_buff_head rx_queue;
73 static struct page **mmap_pages;
74 static inline unsigned long idx_to_kaddr(unsigned int idx)
75 {
76 return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx]));
77 }
79 #define PKT_PROT_LEN 64
81 static struct pending_tx_info {
82 netif_tx_request_t req;
83 netif_t *netif;
84 } pending_tx_info[MAX_PENDING_REQS];
85 static u16 pending_ring[MAX_PENDING_REQS];
86 typedef unsigned int PEND_RING_IDX;
87 #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1))
88 static PEND_RING_IDX pending_prod, pending_cons;
89 #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons)
91 /* Freed TX SKBs get batched on this ring before return to pending_ring. */
92 static u16 dealloc_ring[MAX_PENDING_REQS];
93 static PEND_RING_IDX dealloc_prod, dealloc_cons;
95 static struct sk_buff_head tx_queue;
97 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
98 static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS];
99 static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS];
101 static struct list_head net_schedule_list;
102 static spinlock_t net_schedule_list_lock;
104 #define MAX_MFN_ALLOC 64
105 static unsigned long mfn_list[MAX_MFN_ALLOC];
106 static unsigned int alloc_index = 0;
108 static inline unsigned long alloc_mfn(void)
109 {
110 return mfn_list[--alloc_index];
111 }
113 static int check_mfn(int nr)
114 {
115 struct xen_memory_reservation reservation = {
116 .extent_order = 0,
117 .domid = DOMID_SELF
118 };
120 if (likely(alloc_index >= nr))
121 return 0;
123 set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index);
124 reservation.nr_extents = MAX_MFN_ALLOC - alloc_index;
125 alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation,
126 &reservation);
128 return alloc_index >= nr ? 0 : -ENOMEM;
129 }
131 static inline void maybe_schedule_tx_action(void)
132 {
133 smp_mb();
134 if ((NR_PENDING_REQS < (MAX_PENDING_REQS/2)) &&
135 !list_empty(&net_schedule_list))
136 tasklet_schedule(&net_tx_tasklet);
137 }
139 /*
140 * A gross way of confirming the origin of an skb data page. The slab
141 * allocator abuses a field in the page struct to cache the kmem_cache_t ptr.
142 */
143 static inline int is_xen_skb(struct sk_buff *skb)
144 {
145 extern kmem_cache_t *skbuff_cachep;
146 kmem_cache_t *cp = (kmem_cache_t *)virt_to_page(skb->head)->lru.next;
147 return (cp == skbuff_cachep);
148 }
150 /*
151 * We can flip without copying the packet unless:
152 * 1. The data is not allocated from our special cache; or
153 * 2. The main data area is shared; or
154 * 3. One or more fragments are shared; or
155 * 4. There are chained fragments.
156 */
157 static inline int is_flippable_skb(struct sk_buff *skb)
158 {
159 int frag;
161 if (!is_xen_skb(skb) || skb_cloned(skb))
162 return 0;
164 for (frag = 0; frag < skb_shinfo(skb)->nr_frags; frag++) {
165 if (page_count(skb_shinfo(skb)->frags[frag].page) > 1)
166 return 0;
167 }
169 if (skb_shinfo(skb)->frag_list != NULL)
170 return 0;
172 return 1;
173 }
175 static struct sk_buff *netbk_copy_skb(struct sk_buff *skb)
176 {
177 struct skb_shared_info *ninfo;
178 struct sk_buff *nskb;
179 unsigned long offset;
180 int ret;
181 int len;
182 int headlen;
184 BUG_ON(skb_shinfo(skb)->frag_list != NULL);
186 nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC);
187 if (unlikely(!nskb))
188 goto err;
190 skb_reserve(nskb, 16 + NET_IP_ALIGN);
191 headlen = nskb->end - nskb->data;
192 if (headlen > skb_headlen(skb))
193 headlen = skb_headlen(skb);
194 ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen);
195 BUG_ON(ret);
197 ninfo = skb_shinfo(nskb);
198 ninfo->gso_size = skb_shinfo(skb)->gso_size;
199 ninfo->gso_type = skb_shinfo(skb)->gso_type;
201 offset = headlen;
202 len = skb->len - headlen;
204 nskb->len = skb->len;
205 nskb->data_len = len;
206 nskb->truesize += len;
208 while (len) {
209 struct page *page;
210 int copy;
211 int zero;
213 if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) {
214 dump_stack();
215 goto err_free;
216 }
218 copy = len >= PAGE_SIZE ? PAGE_SIZE : len;
219 zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO;
221 page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | zero);
222 if (unlikely(!page))
223 goto err_free;
225 ret = skb_copy_bits(skb, offset, page_address(page), copy);
226 BUG_ON(ret);
228 ninfo->frags[ninfo->nr_frags].page = page;
229 ninfo->frags[ninfo->nr_frags].page_offset = 0;
230 ninfo->frags[ninfo->nr_frags].size = copy;
231 ninfo->nr_frags++;
233 offset += copy;
234 len -= copy;
235 }
237 offset = nskb->data - skb->data;
239 nskb->h.raw = skb->h.raw + offset;
240 nskb->nh.raw = skb->nh.raw + offset;
241 nskb->mac.raw = skb->mac.raw + offset;
243 return nskb;
245 err_free:
246 kfree_skb(nskb);
247 err:
248 return NULL;
249 }
251 static inline int netbk_max_required_rx_slots(netif_t *netif)
252 {
253 if (netif->features & (NETIF_F_SG|NETIF_F_TSO))
254 return MAX_SKB_FRAGS + 2; /* header + extra_info + frags */
255 return 1; /* all in one */
256 }
258 static inline int netbk_queue_full(netif_t *netif)
259 {
260 RING_IDX peek = netif->rx_req_cons_peek;
261 RING_IDX needed = netbk_max_required_rx_slots(netif);
263 return ((netif->rx.sring->req_prod - peek) < needed) ||
264 ((netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek) < needed);
265 }
267 static void tx_queue_callback(unsigned long data)
268 {
269 netif_t *netif = (netif_t *)data;
270 if (netif_schedulable(netif->dev))
271 netif_wake_queue(netif->dev);
272 }
274 int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev)
275 {
276 netif_t *netif = netdev_priv(dev);
278 BUG_ON(skb->dev != dev);
280 /* Drop the packet if the target domain has no receive buffers. */
281 if (unlikely(!netif_schedulable(dev) || netbk_queue_full(netif)))
282 goto drop;
284 /*
285 * Copy the packet here if it's destined for a flipping interface
286 * but isn't flippable (e.g. extra references to data).
287 */
288 if (!netif->copying_receiver && !is_flippable_skb(skb)) {
289 struct sk_buff *nskb = netbk_copy_skb(skb);
290 if ( unlikely(nskb == NULL) )
291 goto drop;
292 /* Copy only the header fields we use in this driver. */
293 nskb->dev = skb->dev;
294 nskb->ip_summed = skb->ip_summed;
295 nskb->proto_data_valid = skb->proto_data_valid;
296 dev_kfree_skb(skb);
297 skb = nskb;
298 }
300 netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1 +
301 !!skb_shinfo(skb)->gso_size;
302 netif_get(netif);
304 if (netbk_can_queue(dev) && netbk_queue_full(netif)) {
305 netif->rx.sring->req_event = netif->rx_req_cons_peek +
306 netbk_max_required_rx_slots(netif);
307 mb(); /* request notification /then/ check & stop the queue */
308 if (netbk_queue_full(netif)) {
309 netif_stop_queue(dev);
310 /*
311 * Schedule 500ms timeout to restart the queue, thus
312 * ensuring that an inactive queue will be drained.
313 * Packets will be immediately be dropped until more
314 * receive buffers become available (see
315 * netbk_queue_full() check above).
316 */
317 netif->tx_queue_timeout.data = (unsigned long)netif;
318 netif->tx_queue_timeout.function = tx_queue_callback;
319 __mod_timer(&netif->tx_queue_timeout, jiffies + HZ/2);
320 }
321 }
323 skb_queue_tail(&rx_queue, skb);
324 tasklet_schedule(&net_rx_tasklet);
326 return 0;
328 drop:
329 netif->stats.tx_dropped++;
330 dev_kfree_skb(skb);
331 return 0;
332 }
334 #if 0
335 static void xen_network_done_notify(void)
336 {
337 static struct net_device *eth0_dev = NULL;
338 if (unlikely(eth0_dev == NULL))
339 eth0_dev = __dev_get_by_name("eth0");
340 netif_rx_schedule(eth0_dev);
341 }
342 /*
343 * Add following to poll() function in NAPI driver (Tigon3 is example):
344 * if ( xen_network_done() )
345 * tg3_enable_ints(tp);
346 */
347 int xen_network_done(void)
348 {
349 return skb_queue_empty(&rx_queue);
350 }
351 #endif
353 struct netrx_pending_operations {
354 unsigned trans_prod, trans_cons;
355 unsigned mmu_prod, mmu_cons;
356 unsigned mcl_prod, mcl_cons;
357 unsigned copy_prod, copy_cons;
358 unsigned meta_prod, meta_cons;
359 mmu_update_t *mmu;
360 gnttab_transfer_t *trans;
361 gnttab_copy_t *copy;
362 multicall_entry_t *mcl;
363 struct netbk_rx_meta *meta;
364 };
366 /* Set up the grant operations for this fragment. If it's a flipping
367 interface, we also set up the unmap request from here. */
368 static u16 netbk_gop_frag(netif_t *netif, struct netbk_rx_meta *meta,
369 int i, struct netrx_pending_operations *npo,
370 struct page *page, unsigned long size,
371 unsigned long offset)
372 {
373 mmu_update_t *mmu;
374 gnttab_transfer_t *gop;
375 gnttab_copy_t *copy_gop;
376 multicall_entry_t *mcl;
377 netif_rx_request_t *req;
378 unsigned long old_mfn, new_mfn;
380 old_mfn = virt_to_mfn(page_address(page));
382 req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i);
383 if (netif->copying_receiver) {
384 /* The fragment needs to be copied rather than
385 flipped. */
386 meta->copy = 1;
387 copy_gop = npo->copy + npo->copy_prod++;
388 copy_gop->flags = GNTCOPY_dest_gref;
389 if (PageForeign(page)) {
390 struct pending_tx_info *src_pend =
391 &pending_tx_info[page->index];
392 copy_gop->source.domid = src_pend->netif->domid;
393 copy_gop->source.u.ref = src_pend->req.gref;
394 copy_gop->flags |= GNTCOPY_source_gref;
395 } else {
396 copy_gop->source.domid = DOMID_SELF;
397 copy_gop->source.u.gmfn = old_mfn;
398 }
399 copy_gop->source.offset = offset;
400 copy_gop->dest.domid = netif->domid;
401 copy_gop->dest.offset = 0;
402 copy_gop->dest.u.ref = req->gref;
403 copy_gop->len = size;
404 } else {
405 meta->copy = 0;
406 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
407 new_mfn = alloc_mfn();
409 /*
410 * Set the new P2M table entry before
411 * reassigning the old data page. Heed the
412 * comment in pgtable-2level.h:pte_page(). :-)
413 */
414 set_phys_to_machine(page_to_pfn(page), new_mfn);
416 mcl = npo->mcl + npo->mcl_prod++;
417 MULTI_update_va_mapping(mcl,
418 (unsigned long)page_address(page),
419 pfn_pte_ma(new_mfn, PAGE_KERNEL),
420 0);
422 mmu = npo->mmu + npo->mmu_prod++;
423 mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) |
424 MMU_MACHPHYS_UPDATE;
425 mmu->val = page_to_pfn(page);
426 }
428 gop = npo->trans + npo->trans_prod++;
429 gop->mfn = old_mfn;
430 gop->domid = netif->domid;
431 gop->ref = req->gref;
432 }
433 return req->id;
434 }
436 static void netbk_gop_skb(struct sk_buff *skb,
437 struct netrx_pending_operations *npo)
438 {
439 netif_t *netif = netdev_priv(skb->dev);
440 int nr_frags = skb_shinfo(skb)->nr_frags;
441 int i;
442 int extra;
443 struct netbk_rx_meta *head_meta, *meta;
445 head_meta = npo->meta + npo->meta_prod++;
446 head_meta->frag.page_offset = skb_shinfo(skb)->gso_type;
447 head_meta->frag.size = skb_shinfo(skb)->gso_size;
448 extra = !!head_meta->frag.size + 1;
450 for (i = 0; i < nr_frags; i++) {
451 meta = npo->meta + npo->meta_prod++;
452 meta->frag = skb_shinfo(skb)->frags[i];
453 meta->id = netbk_gop_frag(netif, meta, i + extra, npo,
454 meta->frag.page,
455 meta->frag.size,
456 meta->frag.page_offset);
457 }
459 /*
460 * This must occur at the end to ensure that we don't trash
461 * skb_shinfo until we're done.
462 */
463 head_meta->id = netbk_gop_frag(netif, head_meta, 0, npo,
464 virt_to_page(skb->data),
465 skb_headlen(skb),
466 offset_in_page(skb->data));
468 netif->rx.req_cons += nr_frags + extra;
469 }
471 static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta)
472 {
473 int i;
475 for (i = 0; i < nr_frags; i++)
476 put_page(meta[i].frag.page);
477 }
479 /* This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was
480 used to set up the operations on the top of
481 netrx_pending_operations, which have since been done. Check that
482 they didn't give any errors and advance over them. */
483 static int netbk_check_gop(int nr_frags, domid_t domid,
484 struct netrx_pending_operations *npo)
485 {
486 multicall_entry_t *mcl;
487 gnttab_transfer_t *gop;
488 gnttab_copy_t *copy_op;
489 int status = NETIF_RSP_OKAY;
490 int i;
492 for (i = 0; i <= nr_frags; i++) {
493 if (npo->meta[npo->meta_cons + i].copy) {
494 copy_op = npo->copy + npo->copy_cons++;
495 if (copy_op->status != GNTST_okay) {
496 DPRINTK("Bad status %d from copy to DOM%d.\n",
497 gop->status, domid);
498 status = NETIF_RSP_ERROR;
499 }
500 } else {
501 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
502 mcl = npo->mcl + npo->mcl_cons++;
503 /* The update_va_mapping() must not fail. */
504 BUG_ON(mcl->result != 0);
505 }
507 gop = npo->trans + npo->trans_cons++;
508 /* Check the reassignment error code. */
509 if (gop->status != 0) {
510 DPRINTK("Bad status %d from grant transfer to DOM%u\n",
511 gop->status, domid);
512 /*
513 * Page no longer belongs to us unless
514 * GNTST_bad_page, but that should be
515 * a fatal error anyway.
516 */
517 BUG_ON(gop->status == GNTST_bad_page);
518 status = NETIF_RSP_ERROR;
519 }
520 }
521 }
523 return status;
524 }
526 static void netbk_add_frag_responses(netif_t *netif, int status,
527 struct netbk_rx_meta *meta, int nr_frags)
528 {
529 int i;
530 unsigned long offset;
532 for (i = 0; i < nr_frags; i++) {
533 int id = meta[i].id;
534 int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data;
536 if (meta[i].copy)
537 offset = 0;
538 else
539 offset = meta[i].frag.page_offset;
540 make_rx_response(netif, id, status, offset,
541 meta[i].frag.size, flags);
542 }
543 }
545 static void net_rx_action(unsigned long unused)
546 {
547 netif_t *netif = NULL;
548 s8 status;
549 u16 id, irq, flags;
550 netif_rx_response_t *resp;
551 multicall_entry_t *mcl;
552 struct sk_buff_head rxq;
553 struct sk_buff *skb;
554 int notify_nr = 0;
555 int ret;
556 int nr_frags;
557 int count;
558 unsigned long offset;
560 /*
561 * Putting hundreds of bytes on the stack is considered rude.
562 * Static works because a tasklet can only be on one CPU at any time.
563 */
564 static multicall_entry_t rx_mcl[NET_RX_RING_SIZE+3];
565 static mmu_update_t rx_mmu[NET_RX_RING_SIZE];
566 static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE];
567 static gnttab_copy_t grant_copy_op[NET_RX_RING_SIZE];
568 static unsigned char rx_notify[NR_IRQS];
569 static u16 notify_list[NET_RX_RING_SIZE];
570 static struct netbk_rx_meta meta[NET_RX_RING_SIZE];
572 struct netrx_pending_operations npo = {
573 mmu: rx_mmu,
574 trans: grant_trans_op,
575 copy: grant_copy_op,
576 mcl: rx_mcl,
577 meta: meta};
579 skb_queue_head_init(&rxq);
581 count = 0;
583 while ((skb = skb_dequeue(&rx_queue)) != NULL) {
584 nr_frags = skb_shinfo(skb)->nr_frags;
585 *(int *)skb->cb = nr_frags;
587 if (!xen_feature(XENFEAT_auto_translated_physmap) &&
588 check_mfn(nr_frags + 1)) {
589 /* Memory squeeze? Back off for an arbitrary while. */
590 if ( net_ratelimit() )
591 WPRINTK("Memory squeeze in netback "
592 "driver.\n");
593 mod_timer(&net_timer, jiffies + HZ);
594 skb_queue_head(&rx_queue, skb);
595 break;
596 }
598 netbk_gop_skb(skb, &npo);
600 count += nr_frags + 1;
602 __skb_queue_tail(&rxq, skb);
604 /* Filled the batch queue? */
605 if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE)
606 break;
607 }
609 if (npo.mcl_prod &&
610 !xen_feature(XENFEAT_auto_translated_physmap)) {
611 mcl = npo.mcl + npo.mcl_prod++;
613 BUG_ON(mcl[-1].op != __HYPERVISOR_update_va_mapping);
614 mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL;
616 mcl->op = __HYPERVISOR_mmu_update;
617 mcl->args[0] = (unsigned long)rx_mmu;
618 mcl->args[1] = npo.mmu_prod;
619 mcl->args[2] = 0;
620 mcl->args[3] = DOMID_SELF;
621 }
623 if (npo.trans_prod) {
624 mcl = npo.mcl + npo.mcl_prod++;
625 mcl->op = __HYPERVISOR_grant_table_op;
626 mcl->args[0] = GNTTABOP_transfer;
627 mcl->args[1] = (unsigned long)grant_trans_op;
628 mcl->args[2] = npo.trans_prod;
629 }
631 if (npo.copy_prod) {
632 mcl = npo.mcl + npo.mcl_prod++;
633 mcl->op = __HYPERVISOR_grant_table_op;
634 mcl->args[0] = GNTTABOP_copy;
635 mcl->args[1] = (unsigned long)grant_copy_op;
636 mcl->args[2] = npo.copy_prod;
637 }
639 /* Nothing to do? */
640 if (!npo.mcl_prod)
641 return;
643 BUG_ON(npo.copy_prod > NET_RX_RING_SIZE);
644 BUG_ON(npo.mmu_prod > NET_RX_RING_SIZE);
645 BUG_ON(npo.trans_prod > NET_RX_RING_SIZE);
646 BUG_ON(npo.mcl_prod > NET_RX_RING_SIZE+3);
647 BUG_ON(npo.meta_prod > NET_RX_RING_SIZE);
649 ret = HYPERVISOR_multicall(npo.mcl, npo.mcl_prod);
650 BUG_ON(ret != 0);
652 while ((skb = __skb_dequeue(&rxq)) != NULL) {
653 nr_frags = *(int *)skb->cb;
655 netif = netdev_priv(skb->dev);
656 /* We can't rely on skb_release_data to release the
657 pages used by fragments for us, since it tries to
658 touch the pages in the fraglist. If we're in
659 flipping mode, that doesn't work. In copying mode,
660 we still have access to all of the pages, and so
661 it's safe to let release_data deal with it. */
662 /* (Freeing the fragments is safe since we copy
663 non-linear skbs destined for flipping interfaces) */
664 if (!netif->copying_receiver) {
665 atomic_set(&(skb_shinfo(skb)->dataref), 1);
666 skb_shinfo(skb)->frag_list = NULL;
667 skb_shinfo(skb)->nr_frags = 0;
668 netbk_free_pages(nr_frags, meta + npo.meta_cons + 1);
669 }
671 netif->stats.tx_bytes += skb->len;
672 netif->stats.tx_packets++;
674 status = netbk_check_gop(nr_frags, netif->domid, &npo);
676 id = meta[npo.meta_cons].id;
677 flags = nr_frags ? NETRXF_more_data : 0;
679 if (skb->ip_summed == CHECKSUM_HW) /* local packet? */
680 flags |= NETRXF_csum_blank | NETRXF_data_validated;
681 else if (skb->proto_data_valid) /* remote but checksummed? */
682 flags |= NETRXF_data_validated;
684 if (meta[npo.meta_cons].copy)
685 offset = 0;
686 else
687 offset = offset_in_page(skb->data);
688 resp = make_rx_response(netif, id, status, offset,
689 skb_headlen(skb), flags);
691 if (meta[npo.meta_cons].frag.size) {
692 struct netif_extra_info *gso =
693 (struct netif_extra_info *)
694 RING_GET_RESPONSE(&netif->rx,
695 netif->rx.rsp_prod_pvt++);
697 resp->flags |= NETRXF_extra_info;
699 gso->u.gso.size = meta[npo.meta_cons].frag.size;
700 gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
701 gso->u.gso.pad = 0;
702 gso->u.gso.features = 0;
704 gso->type = XEN_NETIF_EXTRA_TYPE_GSO;
705 gso->flags = 0;
706 }
708 netbk_add_frag_responses(netif, status,
709 meta + npo.meta_cons + 1,
710 nr_frags);
712 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret);
713 irq = netif->irq;
714 if (ret && !rx_notify[irq]) {
715 rx_notify[irq] = 1;
716 notify_list[notify_nr++] = irq;
717 }
719 if (netif_queue_stopped(netif->dev) &&
720 netif_schedulable(netif->dev) &&
721 !netbk_queue_full(netif))
722 netif_wake_queue(netif->dev);
724 netif_put(netif);
725 dev_kfree_skb(skb);
726 npo.meta_cons += nr_frags + 1;
727 }
729 while (notify_nr != 0) {
730 irq = notify_list[--notify_nr];
731 rx_notify[irq] = 0;
732 notify_remote_via_irq(irq);
733 }
735 /* More work to do? */
736 if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer))
737 tasklet_schedule(&net_rx_tasklet);
738 #if 0
739 else
740 xen_network_done_notify();
741 #endif
742 }
744 static void net_alarm(unsigned long unused)
745 {
746 tasklet_schedule(&net_rx_tasklet);
747 }
749 struct net_device_stats *netif_be_get_stats(struct net_device *dev)
750 {
751 netif_t *netif = netdev_priv(dev);
752 return &netif->stats;
753 }
755 static int __on_net_schedule_list(netif_t *netif)
756 {
757 return netif->list.next != NULL;
758 }
760 static void remove_from_net_schedule_list(netif_t *netif)
761 {
762 spin_lock_irq(&net_schedule_list_lock);
763 if (likely(__on_net_schedule_list(netif))) {
764 list_del(&netif->list);
765 netif->list.next = NULL;
766 netif_put(netif);
767 }
768 spin_unlock_irq(&net_schedule_list_lock);
769 }
771 static void add_to_net_schedule_list_tail(netif_t *netif)
772 {
773 if (__on_net_schedule_list(netif))
774 return;
776 spin_lock_irq(&net_schedule_list_lock);
777 if (!__on_net_schedule_list(netif) &&
778 likely(netif_schedulable(netif->dev))) {
779 list_add_tail(&netif->list, &net_schedule_list);
780 netif_get(netif);
781 }
782 spin_unlock_irq(&net_schedule_list_lock);
783 }
785 /*
786 * Note on CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER:
787 * If this driver is pipelining transmit requests then we can be very
788 * aggressive in avoiding new-packet notifications -- frontend only needs to
789 * send a notification if there are no outstanding unreceived responses.
790 * If we may be buffer transmit buffers for any reason then we must be rather
791 * more conservative and treat this as the final check for pending work.
792 */
793 void netif_schedule_work(netif_t *netif)
794 {
795 int more_to_do;
797 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
798 more_to_do = RING_HAS_UNCONSUMED_REQUESTS(&netif->tx);
799 #else
800 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
801 #endif
803 if (more_to_do) {
804 add_to_net_schedule_list_tail(netif);
805 maybe_schedule_tx_action();
806 }
807 }
809 void netif_deschedule_work(netif_t *netif)
810 {
811 remove_from_net_schedule_list(netif);
812 }
815 static void tx_add_credit(netif_t *netif)
816 {
817 unsigned long max_burst, max_credit;
819 /*
820 * Allow a burst big enough to transmit a jumbo packet of up to 128kB.
821 * Otherwise the interface can seize up due to insufficient credit.
822 */
823 max_burst = RING_GET_REQUEST(&netif->tx, netif->tx.req_cons)->size;
824 max_burst = min(max_burst, 131072UL);
825 max_burst = max(max_burst, netif->credit_bytes);
827 /* Take care that adding a new chunk of credit doesn't wrap to zero. */
828 max_credit = max(netif->remaining_credit + netif->credit_bytes, ~0UL);
830 netif->remaining_credit = min(max_credit, max_burst);
831 }
833 static void tx_credit_callback(unsigned long data)
834 {
835 netif_t *netif = (netif_t *)data;
836 tx_add_credit(netif);
837 netif_schedule_work(netif);
838 }
840 inline static void net_tx_action_dealloc(void)
841 {
842 gnttab_unmap_grant_ref_t *gop;
843 u16 pending_idx;
844 PEND_RING_IDX dc, dp;
845 netif_t *netif;
846 int ret;
848 dc = dealloc_cons;
849 dp = dealloc_prod;
851 /* Ensure we see all indexes enqueued by netif_idx_release(). */
852 smp_rmb();
854 /*
855 * Free up any grants we have finished using
856 */
857 gop = tx_unmap_ops;
858 while (dc != dp) {
859 pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)];
860 gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx),
861 GNTMAP_host_map,
862 grant_tx_handle[pending_idx]);
863 gop++;
864 }
865 ret = HYPERVISOR_grant_table_op(
866 GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops);
867 BUG_ON(ret);
869 while (dealloc_cons != dp) {
870 pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)];
872 netif = pending_tx_info[pending_idx].netif;
874 make_tx_response(netif, &pending_tx_info[pending_idx].req,
875 NETIF_RSP_OKAY);
877 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
879 netif_put(netif);
880 }
881 }
883 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
884 {
885 RING_IDX cons = netif->tx.req_cons;
887 do {
888 make_tx_response(netif, txp, NETIF_RSP_ERROR);
889 if (cons >= end)
890 break;
891 txp = RING_GET_REQUEST(&netif->tx, cons++);
892 } while (1);
893 netif->tx.req_cons = cons;
894 netif_schedule_work(netif);
895 netif_put(netif);
896 }
898 static int netbk_count_requests(netif_t *netif, netif_tx_request_t *first,
899 netif_tx_request_t *txp, int work_to_do)
900 {
901 RING_IDX cons = netif->tx.req_cons;
902 int frags = 0;
904 if (!(first->flags & NETTXF_more_data))
905 return 0;
907 do {
908 if (frags >= work_to_do) {
909 DPRINTK("Need more frags\n");
910 return -frags;
911 }
913 if (unlikely(frags >= MAX_SKB_FRAGS)) {
914 DPRINTK("Too many frags\n");
915 return -frags;
916 }
918 memcpy(txp, RING_GET_REQUEST(&netif->tx, cons + frags),
919 sizeof(*txp));
920 if (txp->size > first->size) {
921 DPRINTK("Frags galore\n");
922 return -frags;
923 }
925 first->size -= txp->size;
926 frags++;
928 if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) {
929 DPRINTK("txp->offset: %x, size: %u\n",
930 txp->offset, txp->size);
931 return -frags;
932 }
933 } while ((txp++)->flags & NETTXF_more_data);
935 return frags;
936 }
938 static gnttab_map_grant_ref_t *netbk_get_requests(netif_t *netif,
939 struct sk_buff *skb,
940 netif_tx_request_t *txp,
941 gnttab_map_grant_ref_t *mop)
942 {
943 struct skb_shared_info *shinfo = skb_shinfo(skb);
944 skb_frag_t *frags = shinfo->frags;
945 unsigned long pending_idx = *((u16 *)skb->data);
946 int i, start;
948 /* Skip first skb fragment if it is on same page as header fragment. */
949 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
951 for (i = start; i < shinfo->nr_frags; i++, txp++) {
952 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons++)];
954 gnttab_set_map_op(mop++, idx_to_kaddr(pending_idx),
955 GNTMAP_host_map | GNTMAP_readonly,
956 txp->gref, netif->domid);
958 memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp));
959 netif_get(netif);
960 pending_tx_info[pending_idx].netif = netif;
961 frags[i].page = (void *)pending_idx;
962 }
964 return mop;
965 }
967 static int netbk_tx_check_mop(struct sk_buff *skb,
968 gnttab_map_grant_ref_t **mopp)
969 {
970 gnttab_map_grant_ref_t *mop = *mopp;
971 int pending_idx = *((u16 *)skb->data);
972 netif_t *netif = pending_tx_info[pending_idx].netif;
973 netif_tx_request_t *txp;
974 struct skb_shared_info *shinfo = skb_shinfo(skb);
975 int nr_frags = shinfo->nr_frags;
976 int i, err, start;
978 /* Check status of header. */
979 err = mop->status;
980 if (unlikely(err)) {
981 txp = &pending_tx_info[pending_idx].req;
982 make_tx_response(netif, txp, NETIF_RSP_ERROR);
983 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
984 netif_put(netif);
985 } else {
986 set_phys_to_machine(
987 __pa(idx_to_kaddr(pending_idx)) >> PAGE_SHIFT,
988 FOREIGN_FRAME(mop->dev_bus_addr >> PAGE_SHIFT));
989 grant_tx_handle[pending_idx] = mop->handle;
990 }
992 /* Skip first skb fragment if it is on same page as header fragment. */
993 start = ((unsigned long)shinfo->frags[0].page == pending_idx);
995 for (i = start; i < nr_frags; i++) {
996 int j, newerr;
998 pending_idx = (unsigned long)shinfo->frags[i].page;
1000 /* Check error status: if okay then remember grant handle. */
1001 newerr = (++mop)->status;
1002 if (likely(!newerr)) {
1003 set_phys_to_machine(
1004 __pa(idx_to_kaddr(pending_idx))>>PAGE_SHIFT,
1005 FOREIGN_FRAME(mop->dev_bus_addr>>PAGE_SHIFT));
1006 grant_tx_handle[pending_idx] = mop->handle;
1007 /* Had a previous error? Invalidate this fragment. */
1008 if (unlikely(err))
1009 netif_idx_release(pending_idx);
1010 continue;
1013 /* Error on this fragment: respond to client with an error. */
1014 txp = &pending_tx_info[pending_idx].req;
1015 make_tx_response(netif, txp, NETIF_RSP_ERROR);
1016 pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
1017 netif_put(netif);
1019 /* Not the first error? Preceding frags already invalidated. */
1020 if (err)
1021 continue;
1023 /* First error: invalidate header and preceding fragments. */
1024 pending_idx = *((u16 *)skb->data);
1025 netif_idx_release(pending_idx);
1026 for (j = start; j < i; j++) {
1027 pending_idx = (unsigned long)shinfo->frags[i].page;
1028 netif_idx_release(pending_idx);
1031 /* Remember the error: invalidate all subsequent fragments. */
1032 err = newerr;
1035 *mopp = mop + 1;
1036 return err;
1039 static void netbk_fill_frags(struct sk_buff *skb)
1041 struct skb_shared_info *shinfo = skb_shinfo(skb);
1042 int nr_frags = shinfo->nr_frags;
1043 int i;
1045 for (i = 0; i < nr_frags; i++) {
1046 skb_frag_t *frag = shinfo->frags + i;
1047 netif_tx_request_t *txp;
1048 unsigned long pending_idx;
1050 pending_idx = (unsigned long)frag->page;
1051 txp = &pending_tx_info[pending_idx].req;
1052 frag->page = virt_to_page(idx_to_kaddr(pending_idx));
1053 frag->size = txp->size;
1054 frag->page_offset = txp->offset;
1056 skb->len += txp->size;
1057 skb->data_len += txp->size;
1058 skb->truesize += txp->size;
1062 int netbk_get_extras(netif_t *netif, struct netif_extra_info *extras,
1063 int work_to_do)
1065 struct netif_extra_info extra;
1066 RING_IDX cons = netif->tx.req_cons;
1068 do {
1069 if (unlikely(work_to_do-- <= 0)) {
1070 DPRINTK("Missing extra info\n");
1071 return -EBADR;
1074 memcpy(&extra, RING_GET_REQUEST(&netif->tx, cons),
1075 sizeof(extra));
1076 if (unlikely(!extra.type ||
1077 extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) {
1078 netif->tx.req_cons = ++cons;
1079 DPRINTK("Invalid extra type: %d\n", extra.type);
1080 return -EINVAL;
1083 memcpy(&extras[extra.type - 1], &extra, sizeof(extra));
1084 netif->tx.req_cons = ++cons;
1085 } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE);
1087 return work_to_do;
1090 static int netbk_set_skb_gso(struct sk_buff *skb, struct netif_extra_info *gso)
1092 if (!gso->u.gso.size) {
1093 DPRINTK("GSO size must not be zero.\n");
1094 return -EINVAL;
1097 /* Currently only TCPv4 S.O. is supported. */
1098 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) {
1099 DPRINTK("Bad GSO type %d.\n", gso->u.gso.type);
1100 return -EINVAL;
1103 skb_shinfo(skb)->gso_size = gso->u.gso.size;
1104 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1106 /* Header must be checked, and gso_segs computed. */
1107 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1108 skb_shinfo(skb)->gso_segs = 0;
1110 return 0;
1113 /* Called after netfront has transmitted */
1114 static void net_tx_action(unsigned long unused)
1116 struct list_head *ent;
1117 struct sk_buff *skb;
1118 netif_t *netif;
1119 netif_tx_request_t txreq;
1120 netif_tx_request_t txfrags[MAX_SKB_FRAGS];
1121 struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
1122 u16 pending_idx;
1123 RING_IDX i;
1124 gnttab_map_grant_ref_t *mop;
1125 unsigned int data_len;
1126 int ret, work_to_do;
1128 if (dealloc_cons != dealloc_prod)
1129 net_tx_action_dealloc();
1131 mop = tx_map_ops;
1132 while (((NR_PENDING_REQS + MAX_SKB_FRAGS) < MAX_PENDING_REQS) &&
1133 !list_empty(&net_schedule_list)) {
1134 /* Get a netif from the list with work to do. */
1135 ent = net_schedule_list.next;
1136 netif = list_entry(ent, netif_t, list);
1137 netif_get(netif);
1138 remove_from_net_schedule_list(netif);
1140 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do);
1141 if (!work_to_do) {
1142 netif_put(netif);
1143 continue;
1146 i = netif->tx.req_cons;
1147 rmb(); /* Ensure that we see the request before we copy it. */
1148 memcpy(&txreq, RING_GET_REQUEST(&netif->tx, i), sizeof(txreq));
1150 /* Credit-based scheduling. */
1151 if (txreq.size > netif->remaining_credit) {
1152 unsigned long now = jiffies;
1153 unsigned long next_credit =
1154 netif->credit_timeout.expires +
1155 msecs_to_jiffies(netif->credit_usec / 1000);
1157 /* Timer could already be pending in rare cases. */
1158 if (timer_pending(&netif->credit_timeout)) {
1159 netif_put(netif);
1160 continue;
1163 /* Passed the point where we can replenish credit? */
1164 if (time_after_eq(now, next_credit)) {
1165 netif->credit_timeout.expires = now;
1166 tx_add_credit(netif);
1169 /* Still too big to send right now? Set a callback. */
1170 if (txreq.size > netif->remaining_credit) {
1171 netif->credit_timeout.data =
1172 (unsigned long)netif;
1173 netif->credit_timeout.function =
1174 tx_credit_callback;
1175 __mod_timer(&netif->credit_timeout,
1176 next_credit);
1177 netif_put(netif);
1178 continue;
1181 netif->remaining_credit -= txreq.size;
1183 work_to_do--;
1184 netif->tx.req_cons = ++i;
1186 memset(extras, 0, sizeof(extras));
1187 if (txreq.flags & NETTXF_extra_info) {
1188 work_to_do = netbk_get_extras(netif, extras,
1189 work_to_do);
1190 i = netif->tx.req_cons;
1191 if (unlikely(work_to_do < 0)) {
1192 netbk_tx_err(netif, &txreq, i);
1193 continue;
1197 ret = netbk_count_requests(netif, &txreq, txfrags, work_to_do);
1198 if (unlikely(ret < 0)) {
1199 netbk_tx_err(netif, &txreq, i - ret);
1200 continue;
1202 i += ret;
1204 if (unlikely(txreq.size < ETH_HLEN)) {
1205 DPRINTK("Bad packet size: %d\n", txreq.size);
1206 netbk_tx_err(netif, &txreq, i);
1207 continue;
1210 /* No crossing a page as the payload mustn't fragment. */
1211 if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) {
1212 DPRINTK("txreq.offset: %x, size: %u, end: %lu\n",
1213 txreq.offset, txreq.size,
1214 (txreq.offset &~PAGE_MASK) + txreq.size);
1215 netbk_tx_err(netif, &txreq, i);
1216 continue;
1219 pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)];
1221 data_len = (txreq.size > PKT_PROT_LEN &&
1222 ret < MAX_SKB_FRAGS) ?
1223 PKT_PROT_LEN : txreq.size;
1225 skb = alloc_skb(data_len + 16 + NET_IP_ALIGN, GFP_ATOMIC);
1226 if (unlikely(skb == NULL)) {
1227 DPRINTK("Can't allocate a skb in start_xmit.\n");
1228 netbk_tx_err(netif, &txreq, i);
1229 break;
1232 /* Packets passed to netif_rx() must have some headroom. */
1233 skb_reserve(skb, 16 + NET_IP_ALIGN);
1235 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) {
1236 struct netif_extra_info *gso;
1237 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1];
1239 if (netbk_set_skb_gso(skb, gso)) {
1240 kfree_skb(skb);
1241 netbk_tx_err(netif, &txreq, i);
1242 continue;
1246 gnttab_set_map_op(mop, idx_to_kaddr(pending_idx),
1247 GNTMAP_host_map | GNTMAP_readonly,
1248 txreq.gref, netif->domid);
1249 mop++;
1251 memcpy(&pending_tx_info[pending_idx].req,
1252 &txreq, sizeof(txreq));
1253 pending_tx_info[pending_idx].netif = netif;
1254 *((u16 *)skb->data) = pending_idx;
1256 __skb_put(skb, data_len);
1258 skb_shinfo(skb)->nr_frags = ret;
1259 if (data_len < txreq.size) {
1260 skb_shinfo(skb)->nr_frags++;
1261 skb_shinfo(skb)->frags[0].page =
1262 (void *)(unsigned long)pending_idx;
1263 } else {
1264 /* Discriminate from any valid pending_idx value. */
1265 skb_shinfo(skb)->frags[0].page = (void *)~0UL;
1268 __skb_queue_tail(&tx_queue, skb);
1270 pending_cons++;
1272 mop = netbk_get_requests(netif, skb, txfrags, mop);
1274 netif->tx.req_cons = i;
1275 netif_schedule_work(netif);
1277 if ((mop - tx_map_ops) >= ARRAY_SIZE(tx_map_ops))
1278 break;
1281 if (mop == tx_map_ops)
1282 return;
1284 ret = HYPERVISOR_grant_table_op(
1285 GNTTABOP_map_grant_ref, tx_map_ops, mop - tx_map_ops);
1286 BUG_ON(ret);
1288 mop = tx_map_ops;
1289 while ((skb = __skb_dequeue(&tx_queue)) != NULL) {
1290 netif_tx_request_t *txp;
1292 pending_idx = *((u16 *)skb->data);
1293 netif = pending_tx_info[pending_idx].netif;
1294 txp = &pending_tx_info[pending_idx].req;
1296 /* Check the remap error code. */
1297 if (unlikely(netbk_tx_check_mop(skb, &mop))) {
1298 printk(KERN_ALERT "#### netback grant fails\n");
1299 skb_shinfo(skb)->nr_frags = 0;
1300 kfree_skb(skb);
1301 continue;
1304 data_len = skb->len;
1305 memcpy(skb->data,
1306 (void *)(idx_to_kaddr(pending_idx)|txp->offset),
1307 data_len);
1308 if (data_len < txp->size) {
1309 /* Append the packet payload as a fragment. */
1310 txp->offset += data_len;
1311 txp->size -= data_len;
1312 } else {
1313 /* Schedule a response immediately. */
1314 netif_idx_release(pending_idx);
1317 /*
1318 * Old frontends do not assert data_validated but we
1319 * can infer it from csum_blank so test both flags.
1320 */
1321 if (txp->flags & (NETTXF_data_validated|NETTXF_csum_blank)) {
1322 skb->ip_summed = CHECKSUM_UNNECESSARY;
1323 skb->proto_data_valid = 1;
1324 } else {
1325 skb->ip_summed = CHECKSUM_NONE;
1326 skb->proto_data_valid = 0;
1328 skb->proto_csum_blank = !!(txp->flags & NETTXF_csum_blank);
1330 netbk_fill_frags(skb);
1332 skb->dev = netif->dev;
1333 skb->protocol = eth_type_trans(skb, skb->dev);
1335 netif->stats.rx_bytes += skb->len;
1336 netif->stats.rx_packets++;
1338 netif_rx(skb);
1339 netif->dev->last_rx = jiffies;
1343 static void netif_idx_release(u16 pending_idx)
1345 static DEFINE_SPINLOCK(_lock);
1346 unsigned long flags;
1348 spin_lock_irqsave(&_lock, flags);
1349 dealloc_ring[MASK_PEND_IDX(dealloc_prod)] = pending_idx;
1350 /* Sync with net_tx_action_dealloc: insert idx /then/ incr producer. */
1351 smp_wmb();
1352 dealloc_prod++;
1353 spin_unlock_irqrestore(&_lock, flags);
1355 tasklet_schedule(&net_tx_tasklet);
1358 static void netif_page_release(struct page *page)
1360 /* Ready for next use. */
1361 set_page_count(page, 1);
1363 netif_idx_release(page->index);
1366 irqreturn_t netif_be_int(int irq, void *dev_id, struct pt_regs *regs)
1368 netif_t *netif = dev_id;
1370 add_to_net_schedule_list_tail(netif);
1371 maybe_schedule_tx_action();
1373 if (netif_schedulable(netif->dev) && !netbk_queue_full(netif))
1374 netif_wake_queue(netif->dev);
1376 return IRQ_HANDLED;
1379 static void make_tx_response(netif_t *netif,
1380 netif_tx_request_t *txp,
1381 s8 st)
1383 RING_IDX i = netif->tx.rsp_prod_pvt;
1384 netif_tx_response_t *resp;
1385 int notify;
1387 resp = RING_GET_RESPONSE(&netif->tx, i);
1388 resp->id = txp->id;
1389 resp->status = st;
1391 if (txp->flags & NETTXF_extra_info)
1392 RING_GET_RESPONSE(&netif->tx, ++i)->status = NETIF_RSP_NULL;
1394 netif->tx.rsp_prod_pvt = ++i;
1395 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify);
1396 if (notify)
1397 notify_remote_via_irq(netif->irq);
1399 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
1400 if (i == netif->tx.req_cons) {
1401 int more_to_do;
1402 RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do);
1403 if (more_to_do)
1404 add_to_net_schedule_list_tail(netif);
1406 #endif
1409 static netif_rx_response_t *make_rx_response(netif_t *netif,
1410 u16 id,
1411 s8 st,
1412 u16 offset,
1413 u16 size,
1414 u16 flags)
1416 RING_IDX i = netif->rx.rsp_prod_pvt;
1417 netif_rx_response_t *resp;
1419 resp = RING_GET_RESPONSE(&netif->rx, i);
1420 resp->offset = offset;
1421 resp->flags = flags;
1422 resp->id = id;
1423 resp->status = (s16)size;
1424 if (st < 0)
1425 resp->status = (s16)st;
1427 netif->rx.rsp_prod_pvt = ++i;
1429 return resp;
1432 #ifdef NETBE_DEBUG_INTERRUPT
1433 static irqreturn_t netif_be_dbg(int irq, void *dev_id, struct pt_regs *regs)
1435 struct list_head *ent;
1436 netif_t *netif;
1437 int i = 0;
1439 printk(KERN_ALERT "netif_schedule_list:\n");
1440 spin_lock_irq(&net_schedule_list_lock);
1442 list_for_each (ent, &net_schedule_list) {
1443 netif = list_entry(ent, netif_t, list);
1444 printk(KERN_ALERT " %d: private(rx_req_cons=%08x "
1445 "rx_resp_prod=%08x\n",
1446 i, netif->rx.req_cons, netif->rx.rsp_prod_pvt);
1447 printk(KERN_ALERT " tx_req_cons=%08x tx_resp_prod=%08x)\n",
1448 netif->tx.req_cons, netif->tx.rsp_prod_pvt);
1449 printk(KERN_ALERT " shared(rx_req_prod=%08x "
1450 "rx_resp_prod=%08x\n",
1451 netif->rx.sring->req_prod, netif->rx.sring->rsp_prod);
1452 printk(KERN_ALERT " rx_event=%08x tx_req_prod=%08x\n",
1453 netif->rx.sring->rsp_event, netif->tx.sring->req_prod);
1454 printk(KERN_ALERT " tx_resp_prod=%08x, tx_event=%08x)\n",
1455 netif->tx.sring->rsp_prod, netif->tx.sring->rsp_event);
1456 i++;
1459 spin_unlock_irq(&net_schedule_list_lock);
1460 printk(KERN_ALERT " ** End of netif_schedule_list **\n");
1462 return IRQ_HANDLED;
1464 #endif
1466 static int __init netback_init(void)
1468 int i;
1469 struct page *page;
1471 if (!is_running_on_xen())
1472 return -ENODEV;
1474 /* We can increase reservation by this much in net_rx_action(). */
1475 balloon_update_driver_allowance(NET_RX_RING_SIZE);
1477 skb_queue_head_init(&rx_queue);
1478 skb_queue_head_init(&tx_queue);
1480 init_timer(&net_timer);
1481 net_timer.data = 0;
1482 net_timer.function = net_alarm;
1484 mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS);
1485 if (mmap_pages == NULL) {
1486 printk("%s: out of memory\n", __FUNCTION__);
1487 return -ENOMEM;
1490 for (i = 0; i < MAX_PENDING_REQS; i++) {
1491 page = mmap_pages[i];
1492 SetPageForeign(page, netif_page_release);
1493 page->index = i;
1496 pending_cons = 0;
1497 pending_prod = MAX_PENDING_REQS;
1498 for (i = 0; i < MAX_PENDING_REQS; i++)
1499 pending_ring[i] = i;
1501 spin_lock_init(&net_schedule_list_lock);
1502 INIT_LIST_HEAD(&net_schedule_list);
1504 netif_xenbus_init();
1506 #ifdef NETBE_DEBUG_INTERRUPT
1507 (void)bind_virq_to_irqhandler(
1508 VIRQ_DEBUG,
1509 0,
1510 netif_be_dbg,
1511 SA_SHIRQ,
1512 "net-be-dbg",
1513 &netif_be_dbg);
1514 #endif
1516 return 0;
1519 module_init(netback_init);
1521 MODULE_LICENSE("Dual BSD/GPL");