ia64/xen-unstable

view xen/net/skbuff.c @ 922:fc5ae656dd1e

bitkeeper revision 1.582.1.2 (3fafc1a8WtDmHzijhEouIqSThPp1nA)

xen_netwatch.c:
Fixes to xen_netwatch.
author kaf24@scramble.cl.cam.ac.uk
date Mon Nov 10 16:49:44 2003 +0000 (2003-11-10)
parents c66f5e9d6427
children 88ef6048499a
line source
1 /*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
4 * Authors: Alan Cox <iiitac@pyr.swan.ac.uk>
5 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
7 * Version: $Id: skbuff.c,v 1.89 2001/08/06 13:25:02 davem Exp $
8 *
9 * Fixes:
10 * Alan Cox : Fixed the worst of the load balancer bugs.
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 *
23 * NOTE:
24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc).
28 *
29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version.
33 */
35 #include <linux/config.h>
36 #include <linux/lib.h>
37 #include <linux/errno.h>
38 #include <linux/types.h>
39 #include <linux/sched.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/slab.h>
43 #include <linux/netdevice.h>
44 #include <linux/skbuff.h>
45 #include <linux/cache.h>
46 #include <linux/init.h>
48 #include <asm/uaccess.h>
49 #include <asm/system.h>
50 #include <asm/io.h>
52 #define BUG_TRAP ASSERT
54 int sysctl_hot_list_len = 128;
56 static kmem_cache_t *skbuff_head_cache;
58 static union {
59 struct sk_buff_head list;
60 char pad[SMP_CACHE_BYTES];
61 } skb_head_pool[NR_CPUS];
63 /*
64 * Keep out-of-line to prevent kernel bloat.
65 * __builtin_return_address is not used because it is not always
66 * reliable.
67 */
69 /**
70 * skb_over_panic - private function
71 * @skb: buffer
72 * @sz: size
73 * @here: address
74 *
75 * Out of line support code for skb_put(). Not user callable.
76 */
78 void skb_over_panic(struct sk_buff *skb, int sz, void *here)
79 {
80 printk("skput:over: %p:%d put:%d dev:%s",
81 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
82 BUG();
83 }
85 /**
86 * skb_under_panic - private function
87 * @skb: buffer
88 * @sz: size
89 * @here: address
90 *
91 * Out of line support code for skb_push(). Not user callable.
92 */
95 void skb_under_panic(struct sk_buff *skb, int sz, void *here)
96 {
97 printk("skput:under: %p:%d put:%d dev:%s",
98 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
99 BUG();
100 }
102 static __inline__ struct sk_buff *skb_head_from_pool(void)
103 {
104 struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
106 if (skb_queue_len(list)) {
107 struct sk_buff *skb;
108 unsigned long flags;
110 local_irq_save(flags);
111 skb = __skb_dequeue(list);
112 local_irq_restore(flags);
113 return skb;
114 }
115 return NULL;
116 }
118 static __inline__ void skb_head_to_pool(struct sk_buff *skb)
119 {
120 struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
122 if (skb_queue_len(list) < sysctl_hot_list_len) {
123 unsigned long flags;
125 local_irq_save(flags);
126 __skb_queue_head(list, skb);
127 local_irq_restore(flags);
129 return;
130 }
131 kmem_cache_free(skbuff_head_cache, skb);
132 }
134 static inline u8 *alloc_skb_data_page(struct sk_buff *skb)
135 {
136 struct list_head *list_ptr;
137 struct pfn_info *pf;
138 unsigned long flags;
140 spin_lock_irqsave(&free_list_lock, flags);
142 if (!free_pfns) return NULL;
144 list_ptr = free_list.next;
145 pf = list_entry(list_ptr, struct pfn_info, list);
146 pf->flags = 0;
147 list_del(&pf->list);
148 free_pfns--;
150 spin_unlock_irqrestore(&free_list_lock, flags);
152 skb->pf = pf;
153 return (u8 *)((pf - frame_table) << PAGE_SHIFT);
154 }
156 static inline void dealloc_skb_data_page(struct sk_buff *skb)
157 {
158 struct pfn_info *pf;
159 unsigned long flags;
161 pf = skb->pf;
163 spin_lock_irqsave(&free_list_lock, flags);
165 pf->flags = pf->type_count = pf->tot_count = 0;
166 list_add(&pf->list, &free_list);
167 free_pfns++;
169 spin_unlock_irqrestore(&free_list_lock, flags);
171 }
173 static inline void INTERRUPT_CHECK(int gfp_mask)
174 {
175 #if 0 /* Not needed in Xen */
176 if (in_interrupt() && (gfp_mask & __GFP_WAIT)) {
177 printk(KERN_ERR "alloc_skb called nonatomically\n");
178 BUG();
179 }
180 #endif
181 }
184 /**
185 * alloc_skb - allocate a network buffer
186 * @size: size to allocate
187 * @gfp_mask: allocation mask
188 *
189 * Allocate a new &sk_buff. The returned buffer has no headroom and a
190 * tail room of size bytes. The object has a reference count of one.
191 * The return is the buffer. On a failure the return is %NULL.
192 *
193 * Buffers may only be allocated from interrupts using a @gfp_mask of
194 * %GFP_ATOMIC.
195 */
197 struct sk_buff *alloc_skb(unsigned int size,int gfp_mask)
198 {
199 struct sk_buff *skb;
200 u8 *data;
202 INTERRUPT_CHECK(gfp_mask);
204 /* Get the HEAD */
205 skb = skb_head_from_pool();
206 if (skb == NULL) {
207 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
208 if (skb == NULL)
209 goto nohead;
210 }
212 /* Get the DATA. Size must match skb_add_mtu(). */
213 size = SKB_DATA_ALIGN(size);
214 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
215 if (data == NULL)
216 goto nodata;
218 /* Load the data pointers. */
219 skb->head = data;
220 skb->data = data;
221 skb->tail = data;
222 skb->end = data + size;
224 /* Set up other state */
225 skb->len = 0;
226 skb->data_len = 0;
227 skb->skb_type = SKB_NORMAL;
229 skb_shinfo(skb)->nr_frags = 0;
230 return skb;
232 nodata:
233 skb_head_to_pool(skb);
234 nohead:
235 return NULL;
236 }
239 struct sk_buff *alloc_zc_skb(unsigned int size,int gfp_mask)
240 {
241 struct sk_buff *skb;
242 u8 *data;
244 INTERRUPT_CHECK(gfp_mask);
246 /* Get the HEAD */
247 skb = skb_head_from_pool();
248 if (skb == NULL) {
249 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
250 if (skb == NULL)
251 goto nohead;
252 }
254 /* Get the DATA. Size must match skb_add_mtu(). */
255 size = SKB_DATA_ALIGN(size);
256 data = alloc_skb_data_page(skb);
258 if (data == NULL)
259 goto nodata;
261 /* A FAKE virtual address, so that pci_map_xxx dor the right thing. */
262 data = phys_to_virt((unsigned long)data);
264 /* Load the data pointers. */
265 skb->head = data;
266 skb->data = data;
267 skb->tail = data;
268 skb->end = data + size;
270 /* Set up other state */
271 skb->len = 0;
272 skb->data_len = 0;
273 skb->skb_type = SKB_ZERO_COPY;
275 skb_shinfo(skb)->nr_frags = 0;
277 return skb;
279 nodata:
280 skb_head_to_pool(skb);
281 nohead:
282 return NULL;
283 }
286 struct sk_buff *alloc_skb_nodata(int gfp_mask)
287 {
288 struct sk_buff *skb;
290 INTERRUPT_CHECK(gfp_mask);
292 /* Get the HEAD */
293 skb = skb_head_from_pool();
294 if (skb == NULL) {
295 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
296 if (skb == NULL)
297 return NULL;
298 }
300 skb->skb_type = SKB_NODATA;
301 return skb;
302 }
305 /*
306 * Slab constructor for a skb head.
307 */
308 static inline void skb_headerinit(void *p, kmem_cache_t *cache,
309 unsigned long flags)
310 {
311 struct sk_buff *skb = p;
313 skb->src_vif = NULL;
314 skb->dst_vif = NULL;
315 skb->next = NULL;
316 skb->prev = NULL;
317 skb->list = NULL;
318 skb->dev = NULL;
319 skb->pkt_type = PACKET_HOST; /* Default type */
320 skb->ip_summed = 0;
321 skb->destructor = NULL;
322 }
324 static void skb_release_data(struct sk_buff *skb)
325 {
326 if (skb_shinfo(skb)->nr_frags) BUG();
328 switch ( skb->skb_type )
329 {
330 case SKB_NORMAL:
331 kfree(skb->head);
332 break;
333 case SKB_ZERO_COPY:
334 dealloc_skb_data_page(skb);
335 break;
336 case SKB_NODATA:
337 break;
338 default:
339 BUG();
340 }
341 }
343 /*
344 * Free an skbuff by memory without cleaning the state.
345 */
346 void kfree_skbmem(struct sk_buff *skb)
347 {
348 skb_release_data(skb);
349 skb_head_to_pool(skb);
350 }
352 /**
353 * __kfree_skb - private function
354 * @skb: buffer
355 *
356 * Free an sk_buff. Release anything attached to the buffer.
357 * Clean the state. This is an internal helper function. Users should
358 * always call kfree_skb
359 */
361 void __kfree_skb(struct sk_buff *skb)
362 {
363 if ( skb->list )
364 panic(KERN_WARNING "Warning: kfree_skb passed an skb still "
365 "on a list (from %p).\n", NET_CALLER(skb));
367 if ( skb->destructor )
368 skb->destructor(skb);
370 skb_headerinit(skb, NULL, 0); /* clean state */
371 kfree_skbmem(skb);
372 }
374 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
375 {
376 /*
377 * Shift between the two data areas in bytes
378 */
379 unsigned long offset = new->data - old->data;
381 new->list=NULL;
382 new->dev=old->dev;
383 new->protocol=old->protocol;
384 new->h.raw=old->h.raw+offset;
385 new->nh.raw=old->nh.raw+offset;
386 new->mac.raw=old->mac.raw+offset;
387 new->pkt_type=old->pkt_type;
388 new->destructor = NULL;
389 }
391 /**
392 * skb_copy - create private copy of an sk_buff
393 * @skb: buffer to copy
394 * @gfp_mask: allocation priority
395 *
396 * Make a copy of both an &sk_buff and its data. This is used when the
397 * caller wishes to modify the data and needs a private copy of the
398 * data to alter. Returns %NULL on failure or the pointer to the buffer
399 * on success. The returned buffer has a reference count of 1.
400 *
401 * As by-product this function converts non-linear &sk_buff to linear
402 * one, so that &sk_buff becomes completely private and caller is allowed
403 * to modify all the data of returned buffer. This means that this
404 * function is not recommended for use in circumstances when only
405 * header is going to be modified. Use pskb_copy() instead.
406 */
408 struct sk_buff *skb_copy(const struct sk_buff *skb, int gfp_mask)
409 {
410 struct sk_buff *n;
411 int headerlen = skb->data-skb->head;
413 /*
414 * Allocate the copy buffer
415 */
416 n=alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
417 if(n==NULL)
418 return NULL;
420 /* Set the data pointer */
421 skb_reserve(n,headerlen);
422 /* Set the tail pointer and length */
423 skb_put(n,skb->len);
424 n->csum = skb->csum;
425 n->ip_summed = skb->ip_summed;
427 if (skb_copy_bits(skb, -headerlen, n->head, headerlen+skb->len))
428 BUG();
430 copy_skb_header(n, skb);
432 return n;
433 }
435 /* Copy some data bits from skb to kernel buffer. */
437 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
438 {
439 int i, copy;
440 int start = skb->len - skb->data_len;
442 if (offset > (int)skb->len-len)
443 goto fault;
445 /* Copy header. */
446 if ((copy = start-offset) > 0) {
447 if (copy > len)
448 copy = len;
449 memcpy(to, skb->data + offset, copy);
450 if ((len -= copy) == 0)
451 return 0;
452 offset += copy;
453 to += copy;
454 }
456 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
457 int end;
459 BUG_TRAP(start <= offset+len);
461 end = start + skb_shinfo(skb)->frags[i].size;
462 if ((copy = end-offset) > 0) {
463 u8 *vaddr;
465 if (copy > len)
466 copy = len;
468 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
469 memcpy(to, vaddr+skb_shinfo(skb)->frags[i].page_offset+
470 offset-start, copy);
471 kunmap_skb_frag(vaddr);
473 if ((len -= copy) == 0)
474 return 0;
475 offset += copy;
476 to += copy;
477 }
478 start = end;
479 }
481 if (len == 0)
482 return 0;
484 fault:
485 return -EFAULT;
486 }
488 void __init skb_init(void)
489 {
490 int i;
492 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
493 sizeof(struct sk_buff),
494 0,
495 SLAB_HWCACHE_ALIGN,
496 skb_headerinit, NULL);
497 if (!skbuff_head_cache)
498 panic("cannot create skbuff cache");
500 for (i=0; i<NR_CPUS; i++)
501 skb_queue_head_init(&skb_head_pool[i].list);
502 }