direct-io.hg

view xen/net/skbuff.c @ 364:942eb9bcae13

bitkeeper revision 1.170 (3e9c936fXyHEI0NKOWQkP9tHN4sbqw)

Many files:
Finished virtualisation of x86 LDT. Xenolinux now exports this to applications (eg. for use by linuxthreads).
author kaf24@scramble.cl.cam.ac.uk
date Tue Apr 15 23:19:11 2003 +0000 (2003-04-15)
parents 170eb7974e43
children ff1bb22c81a1 95c43a4b6685
line source
1 /*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
4 * Authors: Alan Cox <iiitac@pyr.swan.ac.uk>
5 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
7 * Version: $Id: skbuff.c,v 1.89 2001/08/06 13:25:02 davem Exp $
8 *
9 * Fixes:
10 * Alan Cox : Fixed the worst of the load balancer bugs.
11 * Dave Platt : Interrupt stacking fix.
12 * Richard Kooijman : Timestamp fixes.
13 * Alan Cox : Changed buffer format.
14 * Alan Cox : destructor hook for AF_UNIX etc.
15 * Linus Torvalds : Better skb_clone.
16 * Alan Cox : Added skb_copy.
17 * Alan Cox : Added all the changed routines Linus
18 * only put in the headers
19 * Ray VanTassle : Fixed --skb->lock in free
20 * Alan Cox : skb_copy copy arp field
21 * Andi Kleen : slabified it.
22 *
23 * NOTE:
24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc).
28 *
29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version.
33 */
35 #include <linux/config.h>
36 #include <linux/lib.h>
37 #include <linux/errno.h>
38 #include <linux/types.h>
39 #include <linux/sched.h>
40 #include <linux/mm.h>
41 #include <linux/interrupt.h>
42 #include <linux/slab.h>
43 #include <linux/netdevice.h>
44 #include <linux/skbuff.h>
45 #include <linux/cache.h>
46 #include <linux/init.h>
48 #include <asm/uaccess.h>
49 #include <asm/system.h>
50 #include <asm/io.h>
52 #define BUG_TRAP ASSERT
54 int sysctl_hot_list_len = 128;
56 static kmem_cache_t *skbuff_head_cache;
58 static union {
59 struct sk_buff_head list;
60 char pad[SMP_CACHE_BYTES];
61 } skb_head_pool[NR_CPUS];
63 /*
64 * Keep out-of-line to prevent kernel bloat.
65 * __builtin_return_address is not used because it is not always
66 * reliable.
67 */
69 /**
70 * skb_over_panic - private function
71 * @skb: buffer
72 * @sz: size
73 * @here: address
74 *
75 * Out of line support code for skb_put(). Not user callable.
76 */
78 void skb_over_panic(struct sk_buff *skb, int sz, void *here)
79 {
80 printk("skput:over: %p:%d put:%d dev:%s",
81 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
82 BUG();
83 }
85 /**
86 * skb_under_panic - private function
87 * @skb: buffer
88 * @sz: size
89 * @here: address
90 *
91 * Out of line support code for skb_push(). Not user callable.
92 */
95 void skb_under_panic(struct sk_buff *skb, int sz, void *here)
96 {
97 printk("skput:under: %p:%d put:%d dev:%s",
98 here, skb->len, sz, skb->dev ? skb->dev->name : "<NULL>");
99 BUG();
100 }
102 static __inline__ struct sk_buff *skb_head_from_pool(void)
103 {
104 struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
106 if (skb_queue_len(list)) {
107 struct sk_buff *skb;
108 unsigned long flags;
110 local_irq_save(flags);
111 skb = __skb_dequeue(list);
112 local_irq_restore(flags);
113 return skb;
114 }
115 return NULL;
116 }
118 static __inline__ void skb_head_to_pool(struct sk_buff *skb)
119 {
120 struct sk_buff_head *list = &skb_head_pool[smp_processor_id()].list;
122 if (skb_queue_len(list) < sysctl_hot_list_len) {
123 unsigned long flags;
125 local_irq_save(flags);
126 __skb_queue_head(list, skb);
127 local_irq_restore(flags);
129 return;
130 }
131 kmem_cache_free(skbuff_head_cache, skb);
132 }
134 static inline u8 *alloc_skb_data_page(struct sk_buff *skb)
135 {
136 struct list_head *list_ptr;
137 struct pfn_info *pf;
138 unsigned long flags;
140 spin_lock_irqsave(&free_list_lock, flags);
142 if (!free_pfns) return NULL;
144 list_ptr = free_list.next;
145 pf = list_entry(list_ptr, struct pfn_info, list);
146 pf->flags = 0; /* owned by dom0 */
147 list_del(&pf->list);
148 free_pfns--;
150 spin_unlock_irqrestore(&free_list_lock, flags);
152 skb->pf = pf;
153 return (u8 *)((pf - frame_table) << PAGE_SHIFT);
154 }
156 static inline void dealloc_skb_data_page(struct sk_buff *skb)
157 {
158 struct pfn_info *pf;
159 unsigned long flags;
161 pf = skb->pf;
163 spin_lock_irqsave(&free_list_lock, flags);
165 list_add(&pf->list, &free_list);
166 free_pfns++;
168 spin_unlock_irqrestore(&free_list_lock, flags);
170 }
172 static inline void INTERRUPT_CHECK(int gfp_mask)
173 {
174 if (in_interrupt() && (gfp_mask & __GFP_WAIT)) {
175 printk(KERN_ERR "alloc_skb called nonatomically\n");
176 BUG();
177 }
178 }
181 /**
182 * alloc_skb - allocate a network buffer
183 * @size: size to allocate
184 * @gfp_mask: allocation mask
185 *
186 * Allocate a new &sk_buff. The returned buffer has no headroom and a
187 * tail room of size bytes. The object has a reference count of one.
188 * The return is the buffer. On a failure the return is %NULL.
189 *
190 * Buffers may only be allocated from interrupts using a @gfp_mask of
191 * %GFP_ATOMIC.
192 */
194 struct sk_buff *alloc_skb(unsigned int size,int gfp_mask)
195 {
196 struct sk_buff *skb;
197 u8 *data;
199 INTERRUPT_CHECK(gfp_mask);
201 /* Get the HEAD */
202 skb = skb_head_from_pool();
203 if (skb == NULL) {
204 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
205 if (skb == NULL)
206 goto nohead;
207 }
209 /* Get the DATA. Size must match skb_add_mtu(). */
210 size = SKB_DATA_ALIGN(size);
211 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
212 if (data == NULL)
213 goto nodata;
215 /* Load the data pointers. */
216 skb->head = data;
217 skb->data = data;
218 skb->tail = data;
219 skb->end = data + size;
221 /* Set up other state */
222 skb->len = 0;
223 skb->data_len = 0;
224 skb->src_vif = VIF_UNKNOWN_INTERFACE;
225 skb->dst_vif = VIF_UNKNOWN_INTERFACE;
226 skb->skb_type = SKB_NORMAL;
228 skb_shinfo(skb)->nr_frags = 0;
229 return skb;
231 nodata:
232 skb_head_to_pool(skb);
233 nohead:
234 return NULL;
235 }
238 struct sk_buff *alloc_zc_skb(unsigned int size,int gfp_mask)
239 {
240 struct sk_buff *skb;
241 u8 *data;
243 INTERRUPT_CHECK(gfp_mask);
245 /* Get the HEAD */
246 skb = skb_head_from_pool();
247 if (skb == NULL) {
248 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
249 if (skb == NULL)
250 goto nohead;
251 }
253 /* Get the DATA. Size must match skb_add_mtu(). */
254 size = SKB_DATA_ALIGN(size);
255 data = alloc_skb_data_page(skb);
257 if (data == NULL)
258 goto nodata;
260 /* A FAKE virtual address, so that pci_map_xxx dor the right thing. */
261 data = phys_to_virt((unsigned long)data);
263 /* Load the data pointers. */
264 skb->head = data;
265 skb->data = data;
266 skb->tail = data;
267 skb->end = data + size;
269 /* Set up other state */
270 skb->len = 0;
271 skb->data_len = 0;
272 skb->src_vif = VIF_UNKNOWN_INTERFACE;
273 skb->dst_vif = VIF_UNKNOWN_INTERFACE;
274 skb->skb_type = SKB_ZERO_COPY;
276 skb_shinfo(skb)->nr_frags = 0;
278 return skb;
280 nodata:
281 skb_head_to_pool(skb);
282 nohead:
283 return NULL;
284 }
287 struct sk_buff *alloc_skb_nodata(int gfp_mask)
288 {
289 struct sk_buff *skb;
291 INTERRUPT_CHECK(gfp_mask);
293 /* Get the HEAD */
294 skb = skb_head_from_pool();
295 if (skb == NULL) {
296 skb = kmem_cache_alloc(skbuff_head_cache, gfp_mask & ~__GFP_DMA);
297 if (skb == NULL)
298 return NULL;
299 }
301 skb->skb_type = SKB_NODATA;
302 return skb;
303 }
306 /*
307 * Slab constructor for a skb head.
308 */
309 static inline void skb_headerinit(void *p, kmem_cache_t *cache,
310 unsigned long flags)
311 {
312 struct sk_buff *skb = p;
314 skb->next = NULL;
315 skb->prev = NULL;
316 skb->list = NULL;
317 skb->dev = NULL;
318 skb->pkt_type = PACKET_HOST; /* Default type */
319 skb->ip_summed = 0;
320 skb->destructor = NULL;
321 }
323 static void skb_release_data(struct sk_buff *skb)
324 {
325 if (skb_shinfo(skb)->nr_frags) BUG();
327 switch ( skb->skb_type )
328 {
329 case SKB_NORMAL:
330 kfree(skb->head);
331 break;
332 case SKB_ZERO_COPY:
333 dealloc_skb_data_page(skb);
334 break;
335 case SKB_NODATA:
336 break;
337 default:
338 BUG();
339 }
340 }
342 /*
343 * Free an skbuff by memory without cleaning the state.
344 */
345 void kfree_skbmem(struct sk_buff *skb)
346 {
347 skb_release_data(skb);
348 skb_head_to_pool(skb);
349 }
351 /**
352 * __kfree_skb - private function
353 * @skb: buffer
354 *
355 * Free an sk_buff. Release anything attached to the buffer.
356 * Clean the state. This is an internal helper function. Users should
357 * always call kfree_skb
358 */
360 void __kfree_skb(struct sk_buff *skb)
361 {
362 if ( skb->list )
363 panic(KERN_WARNING "Warning: kfree_skb passed an skb still "
364 "on a list (from %p).\n", NET_CALLER(skb));
366 if ( skb->destructor )
367 skb->destructor(skb);
369 skb_headerinit(skb, NULL, 0); /* clean state */
370 kfree_skbmem(skb);
371 }
373 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
374 {
375 /*
376 * Shift between the two data areas in bytes
377 */
378 unsigned long offset = new->data - old->data;
380 new->list=NULL;
381 new->dev=old->dev;
382 new->protocol=old->protocol;
383 new->h.raw=old->h.raw+offset;
384 new->nh.raw=old->nh.raw+offset;
385 new->mac.raw=old->mac.raw+offset;
386 new->pkt_type=old->pkt_type;
387 new->destructor = NULL;
388 }
390 /**
391 * skb_copy - create private copy of an sk_buff
392 * @skb: buffer to copy
393 * @gfp_mask: allocation priority
394 *
395 * Make a copy of both an &sk_buff and its data. This is used when the
396 * caller wishes to modify the data and needs a private copy of the
397 * data to alter. Returns %NULL on failure or the pointer to the buffer
398 * on success. The returned buffer has a reference count of 1.
399 *
400 * As by-product this function converts non-linear &sk_buff to linear
401 * one, so that &sk_buff becomes completely private and caller is allowed
402 * to modify all the data of returned buffer. This means that this
403 * function is not recommended for use in circumstances when only
404 * header is going to be modified. Use pskb_copy() instead.
405 */
407 struct sk_buff *skb_copy(const struct sk_buff *skb, int gfp_mask)
408 {
409 struct sk_buff *n;
410 int headerlen = skb->data-skb->head;
412 /*
413 * Allocate the copy buffer
414 */
415 n=alloc_skb(skb->end - skb->head + skb->data_len, gfp_mask);
416 if(n==NULL)
417 return NULL;
419 /* Set the data pointer */
420 skb_reserve(n,headerlen);
421 /* Set the tail pointer and length */
422 skb_put(n,skb->len);
423 n->csum = skb->csum;
424 n->ip_summed = skb->ip_summed;
426 if (skb_copy_bits(skb, -headerlen, n->head, headerlen+skb->len))
427 BUG();
429 copy_skb_header(n, skb);
431 return n;
432 }
434 /* Copy some data bits from skb to kernel buffer. */
436 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
437 {
438 int i, copy;
439 int start = skb->len - skb->data_len;
441 if (offset > (int)skb->len-len)
442 goto fault;
444 /* Copy header. */
445 if ((copy = start-offset) > 0) {
446 if (copy > len)
447 copy = len;
448 memcpy(to, skb->data + offset, copy);
449 if ((len -= copy) == 0)
450 return 0;
451 offset += copy;
452 to += copy;
453 }
455 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
456 int end;
458 BUG_TRAP(start <= offset+len);
460 end = start + skb_shinfo(skb)->frags[i].size;
461 if ((copy = end-offset) > 0) {
462 u8 *vaddr;
464 if (copy > len)
465 copy = len;
467 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
468 memcpy(to, vaddr+skb_shinfo(skb)->frags[i].page_offset+
469 offset-start, copy);
470 kunmap_skb_frag(vaddr);
472 if ((len -= copy) == 0)
473 return 0;
474 offset += copy;
475 to += copy;
476 }
477 start = end;
478 }
480 if (len == 0)
481 return 0;
483 fault:
484 return -EFAULT;
485 }
487 void __init skb_init(void)
488 {
489 int i;
491 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
492 sizeof(struct sk_buff),
493 0,
494 SLAB_HWCACHE_ALIGN,
495 skb_headerinit, NULL);
496 if (!skbuff_head_cache)
497 panic("cannot create skbuff cache");
499 for (i=0; i<NR_CPUS; i++)
500 skb_queue_head_init(&skb_head_pool[i].list);
501 }