ia64/xen-unstable

view linux-2.6-xen-sparse/net/core/skbuff.c @ 13665:140afd7a5462

merge with xen-unstable.hg
author awilliam@xenbuild2.aw
date Sat Jan 27 14:07:19 2007 -0700 (2007-01-27)
parents 3c8bb086025e
children 273ec6b70b51
line source
1 /*
2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 *
4 * Authors: Alan Cox <iiitac@pyr.swan.ac.uk>
5 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 *
7 * Version: $Id: skbuff.c,v 1.90 2001/11/07 05:56:19 davem Exp $
8 *
9 * Fixes:
10 * Alan Cox : Fixed the worst of the load
11 * balancer bugs.
12 * Dave Platt : Interrupt stacking fix.
13 * Richard Kooijman : Timestamp fixes.
14 * Alan Cox : Changed buffer format.
15 * Alan Cox : destructor hook for AF_UNIX etc.
16 * Linus Torvalds : Better skb_clone.
17 * Alan Cox : Added skb_copy.
18 * Alan Cox : Added all the changed routines Linus
19 * only put in the headers
20 * Ray VanTassle : Fixed --skb->lock in free
21 * Alan Cox : skb_copy copy arp field
22 * Andi Kleen : slabified it.
23 * Robert Olsson : Removed skb_head_pool
24 *
25 * NOTE:
26 * The __skb_ routines should be called with interrupts
27 * disabled, or you better be *real* sure that the operation is atomic
28 * with respect to whatever list is being frobbed (e.g. via lock_sock()
29 * or via disabling bottom half handlers, etc).
30 *
31 * This program is free software; you can redistribute it and/or
32 * modify it under the terms of the GNU General Public License
33 * as published by the Free Software Foundation; either version
34 * 2 of the License, or (at your option) any later version.
35 */
37 /*
38 * The functions in this file will not compile correctly with gcc 2.4.x
39 */
41 #include <linux/module.h>
42 #include <linux/types.h>
43 #include <linux/kernel.h>
44 #include <linux/sched.h>
45 #include <linux/mm.h>
46 #include <linux/interrupt.h>
47 #include <linux/in.h>
48 #include <linux/inet.h>
49 #include <linux/slab.h>
50 #include <linux/netdevice.h>
51 #ifdef CONFIG_NET_CLS_ACT
52 #include <net/pkt_sched.h>
53 #endif
54 #include <linux/string.h>
55 #include <linux/skbuff.h>
56 #include <linux/cache.h>
57 #include <linux/rtnetlink.h>
58 #include <linux/init.h>
59 #include <linux/highmem.h>
61 #include <net/protocol.h>
62 #include <net/dst.h>
63 #include <net/sock.h>
64 #include <net/checksum.h>
65 #include <net/xfrm.h>
67 #include <asm/uaccess.h>
68 #include <asm/system.h>
70 static kmem_cache_t *skbuff_head_cache __read_mostly;
71 static kmem_cache_t *skbuff_fclone_cache __read_mostly;
73 /*
74 * Keep out-of-line to prevent kernel bloat.
75 * __builtin_return_address is not used because it is not always
76 * reliable.
77 */
79 /**
80 * skb_over_panic - private function
81 * @skb: buffer
82 * @sz: size
83 * @here: address
84 *
85 * Out of line support code for skb_put(). Not user callable.
86 */
87 void skb_over_panic(struct sk_buff *skb, int sz, void *here)
88 {
89 printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
90 "data:%p tail:%p end:%p dev:%s\n",
91 here, skb->len, sz, skb->head, skb->data, skb->tail, skb->end,
92 skb->dev ? skb->dev->name : "<NULL>");
93 BUG();
94 }
96 /**
97 * skb_under_panic - private function
98 * @skb: buffer
99 * @sz: size
100 * @here: address
101 *
102 * Out of line support code for skb_push(). Not user callable.
103 */
105 void skb_under_panic(struct sk_buff *skb, int sz, void *here)
106 {
107 printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
108 "data:%p tail:%p end:%p dev:%s\n",
109 here, skb->len, sz, skb->head, skb->data, skb->tail, skb->end,
110 skb->dev ? skb->dev->name : "<NULL>");
111 BUG();
112 }
114 void skb_truesize_bug(struct sk_buff *skb)
115 {
116 printk(KERN_ERR "SKB BUG: Invalid truesize (%u) "
117 "len=%u, sizeof(sk_buff)=%Zd\n",
118 skb->truesize, skb->len, sizeof(struct sk_buff));
119 }
120 EXPORT_SYMBOL(skb_truesize_bug);
122 /* Allocate a new skbuff. We do this ourselves so we can fill in a few
123 * 'private' fields and also do memory statistics to find all the
124 * [BEEP] leaks.
125 *
126 */
128 /**
129 * __alloc_skb - allocate a network buffer
130 * @size: size to allocate
131 * @gfp_mask: allocation mask
132 * @fclone: allocate from fclone cache instead of head cache
133 * and allocate a cloned (child) skb
134 *
135 * Allocate a new &sk_buff. The returned buffer has no headroom and a
136 * tail room of size bytes. The object has a reference count of one.
137 * The return is the buffer. On a failure the return is %NULL.
138 *
139 * Buffers may only be allocated from interrupts using a @gfp_mask of
140 * %GFP_ATOMIC.
141 */
142 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
143 int fclone)
144 {
145 kmem_cache_t *cache;
146 struct skb_shared_info *shinfo;
147 struct sk_buff *skb;
148 u8 *data;
150 cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
152 /* Get the HEAD */
153 skb = kmem_cache_alloc(cache, gfp_mask & ~__GFP_DMA);
154 if (!skb)
155 goto out;
157 /* Get the DATA. Size must match skb_add_mtu(). */
158 size = SKB_DATA_ALIGN(size);
159 data = ____kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
160 if (!data)
161 goto nodata;
163 memset(skb, 0, offsetof(struct sk_buff, truesize));
164 skb->truesize = size + sizeof(struct sk_buff);
165 atomic_set(&skb->users, 1);
166 skb->head = data;
167 skb->data = data;
168 skb->tail = data;
169 skb->end = data + size;
170 /* make sure we initialize shinfo sequentially */
171 shinfo = skb_shinfo(skb);
172 atomic_set(&shinfo->dataref, 1);
173 shinfo->nr_frags = 0;
174 shinfo->gso_size = 0;
175 shinfo->gso_segs = 0;
176 shinfo->gso_type = 0;
177 shinfo->ip6_frag_id = 0;
178 shinfo->frag_list = NULL;
180 if (fclone) {
181 struct sk_buff *child = skb + 1;
182 atomic_t *fclone_ref = (atomic_t *) (child + 1);
184 skb->fclone = SKB_FCLONE_ORIG;
185 atomic_set(fclone_ref, 1);
187 child->fclone = SKB_FCLONE_UNAVAILABLE;
188 }
189 out:
190 return skb;
191 nodata:
192 kmem_cache_free(cache, skb);
193 skb = NULL;
194 goto out;
195 }
197 /**
198 * alloc_skb_from_cache - allocate a network buffer
199 * @cp: kmem_cache from which to allocate the data area
200 * (object size must be big enough for @size bytes + skb overheads)
201 * @size: size to allocate
202 * @gfp_mask: allocation mask
203 *
204 * Allocate a new &sk_buff. The returned buffer has no headroom and
205 * tail room of size bytes. The object has a reference count of one.
206 * The return is the buffer. On a failure the return is %NULL.
207 *
208 * Buffers may only be allocated from interrupts using a @gfp_mask of
209 * %GFP_ATOMIC.
210 */
211 struct sk_buff *alloc_skb_from_cache(kmem_cache_t *cp,
212 unsigned int size,
213 gfp_t gfp_mask)
214 {
215 struct sk_buff *skb;
216 u8 *data;
218 /* Get the HEAD */
219 skb = kmem_cache_alloc(skbuff_head_cache,
220 gfp_mask & ~__GFP_DMA);
221 if (!skb)
222 goto out;
224 /* Get the DATA. */
225 size = SKB_DATA_ALIGN(size);
226 data = kmem_cache_alloc(cp, gfp_mask);
227 if (!data)
228 goto nodata;
230 memset(skb, 0, offsetof(struct sk_buff, truesize));
231 skb->truesize = size + sizeof(struct sk_buff);
232 atomic_set(&skb->users, 1);
233 skb->head = data;
234 skb->data = data;
235 skb->tail = data;
236 skb->end = data + size;
238 atomic_set(&(skb_shinfo(skb)->dataref), 1);
239 skb_shinfo(skb)->nr_frags = 0;
240 skb_shinfo(skb)->gso_size = 0;
241 skb_shinfo(skb)->gso_segs = 0;
242 skb_shinfo(skb)->gso_type = 0;
243 skb_shinfo(skb)->ip6_frag_id = 0;
244 skb_shinfo(skb)->frag_list = NULL;
245 out:
246 return skb;
247 nodata:
248 kmem_cache_free(skbuff_head_cache, skb);
249 skb = NULL;
250 goto out;
251 }
253 /**
254 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
255 * @dev: network device to receive on
256 * @length: length to allocate
257 * @gfp_mask: get_free_pages mask, passed to alloc_skb
258 *
259 * Allocate a new &sk_buff and assign it a usage count of one. The
260 * buffer has unspecified headroom built in. Users should allocate
261 * the headroom they think they need without accounting for the
262 * built in space. The built in space is used for optimisations.
263 *
264 * %NULL is returned if there is no free memory.
265 */
266 struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
267 unsigned int length, gfp_t gfp_mask)
268 {
269 struct sk_buff *skb;
271 skb = alloc_skb(length + NET_SKB_PAD, gfp_mask);
272 if (likely(skb)) {
273 skb_reserve(skb, NET_SKB_PAD);
274 skb->dev = dev;
275 }
276 return skb;
277 }
279 static void skb_drop_list(struct sk_buff **listp)
280 {
281 struct sk_buff *list = *listp;
283 *listp = NULL;
285 do {
286 struct sk_buff *this = list;
287 list = list->next;
288 kfree_skb(this);
289 } while (list);
290 }
292 static inline void skb_drop_fraglist(struct sk_buff *skb)
293 {
294 skb_drop_list(&skb_shinfo(skb)->frag_list);
295 }
297 static void skb_clone_fraglist(struct sk_buff *skb)
298 {
299 struct sk_buff *list;
301 for (list = skb_shinfo(skb)->frag_list; list; list = list->next)
302 skb_get(list);
303 }
305 static void skb_release_data(struct sk_buff *skb)
306 {
307 if (!skb->cloned ||
308 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
309 &skb_shinfo(skb)->dataref)) {
310 if (skb_shinfo(skb)->nr_frags) {
311 int i;
312 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
313 put_page(skb_shinfo(skb)->frags[i].page);
314 }
316 if (skb_shinfo(skb)->frag_list)
317 skb_drop_fraglist(skb);
319 kfree(skb->head);
320 }
321 }
323 /*
324 * Free an skbuff by memory without cleaning the state.
325 */
326 void kfree_skbmem(struct sk_buff *skb)
327 {
328 struct sk_buff *other;
329 atomic_t *fclone_ref;
331 skb_release_data(skb);
332 switch (skb->fclone) {
333 case SKB_FCLONE_UNAVAILABLE:
334 kmem_cache_free(skbuff_head_cache, skb);
335 break;
337 case SKB_FCLONE_ORIG:
338 fclone_ref = (atomic_t *) (skb + 2);
339 if (atomic_dec_and_test(fclone_ref))
340 kmem_cache_free(skbuff_fclone_cache, skb);
341 break;
343 case SKB_FCLONE_CLONE:
344 fclone_ref = (atomic_t *) (skb + 1);
345 other = skb - 1;
347 /* The clone portion is available for
348 * fast-cloning again.
349 */
350 skb->fclone = SKB_FCLONE_UNAVAILABLE;
352 if (atomic_dec_and_test(fclone_ref))
353 kmem_cache_free(skbuff_fclone_cache, other);
354 break;
355 };
356 }
358 /**
359 * __kfree_skb - private function
360 * @skb: buffer
361 *
362 * Free an sk_buff. Release anything attached to the buffer.
363 * Clean the state. This is an internal helper function. Users should
364 * always call kfree_skb
365 */
367 void __kfree_skb(struct sk_buff *skb)
368 {
369 dst_release(skb->dst);
370 #ifdef CONFIG_XFRM
371 secpath_put(skb->sp);
372 #endif
373 if (skb->destructor) {
374 WARN_ON(in_irq());
375 skb->destructor(skb);
376 }
377 #ifdef CONFIG_NETFILTER
378 nf_conntrack_put(skb->nfct);
379 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
380 nf_conntrack_put_reasm(skb->nfct_reasm);
381 #endif
382 #ifdef CONFIG_BRIDGE_NETFILTER
383 nf_bridge_put(skb->nf_bridge);
384 #endif
385 #endif
386 /* XXX: IS this still necessary? - JHS */
387 #ifdef CONFIG_NET_SCHED
388 skb->tc_index = 0;
389 #ifdef CONFIG_NET_CLS_ACT
390 skb->tc_verd = 0;
391 #endif
392 #endif
394 kfree_skbmem(skb);
395 }
397 /**
398 * kfree_skb - free an sk_buff
399 * @skb: buffer to free
400 *
401 * Drop a reference to the buffer and free it if the usage count has
402 * hit zero.
403 */
404 void kfree_skb(struct sk_buff *skb)
405 {
406 if (unlikely(!skb))
407 return;
408 if (likely(atomic_read(&skb->users) == 1))
409 smp_rmb();
410 else if (likely(!atomic_dec_and_test(&skb->users)))
411 return;
412 __kfree_skb(skb);
413 }
415 /**
416 * skb_clone - duplicate an sk_buff
417 * @skb: buffer to clone
418 * @gfp_mask: allocation priority
419 *
420 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
421 * copies share the same packet data but not structure. The new
422 * buffer has a reference count of 1. If the allocation fails the
423 * function returns %NULL otherwise the new buffer is returned.
424 *
425 * If this function is called from an interrupt gfp_mask() must be
426 * %GFP_ATOMIC.
427 */
429 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
430 {
431 struct sk_buff *n;
433 n = skb + 1;
434 if (skb->fclone == SKB_FCLONE_ORIG &&
435 n->fclone == SKB_FCLONE_UNAVAILABLE) {
436 atomic_t *fclone_ref = (atomic_t *) (n + 1);
437 n->fclone = SKB_FCLONE_CLONE;
438 atomic_inc(fclone_ref);
439 } else {
440 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
441 if (!n)
442 return NULL;
443 n->fclone = SKB_FCLONE_UNAVAILABLE;
444 }
446 #define C(x) n->x = skb->x
448 n->next = n->prev = NULL;
449 n->sk = NULL;
450 C(tstamp);
451 C(dev);
452 C(h);
453 C(nh);
454 C(mac);
455 C(dst);
456 dst_clone(skb->dst);
457 C(sp);
458 #ifdef CONFIG_INET
459 secpath_get(skb->sp);
460 #endif
461 memcpy(n->cb, skb->cb, sizeof(skb->cb));
462 C(len);
463 C(data_len);
464 C(csum);
465 C(local_df);
466 n->cloned = 1;
467 n->nohdr = 0;
468 #ifdef CONFIG_XEN
469 C(proto_data_valid);
470 C(proto_csum_blank);
471 #endif
472 C(pkt_type);
473 C(ip_summed);
474 C(priority);
475 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
476 C(ipvs_property);
477 #endif
478 C(protocol);
479 n->destructor = NULL;
480 #ifdef CONFIG_NETFILTER
481 C(nfmark);
482 C(nfct);
483 nf_conntrack_get(skb->nfct);
484 C(nfctinfo);
485 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
486 C(nfct_reasm);
487 nf_conntrack_get_reasm(skb->nfct_reasm);
488 #endif
489 #ifdef CONFIG_BRIDGE_NETFILTER
490 C(nf_bridge);
491 nf_bridge_get(skb->nf_bridge);
492 #endif
493 #endif /*CONFIG_NETFILTER*/
494 #ifdef CONFIG_NET_SCHED
495 C(tc_index);
496 #ifdef CONFIG_NET_CLS_ACT
497 n->tc_verd = SET_TC_VERD(skb->tc_verd,0);
498 n->tc_verd = CLR_TC_OK2MUNGE(n->tc_verd);
499 n->tc_verd = CLR_TC_MUNGED(n->tc_verd);
500 C(input_dev);
501 #endif
502 skb_copy_secmark(n, skb);
503 #endif
504 C(truesize);
505 atomic_set(&n->users, 1);
506 C(head);
507 C(data);
508 C(tail);
509 C(end);
511 atomic_inc(&(skb_shinfo(skb)->dataref));
512 skb->cloned = 1;
514 return n;
515 }
517 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
518 {
519 /*
520 * Shift between the two data areas in bytes
521 */
522 unsigned long offset = new->data - old->data;
524 new->sk = NULL;
525 new->dev = old->dev;
526 new->priority = old->priority;
527 new->protocol = old->protocol;
528 new->dst = dst_clone(old->dst);
529 #ifdef CONFIG_INET
530 new->sp = secpath_get(old->sp);
531 #endif
532 new->h.raw = old->h.raw + offset;
533 new->nh.raw = old->nh.raw + offset;
534 new->mac.raw = old->mac.raw + offset;
535 memcpy(new->cb, old->cb, sizeof(old->cb));
536 new->local_df = old->local_df;
537 new->fclone = SKB_FCLONE_UNAVAILABLE;
538 new->pkt_type = old->pkt_type;
539 new->tstamp = old->tstamp;
540 new->destructor = NULL;
541 #ifdef CONFIG_NETFILTER
542 new->nfmark = old->nfmark;
543 new->nfct = old->nfct;
544 nf_conntrack_get(old->nfct);
545 new->nfctinfo = old->nfctinfo;
546 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
547 new->nfct_reasm = old->nfct_reasm;
548 nf_conntrack_get_reasm(old->nfct_reasm);
549 #endif
550 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
551 new->ipvs_property = old->ipvs_property;
552 #endif
553 #ifdef CONFIG_BRIDGE_NETFILTER
554 new->nf_bridge = old->nf_bridge;
555 nf_bridge_get(old->nf_bridge);
556 #endif
557 #endif
558 #ifdef CONFIG_NET_SCHED
559 #ifdef CONFIG_NET_CLS_ACT
560 new->tc_verd = old->tc_verd;
561 #endif
562 new->tc_index = old->tc_index;
563 #endif
564 skb_copy_secmark(new, old);
565 atomic_set(&new->users, 1);
566 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
567 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
568 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
569 }
571 /**
572 * skb_copy - create private copy of an sk_buff
573 * @skb: buffer to copy
574 * @gfp_mask: allocation priority
575 *
576 * Make a copy of both an &sk_buff and its data. This is used when the
577 * caller wishes to modify the data and needs a private copy of the
578 * data to alter. Returns %NULL on failure or the pointer to the buffer
579 * on success. The returned buffer has a reference count of 1.
580 *
581 * As by-product this function converts non-linear &sk_buff to linear
582 * one, so that &sk_buff becomes completely private and caller is allowed
583 * to modify all the data of returned buffer. This means that this
584 * function is not recommended for use in circumstances when only
585 * header is going to be modified. Use pskb_copy() instead.
586 */
588 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
589 {
590 int headerlen = skb->data - skb->head;
591 /*
592 * Allocate the copy buffer
593 */
594 struct sk_buff *n = alloc_skb(skb->end - skb->head + skb->data_len,
595 gfp_mask);
596 if (!n)
597 return NULL;
599 /* Set the data pointer */
600 skb_reserve(n, headerlen);
601 /* Set the tail pointer and length */
602 skb_put(n, skb->len);
603 n->csum = skb->csum;
604 n->ip_summed = skb->ip_summed;
606 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
607 BUG();
609 copy_skb_header(n, skb);
610 return n;
611 }
614 /**
615 * pskb_copy - create copy of an sk_buff with private head.
616 * @skb: buffer to copy
617 * @gfp_mask: allocation priority
618 *
619 * Make a copy of both an &sk_buff and part of its data, located
620 * in header. Fragmented data remain shared. This is used when
621 * the caller wishes to modify only header of &sk_buff and needs
622 * private copy of the header to alter. Returns %NULL on failure
623 * or the pointer to the buffer on success.
624 * The returned buffer has a reference count of 1.
625 */
627 struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
628 {
629 /*
630 * Allocate the copy buffer
631 */
632 struct sk_buff *n = alloc_skb(skb->end - skb->head, gfp_mask);
634 if (!n)
635 goto out;
637 /* Set the data pointer */
638 skb_reserve(n, skb->data - skb->head);
639 /* Set the tail pointer and length */
640 skb_put(n, skb_headlen(skb));
641 /* Copy the bytes */
642 memcpy(n->data, skb->data, n->len);
643 n->csum = skb->csum;
644 n->ip_summed = skb->ip_summed;
646 n->data_len = skb->data_len;
647 n->len = skb->len;
649 if (skb_shinfo(skb)->nr_frags) {
650 int i;
652 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
653 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
654 get_page(skb_shinfo(n)->frags[i].page);
655 }
656 skb_shinfo(n)->nr_frags = i;
657 }
659 if (skb_shinfo(skb)->frag_list) {
660 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
661 skb_clone_fraglist(n);
662 }
664 copy_skb_header(n, skb);
665 out:
666 return n;
667 }
669 /**
670 * pskb_expand_head - reallocate header of &sk_buff
671 * @skb: buffer to reallocate
672 * @nhead: room to add at head
673 * @ntail: room to add at tail
674 * @gfp_mask: allocation priority
675 *
676 * Expands (or creates identical copy, if &nhead and &ntail are zero)
677 * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
678 * reference count of 1. Returns zero in the case of success or error,
679 * if expansion failed. In the last case, &sk_buff is not changed.
680 *
681 * All the pointers pointing into skb header may change and must be
682 * reloaded after call to this function.
683 */
685 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
686 gfp_t gfp_mask)
687 {
688 int i;
689 u8 *data;
690 int size = nhead + (skb->end - skb->head) + ntail;
691 long off;
693 if (skb_shared(skb))
694 BUG();
696 size = SKB_DATA_ALIGN(size);
698 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
699 if (!data)
700 goto nodata;
702 /* Copy only real data... and, alas, header. This should be
703 * optimized for the cases when header is void. */
704 memcpy(data + nhead, skb->head, skb->tail - skb->head);
705 memcpy(data + size, skb->end, sizeof(struct skb_shared_info));
707 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
708 get_page(skb_shinfo(skb)->frags[i].page);
710 if (skb_shinfo(skb)->frag_list)
711 skb_clone_fraglist(skb);
713 skb_release_data(skb);
715 off = (data + nhead) - skb->head;
717 skb->head = data;
718 skb->end = data + size;
719 skb->data += off;
720 skb->tail += off;
721 skb->mac.raw += off;
722 skb->h.raw += off;
723 skb->nh.raw += off;
724 skb->cloned = 0;
725 skb->nohdr = 0;
726 atomic_set(&skb_shinfo(skb)->dataref, 1);
727 return 0;
729 nodata:
730 return -ENOMEM;
731 }
733 /* Make private copy of skb with writable head and some headroom */
735 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
736 {
737 struct sk_buff *skb2;
738 int delta = headroom - skb_headroom(skb);
740 if (delta <= 0)
741 skb2 = pskb_copy(skb, GFP_ATOMIC);
742 else {
743 skb2 = skb_clone(skb, GFP_ATOMIC);
744 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
745 GFP_ATOMIC)) {
746 kfree_skb(skb2);
747 skb2 = NULL;
748 }
749 }
750 return skb2;
751 }
754 /**
755 * skb_copy_expand - copy and expand sk_buff
756 * @skb: buffer to copy
757 * @newheadroom: new free bytes at head
758 * @newtailroom: new free bytes at tail
759 * @gfp_mask: allocation priority
760 *
761 * Make a copy of both an &sk_buff and its data and while doing so
762 * allocate additional space.
763 *
764 * This is used when the caller wishes to modify the data and needs a
765 * private copy of the data to alter as well as more space for new fields.
766 * Returns %NULL on failure or the pointer to the buffer
767 * on success. The returned buffer has a reference count of 1.
768 *
769 * You must pass %GFP_ATOMIC as the allocation priority if this function
770 * is called from an interrupt.
771 *
772 * BUG ALERT: ip_summed is not copied. Why does this work? Is it used
773 * only by netfilter in the cases when checksum is recalculated? --ANK
774 */
775 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
776 int newheadroom, int newtailroom,
777 gfp_t gfp_mask)
778 {
779 /*
780 * Allocate the copy buffer
781 */
782 struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
783 gfp_mask);
784 int head_copy_len, head_copy_off;
786 if (!n)
787 return NULL;
789 skb_reserve(n, newheadroom);
791 /* Set the tail pointer and length */
792 skb_put(n, skb->len);
794 head_copy_len = skb_headroom(skb);
795 head_copy_off = 0;
796 if (newheadroom <= head_copy_len)
797 head_copy_len = newheadroom;
798 else
799 head_copy_off = newheadroom - head_copy_len;
801 /* Copy the linear header and data. */
802 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
803 skb->len + head_copy_len))
804 BUG();
806 copy_skb_header(n, skb);
808 return n;
809 }
811 /**
812 * skb_pad - zero pad the tail of an skb
813 * @skb: buffer to pad
814 * @pad: space to pad
815 *
816 * Ensure that a buffer is followed by a padding area that is zero
817 * filled. Used by network drivers which may DMA or transfer data
818 * beyond the buffer end onto the wire.
819 *
820 * May return error in out of memory cases. The skb is freed on error.
821 */
823 int skb_pad(struct sk_buff *skb, int pad)
824 {
825 int err;
826 int ntail;
828 /* If the skbuff is non linear tailroom is always zero.. */
829 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
830 memset(skb->data+skb->len, 0, pad);
831 return 0;
832 }
834 ntail = skb->data_len + pad - (skb->end - skb->tail);
835 if (likely(skb_cloned(skb) || ntail > 0)) {
836 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
837 if (unlikely(err))
838 goto free_skb;
839 }
841 /* FIXME: The use of this function with non-linear skb's really needs
842 * to be audited.
843 */
844 err = skb_linearize(skb);
845 if (unlikely(err))
846 goto free_skb;
848 memset(skb->data + skb->len, 0, pad);
849 return 0;
851 free_skb:
852 kfree_skb(skb);
853 return err;
854 }
856 /* Trims skb to length len. It can change skb pointers.
857 */
859 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
860 {
861 struct sk_buff **fragp;
862 struct sk_buff *frag;
863 int offset = skb_headlen(skb);
864 int nfrags = skb_shinfo(skb)->nr_frags;
865 int i;
866 int err;
868 if (skb_cloned(skb) &&
869 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
870 return err;
872 i = 0;
873 if (offset >= len)
874 goto drop_pages;
876 for (; i < nfrags; i++) {
877 int end = offset + skb_shinfo(skb)->frags[i].size;
879 if (end < len) {
880 offset = end;
881 continue;
882 }
884 skb_shinfo(skb)->frags[i++].size = len - offset;
886 drop_pages:
887 skb_shinfo(skb)->nr_frags = i;
889 for (; i < nfrags; i++)
890 put_page(skb_shinfo(skb)->frags[i].page);
892 if (skb_shinfo(skb)->frag_list)
893 skb_drop_fraglist(skb);
894 goto done;
895 }
897 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
898 fragp = &frag->next) {
899 int end = offset + frag->len;
901 if (skb_shared(frag)) {
902 struct sk_buff *nfrag;
904 nfrag = skb_clone(frag, GFP_ATOMIC);
905 if (unlikely(!nfrag))
906 return -ENOMEM;
908 nfrag->next = frag->next;
909 kfree_skb(frag);
910 frag = nfrag;
911 *fragp = frag;
912 }
914 if (end < len) {
915 offset = end;
916 continue;
917 }
919 if (end > len &&
920 unlikely((err = pskb_trim(frag, len - offset))))
921 return err;
923 if (frag->next)
924 skb_drop_list(&frag->next);
925 break;
926 }
928 done:
929 if (len > skb_headlen(skb)) {
930 skb->data_len -= skb->len - len;
931 skb->len = len;
932 } else {
933 skb->len = len;
934 skb->data_len = 0;
935 skb->tail = skb->data + len;
936 }
938 return 0;
939 }
941 /**
942 * __pskb_pull_tail - advance tail of skb header
943 * @skb: buffer to reallocate
944 * @delta: number of bytes to advance tail
945 *
946 * The function makes a sense only on a fragmented &sk_buff,
947 * it expands header moving its tail forward and copying necessary
948 * data from fragmented part.
949 *
950 * &sk_buff MUST have reference count of 1.
951 *
952 * Returns %NULL (and &sk_buff does not change) if pull failed
953 * or value of new tail of skb in the case of success.
954 *
955 * All the pointers pointing into skb header may change and must be
956 * reloaded after call to this function.
957 */
959 /* Moves tail of skb head forward, copying data from fragmented part,
960 * when it is necessary.
961 * 1. It may fail due to malloc failure.
962 * 2. It may change skb pointers.
963 *
964 * It is pretty complicated. Luckily, it is called only in exceptional cases.
965 */
966 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
967 {
968 /* If skb has not enough free space at tail, get new one
969 * plus 128 bytes for future expansions. If we have enough
970 * room at tail, reallocate without expansion only if skb is cloned.
971 */
972 int i, k, eat = (skb->tail + delta) - skb->end;
974 if (eat > 0 || skb_cloned(skb)) {
975 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
976 GFP_ATOMIC))
977 return NULL;
978 }
980 if (skb_copy_bits(skb, skb_headlen(skb), skb->tail, delta))
981 BUG();
983 /* Optimization: no fragments, no reasons to preestimate
984 * size of pulled pages. Superb.
985 */
986 if (!skb_shinfo(skb)->frag_list)
987 goto pull_pages;
989 /* Estimate size of pulled pages. */
990 eat = delta;
991 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
992 if (skb_shinfo(skb)->frags[i].size >= eat)
993 goto pull_pages;
994 eat -= skb_shinfo(skb)->frags[i].size;
995 }
997 /* If we need update frag list, we are in troubles.
998 * Certainly, it possible to add an offset to skb data,
999 * but taking into account that pulling is expected to
1000 * be very rare operation, it is worth to fight against
1001 * further bloating skb head and crucify ourselves here instead.
1002 * Pure masohism, indeed. 8)8)
1003 */
1004 if (eat) {
1005 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1006 struct sk_buff *clone = NULL;
1007 struct sk_buff *insp = NULL;
1009 do {
1010 BUG_ON(!list);
1012 if (list->len <= eat) {
1013 /* Eaten as whole. */
1014 eat -= list->len;
1015 list = list->next;
1016 insp = list;
1017 } else {
1018 /* Eaten partially. */
1020 if (skb_shared(list)) {
1021 /* Sucks! We need to fork list. :-( */
1022 clone = skb_clone(list, GFP_ATOMIC);
1023 if (!clone)
1024 return NULL;
1025 insp = list->next;
1026 list = clone;
1027 } else {
1028 /* This may be pulled without
1029 * problems. */
1030 insp = list;
1032 if (!pskb_pull(list, eat)) {
1033 if (clone)
1034 kfree_skb(clone);
1035 return NULL;
1037 break;
1039 } while (eat);
1041 /* Free pulled out fragments. */
1042 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1043 skb_shinfo(skb)->frag_list = list->next;
1044 kfree_skb(list);
1046 /* And insert new clone at head. */
1047 if (clone) {
1048 clone->next = list;
1049 skb_shinfo(skb)->frag_list = clone;
1052 /* Success! Now we may commit changes to skb data. */
1054 pull_pages:
1055 eat = delta;
1056 k = 0;
1057 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1058 if (skb_shinfo(skb)->frags[i].size <= eat) {
1059 put_page(skb_shinfo(skb)->frags[i].page);
1060 eat -= skb_shinfo(skb)->frags[i].size;
1061 } else {
1062 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1063 if (eat) {
1064 skb_shinfo(skb)->frags[k].page_offset += eat;
1065 skb_shinfo(skb)->frags[k].size -= eat;
1066 eat = 0;
1068 k++;
1071 skb_shinfo(skb)->nr_frags = k;
1073 skb->tail += delta;
1074 skb->data_len -= delta;
1076 return skb->tail;
1079 /* Copy some data bits from skb to kernel buffer. */
1081 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1083 int i, copy;
1084 int start = skb_headlen(skb);
1086 if (offset > (int)skb->len - len)
1087 goto fault;
1089 /* Copy header. */
1090 if ((copy = start - offset) > 0) {
1091 if (copy > len)
1092 copy = len;
1093 memcpy(to, skb->data + offset, copy);
1094 if ((len -= copy) == 0)
1095 return 0;
1096 offset += copy;
1097 to += copy;
1100 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1101 int end;
1103 BUG_TRAP(start <= offset + len);
1105 end = start + skb_shinfo(skb)->frags[i].size;
1106 if ((copy = end - offset) > 0) {
1107 u8 *vaddr;
1109 if (copy > len)
1110 copy = len;
1112 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
1113 memcpy(to,
1114 vaddr + skb_shinfo(skb)->frags[i].page_offset+
1115 offset - start, copy);
1116 kunmap_skb_frag(vaddr);
1118 if ((len -= copy) == 0)
1119 return 0;
1120 offset += copy;
1121 to += copy;
1123 start = end;
1126 if (skb_shinfo(skb)->frag_list) {
1127 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1129 for (; list; list = list->next) {
1130 int end;
1132 BUG_TRAP(start <= offset + len);
1134 end = start + list->len;
1135 if ((copy = end - offset) > 0) {
1136 if (copy > len)
1137 copy = len;
1138 if (skb_copy_bits(list, offset - start,
1139 to, copy))
1140 goto fault;
1141 if ((len -= copy) == 0)
1142 return 0;
1143 offset += copy;
1144 to += copy;
1146 start = end;
1149 if (!len)
1150 return 0;
1152 fault:
1153 return -EFAULT;
1156 /**
1157 * skb_store_bits - store bits from kernel buffer to skb
1158 * @skb: destination buffer
1159 * @offset: offset in destination
1160 * @from: source buffer
1161 * @len: number of bytes to copy
1163 * Copy the specified number of bytes from the source buffer to the
1164 * destination skb. This function handles all the messy bits of
1165 * traversing fragment lists and such.
1166 */
1168 int skb_store_bits(const struct sk_buff *skb, int offset, void *from, int len)
1170 int i, copy;
1171 int start = skb_headlen(skb);
1173 if (offset > (int)skb->len - len)
1174 goto fault;
1176 if ((copy = start - offset) > 0) {
1177 if (copy > len)
1178 copy = len;
1179 memcpy(skb->data + offset, from, copy);
1180 if ((len -= copy) == 0)
1181 return 0;
1182 offset += copy;
1183 from += copy;
1186 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1187 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1188 int end;
1190 BUG_TRAP(start <= offset + len);
1192 end = start + frag->size;
1193 if ((copy = end - offset) > 0) {
1194 u8 *vaddr;
1196 if (copy > len)
1197 copy = len;
1199 vaddr = kmap_skb_frag(frag);
1200 memcpy(vaddr + frag->page_offset + offset - start,
1201 from, copy);
1202 kunmap_skb_frag(vaddr);
1204 if ((len -= copy) == 0)
1205 return 0;
1206 offset += copy;
1207 from += copy;
1209 start = end;
1212 if (skb_shinfo(skb)->frag_list) {
1213 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1215 for (; list; list = list->next) {
1216 int end;
1218 BUG_TRAP(start <= offset + len);
1220 end = start + list->len;
1221 if ((copy = end - offset) > 0) {
1222 if (copy > len)
1223 copy = len;
1224 if (skb_store_bits(list, offset - start,
1225 from, copy))
1226 goto fault;
1227 if ((len -= copy) == 0)
1228 return 0;
1229 offset += copy;
1230 from += copy;
1232 start = end;
1235 if (!len)
1236 return 0;
1238 fault:
1239 return -EFAULT;
1242 EXPORT_SYMBOL(skb_store_bits);
1244 /* Checksum skb data. */
1246 unsigned int skb_checksum(const struct sk_buff *skb, int offset,
1247 int len, unsigned int csum)
1249 int start = skb_headlen(skb);
1250 int i, copy = start - offset;
1251 int pos = 0;
1253 /* Checksum header. */
1254 if (copy > 0) {
1255 if (copy > len)
1256 copy = len;
1257 csum = csum_partial(skb->data + offset, copy, csum);
1258 if ((len -= copy) == 0)
1259 return csum;
1260 offset += copy;
1261 pos = copy;
1264 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1265 int end;
1267 BUG_TRAP(start <= offset + len);
1269 end = start + skb_shinfo(skb)->frags[i].size;
1270 if ((copy = end - offset) > 0) {
1271 unsigned int csum2;
1272 u8 *vaddr;
1273 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1275 if (copy > len)
1276 copy = len;
1277 vaddr = kmap_skb_frag(frag);
1278 csum2 = csum_partial(vaddr + frag->page_offset +
1279 offset - start, copy, 0);
1280 kunmap_skb_frag(vaddr);
1281 csum = csum_block_add(csum, csum2, pos);
1282 if (!(len -= copy))
1283 return csum;
1284 offset += copy;
1285 pos += copy;
1287 start = end;
1290 if (skb_shinfo(skb)->frag_list) {
1291 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1293 for (; list; list = list->next) {
1294 int end;
1296 BUG_TRAP(start <= offset + len);
1298 end = start + list->len;
1299 if ((copy = end - offset) > 0) {
1300 unsigned int csum2;
1301 if (copy > len)
1302 copy = len;
1303 csum2 = skb_checksum(list, offset - start,
1304 copy, 0);
1305 csum = csum_block_add(csum, csum2, pos);
1306 if ((len -= copy) == 0)
1307 return csum;
1308 offset += copy;
1309 pos += copy;
1311 start = end;
1314 BUG_ON(len);
1316 return csum;
1319 /* Both of above in one bottle. */
1321 unsigned int skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
1322 u8 *to, int len, unsigned int csum)
1324 int start = skb_headlen(skb);
1325 int i, copy = start - offset;
1326 int pos = 0;
1328 /* Copy header. */
1329 if (copy > 0) {
1330 if (copy > len)
1331 copy = len;
1332 csum = csum_partial_copy_nocheck(skb->data + offset, to,
1333 copy, csum);
1334 if ((len -= copy) == 0)
1335 return csum;
1336 offset += copy;
1337 to += copy;
1338 pos = copy;
1341 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1342 int end;
1344 BUG_TRAP(start <= offset + len);
1346 end = start + skb_shinfo(skb)->frags[i].size;
1347 if ((copy = end - offset) > 0) {
1348 unsigned int csum2;
1349 u8 *vaddr;
1350 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1352 if (copy > len)
1353 copy = len;
1354 vaddr = kmap_skb_frag(frag);
1355 csum2 = csum_partial_copy_nocheck(vaddr +
1356 frag->page_offset +
1357 offset - start, to,
1358 copy, 0);
1359 kunmap_skb_frag(vaddr);
1360 csum = csum_block_add(csum, csum2, pos);
1361 if (!(len -= copy))
1362 return csum;
1363 offset += copy;
1364 to += copy;
1365 pos += copy;
1367 start = end;
1370 if (skb_shinfo(skb)->frag_list) {
1371 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1373 for (; list; list = list->next) {
1374 unsigned int csum2;
1375 int end;
1377 BUG_TRAP(start <= offset + len);
1379 end = start + list->len;
1380 if ((copy = end - offset) > 0) {
1381 if (copy > len)
1382 copy = len;
1383 csum2 = skb_copy_and_csum_bits(list,
1384 offset - start,
1385 to, copy, 0);
1386 csum = csum_block_add(csum, csum2, pos);
1387 if ((len -= copy) == 0)
1388 return csum;
1389 offset += copy;
1390 to += copy;
1391 pos += copy;
1393 start = end;
1396 BUG_ON(len);
1397 return csum;
1400 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1402 unsigned int csum;
1403 long csstart;
1405 if (skb->ip_summed == CHECKSUM_HW)
1406 csstart = skb->h.raw - skb->data;
1407 else
1408 csstart = skb_headlen(skb);
1410 BUG_ON(csstart > skb_headlen(skb));
1412 memcpy(to, skb->data, csstart);
1414 csum = 0;
1415 if (csstart != skb->len)
1416 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
1417 skb->len - csstart, 0);
1419 if (skb->ip_summed == CHECKSUM_HW) {
1420 long csstuff = csstart + skb->csum;
1422 *((unsigned short *)(to + csstuff)) = csum_fold(csum);
1426 /**
1427 * skb_dequeue - remove from the head of the queue
1428 * @list: list to dequeue from
1430 * Remove the head of the list. The list lock is taken so the function
1431 * may be used safely with other locking list functions. The head item is
1432 * returned or %NULL if the list is empty.
1433 */
1435 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
1437 unsigned long flags;
1438 struct sk_buff *result;
1440 spin_lock_irqsave(&list->lock, flags);
1441 result = __skb_dequeue(list);
1442 spin_unlock_irqrestore(&list->lock, flags);
1443 return result;
1446 /**
1447 * skb_dequeue_tail - remove from the tail of the queue
1448 * @list: list to dequeue from
1450 * Remove the tail of the list. The list lock is taken so the function
1451 * may be used safely with other locking list functions. The tail item is
1452 * returned or %NULL if the list is empty.
1453 */
1454 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
1456 unsigned long flags;
1457 struct sk_buff *result;
1459 spin_lock_irqsave(&list->lock, flags);
1460 result = __skb_dequeue_tail(list);
1461 spin_unlock_irqrestore(&list->lock, flags);
1462 return result;
1465 /**
1466 * skb_queue_purge - empty a list
1467 * @list: list to empty
1469 * Delete all buffers on an &sk_buff list. Each buffer is removed from
1470 * the list and one reference dropped. This function takes the list
1471 * lock and is atomic with respect to other list locking functions.
1472 */
1473 void skb_queue_purge(struct sk_buff_head *list)
1475 struct sk_buff *skb;
1476 while ((skb = skb_dequeue(list)) != NULL)
1477 kfree_skb(skb);
1480 /**
1481 * skb_queue_head - queue a buffer at the list head
1482 * @list: list to use
1483 * @newsk: buffer to queue
1485 * Queue a buffer at the start of the list. This function takes the
1486 * list lock and can be used safely with other locking &sk_buff functions
1487 * safely.
1489 * A buffer cannot be placed on two lists at the same time.
1490 */
1491 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
1493 unsigned long flags;
1495 spin_lock_irqsave(&list->lock, flags);
1496 __skb_queue_head(list, newsk);
1497 spin_unlock_irqrestore(&list->lock, flags);
1500 /**
1501 * skb_queue_tail - queue a buffer at the list tail
1502 * @list: list to use
1503 * @newsk: buffer to queue
1505 * Queue a buffer at the tail of the list. This function takes the
1506 * list lock and can be used safely with other locking &sk_buff functions
1507 * safely.
1509 * A buffer cannot be placed on two lists at the same time.
1510 */
1511 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
1513 unsigned long flags;
1515 spin_lock_irqsave(&list->lock, flags);
1516 __skb_queue_tail(list, newsk);
1517 spin_unlock_irqrestore(&list->lock, flags);
1520 /**
1521 * skb_unlink - remove a buffer from a list
1522 * @skb: buffer to remove
1523 * @list: list to use
1525 * Remove a packet from a list. The list locks are taken and this
1526 * function is atomic with respect to other list locked calls
1528 * You must know what list the SKB is on.
1529 */
1530 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1532 unsigned long flags;
1534 spin_lock_irqsave(&list->lock, flags);
1535 __skb_unlink(skb, list);
1536 spin_unlock_irqrestore(&list->lock, flags);
1539 /**
1540 * skb_append - append a buffer
1541 * @old: buffer to insert after
1542 * @newsk: buffer to insert
1543 * @list: list to use
1545 * Place a packet after a given packet in a list. The list locks are taken
1546 * and this function is atomic with respect to other list locked calls.
1547 * A buffer cannot be placed on two lists at the same time.
1548 */
1549 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
1551 unsigned long flags;
1553 spin_lock_irqsave(&list->lock, flags);
1554 __skb_append(old, newsk, list);
1555 spin_unlock_irqrestore(&list->lock, flags);
1559 /**
1560 * skb_insert - insert a buffer
1561 * @old: buffer to insert before
1562 * @newsk: buffer to insert
1563 * @list: list to use
1565 * Place a packet before a given packet in a list. The list locks are
1566 * taken and this function is atomic with respect to other list locked
1567 * calls.
1569 * A buffer cannot be placed on two lists at the same time.
1570 */
1571 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
1573 unsigned long flags;
1575 spin_lock_irqsave(&list->lock, flags);
1576 __skb_insert(newsk, old->prev, old, list);
1577 spin_unlock_irqrestore(&list->lock, flags);
1580 #if 0
1581 /*
1582 * Tune the memory allocator for a new MTU size.
1583 */
1584 void skb_add_mtu(int mtu)
1586 /* Must match allocation in alloc_skb */
1587 mtu = SKB_DATA_ALIGN(mtu) + sizeof(struct skb_shared_info);
1589 kmem_add_cache_size(mtu);
1591 #endif
1593 static inline void skb_split_inside_header(struct sk_buff *skb,
1594 struct sk_buff* skb1,
1595 const u32 len, const int pos)
1597 int i;
1599 memcpy(skb_put(skb1, pos - len), skb->data + len, pos - len);
1601 /* And move data appendix as is. */
1602 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
1603 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
1605 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
1606 skb_shinfo(skb)->nr_frags = 0;
1607 skb1->data_len = skb->data_len;
1608 skb1->len += skb1->data_len;
1609 skb->data_len = 0;
1610 skb->len = len;
1611 skb->tail = skb->data + len;
1614 static inline void skb_split_no_header(struct sk_buff *skb,
1615 struct sk_buff* skb1,
1616 const u32 len, int pos)
1618 int i, k = 0;
1619 const int nfrags = skb_shinfo(skb)->nr_frags;
1621 skb_shinfo(skb)->nr_frags = 0;
1622 skb1->len = skb1->data_len = skb->len - len;
1623 skb->len = len;
1624 skb->data_len = len - pos;
1626 for (i = 0; i < nfrags; i++) {
1627 int size = skb_shinfo(skb)->frags[i].size;
1629 if (pos + size > len) {
1630 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
1632 if (pos < len) {
1633 /* Split frag.
1634 * We have two variants in this case:
1635 * 1. Move all the frag to the second
1636 * part, if it is possible. F.e.
1637 * this approach is mandatory for TUX,
1638 * where splitting is expensive.
1639 * 2. Split is accurately. We make this.
1640 */
1641 get_page(skb_shinfo(skb)->frags[i].page);
1642 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
1643 skb_shinfo(skb1)->frags[0].size -= len - pos;
1644 skb_shinfo(skb)->frags[i].size = len - pos;
1645 skb_shinfo(skb)->nr_frags++;
1647 k++;
1648 } else
1649 skb_shinfo(skb)->nr_frags++;
1650 pos += size;
1652 skb_shinfo(skb1)->nr_frags = k;
1655 /**
1656 * skb_split - Split fragmented skb to two parts at length len.
1657 * @skb: the buffer to split
1658 * @skb1: the buffer to receive the second part
1659 * @len: new length for skb
1660 */
1661 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
1663 int pos = skb_headlen(skb);
1665 if (len < pos) /* Split line is inside header. */
1666 skb_split_inside_header(skb, skb1, len, pos);
1667 else /* Second chunk has no header, nothing to copy. */
1668 skb_split_no_header(skb, skb1, len, pos);
1671 /**
1672 * skb_prepare_seq_read - Prepare a sequential read of skb data
1673 * @skb: the buffer to read
1674 * @from: lower offset of data to be read
1675 * @to: upper offset of data to be read
1676 * @st: state variable
1678 * Initializes the specified state variable. Must be called before
1679 * invoking skb_seq_read() for the first time.
1680 */
1681 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
1682 unsigned int to, struct skb_seq_state *st)
1684 st->lower_offset = from;
1685 st->upper_offset = to;
1686 st->root_skb = st->cur_skb = skb;
1687 st->frag_idx = st->stepped_offset = 0;
1688 st->frag_data = NULL;
1691 /**
1692 * skb_seq_read - Sequentially read skb data
1693 * @consumed: number of bytes consumed by the caller so far
1694 * @data: destination pointer for data to be returned
1695 * @st: state variable
1697 * Reads a block of skb data at &consumed relative to the
1698 * lower offset specified to skb_prepare_seq_read(). Assigns
1699 * the head of the data block to &data and returns the length
1700 * of the block or 0 if the end of the skb data or the upper
1701 * offset has been reached.
1703 * The caller is not required to consume all of the data
1704 * returned, i.e. &consumed is typically set to the number
1705 * of bytes already consumed and the next call to
1706 * skb_seq_read() will return the remaining part of the block.
1708 * Note: The size of each block of data returned can be arbitary,
1709 * this limitation is the cost for zerocopy seqeuental
1710 * reads of potentially non linear data.
1712 * Note: Fragment lists within fragments are not implemented
1713 * at the moment, state->root_skb could be replaced with
1714 * a stack for this purpose.
1715 */
1716 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
1717 struct skb_seq_state *st)
1719 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
1720 skb_frag_t *frag;
1722 if (unlikely(abs_offset >= st->upper_offset))
1723 return 0;
1725 next_skb:
1726 block_limit = skb_headlen(st->cur_skb);
1728 if (abs_offset < block_limit) {
1729 *data = st->cur_skb->data + abs_offset;
1730 return block_limit - abs_offset;
1733 if (st->frag_idx == 0 && !st->frag_data)
1734 st->stepped_offset += skb_headlen(st->cur_skb);
1736 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
1737 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
1738 block_limit = frag->size + st->stepped_offset;
1740 if (abs_offset < block_limit) {
1741 if (!st->frag_data)
1742 st->frag_data = kmap_skb_frag(frag);
1744 *data = (u8 *) st->frag_data + frag->page_offset +
1745 (abs_offset - st->stepped_offset);
1747 return block_limit - abs_offset;
1750 if (st->frag_data) {
1751 kunmap_skb_frag(st->frag_data);
1752 st->frag_data = NULL;
1755 st->frag_idx++;
1756 st->stepped_offset += frag->size;
1759 if (st->cur_skb->next) {
1760 st->cur_skb = st->cur_skb->next;
1761 st->frag_idx = 0;
1762 goto next_skb;
1763 } else if (st->root_skb == st->cur_skb &&
1764 skb_shinfo(st->root_skb)->frag_list) {
1765 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
1766 goto next_skb;
1769 return 0;
1772 /**
1773 * skb_abort_seq_read - Abort a sequential read of skb data
1774 * @st: state variable
1776 * Must be called if skb_seq_read() was not called until it
1777 * returned 0.
1778 */
1779 void skb_abort_seq_read(struct skb_seq_state *st)
1781 if (st->frag_data)
1782 kunmap_skb_frag(st->frag_data);
1785 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
1787 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
1788 struct ts_config *conf,
1789 struct ts_state *state)
1791 return skb_seq_read(offset, text, TS_SKB_CB(state));
1794 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
1796 skb_abort_seq_read(TS_SKB_CB(state));
1799 /**
1800 * skb_find_text - Find a text pattern in skb data
1801 * @skb: the buffer to look in
1802 * @from: search offset
1803 * @to: search limit
1804 * @config: textsearch configuration
1805 * @state: uninitialized textsearch state variable
1807 * Finds a pattern in the skb data according to the specified
1808 * textsearch configuration. Use textsearch_next() to retrieve
1809 * subsequent occurrences of the pattern. Returns the offset
1810 * to the first occurrence or UINT_MAX if no match was found.
1811 */
1812 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
1813 unsigned int to, struct ts_config *config,
1814 struct ts_state *state)
1816 unsigned int ret;
1818 config->get_next_block = skb_ts_get_next_block;
1819 config->finish = skb_ts_finish;
1821 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
1823 ret = textsearch_find(config, state);
1824 return (ret <= to - from ? ret : UINT_MAX);
1827 /**
1828 * skb_append_datato_frags: - append the user data to a skb
1829 * @sk: sock structure
1830 * @skb: skb structure to be appened with user data.
1831 * @getfrag: call back function to be used for getting the user data
1832 * @from: pointer to user message iov
1833 * @length: length of the iov message
1835 * Description: This procedure append the user data in the fragment part
1836 * of the skb if any page alloc fails user this procedure returns -ENOMEM
1837 */
1838 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
1839 int (*getfrag)(void *from, char *to, int offset,
1840 int len, int odd, struct sk_buff *skb),
1841 void *from, int length)
1843 int frg_cnt = 0;
1844 skb_frag_t *frag = NULL;
1845 struct page *page = NULL;
1846 int copy, left;
1847 int offset = 0;
1848 int ret;
1850 do {
1851 /* Return error if we don't have space for new frag */
1852 frg_cnt = skb_shinfo(skb)->nr_frags;
1853 if (frg_cnt >= MAX_SKB_FRAGS)
1854 return -EFAULT;
1856 /* allocate a new page for next frag */
1857 page = alloc_pages(sk->sk_allocation, 0);
1859 /* If alloc_page fails just return failure and caller will
1860 * free previous allocated pages by doing kfree_skb()
1861 */
1862 if (page == NULL)
1863 return -ENOMEM;
1865 /* initialize the next frag */
1866 sk->sk_sndmsg_page = page;
1867 sk->sk_sndmsg_off = 0;
1868 skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
1869 skb->truesize += PAGE_SIZE;
1870 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
1872 /* get the new initialized frag */
1873 frg_cnt = skb_shinfo(skb)->nr_frags;
1874 frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
1876 /* copy the user data to page */
1877 left = PAGE_SIZE - frag->page_offset;
1878 copy = (length > left)? left : length;
1880 ret = getfrag(from, (page_address(frag->page) +
1881 frag->page_offset + frag->size),
1882 offset, copy, 0, skb);
1883 if (ret < 0)
1884 return -EFAULT;
1886 /* copy was successful so update the size parameters */
1887 sk->sk_sndmsg_off += copy;
1888 frag->size += copy;
1889 skb->len += copy;
1890 skb->data_len += copy;
1891 offset += copy;
1892 length -= copy;
1894 } while (length > 0);
1896 return 0;
1899 /**
1900 * skb_segment - Perform protocol segmentation on skb.
1901 * @skb: buffer to segment
1902 * @features: features for the output path (see dev->features)
1904 * This function performs segmentation on the given skb. It returns
1905 * the segment at the given position. It returns NULL if there are
1906 * no more segments to generate, or when an error is encountered.
1907 */
1908 struct sk_buff *skb_segment(struct sk_buff *skb, int features)
1910 struct sk_buff *segs = NULL;
1911 struct sk_buff *tail = NULL;
1912 unsigned int mss = skb_shinfo(skb)->gso_size;
1913 unsigned int doffset = skb->data - skb->mac.raw;
1914 unsigned int offset = doffset;
1915 unsigned int headroom;
1916 unsigned int len;
1917 int sg = features & NETIF_F_SG;
1918 int nfrags = skb_shinfo(skb)->nr_frags;
1919 int err = -ENOMEM;
1920 int i = 0;
1921 int pos;
1923 __skb_push(skb, doffset);
1924 headroom = skb_headroom(skb);
1925 pos = skb_headlen(skb);
1927 do {
1928 struct sk_buff *nskb;
1929 skb_frag_t *frag;
1930 int hsize;
1931 int k;
1932 int size;
1934 len = skb->len - offset;
1935 if (len > mss)
1936 len = mss;
1938 hsize = skb_headlen(skb) - offset;
1939 if (hsize < 0)
1940 hsize = 0;
1941 if (hsize > len || !sg)
1942 hsize = len;
1944 nskb = alloc_skb(hsize + doffset + headroom, GFP_ATOMIC);
1945 if (unlikely(!nskb))
1946 goto err;
1948 if (segs)
1949 tail->next = nskb;
1950 else
1951 segs = nskb;
1952 tail = nskb;
1954 nskb->dev = skb->dev;
1955 nskb->priority = skb->priority;
1956 nskb->protocol = skb->protocol;
1957 nskb->dst = dst_clone(skb->dst);
1958 memcpy(nskb->cb, skb->cb, sizeof(skb->cb));
1959 nskb->pkt_type = skb->pkt_type;
1960 nskb->mac_len = skb->mac_len;
1962 skb_reserve(nskb, headroom);
1963 nskb->mac.raw = nskb->data;
1964 nskb->nh.raw = nskb->data + skb->mac_len;
1965 nskb->h.raw = nskb->nh.raw + (skb->h.raw - skb->nh.raw);
1966 memcpy(skb_put(nskb, doffset), skb->data, doffset);
1968 if (!sg) {
1969 nskb->csum = skb_copy_and_csum_bits(skb, offset,
1970 skb_put(nskb, len),
1971 len, 0);
1972 continue;
1975 frag = skb_shinfo(nskb)->frags;
1976 k = 0;
1978 nskb->ip_summed = CHECKSUM_HW;
1979 nskb->csum = skb->csum;
1980 memcpy(skb_put(nskb, hsize), skb->data + offset, hsize);
1982 while (pos < offset + len) {
1983 BUG_ON(i >= nfrags);
1985 *frag = skb_shinfo(skb)->frags[i];
1986 get_page(frag->page);
1987 size = frag->size;
1989 if (pos < offset) {
1990 frag->page_offset += offset - pos;
1991 frag->size -= offset - pos;
1994 k++;
1996 if (pos + size <= offset + len) {
1997 i++;
1998 pos += size;
1999 } else {
2000 frag->size -= pos + size - (offset + len);
2001 break;
2004 frag++;
2007 skb_shinfo(nskb)->nr_frags = k;
2008 nskb->data_len = len - hsize;
2009 nskb->len += nskb->data_len;
2010 nskb->truesize += nskb->data_len;
2011 } while ((offset += len) < skb->len);
2013 return segs;
2015 err:
2016 while ((skb = segs)) {
2017 segs = skb->next;
2018 kfree(skb);
2020 return ERR_PTR(err);
2023 EXPORT_SYMBOL_GPL(skb_segment);
2025 /**
2026 * skb_pull_rcsum - pull skb and update receive checksum
2027 * @skb: buffer to update
2028 * @start: start of data before pull
2029 * @len: length of data pulled
2031 * This function performs an skb_pull on the packet and updates
2032 * update the CHECKSUM_HW checksum. It should be used on receive
2033 * path processing instead of skb_pull unless you know that the
2034 * checksum difference is zero (e.g., a valid IP header) or you
2035 * are setting ip_summed to CHECKSUM_NONE.
2036 */
2037 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2039 BUG_ON(len > skb->len);
2040 skb->len -= len;
2041 BUG_ON(skb->len < skb->data_len);
2042 skb_postpull_rcsum(skb, skb->data, len);
2043 return skb->data += len;
2046 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2048 void __init skb_init(void)
2050 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
2051 sizeof(struct sk_buff),
2052 0,
2053 SLAB_HWCACHE_ALIGN,
2054 NULL, NULL);
2055 if (!skbuff_head_cache)
2056 panic("cannot create skbuff cache");
2058 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
2059 (2*sizeof(struct sk_buff)) +
2060 sizeof(atomic_t),
2061 0,
2062 SLAB_HWCACHE_ALIGN,
2063 NULL, NULL);
2064 if (!skbuff_fclone_cache)
2065 panic("cannot create skbuff cache");
2068 EXPORT_SYMBOL(___pskb_trim);
2069 EXPORT_SYMBOL(__kfree_skb);
2070 EXPORT_SYMBOL(kfree_skb);
2071 EXPORT_SYMBOL(__pskb_pull_tail);
2072 EXPORT_SYMBOL(__alloc_skb);
2073 EXPORT_SYMBOL(__netdev_alloc_skb);
2074 EXPORT_SYMBOL(pskb_copy);
2075 EXPORT_SYMBOL(pskb_expand_head);
2076 EXPORT_SYMBOL(skb_checksum);
2077 EXPORT_SYMBOL(skb_clone);
2078 EXPORT_SYMBOL(skb_clone_fraglist);
2079 EXPORT_SYMBOL(skb_copy);
2080 EXPORT_SYMBOL(skb_copy_and_csum_bits);
2081 EXPORT_SYMBOL(skb_copy_and_csum_dev);
2082 EXPORT_SYMBOL(skb_copy_bits);
2083 EXPORT_SYMBOL(skb_copy_expand);
2084 EXPORT_SYMBOL(skb_over_panic);
2085 EXPORT_SYMBOL(skb_pad);
2086 EXPORT_SYMBOL(skb_realloc_headroom);
2087 EXPORT_SYMBOL(skb_under_panic);
2088 EXPORT_SYMBOL(skb_dequeue);
2089 EXPORT_SYMBOL(skb_dequeue_tail);
2090 EXPORT_SYMBOL(skb_insert);
2091 EXPORT_SYMBOL(skb_queue_purge);
2092 EXPORT_SYMBOL(skb_queue_head);
2093 EXPORT_SYMBOL(skb_queue_tail);
2094 EXPORT_SYMBOL(skb_unlink);
2095 EXPORT_SYMBOL(skb_append);
2096 EXPORT_SYMBOL(skb_split);
2097 EXPORT_SYMBOL(skb_prepare_seq_read);
2098 EXPORT_SYMBOL(skb_seq_read);
2099 EXPORT_SYMBOL(skb_abort_seq_read);
2100 EXPORT_SYMBOL(skb_find_text);
2101 EXPORT_SYMBOL(skb_append_datato_frags);