Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 464143c911df98d4913932534485113a0a14aa74

Authored by David S. Miller
2 parents bb1231052e 2fc72c7b84
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

Merge branch 'master' of git://1984.lsi.us.es/net-2.6

Showing 5 changed files Inline Diff

include/linux/skbuff.h
Diff comments   View file @ 464143c
1 /* 1 /*
2 * Definitions for the 'struct sk_buff' memory handlers. 2 * Definitions for the 'struct sk_buff' memory handlers.
3 * 3 *
4 * Authors: 4 * Authors:
5 * Alan Cox, <gw4pts@gw4pts.ampr.org> 5 * Alan Cox, <gw4pts@gw4pts.ampr.org>
6 * Florian La Roche, <rzsfl@rz.uni-sb.de> 6 * Florian La Roche, <rzsfl@rz.uni-sb.de>
7 * 7 *
8 * This program is free software; you can redistribute it and/or 8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License 9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version. 11 * 2 of the License, or (at your option) any later version.
12 */ 12 */
13 13
14 #ifndef _LINUX_SKBUFF_H 14 #ifndef _LINUX_SKBUFF_H
15 #define _LINUX_SKBUFF_H 15 #define _LINUX_SKBUFF_H
16 16
17 #include <linux/kernel.h> 17 #include <linux/kernel.h>
18 #include <linux/kmemcheck.h> 18 #include <linux/kmemcheck.h>
19 #include <linux/compiler.h> 19 #include <linux/compiler.h>
20 #include <linux/time.h> 20 #include <linux/time.h>
21 #include <linux/cache.h> 21 #include <linux/cache.h>
22 22
23 #include <asm/atomic.h> 23 #include <asm/atomic.h>
24 #include <asm/types.h> 24 #include <asm/types.h>
25 #include <linux/spinlock.h> 25 #include <linux/spinlock.h>
26 #include <linux/net.h> 26 #include <linux/net.h>
27 #include <linux/textsearch.h> 27 #include <linux/textsearch.h>
28 #include <net/checksum.h> 28 #include <net/checksum.h>
29 #include <linux/rcupdate.h> 29 #include <linux/rcupdate.h>
30 #include <linux/dmaengine.h> 30 #include <linux/dmaengine.h>
31 #include <linux/hrtimer.h> 31 #include <linux/hrtimer.h>
32 32
33 /* Don't change this without changing skb_csum_unnecessary! */ 33 /* Don't change this without changing skb_csum_unnecessary! */
34 #define CHECKSUM_NONE 0 34 #define CHECKSUM_NONE 0
35 #define CHECKSUM_UNNECESSARY 1 35 #define CHECKSUM_UNNECESSARY 1
36 #define CHECKSUM_COMPLETE 2 36 #define CHECKSUM_COMPLETE 2
37 #define CHECKSUM_PARTIAL 3 37 #define CHECKSUM_PARTIAL 3
38 38
39 #define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \ 39 #define SKB_DATA_ALIGN(X) (((X) + (SMP_CACHE_BYTES - 1)) & \
40 ~(SMP_CACHE_BYTES - 1)) 40 ~(SMP_CACHE_BYTES - 1))
41 #define SKB_WITH_OVERHEAD(X) \ 41 #define SKB_WITH_OVERHEAD(X) \
42 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 42 ((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
43 #define SKB_MAX_ORDER(X, ORDER) \ 43 #define SKB_MAX_ORDER(X, ORDER) \
44 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X)) 44 SKB_WITH_OVERHEAD((PAGE_SIZE << (ORDER)) - (X))
45 #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0)) 45 #define SKB_MAX_HEAD(X) (SKB_MAX_ORDER((X), 0))
46 #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2)) 46 #define SKB_MAX_ALLOC (SKB_MAX_ORDER(0, 2))
47 47
48 /* A. Checksumming of received packets by device. 48 /* A. Checksumming of received packets by device.
49 * 49 *
50 * NONE: device failed to checksum this packet. 50 * NONE: device failed to checksum this packet.
51 * skb->csum is undefined. 51 * skb->csum is undefined.
52 * 52 *
53 * UNNECESSARY: device parsed packet and wouldbe verified checksum. 53 * UNNECESSARY: device parsed packet and wouldbe verified checksum.
54 * skb->csum is undefined. 54 * skb->csum is undefined.
55 * It is bad option, but, unfortunately, many of vendors do this. 55 * It is bad option, but, unfortunately, many of vendors do this.
56 * Apparently with secret goal to sell you new device, when you 56 * Apparently with secret goal to sell you new device, when you
57 * will add new protocol to your host. F.e. IPv6. 8) 57 * will add new protocol to your host. F.e. IPv6. 8)
58 * 58 *
59 * COMPLETE: the most generic way. Device supplied checksum of _all_ 59 * COMPLETE: the most generic way. Device supplied checksum of _all_
60 * the packet as seen by netif_rx in skb->csum. 60 * the packet as seen by netif_rx in skb->csum.
61 * NOTE: Even if device supports only some protocols, but 61 * NOTE: Even if device supports only some protocols, but
62 * is able to produce some skb->csum, it MUST use COMPLETE, 62 * is able to produce some skb->csum, it MUST use COMPLETE,
63 * not UNNECESSARY. 63 * not UNNECESSARY.
64 * 64 *
65 * PARTIAL: identical to the case for output below. This may occur 65 * PARTIAL: identical to the case for output below. This may occur
66 * on a packet received directly from another Linux OS, e.g., 66 * on a packet received directly from another Linux OS, e.g.,
67 * a virtualised Linux kernel on the same host. The packet can 67 * a virtualised Linux kernel on the same host. The packet can
68 * be treated in the same way as UNNECESSARY except that on 68 * be treated in the same way as UNNECESSARY except that on
69 * output (i.e., forwarding) the checksum must be filled in 69 * output (i.e., forwarding) the checksum must be filled in
70 * by the OS or the hardware. 70 * by the OS or the hardware.
71 * 71 *
72 * B. Checksumming on output. 72 * B. Checksumming on output.
73 * 73 *
74 * NONE: skb is checksummed by protocol or csum is not required. 74 * NONE: skb is checksummed by protocol or csum is not required.
75 * 75 *
76 * PARTIAL: device is required to csum packet as seen by hard_start_xmit 76 * PARTIAL: device is required to csum packet as seen by hard_start_xmit
77 * from skb->csum_start to the end and to record the checksum 77 * from skb->csum_start to the end and to record the checksum
78 * at skb->csum_start + skb->csum_offset. 78 * at skb->csum_start + skb->csum_offset.
79 * 79 *
80 * Device must show its capabilities in dev->features, set 80 * Device must show its capabilities in dev->features, set
81 * at device setup time. 81 * at device setup time.
82 * NETIF_F_HW_CSUM - it is clever device, it is able to checksum 82 * NETIF_F_HW_CSUM - it is clever device, it is able to checksum
83 * everything. 83 * everything.
84 * NETIF_F_NO_CSUM - loopback or reliable single hop media. 84 * NETIF_F_NO_CSUM - loopback or reliable single hop media.
85 * NETIF_F_IP_CSUM - device is dumb. It is able to csum only 85 * NETIF_F_IP_CSUM - device is dumb. It is able to csum only
86 * TCP/UDP over IPv4. Sigh. Vendors like this 86 * TCP/UDP over IPv4. Sigh. Vendors like this
87 * way by an unknown reason. Though, see comment above 87 * way by an unknown reason. Though, see comment above
88 * about CHECKSUM_UNNECESSARY. 8) 88 * about CHECKSUM_UNNECESSARY. 8)
89 * NETIF_F_IPV6_CSUM about as dumb as the last one but does IPv6 instead. 89 * NETIF_F_IPV6_CSUM about as dumb as the last one but does IPv6 instead.
90 * 90 *
91 * Any questions? No questions, good. --ANK 91 * Any questions? No questions, good. --ANK
92 */ 92 */
93 93
94 struct net_device; 94 struct net_device;
95 struct scatterlist; 95 struct scatterlist;
96 struct pipe_inode_info; 96 struct pipe_inode_info;
97 97
98 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 98 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
99 struct nf_conntrack { 99 struct nf_conntrack {
100 atomic_t use; 100 atomic_t use;
101 }; 101 };
102 #endif 102 #endif
103 103
104 #ifdef CONFIG_BRIDGE_NETFILTER 104 #ifdef CONFIG_BRIDGE_NETFILTER
105 struct nf_bridge_info { 105 struct nf_bridge_info {
106 atomic_t use; 106 atomic_t use;
107 struct net_device *physindev; 107 struct net_device *physindev;
108 struct net_device *physoutdev; 108 struct net_device *physoutdev;
109 unsigned int mask; 109 unsigned int mask;
110 unsigned long data[32 / sizeof(unsigned long)]; 110 unsigned long data[32 / sizeof(unsigned long)];
111 }; 111 };
112 #endif 112 #endif
113 113
114 struct sk_buff_head { 114 struct sk_buff_head {
115 /* These two members must be first. */ 115 /* These two members must be first. */
116 struct sk_buff *next; 116 struct sk_buff *next;
117 struct sk_buff *prev; 117 struct sk_buff *prev;
118 118
119 __u32 qlen; 119 __u32 qlen;
120 spinlock_t lock; 120 spinlock_t lock;
121 }; 121 };
122 122
123 struct sk_buff; 123 struct sk_buff;
124 124
125 /* To allow 64K frame to be packed as single skb without frag_list */ 125 /* To allow 64K frame to be packed as single skb without frag_list */
126 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 2) 126 #define MAX_SKB_FRAGS (65536/PAGE_SIZE + 2)
127 127
128 typedef struct skb_frag_struct skb_frag_t; 128 typedef struct skb_frag_struct skb_frag_t;
129 129
130 struct skb_frag_struct { 130 struct skb_frag_struct {
131 struct page *page; 131 struct page *page;
132 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) 132 #if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
133 __u32 page_offset; 133 __u32 page_offset;
134 __u32 size; 134 __u32 size;
135 #else 135 #else
136 __u16 page_offset; 136 __u16 page_offset;
137 __u16 size; 137 __u16 size;
138 #endif 138 #endif
139 }; 139 };
140 140
141 #define HAVE_HW_TIME_STAMP 141 #define HAVE_HW_TIME_STAMP
142 142
143 /** 143 /**
144 * struct skb_shared_hwtstamps - hardware time stamps 144 * struct skb_shared_hwtstamps - hardware time stamps
145 * @hwtstamp: hardware time stamp transformed into duration 145 * @hwtstamp: hardware time stamp transformed into duration
146 * since arbitrary point in time 146 * since arbitrary point in time
147 * @syststamp: hwtstamp transformed to system time base 147 * @syststamp: hwtstamp transformed to system time base
148 * 148 *
149 * Software time stamps generated by ktime_get_real() are stored in 149 * Software time stamps generated by ktime_get_real() are stored in
150 * skb->tstamp. The relation between the different kinds of time 150 * skb->tstamp. The relation between the different kinds of time
151 * stamps is as follows: 151 * stamps is as follows:
152 * 152 *
153 * syststamp and tstamp can be compared against each other in 153 * syststamp and tstamp can be compared against each other in
154 * arbitrary combinations. The accuracy of a 154 * arbitrary combinations. The accuracy of a
155 * syststamp/tstamp/"syststamp from other device" comparison is 155 * syststamp/tstamp/"syststamp from other device" comparison is
156 * limited by the accuracy of the transformation into system time 156 * limited by the accuracy of the transformation into system time
157 * base. This depends on the device driver and its underlying 157 * base. This depends on the device driver and its underlying
158 * hardware. 158 * hardware.
159 * 159 *
160 * hwtstamps can only be compared against other hwtstamps from 160 * hwtstamps can only be compared against other hwtstamps from
161 * the same device. 161 * the same device.
162 * 162 *
163 * This structure is attached to packets as part of the 163 * This structure is attached to packets as part of the
164 * &skb_shared_info. Use skb_hwtstamps() to get a pointer. 164 * &skb_shared_info. Use skb_hwtstamps() to get a pointer.
165 */ 165 */
166 struct skb_shared_hwtstamps { 166 struct skb_shared_hwtstamps {
167 ktime_t hwtstamp; 167 ktime_t hwtstamp;
168 ktime_t syststamp; 168 ktime_t syststamp;
169 }; 169 };
170 170
171 /* Definitions for tx_flags in struct skb_shared_info */ 171 /* Definitions for tx_flags in struct skb_shared_info */
172 enum { 172 enum {
173 /* generate hardware time stamp */ 173 /* generate hardware time stamp */
174 SKBTX_HW_TSTAMP = 1 << 0, 174 SKBTX_HW_TSTAMP = 1 << 0,
175 175
176 /* generate software time stamp */ 176 /* generate software time stamp */
177 SKBTX_SW_TSTAMP = 1 << 1, 177 SKBTX_SW_TSTAMP = 1 << 1,
178 178
179 /* device driver is going to provide hardware time stamp */ 179 /* device driver is going to provide hardware time stamp */
180 SKBTX_IN_PROGRESS = 1 << 2, 180 SKBTX_IN_PROGRESS = 1 << 2,
181 181
182 /* ensure the originating sk reference is available on driver level */ 182 /* ensure the originating sk reference is available on driver level */
183 SKBTX_DRV_NEEDS_SK_REF = 1 << 3, 183 SKBTX_DRV_NEEDS_SK_REF = 1 << 3,
184 }; 184 };
185 185
186 /* This data is invariant across clones and lives at 186 /* This data is invariant across clones and lives at
187 * the end of the header data, ie. at skb->end. 187 * the end of the header data, ie. at skb->end.
188 */ 188 */
189 struct skb_shared_info { 189 struct skb_shared_info {
190 unsigned short nr_frags; 190 unsigned short nr_frags;
191 unsigned short gso_size; 191 unsigned short gso_size;
192 /* Warning: this field is not always filled in (UFO)! */ 192 /* Warning: this field is not always filled in (UFO)! */
193 unsigned short gso_segs; 193 unsigned short gso_segs;
194 unsigned short gso_type; 194 unsigned short gso_type;
195 __be32 ip6_frag_id; 195 __be32 ip6_frag_id;
196 __u8 tx_flags; 196 __u8 tx_flags;
197 struct sk_buff *frag_list; 197 struct sk_buff *frag_list;
198 struct skb_shared_hwtstamps hwtstamps; 198 struct skb_shared_hwtstamps hwtstamps;
199 199
200 /* 200 /*
201 * Warning : all fields before dataref are cleared in __alloc_skb() 201 * Warning : all fields before dataref are cleared in __alloc_skb()
202 */ 202 */
203 atomic_t dataref; 203 atomic_t dataref;
204 204
205 /* Intermediate layers must ensure that destructor_arg 205 /* Intermediate layers must ensure that destructor_arg
206 * remains valid until skb destructor */ 206 * remains valid until skb destructor */
207 void * destructor_arg; 207 void * destructor_arg;
208 /* must be last field, see pskb_expand_head() */ 208 /* must be last field, see pskb_expand_head() */
209 skb_frag_t frags[MAX_SKB_FRAGS]; 209 skb_frag_t frags[MAX_SKB_FRAGS];
210 }; 210 };
211 211
212 /* We divide dataref into two halves. The higher 16 bits hold references 212 /* We divide dataref into two halves. The higher 16 bits hold references
213 * to the payload part of skb->data. The lower 16 bits hold references to 213 * to the payload part of skb->data. The lower 16 bits hold references to
214 * the entire skb->data. A clone of a headerless skb holds the length of 214 * the entire skb->data. A clone of a headerless skb holds the length of
215 * the header in skb->hdr_len. 215 * the header in skb->hdr_len.
216 * 216 *
217 * All users must obey the rule that the skb->data reference count must be 217 * All users must obey the rule that the skb->data reference count must be
218 * greater than or equal to the payload reference count. 218 * greater than or equal to the payload reference count.
219 * 219 *
220 * Holding a reference to the payload part means that the user does not 220 * Holding a reference to the payload part means that the user does not
221 * care about modifications to the header part of skb->data. 221 * care about modifications to the header part of skb->data.
222 */ 222 */
223 #define SKB_DATAREF_SHIFT 16 223 #define SKB_DATAREF_SHIFT 16
224 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1) 224 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1)
225 225
226 226
227 enum { 227 enum {
228 SKB_FCLONE_UNAVAILABLE, 228 SKB_FCLONE_UNAVAILABLE,
229 SKB_FCLONE_ORIG, 229 SKB_FCLONE_ORIG,
230 SKB_FCLONE_CLONE, 230 SKB_FCLONE_CLONE,
231 }; 231 };
232 232
233 enum { 233 enum {
234 SKB_GSO_TCPV4 = 1 << 0, 234 SKB_GSO_TCPV4 = 1 << 0,
235 SKB_GSO_UDP = 1 << 1, 235 SKB_GSO_UDP = 1 << 1,
236 236
237 /* This indicates the skb is from an untrusted source. */ 237 /* This indicates the skb is from an untrusted source. */
238 SKB_GSO_DODGY = 1 << 2, 238 SKB_GSO_DODGY = 1 << 2,
239 239
240 /* This indicates the tcp segment has CWR set. */ 240 /* This indicates the tcp segment has CWR set. */
241 SKB_GSO_TCP_ECN = 1 << 3, 241 SKB_GSO_TCP_ECN = 1 << 3,
242 242
243 SKB_GSO_TCPV6 = 1 << 4, 243 SKB_GSO_TCPV6 = 1 << 4,
244 244
245 SKB_GSO_FCOE = 1 << 5, 245 SKB_GSO_FCOE = 1 << 5,
246 }; 246 };
247 247
248 #if BITS_PER_LONG > 32 248 #if BITS_PER_LONG > 32
249 #define NET_SKBUFF_DATA_USES_OFFSET 1 249 #define NET_SKBUFF_DATA_USES_OFFSET 1
250 #endif 250 #endif
251 251
252 #ifdef NET_SKBUFF_DATA_USES_OFFSET 252 #ifdef NET_SKBUFF_DATA_USES_OFFSET
253 typedef unsigned int sk_buff_data_t; 253 typedef unsigned int sk_buff_data_t;
254 #else 254 #else
255 typedef unsigned char *sk_buff_data_t; 255 typedef unsigned char *sk_buff_data_t;
256 #endif 256 #endif
257 257
258 #if defined(CONFIG_NF_DEFRAG_IPV4) || defined(CONFIG_NF_DEFRAG_IPV4_MODULE) || \
259 defined(CONFIG_NF_DEFRAG_IPV6) || defined(CONFIG_NF_DEFRAG_IPV6_MODULE)
260 #define NET_SKBUFF_NF_DEFRAG_NEEDED 1
261 #endif
262
258 /** 263 /**
259 * struct sk_buff - socket buffer 264 * struct sk_buff - socket buffer
260 * @next: Next buffer in list 265 * @next: Next buffer in list
261 * @prev: Previous buffer in list 266 * @prev: Previous buffer in list
262 * @sk: Socket we are owned by 267 * @sk: Socket we are owned by
263 * @tstamp: Time we arrived 268 * @tstamp: Time we arrived
264 * @dev: Device we arrived on/are leaving by 269 * @dev: Device we arrived on/are leaving by
265 * @transport_header: Transport layer header 270 * @transport_header: Transport layer header
266 * @network_header: Network layer header 271 * @network_header: Network layer header
267 * @mac_header: Link layer header 272 * @mac_header: Link layer header
268 * @_skb_refdst: destination entry (with norefcount bit) 273 * @_skb_refdst: destination entry (with norefcount bit)
269 * @sp: the security path, used for xfrm 274 * @sp: the security path, used for xfrm
270 * @cb: Control buffer. Free for use by every layer. Put private vars here 275 * @cb: Control buffer. Free for use by every layer. Put private vars here
271 * @len: Length of actual data 276 * @len: Length of actual data
272 * @data_len: Data length 277 * @data_len: Data length
273 * @mac_len: Length of link layer header 278 * @mac_len: Length of link layer header
274 * @hdr_len: writable header length of cloned skb 279 * @hdr_len: writable header length of cloned skb
275 * @csum: Checksum (must include start/offset pair) 280 * @csum: Checksum (must include start/offset pair)
276 * @csum_start: Offset from skb->head where checksumming should start 281 * @csum_start: Offset from skb->head where checksumming should start
277 * @csum_offset: Offset from csum_start where checksum should be stored 282 * @csum_offset: Offset from csum_start where checksum should be stored
278 * @local_df: allow local fragmentation 283 * @local_df: allow local fragmentation
279 * @cloned: Head may be cloned (check refcnt to be sure) 284 * @cloned: Head may be cloned (check refcnt to be sure)
280 * @nohdr: Payload reference only, must not modify header 285 * @nohdr: Payload reference only, must not modify header
281 * @pkt_type: Packet class 286 * @pkt_type: Packet class
282 * @fclone: skbuff clone status 287 * @fclone: skbuff clone status
283 * @ip_summed: Driver fed us an IP checksum 288 * @ip_summed: Driver fed us an IP checksum
284 * @priority: Packet queueing priority 289 * @priority: Packet queueing priority
285 * @users: User count - see {datagram,tcp}.c 290 * @users: User count - see {datagram,tcp}.c
286 * @protocol: Packet protocol from driver 291 * @protocol: Packet protocol from driver
287 * @truesize: Buffer size 292 * @truesize: Buffer size
288 * @head: Head of buffer 293 * @head: Head of buffer
289 * @data: Data head pointer 294 * @data: Data head pointer
290 * @tail: Tail pointer 295 * @tail: Tail pointer
291 * @end: End pointer 296 * @end: End pointer
292 * @destructor: Destruct function 297 * @destructor: Destruct function
293 * @mark: Generic packet mark 298 * @mark: Generic packet mark
294 * @nfct: Associated connection, if any 299 * @nfct: Associated connection, if any
295 * @ipvs_property: skbuff is owned by ipvs 300 * @ipvs_property: skbuff is owned by ipvs
296 * @peeked: this packet has been seen already, so stats have been 301 * @peeked: this packet has been seen already, so stats have been
297 * done for it, don't do them again 302 * done for it, don't do them again
298 * @nf_trace: netfilter packet trace flag 303 * @nf_trace: netfilter packet trace flag
299 * @nfctinfo: Relationship of this skb to the connection 304 * @nfctinfo: Relationship of this skb to the connection
300 * @nfct_reasm: netfilter conntrack re-assembly pointer 305 * @nfct_reasm: netfilter conntrack re-assembly pointer
301 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c 306 * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
302 * @skb_iif: ifindex of device we arrived on 307 * @skb_iif: ifindex of device we arrived on
303 * @rxhash: the packet hash computed on receive 308 * @rxhash: the packet hash computed on receive
304 * @queue_mapping: Queue mapping for multiqueue devices 309 * @queue_mapping: Queue mapping for multiqueue devices
305 * @tc_index: Traffic control index 310 * @tc_index: Traffic control index
306 * @tc_verd: traffic control verdict 311 * @tc_verd: traffic control verdict
307 * @ndisc_nodetype: router type (from link layer) 312 * @ndisc_nodetype: router type (from link layer)
308 * @dma_cookie: a cookie to one of several possible DMA operations 313 * @dma_cookie: a cookie to one of several possible DMA operations
309 * done by skb DMA functions 314 * done by skb DMA functions
310 * @secmark: security marking 315 * @secmark: security marking
311 * @vlan_tci: vlan tag control information 316 * @vlan_tci: vlan tag control information
312 */ 317 */
313 318
314 struct sk_buff { 319 struct sk_buff {
315 /* These two members must be first. */ 320 /* These two members must be first. */
316 struct sk_buff *next; 321 struct sk_buff *next;
317 struct sk_buff *prev; 322 struct sk_buff *prev;
318 323
319 ktime_t tstamp; 324 ktime_t tstamp;
320 325
321 struct sock *sk; 326 struct sock *sk;
322 struct net_device *dev; 327 struct net_device *dev;
323 328
324 /* 329 /*
325 * This is the control buffer. It is free to use for every 330 * This is the control buffer. It is free to use for every
326 * layer. Please put your private variables there. If you 331 * layer. Please put your private variables there. If you
327 * want to keep them across layers you have to do a skb_clone() 332 * want to keep them across layers you have to do a skb_clone()
328 * first. This is owned by whoever has the skb queued ATM. 333 * first. This is owned by whoever has the skb queued ATM.
329 */ 334 */
330 char cb[48] __aligned(8); 335 char cb[48] __aligned(8);
331 336
332 unsigned long _skb_refdst; 337 unsigned long _skb_refdst;
333 #ifdef CONFIG_XFRM 338 #ifdef CONFIG_XFRM
334 struct sec_path *sp; 339 struct sec_path *sp;
335 #endif 340 #endif
336 unsigned int len, 341 unsigned int len,
337 data_len; 342 data_len;
338 __u16 mac_len, 343 __u16 mac_len,
339 hdr_len; 344 hdr_len;
340 union { 345 union {
341 __wsum csum; 346 __wsum csum;
342 struct { 347 struct {
343 __u16 csum_start; 348 __u16 csum_start;
344 __u16 csum_offset; 349 __u16 csum_offset;
345 }; 350 };
346 }; 351 };
347 __u32 priority; 352 __u32 priority;
348 kmemcheck_bitfield_begin(flags1); 353 kmemcheck_bitfield_begin(flags1);
349 __u8 local_df:1, 354 __u8 local_df:1,
350 cloned:1, 355 cloned:1,
351 ip_summed:2, 356 ip_summed:2,
352 nohdr:1, 357 nohdr:1,
353 nfctinfo:3; 358 nfctinfo:3;
354 __u8 pkt_type:3, 359 __u8 pkt_type:3,
355 fclone:2, 360 fclone:2,
356 ipvs_property:1, 361 ipvs_property:1,
357 peeked:1, 362 peeked:1,
358 nf_trace:1; 363 nf_trace:1;
359 kmemcheck_bitfield_end(flags1); 364 kmemcheck_bitfield_end(flags1);
360 __be16 protocol; 365 __be16 protocol;
361 366
362 void (*destructor)(struct sk_buff *skb); 367 void (*destructor)(struct sk_buff *skb);
363 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 368 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
364 struct nf_conntrack *nfct; 369 struct nf_conntrack *nfct;
370 #endif
371 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
365 struct sk_buff *nfct_reasm; 372 struct sk_buff *nfct_reasm;
366 #endif 373 #endif
367 #ifdef CONFIG_BRIDGE_NETFILTER 374 #ifdef CONFIG_BRIDGE_NETFILTER
368 struct nf_bridge_info *nf_bridge; 375 struct nf_bridge_info *nf_bridge;
369 #endif 376 #endif
370 377
371 int skb_iif; 378 int skb_iif;
372 #ifdef CONFIG_NET_SCHED 379 #ifdef CONFIG_NET_SCHED
373 __u16 tc_index; /* traffic control index */ 380 __u16 tc_index; /* traffic control index */
374 #ifdef CONFIG_NET_CLS_ACT 381 #ifdef CONFIG_NET_CLS_ACT
375 __u16 tc_verd; /* traffic control verdict */ 382 __u16 tc_verd; /* traffic control verdict */
376 #endif 383 #endif
377 #endif 384 #endif
378 385
379 __u32 rxhash; 386 __u32 rxhash;
380 387
381 kmemcheck_bitfield_begin(flags2); 388 kmemcheck_bitfield_begin(flags2);
382 __u16 queue_mapping:16; 389 __u16 queue_mapping:16;
383 #ifdef CONFIG_IPV6_NDISC_NODETYPE 390 #ifdef CONFIG_IPV6_NDISC_NODETYPE
384 __u8 ndisc_nodetype:2, 391 __u8 ndisc_nodetype:2,
385 deliver_no_wcard:1; 392 deliver_no_wcard:1;
386 #else 393 #else
387 __u8 deliver_no_wcard:1; 394 __u8 deliver_no_wcard:1;
388 #endif 395 #endif
389 __u8 ooo_okay:1; 396 __u8 ooo_okay:1;
390 kmemcheck_bitfield_end(flags2); 397 kmemcheck_bitfield_end(flags2);
391 398
392 /* 0/13 bit hole */ 399 /* 0/13 bit hole */
393 400
394 #ifdef CONFIG_NET_DMA 401 #ifdef CONFIG_NET_DMA
395 dma_cookie_t dma_cookie; 402 dma_cookie_t dma_cookie;
396 #endif 403 #endif
397 #ifdef CONFIG_NETWORK_SECMARK 404 #ifdef CONFIG_NETWORK_SECMARK
398 __u32 secmark; 405 __u32 secmark;
399 #endif 406 #endif
400 union { 407 union {
401 __u32 mark; 408 __u32 mark;
402 __u32 dropcount; 409 __u32 dropcount;
403 }; 410 };
404 411
405 __u16 vlan_tci; 412 __u16 vlan_tci;
406 413
407 sk_buff_data_t transport_header; 414 sk_buff_data_t transport_header;
408 sk_buff_data_t network_header; 415 sk_buff_data_t network_header;
409 sk_buff_data_t mac_header; 416 sk_buff_data_t mac_header;
410 /* These elements must be at the end, see alloc_skb() for details. */ 417 /* These elements must be at the end, see alloc_skb() for details. */
411 sk_buff_data_t tail; 418 sk_buff_data_t tail;
412 sk_buff_data_t end; 419 sk_buff_data_t end;
413 unsigned char *head, 420 unsigned char *head,
414 *data; 421 *data;
415 unsigned int truesize; 422 unsigned int truesize;
416 atomic_t users; 423 atomic_t users;
417 }; 424 };
418 425
419 #ifdef __KERNEL__ 426 #ifdef __KERNEL__
420 /* 427 /*
421 * Handling routines are only of interest to the kernel 428 * Handling routines are only of interest to the kernel
422 */ 429 */
423 #include <linux/slab.h> 430 #include <linux/slab.h>
424 431
425 #include <asm/system.h> 432 #include <asm/system.h>
426 433
427 /* 434 /*
428 * skb might have a dst pointer attached, refcounted or not. 435 * skb might have a dst pointer attached, refcounted or not.
429 * _skb_refdst low order bit is set if refcount was _not_ taken 436 * _skb_refdst low order bit is set if refcount was _not_ taken
430 */ 437 */
431 #define SKB_DST_NOREF 1UL 438 #define SKB_DST_NOREF 1UL
432 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF) 439 #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
433 440
434 /** 441 /**
435 * skb_dst - returns skb dst_entry 442 * skb_dst - returns skb dst_entry
436 * @skb: buffer 443 * @skb: buffer
437 * 444 *
438 * Returns skb dst_entry, regardless of reference taken or not. 445 * Returns skb dst_entry, regardless of reference taken or not.
439 */ 446 */
440 static inline struct dst_entry *skb_dst(const struct sk_buff *skb) 447 static inline struct dst_entry *skb_dst(const struct sk_buff *skb)
441 { 448 {
442 /* If refdst was not refcounted, check we still are in a 449 /* If refdst was not refcounted, check we still are in a
443 * rcu_read_lock section 450 * rcu_read_lock section
444 */ 451 */
445 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) && 452 WARN_ON((skb->_skb_refdst & SKB_DST_NOREF) &&
446 !rcu_read_lock_held() && 453 !rcu_read_lock_held() &&
447 !rcu_read_lock_bh_held()); 454 !rcu_read_lock_bh_held());
448 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK); 455 return (struct dst_entry *)(skb->_skb_refdst & SKB_DST_PTRMASK);
449 } 456 }
450 457
451 /** 458 /**
452 * skb_dst_set - sets skb dst 459 * skb_dst_set - sets skb dst
453 * @skb: buffer 460 * @skb: buffer
454 * @dst: dst entry 461 * @dst: dst entry
455 * 462 *
456 * Sets skb dst, assuming a reference was taken on dst and should 463 * Sets skb dst, assuming a reference was taken on dst and should
457 * be released by skb_dst_drop() 464 * be released by skb_dst_drop()
458 */ 465 */
459 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst) 466 static inline void skb_dst_set(struct sk_buff *skb, struct dst_entry *dst)
460 { 467 {
461 skb->_skb_refdst = (unsigned long)dst; 468 skb->_skb_refdst = (unsigned long)dst;
462 } 469 }
463 470
464 extern void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst); 471 extern void skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst);
465 472
466 /** 473 /**
467 * skb_dst_is_noref - Test if skb dst isnt refcounted 474 * skb_dst_is_noref - Test if skb dst isnt refcounted
468 * @skb: buffer 475 * @skb: buffer
469 */ 476 */
470 static inline bool skb_dst_is_noref(const struct sk_buff *skb) 477 static inline bool skb_dst_is_noref(const struct sk_buff *skb)
471 { 478 {
472 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb); 479 return (skb->_skb_refdst & SKB_DST_NOREF) && skb_dst(skb);
473 } 480 }
474 481
475 static inline struct rtable *skb_rtable(const struct sk_buff *skb) 482 static inline struct rtable *skb_rtable(const struct sk_buff *skb)
476 { 483 {
477 return (struct rtable *)skb_dst(skb); 484 return (struct rtable *)skb_dst(skb);
478 } 485 }
479 486
480 extern void kfree_skb(struct sk_buff *skb); 487 extern void kfree_skb(struct sk_buff *skb);
481 extern void consume_skb(struct sk_buff *skb); 488 extern void consume_skb(struct sk_buff *skb);
482 extern void __kfree_skb(struct sk_buff *skb); 489 extern void __kfree_skb(struct sk_buff *skb);
483 extern struct sk_buff *__alloc_skb(unsigned int size, 490 extern struct sk_buff *__alloc_skb(unsigned int size,
484 gfp_t priority, int fclone, int node); 491 gfp_t priority, int fclone, int node);
485 static inline struct sk_buff *alloc_skb(unsigned int size, 492 static inline struct sk_buff *alloc_skb(unsigned int size,
486 gfp_t priority) 493 gfp_t priority)
487 { 494 {
488 return __alloc_skb(size, priority, 0, NUMA_NO_NODE); 495 return __alloc_skb(size, priority, 0, NUMA_NO_NODE);
489 } 496 }
490 497
491 static inline struct sk_buff *alloc_skb_fclone(unsigned int size, 498 static inline struct sk_buff *alloc_skb_fclone(unsigned int size,
492 gfp_t priority) 499 gfp_t priority)
493 { 500 {
494 return __alloc_skb(size, priority, 1, NUMA_NO_NODE); 501 return __alloc_skb(size, priority, 1, NUMA_NO_NODE);
495 } 502 }
496 503
497 extern bool skb_recycle_check(struct sk_buff *skb, int skb_size); 504 extern bool skb_recycle_check(struct sk_buff *skb, int skb_size);
498 505
499 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src); 506 extern struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src);
500 extern struct sk_buff *skb_clone(struct sk_buff *skb, 507 extern struct sk_buff *skb_clone(struct sk_buff *skb,
501 gfp_t priority); 508 gfp_t priority);
502 extern struct sk_buff *skb_copy(const struct sk_buff *skb, 509 extern struct sk_buff *skb_copy(const struct sk_buff *skb,
503 gfp_t priority); 510 gfp_t priority);
504 extern struct sk_buff *pskb_copy(struct sk_buff *skb, 511 extern struct sk_buff *pskb_copy(struct sk_buff *skb,
505 gfp_t gfp_mask); 512 gfp_t gfp_mask);
506 extern int pskb_expand_head(struct sk_buff *skb, 513 extern int pskb_expand_head(struct sk_buff *skb,
507 int nhead, int ntail, 514 int nhead, int ntail,
508 gfp_t gfp_mask); 515 gfp_t gfp_mask);
509 extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, 516 extern struct sk_buff *skb_realloc_headroom(struct sk_buff *skb,
510 unsigned int headroom); 517 unsigned int headroom);
511 extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 518 extern struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
512 int newheadroom, int newtailroom, 519 int newheadroom, int newtailroom,
513 gfp_t priority); 520 gfp_t priority);
514 extern int skb_to_sgvec(struct sk_buff *skb, 521 extern int skb_to_sgvec(struct sk_buff *skb,
515 struct scatterlist *sg, int offset, 522 struct scatterlist *sg, int offset,
516 int len); 523 int len);
517 extern int skb_cow_data(struct sk_buff *skb, int tailbits, 524 extern int skb_cow_data(struct sk_buff *skb, int tailbits,
518 struct sk_buff **trailer); 525 struct sk_buff **trailer);
519 extern int skb_pad(struct sk_buff *skb, int pad); 526 extern int skb_pad(struct sk_buff *skb, int pad);
520 #define dev_kfree_skb(a) consume_skb(a) 527 #define dev_kfree_skb(a) consume_skb(a)
521 528
522 extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 529 extern int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
523 int getfrag(void *from, char *to, int offset, 530 int getfrag(void *from, char *to, int offset,
524 int len,int odd, struct sk_buff *skb), 531 int len,int odd, struct sk_buff *skb),
525 void *from, int length); 532 void *from, int length);
526 533
527 struct skb_seq_state { 534 struct skb_seq_state {
528 __u32 lower_offset; 535 __u32 lower_offset;
529 __u32 upper_offset; 536 __u32 upper_offset;
530 __u32 frag_idx; 537 __u32 frag_idx;
531 __u32 stepped_offset; 538 __u32 stepped_offset;
532 struct sk_buff *root_skb; 539 struct sk_buff *root_skb;
533 struct sk_buff *cur_skb; 540 struct sk_buff *cur_skb;
534 __u8 *frag_data; 541 __u8 *frag_data;
535 }; 542 };
536 543
537 extern void skb_prepare_seq_read(struct sk_buff *skb, 544 extern void skb_prepare_seq_read(struct sk_buff *skb,
538 unsigned int from, unsigned int to, 545 unsigned int from, unsigned int to,
539 struct skb_seq_state *st); 546 struct skb_seq_state *st);
540 extern unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 547 extern unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
541 struct skb_seq_state *st); 548 struct skb_seq_state *st);
542 extern void skb_abort_seq_read(struct skb_seq_state *st); 549 extern void skb_abort_seq_read(struct skb_seq_state *st);
543 550
544 extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 551 extern unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
545 unsigned int to, struct ts_config *config, 552 unsigned int to, struct ts_config *config,
546 struct ts_state *state); 553 struct ts_state *state);
547 554
548 extern __u32 __skb_get_rxhash(struct sk_buff *skb); 555 extern __u32 __skb_get_rxhash(struct sk_buff *skb);
549 static inline __u32 skb_get_rxhash(struct sk_buff *skb) 556 static inline __u32 skb_get_rxhash(struct sk_buff *skb)
550 { 557 {
551 if (!skb->rxhash) 558 if (!skb->rxhash)
552 skb->rxhash = __skb_get_rxhash(skb); 559 skb->rxhash = __skb_get_rxhash(skb);
553 560
554 return skb->rxhash; 561 return skb->rxhash;
555 } 562 }
556 563
557 #ifdef NET_SKBUFF_DATA_USES_OFFSET 564 #ifdef NET_SKBUFF_DATA_USES_OFFSET
558 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 565 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
559 { 566 {
560 return skb->head + skb->end; 567 return skb->head + skb->end;
561 } 568 }
562 #else 569 #else
563 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb) 570 static inline unsigned char *skb_end_pointer(const struct sk_buff *skb)
564 { 571 {
565 return skb->end; 572 return skb->end;
566 } 573 }
567 #endif 574 #endif
568 575
569 /* Internal */ 576 /* Internal */
570 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB))) 577 #define skb_shinfo(SKB) ((struct skb_shared_info *)(skb_end_pointer(SKB)))
571 578
572 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb) 579 static inline struct skb_shared_hwtstamps *skb_hwtstamps(struct sk_buff *skb)
573 { 580 {
574 return &skb_shinfo(skb)->hwtstamps; 581 return &skb_shinfo(skb)->hwtstamps;
575 } 582 }
576 583
577 /** 584 /**
578 * skb_queue_empty - check if a queue is empty 585 * skb_queue_empty - check if a queue is empty
579 * @list: queue head 586 * @list: queue head
580 * 587 *
581 * Returns true if the queue is empty, false otherwise. 588 * Returns true if the queue is empty, false otherwise.
582 */ 589 */
583 static inline int skb_queue_empty(const struct sk_buff_head *list) 590 static inline int skb_queue_empty(const struct sk_buff_head *list)
584 { 591 {
585 return list->next == (struct sk_buff *)list; 592 return list->next == (struct sk_buff *)list;
586 } 593 }
587 594
588 /** 595 /**
589 * skb_queue_is_last - check if skb is the last entry in the queue 596 * skb_queue_is_last - check if skb is the last entry in the queue
590 * @list: queue head 597 * @list: queue head
591 * @skb: buffer 598 * @skb: buffer
592 * 599 *
593 * Returns true if @skb is the last buffer on the list. 600 * Returns true if @skb is the last buffer on the list.
594 */ 601 */
595 static inline bool skb_queue_is_last(const struct sk_buff_head *list, 602 static inline bool skb_queue_is_last(const struct sk_buff_head *list,
596 const struct sk_buff *skb) 603 const struct sk_buff *skb)
597 { 604 {
598 return skb->next == (struct sk_buff *)list; 605 return skb->next == (struct sk_buff *)list;
599 } 606 }
600 607
601 /** 608 /**
602 * skb_queue_is_first - check if skb is the first entry in the queue 609 * skb_queue_is_first - check if skb is the first entry in the queue
603 * @list: queue head 610 * @list: queue head
604 * @skb: buffer 611 * @skb: buffer
605 * 612 *
606 * Returns true if @skb is the first buffer on the list. 613 * Returns true if @skb is the first buffer on the list.
607 */ 614 */
608 static inline bool skb_queue_is_first(const struct sk_buff_head *list, 615 static inline bool skb_queue_is_first(const struct sk_buff_head *list,
609 const struct sk_buff *skb) 616 const struct sk_buff *skb)
610 { 617 {
611 return skb->prev == (struct sk_buff *)list; 618 return skb->prev == (struct sk_buff *)list;
612 } 619 }
613 620
614 /** 621 /**
615 * skb_queue_next - return the next packet in the queue 622 * skb_queue_next - return the next packet in the queue
616 * @list: queue head 623 * @list: queue head
617 * @skb: current buffer 624 * @skb: current buffer
618 * 625 *
619 * Return the next packet in @list after @skb. It is only valid to 626 * Return the next packet in @list after @skb. It is only valid to
620 * call this if skb_queue_is_last() evaluates to false. 627 * call this if skb_queue_is_last() evaluates to false.
621 */ 628 */
622 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list, 629 static inline struct sk_buff *skb_queue_next(const struct sk_buff_head *list,
623 const struct sk_buff *skb) 630 const struct sk_buff *skb)
624 { 631 {
625 /* This BUG_ON may seem severe, but if we just return then we 632 /* This BUG_ON may seem severe, but if we just return then we
626 * are going to dereference garbage. 633 * are going to dereference garbage.
627 */ 634 */
628 BUG_ON(skb_queue_is_last(list, skb)); 635 BUG_ON(skb_queue_is_last(list, skb));
629 return skb->next; 636 return skb->next;
630 } 637 }
631 638
632 /** 639 /**
633 * skb_queue_prev - return the prev packet in the queue 640 * skb_queue_prev - return the prev packet in the queue
634 * @list: queue head 641 * @list: queue head
635 * @skb: current buffer 642 * @skb: current buffer
636 * 643 *
637 * Return the prev packet in @list before @skb. It is only valid to 644 * Return the prev packet in @list before @skb. It is only valid to
638 * call this if skb_queue_is_first() evaluates to false. 645 * call this if skb_queue_is_first() evaluates to false.
639 */ 646 */
640 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list, 647 static inline struct sk_buff *skb_queue_prev(const struct sk_buff_head *list,
641 const struct sk_buff *skb) 648 const struct sk_buff *skb)
642 { 649 {
643 /* This BUG_ON may seem severe, but if we just return then we 650 /* This BUG_ON may seem severe, but if we just return then we
644 * are going to dereference garbage. 651 * are going to dereference garbage.
645 */ 652 */
646 BUG_ON(skb_queue_is_first(list, skb)); 653 BUG_ON(skb_queue_is_first(list, skb));
647 return skb->prev; 654 return skb->prev;
648 } 655 }
649 656
650 /** 657 /**
651 * skb_get - reference buffer 658 * skb_get - reference buffer
652 * @skb: buffer to reference 659 * @skb: buffer to reference
653 * 660 *
654 * Makes another reference to a socket buffer and returns a pointer 661 * Makes another reference to a socket buffer and returns a pointer
655 * to the buffer. 662 * to the buffer.
656 */ 663 */
657 static inline struct sk_buff *skb_get(struct sk_buff *skb) 664 static inline struct sk_buff *skb_get(struct sk_buff *skb)
658 { 665 {
659 atomic_inc(&skb->users); 666 atomic_inc(&skb->users);
660 return skb; 667 return skb;
661 } 668 }
662 669
663 /* 670 /*
664 * If users == 1, we are the only owner and are can avoid redundant 671 * If users == 1, we are the only owner and are can avoid redundant
665 * atomic change. 672 * atomic change.
666 */ 673 */
667 674
668 /** 675 /**
669 * skb_cloned - is the buffer a clone 676 * skb_cloned - is the buffer a clone
670 * @skb: buffer to check 677 * @skb: buffer to check
671 * 678 *
672 * Returns true if the buffer was generated with skb_clone() and is 679 * Returns true if the buffer was generated with skb_clone() and is
673 * one of multiple shared copies of the buffer. Cloned buffers are 680 * one of multiple shared copies of the buffer. Cloned buffers are
674 * shared data so must not be written to under normal circumstances. 681 * shared data so must not be written to under normal circumstances.
675 */ 682 */
676 static inline int skb_cloned(const struct sk_buff *skb) 683 static inline int skb_cloned(const struct sk_buff *skb)
677 { 684 {
678 return skb->cloned && 685 return skb->cloned &&
679 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1; 686 (atomic_read(&skb_shinfo(skb)->dataref) & SKB_DATAREF_MASK) != 1;
680 } 687 }
681 688
682 /** 689 /**
683 * skb_header_cloned - is the header a clone 690 * skb_header_cloned - is the header a clone
684 * @skb: buffer to check 691 * @skb: buffer to check
685 * 692 *
686 * Returns true if modifying the header part of the buffer requires 693 * Returns true if modifying the header part of the buffer requires
687 * the data to be copied. 694 * the data to be copied.
688 */ 695 */
689 static inline int skb_header_cloned(const struct sk_buff *skb) 696 static inline int skb_header_cloned(const struct sk_buff *skb)
690 { 697 {
691 int dataref; 698 int dataref;
692 699
693 if (!skb->cloned) 700 if (!skb->cloned)
694 return 0; 701 return 0;
695 702
696 dataref = atomic_read(&skb_shinfo(skb)->dataref); 703 dataref = atomic_read(&skb_shinfo(skb)->dataref);
697 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT); 704 dataref = (dataref & SKB_DATAREF_MASK) - (dataref >> SKB_DATAREF_SHIFT);
698 return dataref != 1; 705 return dataref != 1;
699 } 706 }
700 707
701 /** 708 /**
702 * skb_header_release - release reference to header 709 * skb_header_release - release reference to header
703 * @skb: buffer to operate on 710 * @skb: buffer to operate on
704 * 711 *
705 * Drop a reference to the header part of the buffer. This is done 712 * Drop a reference to the header part of the buffer. This is done
706 * by acquiring a payload reference. You must not read from the header 713 * by acquiring a payload reference. You must not read from the header
707 * part of skb->data after this. 714 * part of skb->data after this.
708 */ 715 */
709 static inline void skb_header_release(struct sk_buff *skb) 716 static inline void skb_header_release(struct sk_buff *skb)
710 { 717 {
711 BUG_ON(skb->nohdr); 718 BUG_ON(skb->nohdr);
712 skb->nohdr = 1; 719 skb->nohdr = 1;
713 atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref); 720 atomic_add(1 << SKB_DATAREF_SHIFT, &skb_shinfo(skb)->dataref);
714 } 721 }
715 722
716 /** 723 /**
717 * skb_shared - is the buffer shared 724 * skb_shared - is the buffer shared
718 * @skb: buffer to check 725 * @skb: buffer to check
719 * 726 *
720 * Returns true if more than one person has a reference to this 727 * Returns true if more than one person has a reference to this
721 * buffer. 728 * buffer.
722 */ 729 */
723 static inline int skb_shared(const struct sk_buff *skb) 730 static inline int skb_shared(const struct sk_buff *skb)
724 { 731 {
725 return atomic_read(&skb->users) != 1; 732 return atomic_read(&skb->users) != 1;
726 } 733 }
727 734
728 /** 735 /**
729 * skb_share_check - check if buffer is shared and if so clone it 736 * skb_share_check - check if buffer is shared and if so clone it
730 * @skb: buffer to check 737 * @skb: buffer to check
731 * @pri: priority for memory allocation 738 * @pri: priority for memory allocation
732 * 739 *
733 * If the buffer is shared the buffer is cloned and the old copy 740 * If the buffer is shared the buffer is cloned and the old copy
734 * drops a reference. A new clone with a single reference is returned. 741 * drops a reference. A new clone with a single reference is returned.
735 * If the buffer is not shared the original buffer is returned. When 742 * If the buffer is not shared the original buffer is returned. When
736 * being called from interrupt status or with spinlocks held pri must 743 * being called from interrupt status or with spinlocks held pri must
737 * be GFP_ATOMIC. 744 * be GFP_ATOMIC.
738 * 745 *
739 * NULL is returned on a memory allocation failure. 746 * NULL is returned on a memory allocation failure.
740 */ 747 */
741 static inline struct sk_buff *skb_share_check(struct sk_buff *skb, 748 static inline struct sk_buff *skb_share_check(struct sk_buff *skb,
742 gfp_t pri) 749 gfp_t pri)
743 { 750 {
744 might_sleep_if(pri & __GFP_WAIT); 751 might_sleep_if(pri & __GFP_WAIT);
745 if (skb_shared(skb)) { 752 if (skb_shared(skb)) {
746 struct sk_buff *nskb = skb_clone(skb, pri); 753 struct sk_buff *nskb = skb_clone(skb, pri);
747 kfree_skb(skb); 754 kfree_skb(skb);
748 skb = nskb; 755 skb = nskb;
749 } 756 }
750 return skb; 757 return skb;
751 } 758 }
752 759
753 /* 760 /*
754 * Copy shared buffers into a new sk_buff. We effectively do COW on 761 * Copy shared buffers into a new sk_buff. We effectively do COW on
755 * packets to handle cases where we have a local reader and forward 762 * packets to handle cases where we have a local reader and forward
756 * and a couple of other messy ones. The normal one is tcpdumping 763 * and a couple of other messy ones. The normal one is tcpdumping
757 * a packet thats being forwarded. 764 * a packet thats being forwarded.
758 */ 765 */
759 766
760 /** 767 /**
761 * skb_unshare - make a copy of a shared buffer 768 * skb_unshare - make a copy of a shared buffer
762 * @skb: buffer to check 769 * @skb: buffer to check
763 * @pri: priority for memory allocation 770 * @pri: priority for memory allocation
764 * 771 *
765 * If the socket buffer is a clone then this function creates a new 772 * If the socket buffer is a clone then this function creates a new
766 * copy of the data, drops a reference count on the old copy and returns 773 * copy of the data, drops a reference count on the old copy and returns
767 * the new copy with the reference count at 1. If the buffer is not a clone 774 * the new copy with the reference count at 1. If the buffer is not a clone
768 * the original buffer is returned. When called with a spinlock held or 775 * the original buffer is returned. When called with a spinlock held or
769 * from interrupt state @pri must be %GFP_ATOMIC 776 * from interrupt state @pri must be %GFP_ATOMIC
770 * 777 *
771 * %NULL is returned on a memory allocation failure. 778 * %NULL is returned on a memory allocation failure.
772 */ 779 */
773 static inline struct sk_buff *skb_unshare(struct sk_buff *skb, 780 static inline struct sk_buff *skb_unshare(struct sk_buff *skb,
774 gfp_t pri) 781 gfp_t pri)
775 { 782 {
776 might_sleep_if(pri & __GFP_WAIT); 783 might_sleep_if(pri & __GFP_WAIT);
777 if (skb_cloned(skb)) { 784 if (skb_cloned(skb)) {
778 struct sk_buff *nskb = skb_copy(skb, pri); 785 struct sk_buff *nskb = skb_copy(skb, pri);
779 kfree_skb(skb); /* Free our shared copy */ 786 kfree_skb(skb); /* Free our shared copy */
780 skb = nskb; 787 skb = nskb;
781 } 788 }
782 return skb; 789 return skb;
783 } 790 }
784 791
785 /** 792 /**
786 * skb_peek - peek at the head of an &sk_buff_head 793 * skb_peek - peek at the head of an &sk_buff_head
787 * @list_: list to peek at 794 * @list_: list to peek at
788 * 795 *
789 * Peek an &sk_buff. Unlike most other operations you _MUST_ 796 * Peek an &sk_buff. Unlike most other operations you _MUST_
790 * be careful with this one. A peek leaves the buffer on the 797 * be careful with this one. A peek leaves the buffer on the
791 * list and someone else may run off with it. You must hold 798 * list and someone else may run off with it. You must hold
792 * the appropriate locks or have a private queue to do this. 799 * the appropriate locks or have a private queue to do this.
793 * 800 *
794 * Returns %NULL for an empty list or a pointer to the head element. 801 * Returns %NULL for an empty list or a pointer to the head element.
795 * The reference count is not incremented and the reference is therefore 802 * The reference count is not incremented and the reference is therefore
796 * volatile. Use with caution. 803 * volatile. Use with caution.
797 */ 804 */
798 static inline struct sk_buff *skb_peek(struct sk_buff_head *list_) 805 static inline struct sk_buff *skb_peek(struct sk_buff_head *list_)
799 { 806 {
800 struct sk_buff *list = ((struct sk_buff *)list_)->next; 807 struct sk_buff *list = ((struct sk_buff *)list_)->next;
801 if (list == (struct sk_buff *)list_) 808 if (list == (struct sk_buff *)list_)
802 list = NULL; 809 list = NULL;
803 return list; 810 return list;
804 } 811 }
805 812
806 /** 813 /**
807 * skb_peek_tail - peek at the tail of an &sk_buff_head 814 * skb_peek_tail - peek at the tail of an &sk_buff_head
808 * @list_: list to peek at 815 * @list_: list to peek at
809 * 816 *
810 * Peek an &sk_buff. Unlike most other operations you _MUST_ 817 * Peek an &sk_buff. Unlike most other operations you _MUST_
811 * be careful with this one. A peek leaves the buffer on the 818 * be careful with this one. A peek leaves the buffer on the
812 * list and someone else may run off with it. You must hold 819 * list and someone else may run off with it. You must hold
813 * the appropriate locks or have a private queue to do this. 820 * the appropriate locks or have a private queue to do this.
814 * 821 *
815 * Returns %NULL for an empty list or a pointer to the tail element. 822 * Returns %NULL for an empty list or a pointer to the tail element.
816 * The reference count is not incremented and the reference is therefore 823 * The reference count is not incremented and the reference is therefore
817 * volatile. Use with caution. 824 * volatile. Use with caution.
818 */ 825 */
819 static inline struct sk_buff *skb_peek_tail(struct sk_buff_head *list_) 826 static inline struct sk_buff *skb_peek_tail(struct sk_buff_head *list_)
820 { 827 {
821 struct sk_buff *list = ((struct sk_buff *)list_)->prev; 828 struct sk_buff *list = ((struct sk_buff *)list_)->prev;
822 if (list == (struct sk_buff *)list_) 829 if (list == (struct sk_buff *)list_)
823 list = NULL; 830 list = NULL;
824 return list; 831 return list;
825 } 832 }
826 833
827 /** 834 /**
828 * skb_queue_len - get queue length 835 * skb_queue_len - get queue length
829 * @list_: list to measure 836 * @list_: list to measure
830 * 837 *
831 * Return the length of an &sk_buff queue. 838 * Return the length of an &sk_buff queue.
832 */ 839 */
833 static inline __u32 skb_queue_len(const struct sk_buff_head *list_) 840 static inline __u32 skb_queue_len(const struct sk_buff_head *list_)
834 { 841 {
835 return list_->qlen; 842 return list_->qlen;
836 } 843 }
837 844
838 /** 845 /**
839 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head 846 * __skb_queue_head_init - initialize non-spinlock portions of sk_buff_head
840 * @list: queue to initialize 847 * @list: queue to initialize
841 * 848 *
842 * This initializes only the list and queue length aspects of 849 * This initializes only the list and queue length aspects of
843 * an sk_buff_head object. This allows to initialize the list 850 * an sk_buff_head object. This allows to initialize the list
844 * aspects of an sk_buff_head without reinitializing things like 851 * aspects of an sk_buff_head without reinitializing things like
845 * the spinlock. It can also be used for on-stack sk_buff_head 852 * the spinlock. It can also be used for on-stack sk_buff_head
846 * objects where the spinlock is known to not be used. 853 * objects where the spinlock is known to not be used.
847 */ 854 */
848 static inline void __skb_queue_head_init(struct sk_buff_head *list) 855 static inline void __skb_queue_head_init(struct sk_buff_head *list)
849 { 856 {
850 list->prev = list->next = (struct sk_buff *)list; 857 list->prev = list->next = (struct sk_buff *)list;
851 list->qlen = 0; 858 list->qlen = 0;
852 } 859 }
853 860
854 /* 861 /*
855 * This function creates a split out lock class for each invocation; 862 * This function creates a split out lock class for each invocation;
856 * this is needed for now since a whole lot of users of the skb-queue 863 * this is needed for now since a whole lot of users of the skb-queue
857 * infrastructure in drivers have different locking usage (in hardirq) 864 * infrastructure in drivers have different locking usage (in hardirq)
858 * than the networking core (in softirq only). In the long run either the 865 * than the networking core (in softirq only). In the long run either the
859 * network layer or drivers should need annotation to consolidate the 866 * network layer or drivers should need annotation to consolidate the
860 * main types of usage into 3 classes. 867 * main types of usage into 3 classes.
861 */ 868 */
862 static inline void skb_queue_head_init(struct sk_buff_head *list) 869 static inline void skb_queue_head_init(struct sk_buff_head *list)
863 { 870 {
864 spin_lock_init(&list->lock); 871 spin_lock_init(&list->lock);
865 __skb_queue_head_init(list); 872 __skb_queue_head_init(list);
866 } 873 }
867 874
868 static inline void skb_queue_head_init_class(struct sk_buff_head *list, 875 static inline void skb_queue_head_init_class(struct sk_buff_head *list,
869 struct lock_class_key *class) 876 struct lock_class_key *class)
870 { 877 {
871 skb_queue_head_init(list); 878 skb_queue_head_init(list);
872 lockdep_set_class(&list->lock, class); 879 lockdep_set_class(&list->lock, class);
873 } 880 }
874 881
875 /* 882 /*
876 * Insert an sk_buff on a list. 883 * Insert an sk_buff on a list.
877 * 884 *
878 * The "__skb_xxxx()" functions are the non-atomic ones that 885 * The "__skb_xxxx()" functions are the non-atomic ones that
879 * can only be called with interrupts disabled. 886 * can only be called with interrupts disabled.
880 */ 887 */
881 extern void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list); 888 extern void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list);
882 static inline void __skb_insert(struct sk_buff *newsk, 889 static inline void __skb_insert(struct sk_buff *newsk,
883 struct sk_buff *prev, struct sk_buff *next, 890 struct sk_buff *prev, struct sk_buff *next,
884 struct sk_buff_head *list) 891 struct sk_buff_head *list)
885 { 892 {
886 newsk->next = next; 893 newsk->next = next;
887 newsk->prev = prev; 894 newsk->prev = prev;
888 next->prev = prev->next = newsk; 895 next->prev = prev->next = newsk;
889 list->qlen++; 896 list->qlen++;
890 } 897 }
891 898
892 static inline void __skb_queue_splice(const struct sk_buff_head *list, 899 static inline void __skb_queue_splice(const struct sk_buff_head *list,
893 struct sk_buff *prev, 900 struct sk_buff *prev,
894 struct sk_buff *next) 901 struct sk_buff *next)
895 { 902 {
896 struct sk_buff *first = list->next; 903 struct sk_buff *first = list->next;
897 struct sk_buff *last = list->prev; 904 struct sk_buff *last = list->prev;
898 905
899 first->prev = prev; 906 first->prev = prev;
900 prev->next = first; 907 prev->next = first;
901 908
902 last->next = next; 909 last->next = next;
903 next->prev = last; 910 next->prev = last;
904 } 911 }
905 912
906 /** 913 /**
907 * skb_queue_splice - join two skb lists, this is designed for stacks 914 * skb_queue_splice - join two skb lists, this is designed for stacks
908 * @list: the new list to add 915 * @list: the new list to add
909 * @head: the place to add it in the first list 916 * @head: the place to add it in the first list
910 */ 917 */
911 static inline void skb_queue_splice(const struct sk_buff_head *list, 918 static inline void skb_queue_splice(const struct sk_buff_head *list,
912 struct sk_buff_head *head) 919 struct sk_buff_head *head)
913 { 920 {
914 if (!skb_queue_empty(list)) { 921 if (!skb_queue_empty(list)) {
915 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 922 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
916 head->qlen += list->qlen; 923 head->qlen += list->qlen;
917 } 924 }
918 } 925 }
919 926
920 /** 927 /**
921 * skb_queue_splice - join two skb lists and reinitialise the emptied list 928 * skb_queue_splice - join two skb lists and reinitialise the emptied list
922 * @list: the new list to add 929 * @list: the new list to add
923 * @head: the place to add it in the first list 930 * @head: the place to add it in the first list
924 * 931 *
925 * The list at @list is reinitialised 932 * The list at @list is reinitialised
926 */ 933 */
927 static inline void skb_queue_splice_init(struct sk_buff_head *list, 934 static inline void skb_queue_splice_init(struct sk_buff_head *list,
928 struct sk_buff_head *head) 935 struct sk_buff_head *head)
929 { 936 {
930 if (!skb_queue_empty(list)) { 937 if (!skb_queue_empty(list)) {
931 __skb_queue_splice(list, (struct sk_buff *) head, head->next); 938 __skb_queue_splice(list, (struct sk_buff *) head, head->next);
932 head->qlen += list->qlen; 939 head->qlen += list->qlen;
933 __skb_queue_head_init(list); 940 __skb_queue_head_init(list);
934 } 941 }
935 } 942 }
936 943
937 /** 944 /**
938 * skb_queue_splice_tail - join two skb lists, each list being a queue 945 * skb_queue_splice_tail - join two skb lists, each list being a queue
939 * @list: the new list to add 946 * @list: the new list to add
940 * @head: the place to add it in the first list 947 * @head: the place to add it in the first list
941 */ 948 */
942 static inline void skb_queue_splice_tail(const struct sk_buff_head *list, 949 static inline void skb_queue_splice_tail(const struct sk_buff_head *list,
943 struct sk_buff_head *head) 950 struct sk_buff_head *head)
944 { 951 {
945 if (!skb_queue_empty(list)) { 952 if (!skb_queue_empty(list)) {
946 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 953 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
947 head->qlen += list->qlen; 954 head->qlen += list->qlen;
948 } 955 }
949 } 956 }
950 957
951 /** 958 /**
952 * skb_queue_splice_tail - join two skb lists and reinitialise the emptied list 959 * skb_queue_splice_tail - join two skb lists and reinitialise the emptied list
953 * @list: the new list to add 960 * @list: the new list to add
954 * @head: the place to add it in the first list 961 * @head: the place to add it in the first list
955 * 962 *
956 * Each of the lists is a queue. 963 * Each of the lists is a queue.
957 * The list at @list is reinitialised 964 * The list at @list is reinitialised
958 */ 965 */
959 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list, 966 static inline void skb_queue_splice_tail_init(struct sk_buff_head *list,
960 struct sk_buff_head *head) 967 struct sk_buff_head *head)
961 { 968 {
962 if (!skb_queue_empty(list)) { 969 if (!skb_queue_empty(list)) {
963 __skb_queue_splice(list, head->prev, (struct sk_buff *) head); 970 __skb_queue_splice(list, head->prev, (struct sk_buff *) head);
964 head->qlen += list->qlen; 971 head->qlen += list->qlen;
965 __skb_queue_head_init(list); 972 __skb_queue_head_init(list);
966 } 973 }
967 } 974 }
968 975
969 /** 976 /**
970 * __skb_queue_after - queue a buffer at the list head 977 * __skb_queue_after - queue a buffer at the list head
971 * @list: list to use 978 * @list: list to use
972 * @prev: place after this buffer 979 * @prev: place after this buffer
973 * @newsk: buffer to queue 980 * @newsk: buffer to queue
974 * 981 *
975 * Queue a buffer int the middle of a list. This function takes no locks 982 * Queue a buffer int the middle of a list. This function takes no locks
976 * and you must therefore hold required locks before calling it. 983 * and you must therefore hold required locks before calling it.
977 * 984 *
978 * A buffer cannot be placed on two lists at the same time. 985 * A buffer cannot be placed on two lists at the same time.
979 */ 986 */
980 static inline void __skb_queue_after(struct sk_buff_head *list, 987 static inline void __skb_queue_after(struct sk_buff_head *list,
981 struct sk_buff *prev, 988 struct sk_buff *prev,
982 struct sk_buff *newsk) 989 struct sk_buff *newsk)
983 { 990 {
984 __skb_insert(newsk, prev, prev->next, list); 991 __skb_insert(newsk, prev, prev->next, list);
985 } 992 }
986 993
987 extern void skb_append(struct sk_buff *old, struct sk_buff *newsk, 994 extern void skb_append(struct sk_buff *old, struct sk_buff *newsk,
988 struct sk_buff_head *list); 995 struct sk_buff_head *list);
989 996
990 static inline void __skb_queue_before(struct sk_buff_head *list, 997 static inline void __skb_queue_before(struct sk_buff_head *list,
991 struct sk_buff *next, 998 struct sk_buff *next,
992 struct sk_buff *newsk) 999 struct sk_buff *newsk)
993 { 1000 {
994 __skb_insert(newsk, next->prev, next, list); 1001 __skb_insert(newsk, next->prev, next, list);
995 } 1002 }
996 1003
997 /** 1004 /**
998 * __skb_queue_head - queue a buffer at the list head 1005 * __skb_queue_head - queue a buffer at the list head
999 * @list: list to use 1006 * @list: list to use
1000 * @newsk: buffer to queue 1007 * @newsk: buffer to queue
1001 * 1008 *
1002 * Queue a buffer at the start of a list. This function takes no locks 1009 * Queue a buffer at the start of a list. This function takes no locks
1003 * and you must therefore hold required locks before calling it. 1010 * and you must therefore hold required locks before calling it.
1004 * 1011 *
1005 * A buffer cannot be placed on two lists at the same time. 1012 * A buffer cannot be placed on two lists at the same time.
1006 */ 1013 */
1007 extern void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk); 1014 extern void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk);
1008 static inline void __skb_queue_head(struct sk_buff_head *list, 1015 static inline void __skb_queue_head(struct sk_buff_head *list,
1009 struct sk_buff *newsk) 1016 struct sk_buff *newsk)
1010 { 1017 {
1011 __skb_queue_after(list, (struct sk_buff *)list, newsk); 1018 __skb_queue_after(list, (struct sk_buff *)list, newsk);
1012 } 1019 }
1013 1020
1014 /** 1021 /**
1015 * __skb_queue_tail - queue a buffer at the list tail 1022 * __skb_queue_tail - queue a buffer at the list tail
1016 * @list: list to use 1023 * @list: list to use
1017 * @newsk: buffer to queue 1024 * @newsk: buffer to queue
1018 * 1025 *
1019 * Queue a buffer at the end of a list. This function takes no locks 1026 * Queue a buffer at the end of a list. This function takes no locks
1020 * and you must therefore hold required locks before calling it. 1027 * and you must therefore hold required locks before calling it.
1021 * 1028 *
1022 * A buffer cannot be placed on two lists at the same time. 1029 * A buffer cannot be placed on two lists at the same time.
1023 */ 1030 */
1024 extern void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk); 1031 extern void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk);
1025 static inline void __skb_queue_tail(struct sk_buff_head *list, 1032 static inline void __skb_queue_tail(struct sk_buff_head *list,
1026 struct sk_buff *newsk) 1033 struct sk_buff *newsk)
1027 { 1034 {
1028 __skb_queue_before(list, (struct sk_buff *)list, newsk); 1035 __skb_queue_before(list, (struct sk_buff *)list, newsk);
1029 } 1036 }
1030 1037
1031 /* 1038 /*
1032 * remove sk_buff from list. _Must_ be called atomically, and with 1039 * remove sk_buff from list. _Must_ be called atomically, and with
1033 * the list known.. 1040 * the list known..
1034 */ 1041 */
1035 extern void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list); 1042 extern void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list);
1036 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 1043 static inline void __skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1037 { 1044 {
1038 struct sk_buff *next, *prev; 1045 struct sk_buff *next, *prev;
1039 1046
1040 list->qlen--; 1047 list->qlen--;
1041 next = skb->next; 1048 next = skb->next;
1042 prev = skb->prev; 1049 prev = skb->prev;
1043 skb->next = skb->prev = NULL; 1050 skb->next = skb->prev = NULL;
1044 next->prev = prev; 1051 next->prev = prev;
1045 prev->next = next; 1052 prev->next = next;
1046 } 1053 }
1047 1054
1048 /** 1055 /**
1049 * __skb_dequeue - remove from the head of the queue 1056 * __skb_dequeue - remove from the head of the queue
1050 * @list: list to dequeue from 1057 * @list: list to dequeue from
1051 * 1058 *
1052 * Remove the head of the list. This function does not take any locks 1059 * Remove the head of the list. This function does not take any locks
1053 * so must be used with appropriate locks held only. The head item is 1060 * so must be used with appropriate locks held only. The head item is
1054 * returned or %NULL if the list is empty. 1061 * returned or %NULL if the list is empty.
1055 */ 1062 */
1056 extern struct sk_buff *skb_dequeue(struct sk_buff_head *list); 1063 extern struct sk_buff *skb_dequeue(struct sk_buff_head *list);
1057 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list) 1064 static inline struct sk_buff *__skb_dequeue(struct sk_buff_head *list)
1058 { 1065 {
1059 struct sk_buff *skb = skb_peek(list); 1066 struct sk_buff *skb = skb_peek(list);
1060 if (skb) 1067 if (skb)
1061 __skb_unlink(skb, list); 1068 __skb_unlink(skb, list);
1062 return skb; 1069 return skb;
1063 } 1070 }
1064 1071
1065 /** 1072 /**
1066 * __skb_dequeue_tail - remove from the tail of the queue 1073 * __skb_dequeue_tail - remove from the tail of the queue
1067 * @list: list to dequeue from 1074 * @list: list to dequeue from
1068 * 1075 *
1069 * Remove the tail of the list. This function does not take any locks 1076 * Remove the tail of the list. This function does not take any locks
1070 * so must be used with appropriate locks held only. The tail item is 1077 * so must be used with appropriate locks held only. The tail item is
1071 * returned or %NULL if the list is empty. 1078 * returned or %NULL if the list is empty.
1072 */ 1079 */
1073 extern struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list); 1080 extern struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list);
1074 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list) 1081 static inline struct sk_buff *__skb_dequeue_tail(struct sk_buff_head *list)
1075 { 1082 {
1076 struct sk_buff *skb = skb_peek_tail(list); 1083 struct sk_buff *skb = skb_peek_tail(list);
1077 if (skb) 1084 if (skb)
1078 __skb_unlink(skb, list); 1085 __skb_unlink(skb, list);
1079 return skb; 1086 return skb;
1080 } 1087 }
1081 1088
1082 1089
1083 static inline int skb_is_nonlinear(const struct sk_buff *skb) 1090 static inline int skb_is_nonlinear(const struct sk_buff *skb)
1084 { 1091 {
1085 return skb->data_len; 1092 return skb->data_len;
1086 } 1093 }
1087 1094
1088 static inline unsigned int skb_headlen(const struct sk_buff *skb) 1095 static inline unsigned int skb_headlen(const struct sk_buff *skb)
1089 { 1096 {
1090 return skb->len - skb->data_len; 1097 return skb->len - skb->data_len;
1091 } 1098 }
1092 1099
1093 static inline int skb_pagelen(const struct sk_buff *skb) 1100 static inline int skb_pagelen(const struct sk_buff *skb)
1094 { 1101 {
1095 int i, len = 0; 1102 int i, len = 0;
1096 1103
1097 for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--) 1104 for (i = (int)skb_shinfo(skb)->nr_frags - 1; i >= 0; i--)
1098 len += skb_shinfo(skb)->frags[i].size; 1105 len += skb_shinfo(skb)->frags[i].size;
1099 return len + skb_headlen(skb); 1106 return len + skb_headlen(skb);
1100 } 1107 }
1101 1108
1102 static inline void skb_fill_page_desc(struct sk_buff *skb, int i, 1109 static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
1103 struct page *page, int off, int size) 1110 struct page *page, int off, int size)
1104 { 1111 {
1105 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1112 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1106 1113
1107 frag->page = page; 1114 frag->page = page;
1108 frag->page_offset = off; 1115 frag->page_offset = off;
1109 frag->size = size; 1116 frag->size = size;
1110 skb_shinfo(skb)->nr_frags = i + 1; 1117 skb_shinfo(skb)->nr_frags = i + 1;
1111 } 1118 }
1112 1119
1113 extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, 1120 extern void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page,
1114 int off, int size); 1121 int off, int size);
1115 1122
1116 #define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags) 1123 #define SKB_PAGE_ASSERT(skb) BUG_ON(skb_shinfo(skb)->nr_frags)
1117 #define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb)) 1124 #define SKB_FRAG_ASSERT(skb) BUG_ON(skb_has_frag_list(skb))
1118 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb)) 1125 #define SKB_LINEAR_ASSERT(skb) BUG_ON(skb_is_nonlinear(skb))
1119 1126
1120 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1127 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1121 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 1128 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1122 { 1129 {
1123 return skb->head + skb->tail; 1130 return skb->head + skb->tail;
1124 } 1131 }
1125 1132
1126 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 1133 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1127 { 1134 {
1128 skb->tail = skb->data - skb->head; 1135 skb->tail = skb->data - skb->head;
1129 } 1136 }
1130 1137
1131 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 1138 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1132 { 1139 {
1133 skb_reset_tail_pointer(skb); 1140 skb_reset_tail_pointer(skb);
1134 skb->tail += offset; 1141 skb->tail += offset;
1135 } 1142 }
1136 #else /* NET_SKBUFF_DATA_USES_OFFSET */ 1143 #else /* NET_SKBUFF_DATA_USES_OFFSET */
1137 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb) 1144 static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
1138 { 1145 {
1139 return skb->tail; 1146 return skb->tail;
1140 } 1147 }
1141 1148
1142 static inline void skb_reset_tail_pointer(struct sk_buff *skb) 1149 static inline void skb_reset_tail_pointer(struct sk_buff *skb)
1143 { 1150 {
1144 skb->tail = skb->data; 1151 skb->tail = skb->data;
1145 } 1152 }
1146 1153
1147 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset) 1154 static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
1148 { 1155 {
1149 skb->tail = skb->data + offset; 1156 skb->tail = skb->data + offset;
1150 } 1157 }
1151 1158
1152 #endif /* NET_SKBUFF_DATA_USES_OFFSET */ 1159 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1153 1160
1154 /* 1161 /*
1155 * Add data to an sk_buff 1162 * Add data to an sk_buff
1156 */ 1163 */
1157 extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len); 1164 extern unsigned char *skb_put(struct sk_buff *skb, unsigned int len);
1158 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len) 1165 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
1159 { 1166 {
1160 unsigned char *tmp = skb_tail_pointer(skb); 1167 unsigned char *tmp = skb_tail_pointer(skb);
1161 SKB_LINEAR_ASSERT(skb); 1168 SKB_LINEAR_ASSERT(skb);
1162 skb->tail += len; 1169 skb->tail += len;
1163 skb->len += len; 1170 skb->len += len;
1164 return tmp; 1171 return tmp;
1165 } 1172 }
1166 1173
1167 extern unsigned char *skb_push(struct sk_buff *skb, unsigned int len); 1174 extern unsigned char *skb_push(struct sk_buff *skb, unsigned int len);
1168 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len) 1175 static inline unsigned char *__skb_push(struct sk_buff *skb, unsigned int len)
1169 { 1176 {
1170 skb->data -= len; 1177 skb->data -= len;
1171 skb->len += len; 1178 skb->len += len;
1172 return skb->data; 1179 return skb->data;
1173 } 1180 }
1174 1181
1175 extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len); 1182 extern unsigned char *skb_pull(struct sk_buff *skb, unsigned int len);
1176 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len) 1183 static inline unsigned char *__skb_pull(struct sk_buff *skb, unsigned int len)
1177 { 1184 {
1178 skb->len -= len; 1185 skb->len -= len;
1179 BUG_ON(skb->len < skb->data_len); 1186 BUG_ON(skb->len < skb->data_len);
1180 return skb->data += len; 1187 return skb->data += len;
1181 } 1188 }
1182 1189
1183 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len) 1190 static inline unsigned char *skb_pull_inline(struct sk_buff *skb, unsigned int len)
1184 { 1191 {
1185 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len); 1192 return unlikely(len > skb->len) ? NULL : __skb_pull(skb, len);
1186 } 1193 }
1187 1194
1188 extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta); 1195 extern unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta);
1189 1196
1190 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len) 1197 static inline unsigned char *__pskb_pull(struct sk_buff *skb, unsigned int len)
1191 { 1198 {
1192 if (len > skb_headlen(skb) && 1199 if (len > skb_headlen(skb) &&
1193 !__pskb_pull_tail(skb, len - skb_headlen(skb))) 1200 !__pskb_pull_tail(skb, len - skb_headlen(skb)))
1194 return NULL; 1201 return NULL;
1195 skb->len -= len; 1202 skb->len -= len;
1196 return skb->data += len; 1203 return skb->data += len;
1197 } 1204 }
1198 1205
1199 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len) 1206 static inline unsigned char *pskb_pull(struct sk_buff *skb, unsigned int len)
1200 { 1207 {
1201 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); 1208 return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
1202 } 1209 }
1203 1210
1204 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len) 1211 static inline int pskb_may_pull(struct sk_buff *skb, unsigned int len)
1205 { 1212 {
1206 if (likely(len <= skb_headlen(skb))) 1213 if (likely(len <= skb_headlen(skb)))
1207 return 1; 1214 return 1;
1208 if (unlikely(len > skb->len)) 1215 if (unlikely(len > skb->len))
1209 return 0; 1216 return 0;
1210 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; 1217 return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
1211 } 1218 }
1212 1219
1213 /** 1220 /**
1214 * skb_headroom - bytes at buffer head 1221 * skb_headroom - bytes at buffer head
1215 * @skb: buffer to check 1222 * @skb: buffer to check
1216 * 1223 *
1217 * Return the number of bytes of free space at the head of an &sk_buff. 1224 * Return the number of bytes of free space at the head of an &sk_buff.
1218 */ 1225 */
1219 static inline unsigned int skb_headroom(const struct sk_buff *skb) 1226 static inline unsigned int skb_headroom(const struct sk_buff *skb)
1220 { 1227 {
1221 return skb->data - skb->head; 1228 return skb->data - skb->head;
1222 } 1229 }
1223 1230
1224 /** 1231 /**
1225 * skb_tailroom - bytes at buffer end 1232 * skb_tailroom - bytes at buffer end
1226 * @skb: buffer to check 1233 * @skb: buffer to check
1227 * 1234 *
1228 * Return the number of bytes of free space at the tail of an sk_buff 1235 * Return the number of bytes of free space at the tail of an sk_buff
1229 */ 1236 */
1230 static inline int skb_tailroom(const struct sk_buff *skb) 1237 static inline int skb_tailroom(const struct sk_buff *skb)
1231 { 1238 {
1232 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail; 1239 return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
1233 } 1240 }
1234 1241
1235 /** 1242 /**
1236 * skb_reserve - adjust headroom 1243 * skb_reserve - adjust headroom
1237 * @skb: buffer to alter 1244 * @skb: buffer to alter
1238 * @len: bytes to move 1245 * @len: bytes to move
1239 * 1246 *
1240 * Increase the headroom of an empty &sk_buff by reducing the tail 1247 * Increase the headroom of an empty &sk_buff by reducing the tail
1241 * room. This is only allowed for an empty buffer. 1248 * room. This is only allowed for an empty buffer.
1242 */ 1249 */
1243 static inline void skb_reserve(struct sk_buff *skb, int len) 1250 static inline void skb_reserve(struct sk_buff *skb, int len)
1244 { 1251 {
1245 skb->data += len; 1252 skb->data += len;
1246 skb->tail += len; 1253 skb->tail += len;
1247 } 1254 }
1248 1255
1249 #ifdef NET_SKBUFF_DATA_USES_OFFSET 1256 #ifdef NET_SKBUFF_DATA_USES_OFFSET
1250 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) 1257 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1251 { 1258 {
1252 return skb->head + skb->transport_header; 1259 return skb->head + skb->transport_header;
1253 } 1260 }
1254 1261
1255 static inline void skb_reset_transport_header(struct sk_buff *skb) 1262 static inline void skb_reset_transport_header(struct sk_buff *skb)
1256 { 1263 {
1257 skb->transport_header = skb->data - skb->head; 1264 skb->transport_header = skb->data - skb->head;
1258 } 1265 }
1259 1266
1260 static inline void skb_set_transport_header(struct sk_buff *skb, 1267 static inline void skb_set_transport_header(struct sk_buff *skb,
1261 const int offset) 1268 const int offset)
1262 { 1269 {
1263 skb_reset_transport_header(skb); 1270 skb_reset_transport_header(skb);
1264 skb->transport_header += offset; 1271 skb->transport_header += offset;
1265 } 1272 }
1266 1273
1267 static inline unsigned char *skb_network_header(const struct sk_buff *skb) 1274 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1268 { 1275 {
1269 return skb->head + skb->network_header; 1276 return skb->head + skb->network_header;
1270 } 1277 }
1271 1278
1272 static inline void skb_reset_network_header(struct sk_buff *skb) 1279 static inline void skb_reset_network_header(struct sk_buff *skb)
1273 { 1280 {
1274 skb->network_header = skb->data - skb->head; 1281 skb->network_header = skb->data - skb->head;
1275 } 1282 }
1276 1283
1277 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) 1284 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1278 { 1285 {
1279 skb_reset_network_header(skb); 1286 skb_reset_network_header(skb);
1280 skb->network_header += offset; 1287 skb->network_header += offset;
1281 } 1288 }
1282 1289
1283 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) 1290 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1284 { 1291 {
1285 return skb->head + skb->mac_header; 1292 return skb->head + skb->mac_header;
1286 } 1293 }
1287 1294
1288 static inline int skb_mac_header_was_set(const struct sk_buff *skb) 1295 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1289 { 1296 {
1290 return skb->mac_header != ~0U; 1297 return skb->mac_header != ~0U;
1291 } 1298 }
1292 1299
1293 static inline void skb_reset_mac_header(struct sk_buff *skb) 1300 static inline void skb_reset_mac_header(struct sk_buff *skb)
1294 { 1301 {
1295 skb->mac_header = skb->data - skb->head; 1302 skb->mac_header = skb->data - skb->head;
1296 } 1303 }
1297 1304
1298 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) 1305 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1299 { 1306 {
1300 skb_reset_mac_header(skb); 1307 skb_reset_mac_header(skb);
1301 skb->mac_header += offset; 1308 skb->mac_header += offset;
1302 } 1309 }
1303 1310
1304 #else /* NET_SKBUFF_DATA_USES_OFFSET */ 1311 #else /* NET_SKBUFF_DATA_USES_OFFSET */
1305 1312
1306 static inline unsigned char *skb_transport_header(const struct sk_buff *skb) 1313 static inline unsigned char *skb_transport_header(const struct sk_buff *skb)
1307 { 1314 {
1308 return skb->transport_header; 1315 return skb->transport_header;
1309 } 1316 }
1310 1317
1311 static inline void skb_reset_transport_header(struct sk_buff *skb) 1318 static inline void skb_reset_transport_header(struct sk_buff *skb)
1312 { 1319 {
1313 skb->transport_header = skb->data; 1320 skb->transport_header = skb->data;
1314 } 1321 }
1315 1322
1316 static inline void skb_set_transport_header(struct sk_buff *skb, 1323 static inline void skb_set_transport_header(struct sk_buff *skb,
1317 const int offset) 1324 const int offset)
1318 { 1325 {
1319 skb->transport_header = skb->data + offset; 1326 skb->transport_header = skb->data + offset;
1320 } 1327 }
1321 1328
1322 static inline unsigned char *skb_network_header(const struct sk_buff *skb) 1329 static inline unsigned char *skb_network_header(const struct sk_buff *skb)
1323 { 1330 {
1324 return skb->network_header; 1331 return skb->network_header;
1325 } 1332 }
1326 1333
1327 static inline void skb_reset_network_header(struct sk_buff *skb) 1334 static inline void skb_reset_network_header(struct sk_buff *skb)
1328 { 1335 {
1329 skb->network_header = skb->data; 1336 skb->network_header = skb->data;
1330 } 1337 }
1331 1338
1332 static inline void skb_set_network_header(struct sk_buff *skb, const int offset) 1339 static inline void skb_set_network_header(struct sk_buff *skb, const int offset)
1333 { 1340 {
1334 skb->network_header = skb->data + offset; 1341 skb->network_header = skb->data + offset;
1335 } 1342 }
1336 1343
1337 static inline unsigned char *skb_mac_header(const struct sk_buff *skb) 1344 static inline unsigned char *skb_mac_header(const struct sk_buff *skb)
1338 { 1345 {
1339 return skb->mac_header; 1346 return skb->mac_header;
1340 } 1347 }
1341 1348
1342 static inline int skb_mac_header_was_set(const struct sk_buff *skb) 1349 static inline int skb_mac_header_was_set(const struct sk_buff *skb)
1343 { 1350 {
1344 return skb->mac_header != NULL; 1351 return skb->mac_header != NULL;
1345 } 1352 }
1346 1353
1347 static inline void skb_reset_mac_header(struct sk_buff *skb) 1354 static inline void skb_reset_mac_header(struct sk_buff *skb)
1348 { 1355 {
1349 skb->mac_header = skb->data; 1356 skb->mac_header = skb->data;
1350 } 1357 }
1351 1358
1352 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset) 1359 static inline void skb_set_mac_header(struct sk_buff *skb, const int offset)
1353 { 1360 {
1354 skb->mac_header = skb->data + offset; 1361 skb->mac_header = skb->data + offset;
1355 } 1362 }
1356 #endif /* NET_SKBUFF_DATA_USES_OFFSET */ 1363 #endif /* NET_SKBUFF_DATA_USES_OFFSET */
1357 1364
1358 static inline int skb_checksum_start_offset(const struct sk_buff *skb) 1365 static inline int skb_checksum_start_offset(const struct sk_buff *skb)
1359 { 1366 {
1360 return skb->csum_start - skb_headroom(skb); 1367 return skb->csum_start - skb_headroom(skb);
1361 } 1368 }
1362 1369
1363 static inline int skb_transport_offset(const struct sk_buff *skb) 1370 static inline int skb_transport_offset(const struct sk_buff *skb)
1364 { 1371 {
1365 return skb_transport_header(skb) - skb->data; 1372 return skb_transport_header(skb) - skb->data;
1366 } 1373 }
1367 1374
1368 static inline u32 skb_network_header_len(const struct sk_buff *skb) 1375 static inline u32 skb_network_header_len(const struct sk_buff *skb)
1369 { 1376 {
1370 return skb->transport_header - skb->network_header; 1377 return skb->transport_header - skb->network_header;
1371 } 1378 }
1372 1379
1373 static inline int skb_network_offset(const struct sk_buff *skb) 1380 static inline int skb_network_offset(const struct sk_buff *skb)
1374 { 1381 {
1375 return skb_network_header(skb) - skb->data; 1382 return skb_network_header(skb) - skb->data;
1376 } 1383 }
1377 1384
1378 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) 1385 static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len)
1379 { 1386 {
1380 return pskb_may_pull(skb, skb_network_offset(skb) + len); 1387 return pskb_may_pull(skb, skb_network_offset(skb) + len);
1381 } 1388 }
1382 1389
1383 /* 1390 /*
1384 * CPUs often take a performance hit when accessing unaligned memory 1391 * CPUs often take a performance hit when accessing unaligned memory
1385 * locations. The actual performance hit varies, it can be small if the 1392 * locations. The actual performance hit varies, it can be small if the
1386 * hardware handles it or large if we have to take an exception and fix it 1393 * hardware handles it or large if we have to take an exception and fix it
1387 * in software. 1394 * in software.
1388 * 1395 *
1389 * Since an ethernet header is 14 bytes network drivers often end up with 1396 * Since an ethernet header is 14 bytes network drivers often end up with
1390 * the IP header at an unaligned offset. The IP header can be aligned by 1397 * the IP header at an unaligned offset. The IP header can be aligned by
1391 * shifting the start of the packet by 2 bytes. Drivers should do this 1398 * shifting the start of the packet by 2 bytes. Drivers should do this
1392 * with: 1399 * with:
1393 * 1400 *
1394 * skb_reserve(skb, NET_IP_ALIGN); 1401 * skb_reserve(skb, NET_IP_ALIGN);
1395 * 1402 *
1396 * The downside to this alignment of the IP header is that the DMA is now 1403 * The downside to this alignment of the IP header is that the DMA is now
1397 * unaligned. On some architectures the cost of an unaligned DMA is high 1404 * unaligned. On some architectures the cost of an unaligned DMA is high
1398 * and this cost outweighs the gains made by aligning the IP header. 1405 * and this cost outweighs the gains made by aligning the IP header.
1399 * 1406 *
1400 * Since this trade off varies between architectures, we allow NET_IP_ALIGN 1407 * Since this trade off varies between architectures, we allow NET_IP_ALIGN
1401 * to be overridden. 1408 * to be overridden.
1402 */ 1409 */
1403 #ifndef NET_IP_ALIGN 1410 #ifndef NET_IP_ALIGN
1404 #define NET_IP_ALIGN 2 1411 #define NET_IP_ALIGN 2
1405 #endif 1412 #endif
1406 1413
1407 /* 1414 /*
1408 * The networking layer reserves some headroom in skb data (via 1415 * The networking layer reserves some headroom in skb data (via
1409 * dev_alloc_skb). This is used to avoid having to reallocate skb data when 1416 * dev_alloc_skb). This is used to avoid having to reallocate skb data when
1410 * the header has to grow. In the default case, if the header has to grow 1417 * the header has to grow. In the default case, if the header has to grow
1411 * 32 bytes or less we avoid the reallocation. 1418 * 32 bytes or less we avoid the reallocation.
1412 * 1419 *
1413 * Unfortunately this headroom changes the DMA alignment of the resulting 1420 * Unfortunately this headroom changes the DMA alignment of the resulting
1414 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive 1421 * network packet. As for NET_IP_ALIGN, this unaligned DMA is expensive
1415 * on some architectures. An architecture can override this value, 1422 * on some architectures. An architecture can override this value,
1416 * perhaps setting it to a cacheline in size (since that will maintain 1423 * perhaps setting it to a cacheline in size (since that will maintain
1417 * cacheline alignment of the DMA). It must be a power of 2. 1424 * cacheline alignment of the DMA). It must be a power of 2.
1418 * 1425 *
1419 * Various parts of the networking layer expect at least 32 bytes of 1426 * Various parts of the networking layer expect at least 32 bytes of
1420 * headroom, you should not reduce this. 1427 * headroom, you should not reduce this.
1421 * 1428 *
1422 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS) 1429 * Using max(32, L1_CACHE_BYTES) makes sense (especially with RPS)
1423 * to reduce average number of cache lines per packet. 1430 * to reduce average number of cache lines per packet.
1424 * get_rps_cpus() for example only access one 64 bytes aligned block : 1431 * get_rps_cpus() for example only access one 64 bytes aligned block :
1425 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8) 1432 * NET_IP_ALIGN(2) + ethernet_header(14) + IP_header(20/40) + ports(8)
1426 */ 1433 */
1427 #ifndef NET_SKB_PAD 1434 #ifndef NET_SKB_PAD
1428 #define NET_SKB_PAD max(32, L1_CACHE_BYTES) 1435 #define NET_SKB_PAD max(32, L1_CACHE_BYTES)
1429 #endif 1436 #endif
1430 1437
1431 extern int ___pskb_trim(struct sk_buff *skb, unsigned int len); 1438 extern int ___pskb_trim(struct sk_buff *skb, unsigned int len);
1432 1439
1433 static inline void __skb_trim(struct sk_buff *skb, unsigned int len) 1440 static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
1434 { 1441 {
1435 if (unlikely(skb->data_len)) { 1442 if (unlikely(skb->data_len)) {
1436 WARN_ON(1); 1443 WARN_ON(1);
1437 return; 1444 return;
1438 } 1445 }
1439 skb->len = len; 1446 skb->len = len;
1440 skb_set_tail_pointer(skb, len); 1447 skb_set_tail_pointer(skb, len);
1441 } 1448 }
1442 1449
1443 extern void skb_trim(struct sk_buff *skb, unsigned int len); 1450 extern void skb_trim(struct sk_buff *skb, unsigned int len);
1444 1451
1445 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) 1452 static inline int __pskb_trim(struct sk_buff *skb, unsigned int len)
1446 { 1453 {
1447 if (skb->data_len) 1454 if (skb->data_len)
1448 return ___pskb_trim(skb, len); 1455 return ___pskb_trim(skb, len);
1449 __skb_trim(skb, len); 1456 __skb_trim(skb, len);
1450 return 0; 1457 return 0;
1451 } 1458 }
1452 1459
1453 static inline int pskb_trim(struct sk_buff *skb, unsigned int len) 1460 static inline int pskb_trim(struct sk_buff *skb, unsigned int len)
1454 { 1461 {
1455 return (len < skb->len) ? __pskb_trim(skb, len) : 0; 1462 return (len < skb->len) ? __pskb_trim(skb, len) : 0;
1456 } 1463 }
1457 1464
1458 /** 1465 /**
1459 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer 1466 * pskb_trim_unique - remove end from a paged unique (not cloned) buffer
1460 * @skb: buffer to alter 1467 * @skb: buffer to alter
1461 * @len: new length 1468 * @len: new length
1462 * 1469 *
1463 * This is identical to pskb_trim except that the caller knows that 1470 * This is identical to pskb_trim except that the caller knows that
1464 * the skb is not cloned so we should never get an error due to out- 1471 * the skb is not cloned so we should never get an error due to out-
1465 * of-memory. 1472 * of-memory.
1466 */ 1473 */
1467 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len) 1474 static inline void pskb_trim_unique(struct sk_buff *skb, unsigned int len)
1468 { 1475 {
1469 int err = pskb_trim(skb, len); 1476 int err = pskb_trim(skb, len);
1470 BUG_ON(err); 1477 BUG_ON(err);
1471 } 1478 }
1472 1479
1473 /** 1480 /**
1474 * skb_orphan - orphan a buffer 1481 * skb_orphan - orphan a buffer
1475 * @skb: buffer to orphan 1482 * @skb: buffer to orphan
1476 * 1483 *
1477 * If a buffer currently has an owner then we call the owner's 1484 * If a buffer currently has an owner then we call the owner's
1478 * destructor function and make the @skb unowned. The buffer continues 1485 * destructor function and make the @skb unowned. The buffer continues
1479 * to exist but is no longer charged to its former owner. 1486 * to exist but is no longer charged to its former owner.
1480 */ 1487 */
1481 static inline void skb_orphan(struct sk_buff *skb) 1488 static inline void skb_orphan(struct sk_buff *skb)
1482 { 1489 {
1483 if (skb->destructor) 1490 if (skb->destructor)
1484 skb->destructor(skb); 1491 skb->destructor(skb);
1485 skb->destructor = NULL; 1492 skb->destructor = NULL;
1486 skb->sk = NULL; 1493 skb->sk = NULL;
1487 } 1494 }
1488 1495
1489 /** 1496 /**
1490 * __skb_queue_purge - empty a list 1497 * __skb_queue_purge - empty a list
1491 * @list: list to empty 1498 * @list: list to empty
1492 * 1499 *
1493 * Delete all buffers on an &sk_buff list. Each buffer is removed from 1500 * Delete all buffers on an &sk_buff list. Each buffer is removed from
1494 * the list and one reference dropped. This function does not take the 1501 * the list and one reference dropped. This function does not take the
1495 * list lock and the caller must hold the relevant locks to use it. 1502 * list lock and the caller must hold the relevant locks to use it.
1496 */ 1503 */
1497 extern void skb_queue_purge(struct sk_buff_head *list); 1504 extern void skb_queue_purge(struct sk_buff_head *list);
1498 static inline void __skb_queue_purge(struct sk_buff_head *list) 1505 static inline void __skb_queue_purge(struct sk_buff_head *list)
1499 { 1506 {
1500 struct sk_buff *skb; 1507 struct sk_buff *skb;
1501 while ((skb = __skb_dequeue(list)) != NULL) 1508 while ((skb = __skb_dequeue(list)) != NULL)
1502 kfree_skb(skb); 1509 kfree_skb(skb);
1503 } 1510 }
1504 1511
1505 /** 1512 /**
1506 * __dev_alloc_skb - allocate an skbuff for receiving 1513 * __dev_alloc_skb - allocate an skbuff for receiving
1507 * @length: length to allocate 1514 * @length: length to allocate
1508 * @gfp_mask: get_free_pages mask, passed to alloc_skb 1515 * @gfp_mask: get_free_pages mask, passed to alloc_skb
1509 * 1516 *
1510 * Allocate a new &sk_buff and assign it a usage count of one. The 1517 * Allocate a new &sk_buff and assign it a usage count of one. The
1511 * buffer has unspecified headroom built in. Users should allocate 1518 * buffer has unspecified headroom built in. Users should allocate
1512 * the headroom they think they need without accounting for the 1519 * the headroom they think they need without accounting for the
1513 * built in space. The built in space is used for optimisations. 1520 * built in space. The built in space is used for optimisations.
1514 * 1521 *
1515 * %NULL is returned if there is no free memory. 1522 * %NULL is returned if there is no free memory.
1516 */ 1523 */
1517 static inline struct sk_buff *__dev_alloc_skb(unsigned int length, 1524 static inline struct sk_buff *__dev_alloc_skb(unsigned int length,
1518 gfp_t gfp_mask) 1525 gfp_t gfp_mask)
1519 { 1526 {
1520 struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask); 1527 struct sk_buff *skb = alloc_skb(length + NET_SKB_PAD, gfp_mask);
1521 if (likely(skb)) 1528 if (likely(skb))
1522 skb_reserve(skb, NET_SKB_PAD); 1529 skb_reserve(skb, NET_SKB_PAD);
1523 return skb; 1530 return skb;
1524 } 1531 }
1525 1532
1526 extern struct sk_buff *dev_alloc_skb(unsigned int length); 1533 extern struct sk_buff *dev_alloc_skb(unsigned int length);
1527 1534
1528 extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 1535 extern struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
1529 unsigned int length, gfp_t gfp_mask); 1536 unsigned int length, gfp_t gfp_mask);
1530 1537
1531 /** 1538 /**
1532 * netdev_alloc_skb - allocate an skbuff for rx on a specific device 1539 * netdev_alloc_skb - allocate an skbuff for rx on a specific device
1533 * @dev: network device to receive on 1540 * @dev: network device to receive on
1534 * @length: length to allocate 1541 * @length: length to allocate
1535 * 1542 *
1536 * Allocate a new &sk_buff and assign it a usage count of one. The 1543 * Allocate a new &sk_buff and assign it a usage count of one. The
1537 * buffer has unspecified headroom built in. Users should allocate 1544 * buffer has unspecified headroom built in. Users should allocate
1538 * the headroom they think they need without accounting for the 1545 * the headroom they think they need without accounting for the
1539 * built in space. The built in space is used for optimisations. 1546 * built in space. The built in space is used for optimisations.
1540 * 1547 *
1541 * %NULL is returned if there is no free memory. Although this function 1548 * %NULL is returned if there is no free memory. Although this function
1542 * allocates memory it can be called from an interrupt. 1549 * allocates memory it can be called from an interrupt.
1543 */ 1550 */
1544 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev, 1551 static inline struct sk_buff *netdev_alloc_skb(struct net_device *dev,
1545 unsigned int length) 1552 unsigned int length)
1546 { 1553 {
1547 return __netdev_alloc_skb(dev, length, GFP_ATOMIC); 1554 return __netdev_alloc_skb(dev, length, GFP_ATOMIC);
1548 } 1555 }
1549 1556
1550 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev, 1557 static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
1551 unsigned int length) 1558 unsigned int length)
1552 { 1559 {
1553 struct sk_buff *skb = netdev_alloc_skb(dev, length + NET_IP_ALIGN); 1560 struct sk_buff *skb = netdev_alloc_skb(dev, length + NET_IP_ALIGN);
1554 1561
1555 if (NET_IP_ALIGN && skb) 1562 if (NET_IP_ALIGN && skb)
1556 skb_reserve(skb, NET_IP_ALIGN); 1563 skb_reserve(skb, NET_IP_ALIGN);
1557 return skb; 1564 return skb;
1558 } 1565 }
1559 1566
1560 /** 1567 /**
1561 * __netdev_alloc_page - allocate a page for ps-rx on a specific device 1568 * __netdev_alloc_page - allocate a page for ps-rx on a specific device
1562 * @dev: network device to receive on 1569 * @dev: network device to receive on
1563 * @gfp_mask: alloc_pages_node mask 1570 * @gfp_mask: alloc_pages_node mask
1564 * 1571 *
1565 * Allocate a new page. dev currently unused. 1572 * Allocate a new page. dev currently unused.
1566 * 1573 *
1567 * %NULL is returned if there is no free memory. 1574 * %NULL is returned if there is no free memory.
1568 */ 1575 */
1569 static inline struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask) 1576 static inline struct page *__netdev_alloc_page(struct net_device *dev, gfp_t gfp_mask)
1570 { 1577 {
1571 return alloc_pages_node(NUMA_NO_NODE, gfp_mask, 0); 1578 return alloc_pages_node(NUMA_NO_NODE, gfp_mask, 0);
1572 } 1579 }
1573 1580
1574 /** 1581 /**
1575 * netdev_alloc_page - allocate a page for ps-rx on a specific device 1582 * netdev_alloc_page - allocate a page for ps-rx on a specific device
1576 * @dev: network device to receive on 1583 * @dev: network device to receive on
1577 * 1584 *
1578 * Allocate a new page. dev currently unused. 1585 * Allocate a new page. dev currently unused.
1579 * 1586 *
1580 * %NULL is returned if there is no free memory. 1587 * %NULL is returned if there is no free memory.
1581 */ 1588 */
1582 static inline struct page *netdev_alloc_page(struct net_device *dev) 1589 static inline struct page *netdev_alloc_page(struct net_device *dev)
1583 { 1590 {
1584 return __netdev_alloc_page(dev, GFP_ATOMIC); 1591 return __netdev_alloc_page(dev, GFP_ATOMIC);
1585 } 1592 }
1586 1593
1587 static inline void netdev_free_page(struct net_device *dev, struct page *page) 1594 static inline void netdev_free_page(struct net_device *dev, struct page *page)
1588 { 1595 {
1589 __free_page(page); 1596 __free_page(page);
1590 } 1597 }
1591 1598
1592 /** 1599 /**
1593 * skb_clone_writable - is the header of a clone writable 1600 * skb_clone_writable - is the header of a clone writable
1594 * @skb: buffer to check 1601 * @skb: buffer to check
1595 * @len: length up to which to write 1602 * @len: length up to which to write
1596 * 1603 *
1597 * Returns true if modifying the header part of the cloned buffer 1604 * Returns true if modifying the header part of the cloned buffer
1598 * does not requires the data to be copied. 1605 * does not requires the data to be copied.
1599 */ 1606 */
1600 static inline int skb_clone_writable(struct sk_buff *skb, unsigned int len) 1607 static inline int skb_clone_writable(struct sk_buff *skb, unsigned int len)
1601 { 1608 {
1602 return !skb_header_cloned(skb) && 1609 return !skb_header_cloned(skb) &&
1603 skb_headroom(skb) + len <= skb->hdr_len; 1610 skb_headroom(skb) + len <= skb->hdr_len;
1604 } 1611 }
1605 1612
1606 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom, 1613 static inline int __skb_cow(struct sk_buff *skb, unsigned int headroom,
1607 int cloned) 1614 int cloned)
1608 { 1615 {
1609 int delta = 0; 1616 int delta = 0;
1610 1617
1611 if (headroom < NET_SKB_PAD) 1618 if (headroom < NET_SKB_PAD)
1612 headroom = NET_SKB_PAD; 1619 headroom = NET_SKB_PAD;
1613 if (headroom > skb_headroom(skb)) 1620 if (headroom > skb_headroom(skb))
1614 delta = headroom - skb_headroom(skb); 1621 delta = headroom - skb_headroom(skb);
1615 1622
1616 if (delta || cloned) 1623 if (delta || cloned)
1617 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0, 1624 return pskb_expand_head(skb, ALIGN(delta, NET_SKB_PAD), 0,
1618 GFP_ATOMIC); 1625 GFP_ATOMIC);
1619 return 0; 1626 return 0;
1620 } 1627 }
1621 1628
1622 /** 1629 /**
1623 * skb_cow - copy header of skb when it is required 1630 * skb_cow - copy header of skb when it is required
1624 * @skb: buffer to cow 1631 * @skb: buffer to cow
1625 * @headroom: needed headroom 1632 * @headroom: needed headroom
1626 * 1633 *
1627 * If the skb passed lacks sufficient headroom or its data part 1634 * If the skb passed lacks sufficient headroom or its data part
1628 * is shared, data is reallocated. If reallocation fails, an error 1635 * is shared, data is reallocated. If reallocation fails, an error
1629 * is returned and original skb is not changed. 1636 * is returned and original skb is not changed.
1630 * 1637 *
1631 * The result is skb with writable area skb->head...skb->tail 1638 * The result is skb with writable area skb->head...skb->tail
1632 * and at least @headroom of space at head. 1639 * and at least @headroom of space at head.
1633 */ 1640 */
1634 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom) 1641 static inline int skb_cow(struct sk_buff *skb, unsigned int headroom)
1635 { 1642 {
1636 return __skb_cow(skb, headroom, skb_cloned(skb)); 1643 return __skb_cow(skb, headroom, skb_cloned(skb));
1637 } 1644 }
1638 1645
1639 /** 1646 /**
1640 * skb_cow_head - skb_cow but only making the head writable 1647 * skb_cow_head - skb_cow but only making the head writable
1641 * @skb: buffer to cow 1648 * @skb: buffer to cow
1642 * @headroom: needed headroom 1649 * @headroom: needed headroom
1643 * 1650 *
1644 * This function is identical to skb_cow except that we replace the 1651 * This function is identical to skb_cow except that we replace the
1645 * skb_cloned check by skb_header_cloned. It should be used when 1652 * skb_cloned check by skb_header_cloned. It should be used when
1646 * you only need to push on some header and do not need to modify 1653 * you only need to push on some header and do not need to modify
1647 * the data. 1654 * the data.
1648 */ 1655 */
1649 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom) 1656 static inline int skb_cow_head(struct sk_buff *skb, unsigned int headroom)
1650 { 1657 {
1651 return __skb_cow(skb, headroom, skb_header_cloned(skb)); 1658 return __skb_cow(skb, headroom, skb_header_cloned(skb));
1652 } 1659 }
1653 1660
1654 /** 1661 /**
1655 * skb_padto - pad an skbuff up to a minimal size 1662 * skb_padto - pad an skbuff up to a minimal size
1656 * @skb: buffer to pad 1663 * @skb: buffer to pad
1657 * @len: minimal length 1664 * @len: minimal length
1658 * 1665 *
1659 * Pads up a buffer to ensure the trailing bytes exist and are 1666 * Pads up a buffer to ensure the trailing bytes exist and are
1660 * blanked. If the buffer already contains sufficient data it 1667 * blanked. If the buffer already contains sufficient data it
1661 * is untouched. Otherwise it is extended. Returns zero on 1668 * is untouched. Otherwise it is extended. Returns zero on
1662 * success. The skb is freed on error. 1669 * success. The skb is freed on error.
1663 */ 1670 */
1664 1671
1665 static inline int skb_padto(struct sk_buff *skb, unsigned int len) 1672 static inline int skb_padto(struct sk_buff *skb, unsigned int len)
1666 { 1673 {
1667 unsigned int size = skb->len; 1674 unsigned int size = skb->len;
1668 if (likely(size >= len)) 1675 if (likely(size >= len))
1669 return 0; 1676 return 0;
1670 return skb_pad(skb, len - size); 1677 return skb_pad(skb, len - size);
1671 } 1678 }
1672 1679
1673 static inline int skb_add_data(struct sk_buff *skb, 1680 static inline int skb_add_data(struct sk_buff *skb,
1674 char __user *from, int copy) 1681 char __user *from, int copy)
1675 { 1682 {
1676 const int off = skb->len; 1683 const int off = skb->len;
1677 1684
1678 if (skb->ip_summed == CHECKSUM_NONE) { 1685 if (skb->ip_summed == CHECKSUM_NONE) {
1679 int err = 0; 1686 int err = 0;
1680 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy), 1687 __wsum csum = csum_and_copy_from_user(from, skb_put(skb, copy),
1681 copy, 0, &err); 1688 copy, 0, &err);
1682 if (!err) { 1689 if (!err) {
1683 skb->csum = csum_block_add(skb->csum, csum, off); 1690 skb->csum = csum_block_add(skb->csum, csum, off);
1684 return 0; 1691 return 0;
1685 } 1692 }
1686 } else if (!copy_from_user(skb_put(skb, copy), from, copy)) 1693 } else if (!copy_from_user(skb_put(skb, copy), from, copy))
1687 return 0; 1694 return 0;
1688 1695
1689 __skb_trim(skb, off); 1696 __skb_trim(skb, off);
1690 return -EFAULT; 1697 return -EFAULT;
1691 } 1698 }
1692 1699
1693 static inline int skb_can_coalesce(struct sk_buff *skb, int i, 1700 static inline int skb_can_coalesce(struct sk_buff *skb, int i,
1694 struct page *page, int off) 1701 struct page *page, int off)
1695 { 1702 {
1696 if (i) { 1703 if (i) {
1697 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1]; 1704 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i - 1];
1698 1705
1699 return page == frag->page && 1706 return page == frag->page &&
1700 off == frag->page_offset + frag->size; 1707 off == frag->page_offset + frag->size;
1701 } 1708 }
1702 return 0; 1709 return 0;
1703 } 1710 }
1704 1711
1705 static inline int __skb_linearize(struct sk_buff *skb) 1712 static inline int __skb_linearize(struct sk_buff *skb)
1706 { 1713 {
1707 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM; 1714 return __pskb_pull_tail(skb, skb->data_len) ? 0 : -ENOMEM;
1708 } 1715 }
1709 1716
1710 /** 1717 /**
1711 * skb_linearize - convert paged skb to linear one 1718 * skb_linearize - convert paged skb to linear one
1712 * @skb: buffer to linarize 1719 * @skb: buffer to linarize
1713 * 1720 *
1714 * If there is no free memory -ENOMEM is returned, otherwise zero 1721 * If there is no free memory -ENOMEM is returned, otherwise zero
1715 * is returned and the old skb data released. 1722 * is returned and the old skb data released.
1716 */ 1723 */
1717 static inline int skb_linearize(struct sk_buff *skb) 1724 static inline int skb_linearize(struct sk_buff *skb)
1718 { 1725 {
1719 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0; 1726 return skb_is_nonlinear(skb) ? __skb_linearize(skb) : 0;
1720 } 1727 }
1721 1728
1722 /** 1729 /**
1723 * skb_linearize_cow - make sure skb is linear and writable 1730 * skb_linearize_cow - make sure skb is linear and writable
1724 * @skb: buffer to process 1731 * @skb: buffer to process
1725 * 1732 *
1726 * If there is no free memory -ENOMEM is returned, otherwise zero 1733 * If there is no free memory -ENOMEM is returned, otherwise zero
1727 * is returned and the old skb data released. 1734 * is returned and the old skb data released.
1728 */ 1735 */
1729 static inline int skb_linearize_cow(struct sk_buff *skb) 1736 static inline int skb_linearize_cow(struct sk_buff *skb)
1730 { 1737 {
1731 return skb_is_nonlinear(skb) || skb_cloned(skb) ? 1738 return skb_is_nonlinear(skb) || skb_cloned(skb) ?
1732 __skb_linearize(skb) : 0; 1739 __skb_linearize(skb) : 0;
1733 } 1740 }
1734 1741
1735 /** 1742 /**
1736 * skb_postpull_rcsum - update checksum for received skb after pull 1743 * skb_postpull_rcsum - update checksum for received skb after pull
1737 * @skb: buffer to update 1744 * @skb: buffer to update
1738 * @start: start of data before pull 1745 * @start: start of data before pull
1739 * @len: length of data pulled 1746 * @len: length of data pulled
1740 * 1747 *
1741 * After doing a pull on a received packet, you need to call this to 1748 * After doing a pull on a received packet, you need to call this to
1742 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to 1749 * update the CHECKSUM_COMPLETE checksum, or set ip_summed to
1743 * CHECKSUM_NONE so that it can be recomputed from scratch. 1750 * CHECKSUM_NONE so that it can be recomputed from scratch.
1744 */ 1751 */
1745 1752
1746 static inline void skb_postpull_rcsum(struct sk_buff *skb, 1753 static inline void skb_postpull_rcsum(struct sk_buff *skb,
1747 const void *start, unsigned int len) 1754 const void *start, unsigned int len)
1748 { 1755 {
1749 if (skb->ip_summed == CHECKSUM_COMPLETE) 1756 if (skb->ip_summed == CHECKSUM_COMPLETE)
1750 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0)); 1757 skb->csum = csum_sub(skb->csum, csum_partial(start, len, 0));
1751 } 1758 }
1752 1759
1753 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len); 1760 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len);
1754 1761
1755 /** 1762 /**
1756 * pskb_trim_rcsum - trim received skb and update checksum 1763 * pskb_trim_rcsum - trim received skb and update checksum
1757 * @skb: buffer to trim 1764 * @skb: buffer to trim
1758 * @len: new length 1765 * @len: new length
1759 * 1766 *
1760 * This is exactly the same as pskb_trim except that it ensures the 1767 * This is exactly the same as pskb_trim except that it ensures the
1761 * checksum of received packets are still valid after the operation. 1768 * checksum of received packets are still valid after the operation.
1762 */ 1769 */
1763 1770
1764 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len) 1771 static inline int pskb_trim_rcsum(struct sk_buff *skb, unsigned int len)
1765 { 1772 {
1766 if (likely(len >= skb->len)) 1773 if (likely(len >= skb->len))
1767 return 0; 1774 return 0;
1768 if (skb->ip_summed == CHECKSUM_COMPLETE) 1775 if (skb->ip_summed == CHECKSUM_COMPLETE)
1769 skb->ip_summed = CHECKSUM_NONE; 1776 skb->ip_summed = CHECKSUM_NONE;
1770 return __pskb_trim(skb, len); 1777 return __pskb_trim(skb, len);
1771 } 1778 }
1772 1779
1773 #define skb_queue_walk(queue, skb) \ 1780 #define skb_queue_walk(queue, skb) \
1774 for (skb = (queue)->next; \ 1781 for (skb = (queue)->next; \
1775 prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ 1782 prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \
1776 skb = skb->next) 1783 skb = skb->next)
1777 1784
1778 #define skb_queue_walk_safe(queue, skb, tmp) \ 1785 #define skb_queue_walk_safe(queue, skb, tmp) \
1779 for (skb = (queue)->next, tmp = skb->next; \ 1786 for (skb = (queue)->next, tmp = skb->next; \
1780 skb != (struct sk_buff *)(queue); \ 1787 skb != (struct sk_buff *)(queue); \
1781 skb = tmp, tmp = skb->next) 1788 skb = tmp, tmp = skb->next)
1782 1789
1783 #define skb_queue_walk_from(queue, skb) \ 1790 #define skb_queue_walk_from(queue, skb) \
1784 for (; prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \ 1791 for (; prefetch(skb->next), (skb != (struct sk_buff *)(queue)); \
1785 skb = skb->next) 1792 skb = skb->next)
1786 1793
1787 #define skb_queue_walk_from_safe(queue, skb, tmp) \ 1794 #define skb_queue_walk_from_safe(queue, skb, tmp) \
1788 for (tmp = skb->next; \ 1795 for (tmp = skb->next; \
1789 skb != (struct sk_buff *)(queue); \ 1796 skb != (struct sk_buff *)(queue); \
1790 skb = tmp, tmp = skb->next) 1797 skb = tmp, tmp = skb->next)
1791 1798
1792 #define skb_queue_reverse_walk(queue, skb) \ 1799 #define skb_queue_reverse_walk(queue, skb) \
1793 for (skb = (queue)->prev; \ 1800 for (skb = (queue)->prev; \
1794 prefetch(skb->prev), (skb != (struct sk_buff *)(queue)); \ 1801 prefetch(skb->prev), (skb != (struct sk_buff *)(queue)); \
1795 skb = skb->prev) 1802 skb = skb->prev)
1796 1803
1797 1804
1798 static inline bool skb_has_frag_list(const struct sk_buff *skb) 1805 static inline bool skb_has_frag_list(const struct sk_buff *skb)
1799 { 1806 {
1800 return skb_shinfo(skb)->frag_list != NULL; 1807 return skb_shinfo(skb)->frag_list != NULL;
1801 } 1808 }
1802 1809
1803 static inline void skb_frag_list_init(struct sk_buff *skb) 1810 static inline void skb_frag_list_init(struct sk_buff *skb)
1804 { 1811 {
1805 skb_shinfo(skb)->frag_list = NULL; 1812 skb_shinfo(skb)->frag_list = NULL;
1806 } 1813 }
1807 1814
1808 static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag) 1815 static inline void skb_frag_add_head(struct sk_buff *skb, struct sk_buff *frag)
1809 { 1816 {
1810 frag->next = skb_shinfo(skb)->frag_list; 1817 frag->next = skb_shinfo(skb)->frag_list;
1811 skb_shinfo(skb)->frag_list = frag; 1818 skb_shinfo(skb)->frag_list = frag;
1812 } 1819 }
1813 1820
1814 #define skb_walk_frags(skb, iter) \ 1821 #define skb_walk_frags(skb, iter) \
1815 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next) 1822 for (iter = skb_shinfo(skb)->frag_list; iter; iter = iter->next)
1816 1823
1817 extern struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags, 1824 extern struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned flags,
1818 int *peeked, int *err); 1825 int *peeked, int *err);
1819 extern struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags, 1826 extern struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned flags,
1820 int noblock, int *err); 1827 int noblock, int *err);
1821 extern unsigned int datagram_poll(struct file *file, struct socket *sock, 1828 extern unsigned int datagram_poll(struct file *file, struct socket *sock,
1822 struct poll_table_struct *wait); 1829 struct poll_table_struct *wait);
1823 extern int skb_copy_datagram_iovec(const struct sk_buff *from, 1830 extern int skb_copy_datagram_iovec(const struct sk_buff *from,
1824 int offset, struct iovec *to, 1831 int offset, struct iovec *to,
1825 int size); 1832 int size);
1826 extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, 1833 extern int skb_copy_and_csum_datagram_iovec(struct sk_buff *skb,
1827 int hlen, 1834 int hlen,
1828 struct iovec *iov); 1835 struct iovec *iov);
1829 extern int skb_copy_datagram_from_iovec(struct sk_buff *skb, 1836 extern int skb_copy_datagram_from_iovec(struct sk_buff *skb,
1830 int offset, 1837 int offset,
1831 const struct iovec *from, 1838 const struct iovec *from,
1832 int from_offset, 1839 int from_offset,
1833 int len); 1840 int len);
1834 extern int skb_copy_datagram_const_iovec(const struct sk_buff *from, 1841 extern int skb_copy_datagram_const_iovec(const struct sk_buff *from,
1835 int offset, 1842 int offset,
1836 const struct iovec *to, 1843 const struct iovec *to,
1837 int to_offset, 1844 int to_offset,
1838 int size); 1845 int size);
1839 extern void skb_free_datagram(struct sock *sk, struct sk_buff *skb); 1846 extern void skb_free_datagram(struct sock *sk, struct sk_buff *skb);
1840 extern void skb_free_datagram_locked(struct sock *sk, 1847 extern void skb_free_datagram_locked(struct sock *sk,
1841 struct sk_buff *skb); 1848 struct sk_buff *skb);
1842 extern int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, 1849 extern int skb_kill_datagram(struct sock *sk, struct sk_buff *skb,
1843 unsigned int flags); 1850 unsigned int flags);
1844 extern __wsum skb_checksum(const struct sk_buff *skb, int offset, 1851 extern __wsum skb_checksum(const struct sk_buff *skb, int offset,
1845 int len, __wsum csum); 1852 int len, __wsum csum);
1846 extern int skb_copy_bits(const struct sk_buff *skb, int offset, 1853 extern int skb_copy_bits(const struct sk_buff *skb, int offset,
1847 void *to, int len); 1854 void *to, int len);
1848 extern int skb_store_bits(struct sk_buff *skb, int offset, 1855 extern int skb_store_bits(struct sk_buff *skb, int offset,
1849 const void *from, int len); 1856 const void *from, int len);
1850 extern __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, 1857 extern __wsum skb_copy_and_csum_bits(const struct sk_buff *skb,
1851 int offset, u8 *to, int len, 1858 int offset, u8 *to, int len,
1852 __wsum csum); 1859 __wsum csum);
1853 extern int skb_splice_bits(struct sk_buff *skb, 1860 extern int skb_splice_bits(struct sk_buff *skb,
1854 unsigned int offset, 1861 unsigned int offset,
1855 struct pipe_inode_info *pipe, 1862 struct pipe_inode_info *pipe,
1856 unsigned int len, 1863 unsigned int len,
1857 unsigned int flags); 1864 unsigned int flags);
1858 extern void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to); 1865 extern void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to);
1859 extern void skb_split(struct sk_buff *skb, 1866 extern void skb_split(struct sk_buff *skb,
1860 struct sk_buff *skb1, const u32 len); 1867 struct sk_buff *skb1, const u32 len);
1861 extern int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, 1868 extern int skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
1862 int shiftlen); 1869 int shiftlen);
1863 1870
1864 extern struct sk_buff *skb_segment(struct sk_buff *skb, int features); 1871 extern struct sk_buff *skb_segment(struct sk_buff *skb, int features);
1865 1872
1866 static inline void *skb_header_pointer(const struct sk_buff *skb, int offset, 1873 static inline void *skb_header_pointer(const struct sk_buff *skb, int offset,
1867 int len, void *buffer) 1874 int len, void *buffer)
1868 { 1875 {
1869 int hlen = skb_headlen(skb); 1876 int hlen = skb_headlen(skb);
1870 1877
1871 if (hlen - offset >= len) 1878 if (hlen - offset >= len)
1872 return skb->data + offset; 1879 return skb->data + offset;
1873 1880
1874 if (skb_copy_bits(skb, offset, buffer, len) < 0) 1881 if (skb_copy_bits(skb, offset, buffer, len) < 0)
1875 return NULL; 1882 return NULL;
1876 1883
1877 return buffer; 1884 return buffer;
1878 } 1885 }
1879 1886
1880 static inline void skb_copy_from_linear_data(const struct sk_buff *skb, 1887 static inline void skb_copy_from_linear_data(const struct sk_buff *skb,
1881 void *to, 1888 void *to,
1882 const unsigned int len) 1889 const unsigned int len)
1883 { 1890 {
1884 memcpy(to, skb->data, len); 1891 memcpy(to, skb->data, len);
1885 } 1892 }
1886 1893
1887 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb, 1894 static inline void skb_copy_from_linear_data_offset(const struct sk_buff *skb,
1888 const int offset, void *to, 1895 const int offset, void *to,
1889 const unsigned int len) 1896 const unsigned int len)
1890 { 1897 {
1891 memcpy(to, skb->data + offset, len); 1898 memcpy(to, skb->data + offset, len);
1892 } 1899 }
1893 1900
1894 static inline void skb_copy_to_linear_data(struct sk_buff *skb, 1901 static inline void skb_copy_to_linear_data(struct sk_buff *skb,
1895 const void *from, 1902 const void *from,
1896 const unsigned int len) 1903 const unsigned int len)
1897 { 1904 {
1898 memcpy(skb->data, from, len); 1905 memcpy(skb->data, from, len);
1899 } 1906 }
1900 1907
1901 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb, 1908 static inline void skb_copy_to_linear_data_offset(struct sk_buff *skb,
1902 const int offset, 1909 const int offset,
1903 const void *from, 1910 const void *from,
1904 const unsigned int len) 1911 const unsigned int len)
1905 { 1912 {
1906 memcpy(skb->data + offset, from, len); 1913 memcpy(skb->data + offset, from, len);
1907 } 1914 }
1908 1915
1909 extern void skb_init(void); 1916 extern void skb_init(void);
1910 1917
1911 static inline ktime_t skb_get_ktime(const struct sk_buff *skb) 1918 static inline ktime_t skb_get_ktime(const struct sk_buff *skb)
1912 { 1919 {
1913 return skb->tstamp; 1920 return skb->tstamp;
1914 } 1921 }
1915 1922
1916 /** 1923 /**
1917 * skb_get_timestamp - get timestamp from a skb 1924 * skb_get_timestamp - get timestamp from a skb
1918 * @skb: skb to get stamp from 1925 * @skb: skb to get stamp from
1919 * @stamp: pointer to struct timeval to store stamp in 1926 * @stamp: pointer to struct timeval to store stamp in
1920 * 1927 *
1921 * Timestamps are stored in the skb as offsets to a base timestamp. 1928 * Timestamps are stored in the skb as offsets to a base timestamp.
1922 * This function converts the offset back to a struct timeval and stores 1929 * This function converts the offset back to a struct timeval and stores
1923 * it in stamp. 1930 * it in stamp.
1924 */ 1931 */
1925 static inline void skb_get_timestamp(const struct sk_buff *skb, 1932 static inline void skb_get_timestamp(const struct sk_buff *skb,
1926 struct timeval *stamp) 1933 struct timeval *stamp)
1927 { 1934 {
1928 *stamp = ktime_to_timeval(skb->tstamp); 1935 *stamp = ktime_to_timeval(skb->tstamp);
1929 } 1936 }
1930 1937
1931 static inline void skb_get_timestampns(const struct sk_buff *skb, 1938 static inline void skb_get_timestampns(const struct sk_buff *skb,
1932 struct timespec *stamp) 1939 struct timespec *stamp)
1933 { 1940 {
1934 *stamp = ktime_to_timespec(skb->tstamp); 1941 *stamp = ktime_to_timespec(skb->tstamp);
1935 } 1942 }
1936 1943
1937 static inline void __net_timestamp(struct sk_buff *skb) 1944 static inline void __net_timestamp(struct sk_buff *skb)
1938 { 1945 {
1939 skb->tstamp = ktime_get_real(); 1946 skb->tstamp = ktime_get_real();
1940 } 1947 }
1941 1948
1942 static inline ktime_t net_timedelta(ktime_t t) 1949 static inline ktime_t net_timedelta(ktime_t t)
1943 { 1950 {
1944 return ktime_sub(ktime_get_real(), t); 1951 return ktime_sub(ktime_get_real(), t);
1945 } 1952 }
1946 1953
1947 static inline ktime_t net_invalid_timestamp(void) 1954 static inline ktime_t net_invalid_timestamp(void)
1948 { 1955 {
1949 return ktime_set(0, 0); 1956 return ktime_set(0, 0);
1950 } 1957 }
1951 1958
1952 extern void skb_timestamping_init(void); 1959 extern void skb_timestamping_init(void);
1953 1960
1954 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING 1961 #ifdef CONFIG_NETWORK_PHY_TIMESTAMPING
1955 1962
1956 extern void skb_clone_tx_timestamp(struct sk_buff *skb); 1963 extern void skb_clone_tx_timestamp(struct sk_buff *skb);
1957 extern bool skb_defer_rx_timestamp(struct sk_buff *skb); 1964 extern bool skb_defer_rx_timestamp(struct sk_buff *skb);
1958 1965
1959 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */ 1966 #else /* CONFIG_NETWORK_PHY_TIMESTAMPING */
1960 1967
1961 static inline void skb_clone_tx_timestamp(struct sk_buff *skb) 1968 static inline void skb_clone_tx_timestamp(struct sk_buff *skb)
1962 { 1969 {
1963 } 1970 }
1964 1971
1965 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb) 1972 static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
1966 { 1973 {
1967 return false; 1974 return false;
1968 } 1975 }
1969 1976
1970 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */ 1977 #endif /* !CONFIG_NETWORK_PHY_TIMESTAMPING */
1971 1978
1972 /** 1979 /**
1973 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps 1980 * skb_complete_tx_timestamp() - deliver cloned skb with tx timestamps
1974 * 1981 *
1975 * @skb: clone of the the original outgoing packet 1982 * @skb: clone of the the original outgoing packet
1976 * @hwtstamps: hardware time stamps 1983 * @hwtstamps: hardware time stamps
1977 * 1984 *
1978 */ 1985 */
1979 void skb_complete_tx_timestamp(struct sk_buff *skb, 1986 void skb_complete_tx_timestamp(struct sk_buff *skb,
1980 struct skb_shared_hwtstamps *hwtstamps); 1987 struct skb_shared_hwtstamps *hwtstamps);
1981 1988
1982 /** 1989 /**
1983 * skb_tstamp_tx - queue clone of skb with send time stamps 1990 * skb_tstamp_tx - queue clone of skb with send time stamps
1984 * @orig_skb: the original outgoing packet 1991 * @orig_skb: the original outgoing packet
1985 * @hwtstamps: hardware time stamps, may be NULL if not available 1992 * @hwtstamps: hardware time stamps, may be NULL if not available
1986 * 1993 *
1987 * If the skb has a socket associated, then this function clones the 1994 * If the skb has a socket associated, then this function clones the
1988 * skb (thus sharing the actual data and optional structures), stores 1995 * skb (thus sharing the actual data and optional structures), stores
1989 * the optional hardware time stamping information (if non NULL) or 1996 * the optional hardware time stamping information (if non NULL) or
1990 * generates a software time stamp (otherwise), then queues the clone 1997 * generates a software time stamp (otherwise), then queues the clone
1991 * to the error queue of the socket. Errors are silently ignored. 1998 * to the error queue of the socket. Errors are silently ignored.
1992 */ 1999 */
1993 extern void skb_tstamp_tx(struct sk_buff *orig_skb, 2000 extern void skb_tstamp_tx(struct sk_buff *orig_skb,
1994 struct skb_shared_hwtstamps *hwtstamps); 2001 struct skb_shared_hwtstamps *hwtstamps);
1995 2002
1996 static inline void sw_tx_timestamp(struct sk_buff *skb) 2003 static inline void sw_tx_timestamp(struct sk_buff *skb)
1997 { 2004 {
1998 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP && 2005 if (skb_shinfo(skb)->tx_flags & SKBTX_SW_TSTAMP &&
1999 !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) 2006 !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))
2000 skb_tstamp_tx(skb, NULL); 2007 skb_tstamp_tx(skb, NULL);
2001 } 2008 }
2002 2009
2003 /** 2010 /**
2004 * skb_tx_timestamp() - Driver hook for transmit timestamping 2011 * skb_tx_timestamp() - Driver hook for transmit timestamping
2005 * 2012 *
2006 * Ethernet MAC Drivers should call this function in their hard_xmit() 2013 * Ethernet MAC Drivers should call this function in their hard_xmit()
2007 * function as soon as possible after giving the sk_buff to the MAC 2014 * function as soon as possible after giving the sk_buff to the MAC
2008 * hardware, but before freeing the sk_buff. 2015 * hardware, but before freeing the sk_buff.
2009 * 2016 *
2010 * @skb: A socket buffer. 2017 * @skb: A socket buffer.
2011 */ 2018 */
2012 static inline void skb_tx_timestamp(struct sk_buff *skb) 2019 static inline void skb_tx_timestamp(struct sk_buff *skb)
2013 { 2020 {
2014 skb_clone_tx_timestamp(skb); 2021 skb_clone_tx_timestamp(skb);
2015 sw_tx_timestamp(skb); 2022 sw_tx_timestamp(skb);
2016 } 2023 }
2017 2024
2018 extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len); 2025 extern __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len);
2019 extern __sum16 __skb_checksum_complete(struct sk_buff *skb); 2026 extern __sum16 __skb_checksum_complete(struct sk_buff *skb);
2020 2027
2021 static inline int skb_csum_unnecessary(const struct sk_buff *skb) 2028 static inline int skb_csum_unnecessary(const struct sk_buff *skb)
2022 { 2029 {
2023 return skb->ip_summed & CHECKSUM_UNNECESSARY; 2030 return skb->ip_summed & CHECKSUM_UNNECESSARY;
2024 } 2031 }
2025 2032
2026 /** 2033 /**
2027 * skb_checksum_complete - Calculate checksum of an entire packet 2034 * skb_checksum_complete - Calculate checksum of an entire packet
2028 * @skb: packet to process 2035 * @skb: packet to process
2029 * 2036 *
2030 * This function calculates the checksum over the entire packet plus 2037 * This function calculates the checksum over the entire packet plus
2031 * the value of skb->csum. The latter can be used to supply the 2038 * the value of skb->csum. The latter can be used to supply the
2032 * checksum of a pseudo header as used by TCP/UDP. It returns the 2039 * checksum of a pseudo header as used by TCP/UDP. It returns the
2033 * checksum. 2040 * checksum.
2034 * 2041 *
2035 * For protocols that contain complete checksums such as ICMP/TCP/UDP, 2042 * For protocols that contain complete checksums such as ICMP/TCP/UDP,
2036 * this function can be used to verify that checksum on received 2043 * this function can be used to verify that checksum on received
2037 * packets. In that case the function should return zero if the 2044 * packets. In that case the function should return zero if the
2038 * checksum is correct. In particular, this function will return zero 2045 * checksum is correct. In particular, this function will return zero
2039 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the 2046 * if skb->ip_summed is CHECKSUM_UNNECESSARY which indicates that the
2040 * hardware has already verified the correctness of the checksum. 2047 * hardware has already verified the correctness of the checksum.
2041 */ 2048 */
2042 static inline __sum16 skb_checksum_complete(struct sk_buff *skb) 2049 static inline __sum16 skb_checksum_complete(struct sk_buff *skb)
2043 { 2050 {
2044 return skb_csum_unnecessary(skb) ? 2051 return skb_csum_unnecessary(skb) ?
2045 0 : __skb_checksum_complete(skb); 2052 0 : __skb_checksum_complete(skb);
2046 } 2053 }
2047 2054
2048 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2055 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2049 extern void nf_conntrack_destroy(struct nf_conntrack *nfct); 2056 extern void nf_conntrack_destroy(struct nf_conntrack *nfct);
2050 static inline void nf_conntrack_put(struct nf_conntrack *nfct) 2057 static inline void nf_conntrack_put(struct nf_conntrack *nfct)
2051 { 2058 {
2052 if (nfct && atomic_dec_and_test(&nfct->use)) 2059 if (nfct && atomic_dec_and_test(&nfct->use))
2053 nf_conntrack_destroy(nfct); 2060 nf_conntrack_destroy(nfct);
2054 } 2061 }
2055 static inline void nf_conntrack_get(struct nf_conntrack *nfct) 2062 static inline void nf_conntrack_get(struct nf_conntrack *nfct)
2056 { 2063 {
2057 if (nfct) 2064 if (nfct)
2058 atomic_inc(&nfct->use); 2065 atomic_inc(&nfct->use);
2059 } 2066 }
2067 #endif
2068 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2060 static inline void nf_conntrack_get_reasm(struct sk_buff *skb) 2069 static inline void nf_conntrack_get_reasm(struct sk_buff *skb)
2061 { 2070 {
2062 if (skb) 2071 if (skb)
2063 atomic_inc(&skb->users); 2072 atomic_inc(&skb->users);
2064 } 2073 }
2065 static inline void nf_conntrack_put_reasm(struct sk_buff *skb) 2074 static inline void nf_conntrack_put_reasm(struct sk_buff *skb)
2066 { 2075 {
2067 if (skb) 2076 if (skb)
2068 kfree_skb(skb); 2077 kfree_skb(skb);
2069 } 2078 }
2070 #endif 2079 #endif
2071 #ifdef CONFIG_BRIDGE_NETFILTER 2080 #ifdef CONFIG_BRIDGE_NETFILTER
2072 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge) 2081 static inline void nf_bridge_put(struct nf_bridge_info *nf_bridge)
2073 { 2082 {
2074 if (nf_bridge && atomic_dec_and_test(&nf_bridge->use)) 2083 if (nf_bridge && atomic_dec_and_test(&nf_bridge->use))
2075 kfree(nf_bridge); 2084 kfree(nf_bridge);
2076 } 2085 }
2077 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge) 2086 static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
2078 { 2087 {
2079 if (nf_bridge) 2088 if (nf_bridge)
2080 atomic_inc(&nf_bridge->use); 2089 atomic_inc(&nf_bridge->use);
2081 } 2090 }
2082 #endif /* CONFIG_BRIDGE_NETFILTER */ 2091 #endif /* CONFIG_BRIDGE_NETFILTER */
2083 static inline void nf_reset(struct sk_buff *skb) 2092 static inline void nf_reset(struct sk_buff *skb)
2084 { 2093 {
2085 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2094 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2086 nf_conntrack_put(skb->nfct); 2095 nf_conntrack_put(skb->nfct);
2087 skb->nfct = NULL; 2096 skb->nfct = NULL;
2097 #endif
2098 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2088 nf_conntrack_put_reasm(skb->nfct_reasm); 2099 nf_conntrack_put_reasm(skb->nfct_reasm);
2089 skb->nfct_reasm = NULL; 2100 skb->nfct_reasm = NULL;
2090 #endif 2101 #endif
2091 #ifdef CONFIG_BRIDGE_NETFILTER 2102 #ifdef CONFIG_BRIDGE_NETFILTER
2092 nf_bridge_put(skb->nf_bridge); 2103 nf_bridge_put(skb->nf_bridge);
2093 skb->nf_bridge = NULL; 2104 skb->nf_bridge = NULL;
2094 #endif 2105 #endif
2095 } 2106 }
2096 2107
2097 /* Note: This doesn't put any conntrack and bridge info in dst. */ 2108 /* Note: This doesn't put any conntrack and bridge info in dst. */
2098 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src) 2109 static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2099 { 2110 {
2100 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2111 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2101 dst->nfct = src->nfct; 2112 dst->nfct = src->nfct;
2102 nf_conntrack_get(src->nfct); 2113 nf_conntrack_get(src->nfct);
2103 dst->nfctinfo = src->nfctinfo; 2114 dst->nfctinfo = src->nfctinfo;
2115 #endif
2116 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2104 dst->nfct_reasm = src->nfct_reasm; 2117 dst->nfct_reasm = src->nfct_reasm;
2105 nf_conntrack_get_reasm(src->nfct_reasm); 2118 nf_conntrack_get_reasm(src->nfct_reasm);
2106 #endif 2119 #endif
2107 #ifdef CONFIG_BRIDGE_NETFILTER 2120 #ifdef CONFIG_BRIDGE_NETFILTER
2108 dst->nf_bridge = src->nf_bridge; 2121 dst->nf_bridge = src->nf_bridge;
2109 nf_bridge_get(src->nf_bridge); 2122 nf_bridge_get(src->nf_bridge);
2110 #endif 2123 #endif
2111 } 2124 }
2112 2125
2113 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src) 2126 static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
2114 { 2127 {
2115 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2128 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
2116 nf_conntrack_put(dst->nfct); 2129 nf_conntrack_put(dst->nfct);
2130 #endif
2131 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
2117 nf_conntrack_put_reasm(dst->nfct_reasm); 2132 nf_conntrack_put_reasm(dst->nfct_reasm);
2118 #endif 2133 #endif
2119 #ifdef CONFIG_BRIDGE_NETFILTER 2134 #ifdef CONFIG_BRIDGE_NETFILTER
2120 nf_bridge_put(dst->nf_bridge); 2135 nf_bridge_put(dst->nf_bridge);
2121 #endif 2136 #endif
2122 __nf_copy(dst, src); 2137 __nf_copy(dst, src);
2123 } 2138 }
2124 2139
2125 #ifdef CONFIG_NETWORK_SECMARK 2140 #ifdef CONFIG_NETWORK_SECMARK
2126 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 2141 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2127 { 2142 {
2128 to->secmark = from->secmark; 2143 to->secmark = from->secmark;
2129 } 2144 }
2130 2145
2131 static inline void skb_init_secmark(struct sk_buff *skb) 2146 static inline void skb_init_secmark(struct sk_buff *skb)
2132 { 2147 {
2133 skb->secmark = 0; 2148 skb->secmark = 0;
2134 } 2149 }
2135 #else 2150 #else
2136 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from) 2151 static inline void skb_copy_secmark(struct sk_buff *to, const struct sk_buff *from)
2137 { } 2152 { }
2138 2153
2139 static inline void skb_init_secmark(struct sk_buff *skb) 2154 static inline void skb_init_secmark(struct sk_buff *skb)
2140 { } 2155 { }
2141 #endif 2156 #endif
2142 2157
2143 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping) 2158 static inline void skb_set_queue_mapping(struct sk_buff *skb, u16 queue_mapping)
2144 { 2159 {
2145 skb->queue_mapping = queue_mapping; 2160 skb->queue_mapping = queue_mapping;
2146 } 2161 }
2147 2162
2148 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb) 2163 static inline u16 skb_get_queue_mapping(const struct sk_buff *skb)
2149 { 2164 {
2150 return skb->queue_mapping; 2165 return skb->queue_mapping;
2151 } 2166 }
2152 2167
2153 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from) 2168 static inline void skb_copy_queue_mapping(struct sk_buff *to, const struct sk_buff *from)
2154 { 2169 {
2155 to->queue_mapping = from->queue_mapping; 2170 to->queue_mapping = from->queue_mapping;
2156 } 2171 }
2157 2172
2158 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue) 2173 static inline void skb_record_rx_queue(struct sk_buff *skb, u16 rx_queue)
2159 { 2174 {
2160 skb->queue_mapping = rx_queue + 1; 2175 skb->queue_mapping = rx_queue + 1;
2161 } 2176 }
2162 2177
2163 static inline u16 skb_get_rx_queue(const struct sk_buff *skb) 2178 static inline u16 skb_get_rx_queue(const struct sk_buff *skb)
2164 { 2179 {
2165 return skb->queue_mapping - 1; 2180 return skb->queue_mapping - 1;
2166 } 2181 }
2167 2182
2168 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb) 2183 static inline bool skb_rx_queue_recorded(const struct sk_buff *skb)
2169 { 2184 {
2170 return skb->queue_mapping != 0; 2185 return skb->queue_mapping != 0;
2171 } 2186 }
2172 2187
2173 extern u16 __skb_tx_hash(const struct net_device *dev, 2188 extern u16 __skb_tx_hash(const struct net_device *dev,
2174 const struct sk_buff *skb, 2189 const struct sk_buff *skb,
2175 unsigned int num_tx_queues); 2190 unsigned int num_tx_queues);
2176 2191
2177 #ifdef CONFIG_XFRM 2192 #ifdef CONFIG_XFRM
2178 static inline struct sec_path *skb_sec_path(struct sk_buff *skb) 2193 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2179 { 2194 {
2180 return skb->sp; 2195 return skb->sp;
2181 } 2196 }
2182 #else 2197 #else
2183 static inline struct sec_path *skb_sec_path(struct sk_buff *skb) 2198 static inline struct sec_path *skb_sec_path(struct sk_buff *skb)
2184 { 2199 {
2185 return NULL; 2200 return NULL;
2186 } 2201 }
2187 #endif 2202 #endif
2188 2203
2189 static inline int skb_is_gso(const struct sk_buff *skb) 2204 static inline int skb_is_gso(const struct sk_buff *skb)
2190 { 2205 {
2191 return skb_shinfo(skb)->gso_size; 2206 return skb_shinfo(skb)->gso_size;
2192 } 2207 }
2193 2208
2194 static inline int skb_is_gso_v6(const struct sk_buff *skb) 2209 static inline int skb_is_gso_v6(const struct sk_buff *skb)
2195 { 2210 {
2196 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6; 2211 return skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6;
2197 } 2212 }
2198 2213
2199 extern void __skb_warn_lro_forwarding(const struct sk_buff *skb); 2214 extern void __skb_warn_lro_forwarding(const struct sk_buff *skb);
2200 2215
2201 static inline bool skb_warn_if_lro(const struct sk_buff *skb) 2216 static inline bool skb_warn_if_lro(const struct sk_buff *skb)
2202 { 2217 {
2203 /* LRO sets gso_size but not gso_type, whereas if GSO is really 2218 /* LRO sets gso_size but not gso_type, whereas if GSO is really
2204 * wanted then gso_type will be set. */ 2219 * wanted then gso_type will be set. */
2205 struct skb_shared_info *shinfo = skb_shinfo(skb); 2220 struct skb_shared_info *shinfo = skb_shinfo(skb);
2206 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 && 2221 if (skb_is_nonlinear(skb) && shinfo->gso_size != 0 &&
2207 unlikely(shinfo->gso_type == 0)) { 2222 unlikely(shinfo->gso_type == 0)) {
2208 __skb_warn_lro_forwarding(skb); 2223 __skb_warn_lro_forwarding(skb);
2209 return true; 2224 return true;
2210 } 2225 }
2211 return false; 2226 return false;
2212 } 2227 }
2213 2228
2214 static inline void skb_forward_csum(struct sk_buff *skb) 2229 static inline void skb_forward_csum(struct sk_buff *skb)
2215 { 2230 {
2216 /* Unfortunately we don't support this one. Any brave souls? */ 2231 /* Unfortunately we don't support this one. Any brave souls? */
2217 if (skb->ip_summed == CHECKSUM_COMPLETE) 2232 if (skb->ip_summed == CHECKSUM_COMPLETE)
2218 skb->ip_summed = CHECKSUM_NONE; 2233 skb->ip_summed = CHECKSUM_NONE;
2219 } 2234 }
2220 2235
2221 /** 2236 /**
2222 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE 2237 * skb_checksum_none_assert - make sure skb ip_summed is CHECKSUM_NONE
2223 * @skb: skb to check 2238 * @skb: skb to check
2224 * 2239 *
2225 * fresh skbs have their ip_summed set to CHECKSUM_NONE. 2240 * fresh skbs have their ip_summed set to CHECKSUM_NONE.
2226 * Instead of forcing ip_summed to CHECKSUM_NONE, we can 2241 * Instead of forcing ip_summed to CHECKSUM_NONE, we can
2227 * use this helper, to document places where we make this assertion. 2242 * use this helper, to document places where we make this assertion.
2228 */ 2243 */
2229 static inline void skb_checksum_none_assert(struct sk_buff *skb) 2244 static inline void skb_checksum_none_assert(struct sk_buff *skb)
2230 { 2245 {
2231 #ifdef DEBUG 2246 #ifdef DEBUG
2232 BUG_ON(skb->ip_summed != CHECKSUM_NONE); 2247 BUG_ON(skb->ip_summed != CHECKSUM_NONE);
2233 #endif 2248 #endif
2234 } 2249 }
2235 2250
2236 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off); 2251 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off);
2237 #endif /* __KERNEL__ */ 2252 #endif /* __KERNEL__ */
2238 #endif /* _LINUX_SKBUFF_H */ 2253 #endif /* _LINUX_SKBUFF_H */
2239 2254
include/net/netfilter/ipv6/nf_conntrack_ipv6.h
Diff comments   View file @ 464143c
1 #ifndef _NF_CONNTRACK_IPV6_H 1 #ifndef _NF_CONNTRACK_IPV6_H
2 #define _NF_CONNTRACK_IPV6_H 2 #define _NF_CONNTRACK_IPV6_H
3 3
4 extern struct nf_conntrack_l3proto nf_conntrack_l3proto_ipv6; 4 extern struct nf_conntrack_l3proto nf_conntrack_l3proto_ipv6;
5 5
6 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_tcp6; 6 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_tcp6;
7 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_udp6; 7 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_udp6;
8 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_icmpv6; 8 extern struct nf_conntrack_l4proto nf_conntrack_l4proto_icmpv6;
9 9
10 extern int nf_ct_frag6_init(void);
11 extern void nf_ct_frag6_cleanup(void);
12 extern struct sk_buff *nf_ct_frag6_gather(struct sk_buff *skb, u32 user);
13 extern void nf_ct_frag6_output(unsigned int hooknum, struct sk_buff *skb,
14 struct net_device *in,
15 struct net_device *out,
16 int (*okfn)(struct sk_buff *));
17
18 struct inet_frags_ctl;
19
20 #include <linux/sysctl.h> 10 #include <linux/sysctl.h>
21 extern struct ctl_table nf_ct_ipv6_sysctl_table[]; 11 extern struct ctl_table nf_ct_ipv6_sysctl_table[];
22 12
23 #endif /* _NF_CONNTRACK_IPV6_H*/ 13 #endif /* _NF_CONNTRACK_IPV6_H*/
24 14
include/net/netfilter/ipv6/nf_defrag_ipv6.h
Diff comments   View file @ 464143c
1 #ifndef _NF_DEFRAG_IPV6_H 1 #ifndef _NF_DEFRAG_IPV6_H
2 #define _NF_DEFRAG_IPV6_H 2 #define _NF_DEFRAG_IPV6_H
3 3
4 extern void nf_defrag_ipv6_enable(void); 4 extern void nf_defrag_ipv6_enable(void);
5 5
6 extern int nf_ct_frag6_init(void);
7 extern void nf_ct_frag6_cleanup(void);
8 extern struct sk_buff *nf_ct_frag6_gather(struct sk_buff *skb, u32 user);
9 extern void nf_ct_frag6_output(unsigned int hooknum, struct sk_buff *skb,
10 struct net_device *in,
11 struct net_device *out,
12 int (*okfn)(struct sk_buff *));
13
14 struct inet_frags_ctl;
15
6 #endif /* _NF_DEFRAG_IPV6_H */ 16 #endif /* _NF_DEFRAG_IPV6_H */
7 17
1 /* 1 /*
2 * Routines having to do with the 'struct sk_buff' memory handlers. 2 * Routines having to do with the 'struct sk_buff' memory handlers.
3 * 3 *
4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>
5 * Florian La Roche <rzsfl@rz.uni-sb.de> 5 * Florian La Roche <rzsfl@rz.uni-sb.de>
6 * 6 *
7 * Fixes: 7 * Fixes:
8 * Alan Cox : Fixed the worst of the load 8 * Alan Cox : Fixed the worst of the load
9 * balancer bugs. 9 * balancer bugs.
10 * Dave Platt : Interrupt stacking fix. 10 * Dave Platt : Interrupt stacking fix.
11 * Richard Kooijman : Timestamp fixes. 11 * Richard Kooijman : Timestamp fixes.
12 * Alan Cox : Changed buffer format. 12 * Alan Cox : Changed buffer format.
13 * Alan Cox : destructor hook for AF_UNIX etc. 13 * Alan Cox : destructor hook for AF_UNIX etc.
14 * Linus Torvalds : Better skb_clone. 14 * Linus Torvalds : Better skb_clone.
15 * Alan Cox : Added skb_copy. 15 * Alan Cox : Added skb_copy.
16 * Alan Cox : Added all the changed routines Linus 16 * Alan Cox : Added all the changed routines Linus
17 * only put in the headers 17 * only put in the headers
18 * Ray VanTassle : Fixed --skb->lock in free 18 * Ray VanTassle : Fixed --skb->lock in free
19 * Alan Cox : skb_copy copy arp field 19 * Alan Cox : skb_copy copy arp field
20 * Andi Kleen : slabified it. 20 * Andi Kleen : slabified it.
21 * Robert Olsson : Removed skb_head_pool 21 * Robert Olsson : Removed skb_head_pool
22 * 22 *
23 * NOTE: 23 * NOTE:
24 * The __skb_ routines should be called with interrupts 24 * The __skb_ routines should be called with interrupts
25 * disabled, or you better be *real* sure that the operation is atomic 25 * disabled, or you better be *real* sure that the operation is atomic
26 * with respect to whatever list is being frobbed (e.g. via lock_sock() 26 * with respect to whatever list is being frobbed (e.g. via lock_sock()
27 * or via disabling bottom half handlers, etc). 27 * or via disabling bottom half handlers, etc).
28 * 28 *
29 * This program is free software; you can redistribute it and/or 29 * This program is free software; you can redistribute it and/or
30 * modify it under the terms of the GNU General Public License 30 * modify it under the terms of the GNU General Public License
31 * as published by the Free Software Foundation; either version 31 * as published by the Free Software Foundation; either version
32 * 2 of the License, or (at your option) any later version. 32 * 2 of the License, or (at your option) any later version.
33 */ 33 */
34 34
35 /* 35 /*
36 * The functions in this file will not compile correctly with gcc 2.4.x 36 * The functions in this file will not compile correctly with gcc 2.4.x
37 */ 37 */
38 38
39 #include <linux/module.h> 39 #include <linux/module.h>
40 #include <linux/types.h> 40 #include <linux/types.h>
41 #include <linux/kernel.h> 41 #include <linux/kernel.h>
42 #include <linux/kmemcheck.h> 42 #include <linux/kmemcheck.h>
43 #include <linux/mm.h> 43 #include <linux/mm.h>
44 #include <linux/interrupt.h> 44 #include <linux/interrupt.h>
45 #include <linux/in.h> 45 #include <linux/in.h>
46 #include <linux/inet.h> 46 #include <linux/inet.h>
47 #include <linux/slab.h> 47 #include <linux/slab.h>
48 #include <linux/netdevice.h> 48 #include <linux/netdevice.h>
49 #ifdef CONFIG_NET_CLS_ACT 49 #ifdef CONFIG_NET_CLS_ACT
50 #include <net/pkt_sched.h> 50 #include <net/pkt_sched.h>
51 #endif 51 #endif
52 #include <linux/string.h> 52 #include <linux/string.h>
53 #include <linux/skbuff.h> 53 #include <linux/skbuff.h>
54 #include <linux/splice.h> 54 #include <linux/splice.h>
55 #include <linux/cache.h> 55 #include <linux/cache.h>
56 #include <linux/rtnetlink.h> 56 #include <linux/rtnetlink.h>
57 #include <linux/init.h> 57 #include <linux/init.h>
58 #include <linux/scatterlist.h> 58 #include <linux/scatterlist.h>
59 #include <linux/errqueue.h> 59 #include <linux/errqueue.h>
60 60
61 #include <net/protocol.h> 61 #include <net/protocol.h>
62 #include <net/dst.h> 62 #include <net/dst.h>
63 #include <net/sock.h> 63 #include <net/sock.h>
64 #include <net/checksum.h> 64 #include <net/checksum.h>
65 #include <net/xfrm.h> 65 #include <net/xfrm.h>
66 66
67 #include <asm/uaccess.h> 67 #include <asm/uaccess.h>
68 #include <asm/system.h> 68 #include <asm/system.h>
69 #include <trace/events/skb.h> 69 #include <trace/events/skb.h>
70 70
71 #include "kmap_skb.h" 71 #include "kmap_skb.h"
72 72
73 static struct kmem_cache *skbuff_head_cache __read_mostly; 73 static struct kmem_cache *skbuff_head_cache __read_mostly;
74 static struct kmem_cache *skbuff_fclone_cache __read_mostly; 74 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
75 75
76 static void sock_pipe_buf_release(struct pipe_inode_info *pipe, 76 static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
77 struct pipe_buffer *buf) 77 struct pipe_buffer *buf)
78 { 78 {
79 put_page(buf->page); 79 put_page(buf->page);
80 } 80 }
81 81
82 static void sock_pipe_buf_get(struct pipe_inode_info *pipe, 82 static void sock_pipe_buf_get(struct pipe_inode_info *pipe,
83 struct pipe_buffer *buf) 83 struct pipe_buffer *buf)
84 { 84 {
85 get_page(buf->page); 85 get_page(buf->page);
86 } 86 }
87 87
88 static int sock_pipe_buf_steal(struct pipe_inode_info *pipe, 88 static int sock_pipe_buf_steal(struct pipe_inode_info *pipe,
89 struct pipe_buffer *buf) 89 struct pipe_buffer *buf)
90 { 90 {
91 return 1; 91 return 1;
92 } 92 }
93 93
94 94
95 /* Pipe buffer operations for a socket. */ 95 /* Pipe buffer operations for a socket. */
96 static const struct pipe_buf_operations sock_pipe_buf_ops = { 96 static const struct pipe_buf_operations sock_pipe_buf_ops = {
97 .can_merge = 0, 97 .can_merge = 0,
98 .map = generic_pipe_buf_map, 98 .map = generic_pipe_buf_map,
99 .unmap = generic_pipe_buf_unmap, 99 .unmap = generic_pipe_buf_unmap,
100 .confirm = generic_pipe_buf_confirm, 100 .confirm = generic_pipe_buf_confirm,
101 .release = sock_pipe_buf_release, 101 .release = sock_pipe_buf_release,
102 .steal = sock_pipe_buf_steal, 102 .steal = sock_pipe_buf_steal,
103 .get = sock_pipe_buf_get, 103 .get = sock_pipe_buf_get,
104 }; 104 };
105 105
106 /* 106 /*
107 * Keep out-of-line to prevent kernel bloat. 107 * Keep out-of-line to prevent kernel bloat.
108 * __builtin_return_address is not used because it is not always 108 * __builtin_return_address is not used because it is not always
109 * reliable. 109 * reliable.
110 */ 110 */
111 111
112 /** 112 /**
113 * skb_over_panic - private function 113 * skb_over_panic - private function
114 * @skb: buffer 114 * @skb: buffer
115 * @sz: size 115 * @sz: size
116 * @here: address 116 * @here: address
117 * 117 *
118 * Out of line support code for skb_put(). Not user callable. 118 * Out of line support code for skb_put(). Not user callable.
119 */ 119 */
120 static void skb_over_panic(struct sk_buff *skb, int sz, void *here) 120 static void skb_over_panic(struct sk_buff *skb, int sz, void *here)
121 { 121 {
122 printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p " 122 printk(KERN_EMERG "skb_over_panic: text:%p len:%d put:%d head:%p "
123 "data:%p tail:%#lx end:%#lx dev:%s\n", 123 "data:%p tail:%#lx end:%#lx dev:%s\n",
124 here, skb->len, sz, skb->head, skb->data, 124 here, skb->len, sz, skb->head, skb->data,
125 (unsigned long)skb->tail, (unsigned long)skb->end, 125 (unsigned long)skb->tail, (unsigned long)skb->end,
126 skb->dev ? skb->dev->name : "<NULL>"); 126 skb->dev ? skb->dev->name : "<NULL>");
127 BUG(); 127 BUG();
128 } 128 }
129 129
130 /** 130 /**
131 * skb_under_panic - private function 131 * skb_under_panic - private function
132 * @skb: buffer 132 * @skb: buffer
133 * @sz: size 133 * @sz: size
134 * @here: address 134 * @here: address
135 * 135 *
136 * Out of line support code for skb_push(). Not user callable. 136 * Out of line support code for skb_push(). Not user callable.
137 */ 137 */
138 138
139 static void skb_under_panic(struct sk_buff *skb, int sz, void *here) 139 static void skb_under_panic(struct sk_buff *skb, int sz, void *here)
140 { 140 {
141 printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p " 141 printk(KERN_EMERG "skb_under_panic: text:%p len:%d put:%d head:%p "
142 "data:%p tail:%#lx end:%#lx dev:%s\n", 142 "data:%p tail:%#lx end:%#lx dev:%s\n",
143 here, skb->len, sz, skb->head, skb->data, 143 here, skb->len, sz, skb->head, skb->data,
144 (unsigned long)skb->tail, (unsigned long)skb->end, 144 (unsigned long)skb->tail, (unsigned long)skb->end,
145 skb->dev ? skb->dev->name : "<NULL>"); 145 skb->dev ? skb->dev->name : "<NULL>");
146 BUG(); 146 BUG();
147 } 147 }
148 148
149 /* Allocate a new skbuff. We do this ourselves so we can fill in a few 149 /* Allocate a new skbuff. We do this ourselves so we can fill in a few
150 * 'private' fields and also do memory statistics to find all the 150 * 'private' fields and also do memory statistics to find all the
151 * [BEEP] leaks. 151 * [BEEP] leaks.
152 * 152 *
153 */ 153 */
154 154
155 /** 155 /**
156 * __alloc_skb - allocate a network buffer 156 * __alloc_skb - allocate a network buffer
157 * @size: size to allocate 157 * @size: size to allocate
158 * @gfp_mask: allocation mask 158 * @gfp_mask: allocation mask
159 * @fclone: allocate from fclone cache instead of head cache 159 * @fclone: allocate from fclone cache instead of head cache
160 * and allocate a cloned (child) skb 160 * and allocate a cloned (child) skb
161 * @node: numa node to allocate memory on 161 * @node: numa node to allocate memory on
162 * 162 *
163 * Allocate a new &sk_buff. The returned buffer has no headroom and a 163 * Allocate a new &sk_buff. The returned buffer has no headroom and a
164 * tail room of size bytes. The object has a reference count of one. 164 * tail room of size bytes. The object has a reference count of one.
165 * The return is the buffer. On a failure the return is %NULL. 165 * The return is the buffer. On a failure the return is %NULL.
166 * 166 *
167 * Buffers may only be allocated from interrupts using a @gfp_mask of 167 * Buffers may only be allocated from interrupts using a @gfp_mask of
168 * %GFP_ATOMIC. 168 * %GFP_ATOMIC.
169 */ 169 */
170 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 170 struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
171 int fclone, int node) 171 int fclone, int node)
172 { 172 {
173 struct kmem_cache *cache; 173 struct kmem_cache *cache;
174 struct skb_shared_info *shinfo; 174 struct skb_shared_info *shinfo;
175 struct sk_buff *skb; 175 struct sk_buff *skb;
176 u8 *data; 176 u8 *data;
177 177
178 cache = fclone ? skbuff_fclone_cache : skbuff_head_cache; 178 cache = fclone ? skbuff_fclone_cache : skbuff_head_cache;
179 179
180 /* Get the HEAD */ 180 /* Get the HEAD */
181 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 181 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node);
182 if (!skb) 182 if (!skb)
183 goto out; 183 goto out;
184 prefetchw(skb); 184 prefetchw(skb);
185 185
186 size = SKB_DATA_ALIGN(size); 186 size = SKB_DATA_ALIGN(size);
187 data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info), 187 data = kmalloc_node_track_caller(size + sizeof(struct skb_shared_info),
188 gfp_mask, node); 188 gfp_mask, node);
189 if (!data) 189 if (!data)
190 goto nodata; 190 goto nodata;
191 prefetchw(data + size); 191 prefetchw(data + size);
192 192
193 /* 193 /*
194 * Only clear those fields we need to clear, not those that we will 194 * Only clear those fields we need to clear, not those that we will
195 * actually initialise below. Hence, don't put any more fields after 195 * actually initialise below. Hence, don't put any more fields after
196 * the tail pointer in struct sk_buff! 196 * the tail pointer in struct sk_buff!
197 */ 197 */
198 memset(skb, 0, offsetof(struct sk_buff, tail)); 198 memset(skb, 0, offsetof(struct sk_buff, tail));
199 skb->truesize = size + sizeof(struct sk_buff); 199 skb->truesize = size + sizeof(struct sk_buff);
200 atomic_set(&skb->users, 1); 200 atomic_set(&skb->users, 1);
201 skb->head = data; 201 skb->head = data;
202 skb->data = data; 202 skb->data = data;
203 skb_reset_tail_pointer(skb); 203 skb_reset_tail_pointer(skb);
204 skb->end = skb->tail + size; 204 skb->end = skb->tail + size;
205 #ifdef NET_SKBUFF_DATA_USES_OFFSET 205 #ifdef NET_SKBUFF_DATA_USES_OFFSET
206 skb->mac_header = ~0U; 206 skb->mac_header = ~0U;
207 #endif 207 #endif
208 208
209 /* make sure we initialize shinfo sequentially */ 209 /* make sure we initialize shinfo sequentially */
210 shinfo = skb_shinfo(skb); 210 shinfo = skb_shinfo(skb);
211 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 211 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
212 atomic_set(&shinfo->dataref, 1); 212 atomic_set(&shinfo->dataref, 1);
213 213
214 if (fclone) { 214 if (fclone) {
215 struct sk_buff *child = skb + 1; 215 struct sk_buff *child = skb + 1;
216 atomic_t *fclone_ref = (atomic_t *) (child + 1); 216 atomic_t *fclone_ref = (atomic_t *) (child + 1);
217 217
218 kmemcheck_annotate_bitfield(child, flags1); 218 kmemcheck_annotate_bitfield(child, flags1);
219 kmemcheck_annotate_bitfield(child, flags2); 219 kmemcheck_annotate_bitfield(child, flags2);
220 skb->fclone = SKB_FCLONE_ORIG; 220 skb->fclone = SKB_FCLONE_ORIG;
221 atomic_set(fclone_ref, 1); 221 atomic_set(fclone_ref, 1);
222 222
223 child->fclone = SKB_FCLONE_UNAVAILABLE; 223 child->fclone = SKB_FCLONE_UNAVAILABLE;
224 } 224 }
225 out: 225 out:
226 return skb; 226 return skb;
227 nodata: 227 nodata:
228 kmem_cache_free(cache, skb); 228 kmem_cache_free(cache, skb);
229 skb = NULL; 229 skb = NULL;
230 goto out; 230 goto out;
231 } 231 }
232 EXPORT_SYMBOL(__alloc_skb); 232 EXPORT_SYMBOL(__alloc_skb);
233 233
234 /** 234 /**
235 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 235 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device
236 * @dev: network device to receive on 236 * @dev: network device to receive on
237 * @length: length to allocate 237 * @length: length to allocate
238 * @gfp_mask: get_free_pages mask, passed to alloc_skb 238 * @gfp_mask: get_free_pages mask, passed to alloc_skb
239 * 239 *
240 * Allocate a new &sk_buff and assign it a usage count of one. The 240 * Allocate a new &sk_buff and assign it a usage count of one. The
241 * buffer has unspecified headroom built in. Users should allocate 241 * buffer has unspecified headroom built in. Users should allocate
242 * the headroom they think they need without accounting for the 242 * the headroom they think they need without accounting for the
243 * built in space. The built in space is used for optimisations. 243 * built in space. The built in space is used for optimisations.
244 * 244 *
245 * %NULL is returned if there is no free memory. 245 * %NULL is returned if there is no free memory.
246 */ 246 */
247 struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 247 struct sk_buff *__netdev_alloc_skb(struct net_device *dev,
248 unsigned int length, gfp_t gfp_mask) 248 unsigned int length, gfp_t gfp_mask)
249 { 249 {
250 struct sk_buff *skb; 250 struct sk_buff *skb;
251 251
252 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, NUMA_NO_NODE); 252 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 0, NUMA_NO_NODE);
253 if (likely(skb)) { 253 if (likely(skb)) {
254 skb_reserve(skb, NET_SKB_PAD); 254 skb_reserve(skb, NET_SKB_PAD);
255 skb->dev = dev; 255 skb->dev = dev;
256 } 256 }
257 return skb; 257 return skb;
258 } 258 }
259 EXPORT_SYMBOL(__netdev_alloc_skb); 259 EXPORT_SYMBOL(__netdev_alloc_skb);
260 260
261 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 261 void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off,
262 int size) 262 int size)
263 { 263 {
264 skb_fill_page_desc(skb, i, page, off, size); 264 skb_fill_page_desc(skb, i, page, off, size);
265 skb->len += size; 265 skb->len += size;
266 skb->data_len += size; 266 skb->data_len += size;
267 skb->truesize += size; 267 skb->truesize += size;
268 } 268 }
269 EXPORT_SYMBOL(skb_add_rx_frag); 269 EXPORT_SYMBOL(skb_add_rx_frag);
270 270
271 /** 271 /**
272 * dev_alloc_skb - allocate an skbuff for receiving 272 * dev_alloc_skb - allocate an skbuff for receiving
273 * @length: length to allocate 273 * @length: length to allocate
274 * 274 *
275 * Allocate a new &sk_buff and assign it a usage count of one. The 275 * Allocate a new &sk_buff and assign it a usage count of one. The
276 * buffer has unspecified headroom built in. Users should allocate 276 * buffer has unspecified headroom built in. Users should allocate
277 * the headroom they think they need without accounting for the 277 * the headroom they think they need without accounting for the
278 * built in space. The built in space is used for optimisations. 278 * built in space. The built in space is used for optimisations.
279 * 279 *
280 * %NULL is returned if there is no free memory. Although this function 280 * %NULL is returned if there is no free memory. Although this function
281 * allocates memory it can be called from an interrupt. 281 * allocates memory it can be called from an interrupt.
282 */ 282 */
283 struct sk_buff *dev_alloc_skb(unsigned int length) 283 struct sk_buff *dev_alloc_skb(unsigned int length)
284 { 284 {
285 /* 285 /*
286 * There is more code here than it seems: 286 * There is more code here than it seems:
287 * __dev_alloc_skb is an inline 287 * __dev_alloc_skb is an inline
288 */ 288 */
289 return __dev_alloc_skb(length, GFP_ATOMIC); 289 return __dev_alloc_skb(length, GFP_ATOMIC);
290 } 290 }
291 EXPORT_SYMBOL(dev_alloc_skb); 291 EXPORT_SYMBOL(dev_alloc_skb);
292 292
293 static void skb_drop_list(struct sk_buff **listp) 293 static void skb_drop_list(struct sk_buff **listp)
294 { 294 {
295 struct sk_buff *list = *listp; 295 struct sk_buff *list = *listp;
296 296
297 *listp = NULL; 297 *listp = NULL;
298 298
299 do { 299 do {
300 struct sk_buff *this = list; 300 struct sk_buff *this = list;
301 list = list->next; 301 list = list->next;
302 kfree_skb(this); 302 kfree_skb(this);
303 } while (list); 303 } while (list);
304 } 304 }
305 305
306 static inline void skb_drop_fraglist(struct sk_buff *skb) 306 static inline void skb_drop_fraglist(struct sk_buff *skb)
307 { 307 {
308 skb_drop_list(&skb_shinfo(skb)->frag_list); 308 skb_drop_list(&skb_shinfo(skb)->frag_list);
309 } 309 }
310 310
311 static void skb_clone_fraglist(struct sk_buff *skb) 311 static void skb_clone_fraglist(struct sk_buff *skb)
312 { 312 {
313 struct sk_buff *list; 313 struct sk_buff *list;
314 314
315 skb_walk_frags(skb, list) 315 skb_walk_frags(skb, list)
316 skb_get(list); 316 skb_get(list);
317 } 317 }
318 318
319 static void skb_release_data(struct sk_buff *skb) 319 static void skb_release_data(struct sk_buff *skb)
320 { 320 {
321 if (!skb->cloned || 321 if (!skb->cloned ||
322 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 322 !atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1,
323 &skb_shinfo(skb)->dataref)) { 323 &skb_shinfo(skb)->dataref)) {
324 if (skb_shinfo(skb)->nr_frags) { 324 if (skb_shinfo(skb)->nr_frags) {
325 int i; 325 int i;
326 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 326 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
327 put_page(skb_shinfo(skb)->frags[i].page); 327 put_page(skb_shinfo(skb)->frags[i].page);
328 } 328 }
329 329
330 if (skb_has_frag_list(skb)) 330 if (skb_has_frag_list(skb))
331 skb_drop_fraglist(skb); 331 skb_drop_fraglist(skb);
332 332
333 kfree(skb->head); 333 kfree(skb->head);
334 } 334 }
335 } 335 }
336 336
337 /* 337 /*
338 * Free an skbuff by memory without cleaning the state. 338 * Free an skbuff by memory without cleaning the state.
339 */ 339 */
340 static void kfree_skbmem(struct sk_buff *skb) 340 static void kfree_skbmem(struct sk_buff *skb)
341 { 341 {
342 struct sk_buff *other; 342 struct sk_buff *other;
343 atomic_t *fclone_ref; 343 atomic_t *fclone_ref;
344 344
345 switch (skb->fclone) { 345 switch (skb->fclone) {
346 case SKB_FCLONE_UNAVAILABLE: 346 case SKB_FCLONE_UNAVAILABLE:
347 kmem_cache_free(skbuff_head_cache, skb); 347 kmem_cache_free(skbuff_head_cache, skb);
348 break; 348 break;
349 349
350 case SKB_FCLONE_ORIG: 350 case SKB_FCLONE_ORIG:
351 fclone_ref = (atomic_t *) (skb + 2); 351 fclone_ref = (atomic_t *) (skb + 2);
352 if (atomic_dec_and_test(fclone_ref)) 352 if (atomic_dec_and_test(fclone_ref))
353 kmem_cache_free(skbuff_fclone_cache, skb); 353 kmem_cache_free(skbuff_fclone_cache, skb);
354 break; 354 break;
355 355
356 case SKB_FCLONE_CLONE: 356 case SKB_FCLONE_CLONE:
357 fclone_ref = (atomic_t *) (skb + 1); 357 fclone_ref = (atomic_t *) (skb + 1);
358 other = skb - 1; 358 other = skb - 1;
359 359
360 /* The clone portion is available for 360 /* The clone portion is available for
361 * fast-cloning again. 361 * fast-cloning again.
362 */ 362 */
363 skb->fclone = SKB_FCLONE_UNAVAILABLE; 363 skb->fclone = SKB_FCLONE_UNAVAILABLE;
364 364
365 if (atomic_dec_and_test(fclone_ref)) 365 if (atomic_dec_and_test(fclone_ref))
366 kmem_cache_free(skbuff_fclone_cache, other); 366 kmem_cache_free(skbuff_fclone_cache, other);
367 break; 367 break;
368 } 368 }
369 } 369 }
370 370
371 static void skb_release_head_state(struct sk_buff *skb) 371 static void skb_release_head_state(struct sk_buff *skb)
372 { 372 {
373 skb_dst_drop(skb); 373 skb_dst_drop(skb);
374 #ifdef CONFIG_XFRM 374 #ifdef CONFIG_XFRM
375 secpath_put(skb->sp); 375 secpath_put(skb->sp);
376 #endif 376 #endif
377 if (skb->destructor) { 377 if (skb->destructor) {
378 WARN_ON(in_irq()); 378 WARN_ON(in_irq());
379 skb->destructor(skb); 379 skb->destructor(skb);
380 } 380 }
381 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 381 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
382 nf_conntrack_put(skb->nfct); 382 nf_conntrack_put(skb->nfct);
383 #endif
384 #ifdef NET_SKBUFF_NF_DEFRAG_NEEDED
383 nf_conntrack_put_reasm(skb->nfct_reasm); 385 nf_conntrack_put_reasm(skb->nfct_reasm);
384 #endif 386 #endif
385 #ifdef CONFIG_BRIDGE_NETFILTER 387 #ifdef CONFIG_BRIDGE_NETFILTER
386 nf_bridge_put(skb->nf_bridge); 388 nf_bridge_put(skb->nf_bridge);
387 #endif 389 #endif
388 /* XXX: IS this still necessary? - JHS */ 390 /* XXX: IS this still necessary? - JHS */
389 #ifdef CONFIG_NET_SCHED 391 #ifdef CONFIG_NET_SCHED
390 skb->tc_index = 0; 392 skb->tc_index = 0;
391 #ifdef CONFIG_NET_CLS_ACT 393 #ifdef CONFIG_NET_CLS_ACT
392 skb->tc_verd = 0; 394 skb->tc_verd = 0;
393 #endif 395 #endif
394 #endif 396 #endif
395 } 397 }
396 398
397 /* Free everything but the sk_buff shell. */ 399 /* Free everything but the sk_buff shell. */
398 static void skb_release_all(struct sk_buff *skb) 400 static void skb_release_all(struct sk_buff *skb)
399 { 401 {
400 skb_release_head_state(skb); 402 skb_release_head_state(skb);
401 skb_release_data(skb); 403 skb_release_data(skb);
402 } 404 }
403 405
404 /** 406 /**
405 * __kfree_skb - private function 407 * __kfree_skb - private function
406 * @skb: buffer 408 * @skb: buffer
407 * 409 *
408 * Free an sk_buff. Release anything attached to the buffer. 410 * Free an sk_buff. Release anything attached to the buffer.
409 * Clean the state. This is an internal helper function. Users should 411 * Clean the state. This is an internal helper function. Users should
410 * always call kfree_skb 412 * always call kfree_skb
411 */ 413 */
412 414
413 void __kfree_skb(struct sk_buff *skb) 415 void __kfree_skb(struct sk_buff *skb)
414 { 416 {
415 skb_release_all(skb); 417 skb_release_all(skb);
416 kfree_skbmem(skb); 418 kfree_skbmem(skb);
417 } 419 }
418 EXPORT_SYMBOL(__kfree_skb); 420 EXPORT_SYMBOL(__kfree_skb);
419 421
420 /** 422 /**
421 * kfree_skb - free an sk_buff 423 * kfree_skb - free an sk_buff
422 * @skb: buffer to free 424 * @skb: buffer to free
423 * 425 *
424 * Drop a reference to the buffer and free it if the usage count has 426 * Drop a reference to the buffer and free it if the usage count has
425 * hit zero. 427 * hit zero.
426 */ 428 */
427 void kfree_skb(struct sk_buff *skb) 429 void kfree_skb(struct sk_buff *skb)
428 { 430 {
429 if (unlikely(!skb)) 431 if (unlikely(!skb))
430 return; 432 return;
431 if (likely(atomic_read(&skb->users) == 1)) 433 if (likely(atomic_read(&skb->users) == 1))
432 smp_rmb(); 434 smp_rmb();
433 else if (likely(!atomic_dec_and_test(&skb->users))) 435 else if (likely(!atomic_dec_and_test(&skb->users)))
434 return; 436 return;
435 trace_kfree_skb(skb, __builtin_return_address(0)); 437 trace_kfree_skb(skb, __builtin_return_address(0));
436 __kfree_skb(skb); 438 __kfree_skb(skb);
437 } 439 }
438 EXPORT_SYMBOL(kfree_skb); 440 EXPORT_SYMBOL(kfree_skb);
439 441
440 /** 442 /**
441 * consume_skb - free an skbuff 443 * consume_skb - free an skbuff
442 * @skb: buffer to free 444 * @skb: buffer to free
443 * 445 *
444 * Drop a ref to the buffer and free it if the usage count has hit zero 446 * Drop a ref to the buffer and free it if the usage count has hit zero
445 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 447 * Functions identically to kfree_skb, but kfree_skb assumes that the frame
446 * is being dropped after a failure and notes that 448 * is being dropped after a failure and notes that
447 */ 449 */
448 void consume_skb(struct sk_buff *skb) 450 void consume_skb(struct sk_buff *skb)
449 { 451 {
450 if (unlikely(!skb)) 452 if (unlikely(!skb))
451 return; 453 return;
452 if (likely(atomic_read(&skb->users) == 1)) 454 if (likely(atomic_read(&skb->users) == 1))
453 smp_rmb(); 455 smp_rmb();
454 else if (likely(!atomic_dec_and_test(&skb->users))) 456 else if (likely(!atomic_dec_and_test(&skb->users)))
455 return; 457 return;
456 trace_consume_skb(skb); 458 trace_consume_skb(skb);
457 __kfree_skb(skb); 459 __kfree_skb(skb);
458 } 460 }
459 EXPORT_SYMBOL(consume_skb); 461 EXPORT_SYMBOL(consume_skb);
460 462
461 /** 463 /**
462 * skb_recycle_check - check if skb can be reused for receive 464 * skb_recycle_check - check if skb can be reused for receive
463 * @skb: buffer 465 * @skb: buffer
464 * @skb_size: minimum receive buffer size 466 * @skb_size: minimum receive buffer size
465 * 467 *
466 * Checks that the skb passed in is not shared or cloned, and 468 * Checks that the skb passed in is not shared or cloned, and
467 * that it is linear and its head portion at least as large as 469 * that it is linear and its head portion at least as large as
468 * skb_size so that it can be recycled as a receive buffer. 470 * skb_size so that it can be recycled as a receive buffer.
469 * If these conditions are met, this function does any necessary 471 * If these conditions are met, this function does any necessary
470 * reference count dropping and cleans up the skbuff as if it 472 * reference count dropping and cleans up the skbuff as if it
471 * just came from __alloc_skb(). 473 * just came from __alloc_skb().
472 */ 474 */
473 bool skb_recycle_check(struct sk_buff *skb, int skb_size) 475 bool skb_recycle_check(struct sk_buff *skb, int skb_size)
474 { 476 {
475 struct skb_shared_info *shinfo; 477 struct skb_shared_info *shinfo;
476 478
477 if (irqs_disabled()) 479 if (irqs_disabled())
478 return false; 480 return false;
479 481
480 if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE) 482 if (skb_is_nonlinear(skb) || skb->fclone != SKB_FCLONE_UNAVAILABLE)
481 return false; 483 return false;
482 484
483 skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD); 485 skb_size = SKB_DATA_ALIGN(skb_size + NET_SKB_PAD);
484 if (skb_end_pointer(skb) - skb->head < skb_size) 486 if (skb_end_pointer(skb) - skb->head < skb_size)
485 return false; 487 return false;
486 488
487 if (skb_shared(skb) || skb_cloned(skb)) 489 if (skb_shared(skb) || skb_cloned(skb))
488 return false; 490 return false;
489 491
490 skb_release_head_state(skb); 492 skb_release_head_state(skb);
491 493
492 shinfo = skb_shinfo(skb); 494 shinfo = skb_shinfo(skb);
493 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 495 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
494 atomic_set(&shinfo->dataref, 1); 496 atomic_set(&shinfo->dataref, 1);
495 497
496 memset(skb, 0, offsetof(struct sk_buff, tail)); 498 memset(skb, 0, offsetof(struct sk_buff, tail));
497 skb->data = skb->head + NET_SKB_PAD; 499 skb->data = skb->head + NET_SKB_PAD;
498 skb_reset_tail_pointer(skb); 500 skb_reset_tail_pointer(skb);
499 501
500 return true; 502 return true;
501 } 503 }
502 EXPORT_SYMBOL(skb_recycle_check); 504 EXPORT_SYMBOL(skb_recycle_check);
503 505
504 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 506 static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
505 { 507 {
506 new->tstamp = old->tstamp; 508 new->tstamp = old->tstamp;
507 new->dev = old->dev; 509 new->dev = old->dev;
508 new->transport_header = old->transport_header; 510 new->transport_header = old->transport_header;
509 new->network_header = old->network_header; 511 new->network_header = old->network_header;
510 new->mac_header = old->mac_header; 512 new->mac_header = old->mac_header;
511 skb_dst_copy(new, old); 513 skb_dst_copy(new, old);
512 new->rxhash = old->rxhash; 514 new->rxhash = old->rxhash;
513 #ifdef CONFIG_XFRM 515 #ifdef CONFIG_XFRM
514 new->sp = secpath_get(old->sp); 516 new->sp = secpath_get(old->sp);
515 #endif 517 #endif
516 memcpy(new->cb, old->cb, sizeof(old->cb)); 518 memcpy(new->cb, old->cb, sizeof(old->cb));
517 new->csum = old->csum; 519 new->csum = old->csum;
518 new->local_df = old->local_df; 520 new->local_df = old->local_df;
519 new->pkt_type = old->pkt_type; 521 new->pkt_type = old->pkt_type;
520 new->ip_summed = old->ip_summed; 522 new->ip_summed = old->ip_summed;
521 skb_copy_queue_mapping(new, old); 523 skb_copy_queue_mapping(new, old);
522 new->priority = old->priority; 524 new->priority = old->priority;
523 new->deliver_no_wcard = old->deliver_no_wcard; 525 new->deliver_no_wcard = old->deliver_no_wcard;
524 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE) 526 #if defined(CONFIG_IP_VS) || defined(CONFIG_IP_VS_MODULE)
525 new->ipvs_property = old->ipvs_property; 527 new->ipvs_property = old->ipvs_property;
526 #endif 528 #endif
527 new->protocol = old->protocol; 529 new->protocol = old->protocol;
528 new->mark = old->mark; 530 new->mark = old->mark;
529 new->skb_iif = old->skb_iif; 531 new->skb_iif = old->skb_iif;
530 __nf_copy(new, old); 532 __nf_copy(new, old);
531 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \ 533 #if defined(CONFIG_NETFILTER_XT_TARGET_TRACE) || \
532 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE) 534 defined(CONFIG_NETFILTER_XT_TARGET_TRACE_MODULE)
533 new->nf_trace = old->nf_trace; 535 new->nf_trace = old->nf_trace;
534 #endif 536 #endif
535 #ifdef CONFIG_NET_SCHED 537 #ifdef CONFIG_NET_SCHED
536 new->tc_index = old->tc_index; 538 new->tc_index = old->tc_index;
537 #ifdef CONFIG_NET_CLS_ACT 539 #ifdef CONFIG_NET_CLS_ACT
538 new->tc_verd = old->tc_verd; 540 new->tc_verd = old->tc_verd;
539 #endif 541 #endif
540 #endif 542 #endif
541 new->vlan_tci = old->vlan_tci; 543 new->vlan_tci = old->vlan_tci;
542 544
543 skb_copy_secmark(new, old); 545 skb_copy_secmark(new, old);
544 } 546 }
545 547
546 /* 548 /*
547 * You should not add any new code to this function. Add it to 549 * You should not add any new code to this function. Add it to
548 * __copy_skb_header above instead. 550 * __copy_skb_header above instead.
549 */ 551 */
550 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 552 static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
551 { 553 {
552 #define C(x) n->x = skb->x 554 #define C(x) n->x = skb->x
553 555
554 n->next = n->prev = NULL; 556 n->next = n->prev = NULL;
555 n->sk = NULL; 557 n->sk = NULL;
556 __copy_skb_header(n, skb); 558 __copy_skb_header(n, skb);
557 559
558 C(len); 560 C(len);
559 C(data_len); 561 C(data_len);
560 C(mac_len); 562 C(mac_len);
561 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 563 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len;
562 n->cloned = 1; 564 n->cloned = 1;
563 n->nohdr = 0; 565 n->nohdr = 0;
564 n->destructor = NULL; 566 n->destructor = NULL;
565 C(tail); 567 C(tail);
566 C(end); 568 C(end);
567 C(head); 569 C(head);
568 C(data); 570 C(data);
569 C(truesize); 571 C(truesize);
570 atomic_set(&n->users, 1); 572 atomic_set(&n->users, 1);
571 573
572 atomic_inc(&(skb_shinfo(skb)->dataref)); 574 atomic_inc(&(skb_shinfo(skb)->dataref));
573 skb->cloned = 1; 575 skb->cloned = 1;
574 576
575 return n; 577 return n;
576 #undef C 578 #undef C
577 } 579 }
578 580
579 /** 581 /**
580 * skb_morph - morph one skb into another 582 * skb_morph - morph one skb into another
581 * @dst: the skb to receive the contents 583 * @dst: the skb to receive the contents
582 * @src: the skb to supply the contents 584 * @src: the skb to supply the contents
583 * 585 *
584 * This is identical to skb_clone except that the target skb is 586 * This is identical to skb_clone except that the target skb is
585 * supplied by the user. 587 * supplied by the user.
586 * 588 *
587 * The target skb is returned upon exit. 589 * The target skb is returned upon exit.
588 */ 590 */
589 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 591 struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src)
590 { 592 {
591 skb_release_all(dst); 593 skb_release_all(dst);
592 return __skb_clone(dst, src); 594 return __skb_clone(dst, src);
593 } 595 }
594 EXPORT_SYMBOL_GPL(skb_morph); 596 EXPORT_SYMBOL_GPL(skb_morph);
595 597
596 /** 598 /**
597 * skb_clone - duplicate an sk_buff 599 * skb_clone - duplicate an sk_buff
598 * @skb: buffer to clone 600 * @skb: buffer to clone
599 * @gfp_mask: allocation priority 601 * @gfp_mask: allocation priority
600 * 602 *
601 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 603 * Duplicate an &sk_buff. The new one is not owned by a socket. Both
602 * copies share the same packet data but not structure. The new 604 * copies share the same packet data but not structure. The new
603 * buffer has a reference count of 1. If the allocation fails the 605 * buffer has a reference count of 1. If the allocation fails the
604 * function returns %NULL otherwise the new buffer is returned. 606 * function returns %NULL otherwise the new buffer is returned.
605 * 607 *
606 * If this function is called from an interrupt gfp_mask() must be 608 * If this function is called from an interrupt gfp_mask() must be
607 * %GFP_ATOMIC. 609 * %GFP_ATOMIC.
608 */ 610 */
609 611
610 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 612 struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
611 { 613 {
612 struct sk_buff *n; 614 struct sk_buff *n;
613 615
614 n = skb + 1; 616 n = skb + 1;
615 if (skb->fclone == SKB_FCLONE_ORIG && 617 if (skb->fclone == SKB_FCLONE_ORIG &&
616 n->fclone == SKB_FCLONE_UNAVAILABLE) { 618 n->fclone == SKB_FCLONE_UNAVAILABLE) {
617 atomic_t *fclone_ref = (atomic_t *) (n + 1); 619 atomic_t *fclone_ref = (atomic_t *) (n + 1);
618 n->fclone = SKB_FCLONE_CLONE; 620 n->fclone = SKB_FCLONE_CLONE;
619 atomic_inc(fclone_ref); 621 atomic_inc(fclone_ref);
620 } else { 622 } else {
621 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 623 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask);
622 if (!n) 624 if (!n)
623 return NULL; 625 return NULL;
624 626
625 kmemcheck_annotate_bitfield(n, flags1); 627 kmemcheck_annotate_bitfield(n, flags1);
626 kmemcheck_annotate_bitfield(n, flags2); 628 kmemcheck_annotate_bitfield(n, flags2);
627 n->fclone = SKB_FCLONE_UNAVAILABLE; 629 n->fclone = SKB_FCLONE_UNAVAILABLE;
628 } 630 }
629 631
630 return __skb_clone(n, skb); 632 return __skb_clone(n, skb);
631 } 633 }
632 EXPORT_SYMBOL(skb_clone); 634 EXPORT_SYMBOL(skb_clone);
633 635
634 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 636 static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old)
635 { 637 {
636 #ifndef NET_SKBUFF_DATA_USES_OFFSET 638 #ifndef NET_SKBUFF_DATA_USES_OFFSET
637 /* 639 /*
638 * Shift between the two data areas in bytes 640 * Shift between the two data areas in bytes
639 */ 641 */
640 unsigned long offset = new->data - old->data; 642 unsigned long offset = new->data - old->data;
641 #endif 643 #endif
642 644
643 __copy_skb_header(new, old); 645 __copy_skb_header(new, old);
644 646
645 #ifndef NET_SKBUFF_DATA_USES_OFFSET 647 #ifndef NET_SKBUFF_DATA_USES_OFFSET
646 /* {transport,network,mac}_header are relative to skb->head */ 648 /* {transport,network,mac}_header are relative to skb->head */
647 new->transport_header += offset; 649 new->transport_header += offset;
648 new->network_header += offset; 650 new->network_header += offset;
649 if (skb_mac_header_was_set(new)) 651 if (skb_mac_header_was_set(new))
650 new->mac_header += offset; 652 new->mac_header += offset;
651 #endif 653 #endif
652 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 654 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size;
653 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 655 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs;
654 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 656 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type;
655 } 657 }
656 658
657 /** 659 /**
658 * skb_copy - create private copy of an sk_buff 660 * skb_copy - create private copy of an sk_buff
659 * @skb: buffer to copy 661 * @skb: buffer to copy
660 * @gfp_mask: allocation priority 662 * @gfp_mask: allocation priority
661 * 663 *
662 * Make a copy of both an &sk_buff and its data. This is used when the 664 * Make a copy of both an &sk_buff and its data. This is used when the
663 * caller wishes to modify the data and needs a private copy of the 665 * caller wishes to modify the data and needs a private copy of the
664 * data to alter. Returns %NULL on failure or the pointer to the buffer 666 * data to alter. Returns %NULL on failure or the pointer to the buffer
665 * on success. The returned buffer has a reference count of 1. 667 * on success. The returned buffer has a reference count of 1.
666 * 668 *
667 * As by-product this function converts non-linear &sk_buff to linear 669 * As by-product this function converts non-linear &sk_buff to linear
668 * one, so that &sk_buff becomes completely private and caller is allowed 670 * one, so that &sk_buff becomes completely private and caller is allowed
669 * to modify all the data of returned buffer. This means that this 671 * to modify all the data of returned buffer. This means that this
670 * function is not recommended for use in circumstances when only 672 * function is not recommended for use in circumstances when only
671 * header is going to be modified. Use pskb_copy() instead. 673 * header is going to be modified. Use pskb_copy() instead.
672 */ 674 */
673 675
674 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 676 struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask)
675 { 677 {
676 int headerlen = skb_headroom(skb); 678 int headerlen = skb_headroom(skb);
677 unsigned int size = (skb_end_pointer(skb) - skb->head) + skb->data_len; 679 unsigned int size = (skb_end_pointer(skb) - skb->head) + skb->data_len;
678 struct sk_buff *n = alloc_skb(size, gfp_mask); 680 struct sk_buff *n = alloc_skb(size, gfp_mask);
679 681
680 if (!n) 682 if (!n)
681 return NULL; 683 return NULL;
682 684
683 /* Set the data pointer */ 685 /* Set the data pointer */
684 skb_reserve(n, headerlen); 686 skb_reserve(n, headerlen);
685 /* Set the tail pointer and length */ 687 /* Set the tail pointer and length */
686 skb_put(n, skb->len); 688 skb_put(n, skb->len);
687 689
688 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)) 690 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len))
689 BUG(); 691 BUG();
690 692
691 copy_skb_header(n, skb); 693 copy_skb_header(n, skb);
692 return n; 694 return n;
693 } 695 }
694 EXPORT_SYMBOL(skb_copy); 696 EXPORT_SYMBOL(skb_copy);
695 697
696 /** 698 /**
697 * pskb_copy - create copy of an sk_buff with private head. 699 * pskb_copy - create copy of an sk_buff with private head.
698 * @skb: buffer to copy 700 * @skb: buffer to copy
699 * @gfp_mask: allocation priority 701 * @gfp_mask: allocation priority
700 * 702 *
701 * Make a copy of both an &sk_buff and part of its data, located 703 * Make a copy of both an &sk_buff and part of its data, located
702 * in header. Fragmented data remain shared. This is used when 704 * in header. Fragmented data remain shared. This is used when
703 * the caller wishes to modify only header of &sk_buff and needs 705 * the caller wishes to modify only header of &sk_buff and needs
704 * private copy of the header to alter. Returns %NULL on failure 706 * private copy of the header to alter. Returns %NULL on failure
705 * or the pointer to the buffer on success. 707 * or the pointer to the buffer on success.
706 * The returned buffer has a reference count of 1. 708 * The returned buffer has a reference count of 1.
707 */ 709 */
708 710
709 struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask) 711 struct sk_buff *pskb_copy(struct sk_buff *skb, gfp_t gfp_mask)
710 { 712 {
711 unsigned int size = skb_end_pointer(skb) - skb->head; 713 unsigned int size = skb_end_pointer(skb) - skb->head;
712 struct sk_buff *n = alloc_skb(size, gfp_mask); 714 struct sk_buff *n = alloc_skb(size, gfp_mask);
713 715
714 if (!n) 716 if (!n)
715 goto out; 717 goto out;
716 718
717 /* Set the data pointer */ 719 /* Set the data pointer */
718 skb_reserve(n, skb_headroom(skb)); 720 skb_reserve(n, skb_headroom(skb));
719 /* Set the tail pointer and length */ 721 /* Set the tail pointer and length */
720 skb_put(n, skb_headlen(skb)); 722 skb_put(n, skb_headlen(skb));
721 /* Copy the bytes */ 723 /* Copy the bytes */
722 skb_copy_from_linear_data(skb, n->data, n->len); 724 skb_copy_from_linear_data(skb, n->data, n->len);
723 725
724 n->truesize += skb->data_len; 726 n->truesize += skb->data_len;
725 n->data_len = skb->data_len; 727 n->data_len = skb->data_len;
726 n->len = skb->len; 728 n->len = skb->len;
727 729
728 if (skb_shinfo(skb)->nr_frags) { 730 if (skb_shinfo(skb)->nr_frags) {
729 int i; 731 int i;
730 732
731 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 733 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
732 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 734 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i];
733 get_page(skb_shinfo(n)->frags[i].page); 735 get_page(skb_shinfo(n)->frags[i].page);
734 } 736 }
735 skb_shinfo(n)->nr_frags = i; 737 skb_shinfo(n)->nr_frags = i;
736 } 738 }
737 739
738 if (skb_has_frag_list(skb)) { 740 if (skb_has_frag_list(skb)) {
739 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 741 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list;
740 skb_clone_fraglist(n); 742 skb_clone_fraglist(n);
741 } 743 }
742 744
743 copy_skb_header(n, skb); 745 copy_skb_header(n, skb);
744 out: 746 out:
745 return n; 747 return n;
746 } 748 }
747 EXPORT_SYMBOL(pskb_copy); 749 EXPORT_SYMBOL(pskb_copy);
748 750
749 /** 751 /**
750 * pskb_expand_head - reallocate header of &sk_buff 752 * pskb_expand_head - reallocate header of &sk_buff
751 * @skb: buffer to reallocate 753 * @skb: buffer to reallocate
752 * @nhead: room to add at head 754 * @nhead: room to add at head
753 * @ntail: room to add at tail 755 * @ntail: room to add at tail
754 * @gfp_mask: allocation priority 756 * @gfp_mask: allocation priority
755 * 757 *
756 * Expands (or creates identical copy, if &nhead and &ntail are zero) 758 * Expands (or creates identical copy, if &nhead and &ntail are zero)
757 * header of skb. &sk_buff itself is not changed. &sk_buff MUST have 759 * header of skb. &sk_buff itself is not changed. &sk_buff MUST have
758 * reference count of 1. Returns zero in the case of success or error, 760 * reference count of 1. Returns zero in the case of success or error,
759 * if expansion failed. In the last case, &sk_buff is not changed. 761 * if expansion failed. In the last case, &sk_buff is not changed.
760 * 762 *
761 * All the pointers pointing into skb header may change and must be 763 * All the pointers pointing into skb header may change and must be
762 * reloaded after call to this function. 764 * reloaded after call to this function.
763 */ 765 */
764 766
765 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 767 int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail,
766 gfp_t gfp_mask) 768 gfp_t gfp_mask)
767 { 769 {
768 int i; 770 int i;
769 u8 *data; 771 u8 *data;
770 int size = nhead + (skb_end_pointer(skb) - skb->head) + ntail; 772 int size = nhead + (skb_end_pointer(skb) - skb->head) + ntail;
771 long off; 773 long off;
772 bool fastpath; 774 bool fastpath;
773 775
774 BUG_ON(nhead < 0); 776 BUG_ON(nhead < 0);
775 777
776 if (skb_shared(skb)) 778 if (skb_shared(skb))
777 BUG(); 779 BUG();
778 780
779 size = SKB_DATA_ALIGN(size); 781 size = SKB_DATA_ALIGN(size);
780 782
781 /* Check if we can avoid taking references on fragments if we own 783 /* Check if we can avoid taking references on fragments if we own
782 * the last reference on skb->head. (see skb_release_data()) 784 * the last reference on skb->head. (see skb_release_data())
783 */ 785 */
784 if (!skb->cloned) 786 if (!skb->cloned)
785 fastpath = true; 787 fastpath = true;
786 else { 788 else {
787 int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1; 789 int delta = skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1;
788 790
789 fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta; 791 fastpath = atomic_read(&skb_shinfo(skb)->dataref) == delta;
790 } 792 }
791 793
792 if (fastpath && 794 if (fastpath &&
793 size + sizeof(struct skb_shared_info) <= ksize(skb->head)) { 795 size + sizeof(struct skb_shared_info) <= ksize(skb->head)) {
794 memmove(skb->head + size, skb_shinfo(skb), 796 memmove(skb->head + size, skb_shinfo(skb),
795 offsetof(struct skb_shared_info, 797 offsetof(struct skb_shared_info,
796 frags[skb_shinfo(skb)->nr_frags])); 798 frags[skb_shinfo(skb)->nr_frags]));
797 memmove(skb->head + nhead, skb->head, 799 memmove(skb->head + nhead, skb->head,
798 skb_tail_pointer(skb) - skb->head); 800 skb_tail_pointer(skb) - skb->head);
799 off = nhead; 801 off = nhead;
800 goto adjust_others; 802 goto adjust_others;
801 } 803 }
802 804
803 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask); 805 data = kmalloc(size + sizeof(struct skb_shared_info), gfp_mask);
804 if (!data) 806 if (!data)
805 goto nodata; 807 goto nodata;
806 808
807 /* Copy only real data... and, alas, header. This should be 809 /* Copy only real data... and, alas, header. This should be
808 * optimized for the cases when header is void. 810 * optimized for the cases when header is void.
809 */ 811 */
810 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 812 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head);
811 813
812 memcpy((struct skb_shared_info *)(data + size), 814 memcpy((struct skb_shared_info *)(data + size),
813 skb_shinfo(skb), 815 skb_shinfo(skb),
814 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 816 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags]));
815 817
816 if (fastpath) { 818 if (fastpath) {
817 kfree(skb->head); 819 kfree(skb->head);
818 } else { 820 } else {
819 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 821 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
820 get_page(skb_shinfo(skb)->frags[i].page); 822 get_page(skb_shinfo(skb)->frags[i].page);
821 823
822 if (skb_has_frag_list(skb)) 824 if (skb_has_frag_list(skb))
823 skb_clone_fraglist(skb); 825 skb_clone_fraglist(skb);
824 826
825 skb_release_data(skb); 827 skb_release_data(skb);
826 } 828 }
827 off = (data + nhead) - skb->head; 829 off = (data + nhead) - skb->head;
828 830
829 skb->head = data; 831 skb->head = data;
830 adjust_others: 832 adjust_others:
831 skb->data += off; 833 skb->data += off;
832 #ifdef NET_SKBUFF_DATA_USES_OFFSET 834 #ifdef NET_SKBUFF_DATA_USES_OFFSET
833 skb->end = size; 835 skb->end = size;
834 off = nhead; 836 off = nhead;
835 #else 837 #else
836 skb->end = skb->head + size; 838 skb->end = skb->head + size;
837 #endif 839 #endif
838 /* {transport,network,mac}_header and tail are relative to skb->head */ 840 /* {transport,network,mac}_header and tail are relative to skb->head */
839 skb->tail += off; 841 skb->tail += off;
840 skb->transport_header += off; 842 skb->transport_header += off;
841 skb->network_header += off; 843 skb->network_header += off;
842 if (skb_mac_header_was_set(skb)) 844 if (skb_mac_header_was_set(skb))
843 skb->mac_header += off; 845 skb->mac_header += off;
844 /* Only adjust this if it actually is csum_start rather than csum */ 846 /* Only adjust this if it actually is csum_start rather than csum */
845 if (skb->ip_summed == CHECKSUM_PARTIAL) 847 if (skb->ip_summed == CHECKSUM_PARTIAL)
846 skb->csum_start += nhead; 848 skb->csum_start += nhead;
847 skb->cloned = 0; 849 skb->cloned = 0;
848 skb->hdr_len = 0; 850 skb->hdr_len = 0;
849 skb->nohdr = 0; 851 skb->nohdr = 0;
850 atomic_set(&skb_shinfo(skb)->dataref, 1); 852 atomic_set(&skb_shinfo(skb)->dataref, 1);
851 return 0; 853 return 0;
852 854
853 nodata: 855 nodata:
854 return -ENOMEM; 856 return -ENOMEM;
855 } 857 }
856 EXPORT_SYMBOL(pskb_expand_head); 858 EXPORT_SYMBOL(pskb_expand_head);
857 859
858 /* Make private copy of skb with writable head and some headroom */ 860 /* Make private copy of skb with writable head and some headroom */
859 861
860 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 862 struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom)
861 { 863 {
862 struct sk_buff *skb2; 864 struct sk_buff *skb2;
863 int delta = headroom - skb_headroom(skb); 865 int delta = headroom - skb_headroom(skb);
864 866
865 if (delta <= 0) 867 if (delta <= 0)
866 skb2 = pskb_copy(skb, GFP_ATOMIC); 868 skb2 = pskb_copy(skb, GFP_ATOMIC);
867 else { 869 else {
868 skb2 = skb_clone(skb, GFP_ATOMIC); 870 skb2 = skb_clone(skb, GFP_ATOMIC);
869 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 871 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0,
870 GFP_ATOMIC)) { 872 GFP_ATOMIC)) {
871 kfree_skb(skb2); 873 kfree_skb(skb2);
872 skb2 = NULL; 874 skb2 = NULL;
873 } 875 }
874 } 876 }
875 return skb2; 877 return skb2;
876 } 878 }
877 EXPORT_SYMBOL(skb_realloc_headroom); 879 EXPORT_SYMBOL(skb_realloc_headroom);
878 880
879 /** 881 /**
880 * skb_copy_expand - copy and expand sk_buff 882 * skb_copy_expand - copy and expand sk_buff
881 * @skb: buffer to copy 883 * @skb: buffer to copy
882 * @newheadroom: new free bytes at head 884 * @newheadroom: new free bytes at head
883 * @newtailroom: new free bytes at tail 885 * @newtailroom: new free bytes at tail
884 * @gfp_mask: allocation priority 886 * @gfp_mask: allocation priority
885 * 887 *
886 * Make a copy of both an &sk_buff and its data and while doing so 888 * Make a copy of both an &sk_buff and its data and while doing so
887 * allocate additional space. 889 * allocate additional space.
888 * 890 *
889 * This is used when the caller wishes to modify the data and needs a 891 * This is used when the caller wishes to modify the data and needs a
890 * private copy of the data to alter as well as more space for new fields. 892 * private copy of the data to alter as well as more space for new fields.
891 * Returns %NULL on failure or the pointer to the buffer 893 * Returns %NULL on failure or the pointer to the buffer
892 * on success. The returned buffer has a reference count of 1. 894 * on success. The returned buffer has a reference count of 1.
893 * 895 *
894 * You must pass %GFP_ATOMIC as the allocation priority if this function 896 * You must pass %GFP_ATOMIC as the allocation priority if this function
895 * is called from an interrupt. 897 * is called from an interrupt.
896 */ 898 */
897 struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 899 struct sk_buff *skb_copy_expand(const struct sk_buff *skb,
898 int newheadroom, int newtailroom, 900 int newheadroom, int newtailroom,
899 gfp_t gfp_mask) 901 gfp_t gfp_mask)
900 { 902 {
901 /* 903 /*
902 * Allocate the copy buffer 904 * Allocate the copy buffer
903 */ 905 */
904 struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom, 906 struct sk_buff *n = alloc_skb(newheadroom + skb->len + newtailroom,
905 gfp_mask); 907 gfp_mask);
906 int oldheadroom = skb_headroom(skb); 908 int oldheadroom = skb_headroom(skb);
907 int head_copy_len, head_copy_off; 909 int head_copy_len, head_copy_off;
908 int off; 910 int off;
909 911
910 if (!n) 912 if (!n)
911 return NULL; 913 return NULL;
912 914
913 skb_reserve(n, newheadroom); 915 skb_reserve(n, newheadroom);
914 916
915 /* Set the tail pointer and length */ 917 /* Set the tail pointer and length */
916 skb_put(n, skb->len); 918 skb_put(n, skb->len);
917 919
918 head_copy_len = oldheadroom; 920 head_copy_len = oldheadroom;
919 head_copy_off = 0; 921 head_copy_off = 0;
920 if (newheadroom <= head_copy_len) 922 if (newheadroom <= head_copy_len)
921 head_copy_len = newheadroom; 923 head_copy_len = newheadroom;
922 else 924 else
923 head_copy_off = newheadroom - head_copy_len; 925 head_copy_off = newheadroom - head_copy_len;
924 926
925 /* Copy the linear header and data. */ 927 /* Copy the linear header and data. */
926 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 928 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off,
927 skb->len + head_copy_len)) 929 skb->len + head_copy_len))
928 BUG(); 930 BUG();
929 931
930 copy_skb_header(n, skb); 932 copy_skb_header(n, skb);
931 933
932 off = newheadroom - oldheadroom; 934 off = newheadroom - oldheadroom;
933 if (n->ip_summed == CHECKSUM_PARTIAL) 935 if (n->ip_summed == CHECKSUM_PARTIAL)
934 n->csum_start += off; 936 n->csum_start += off;
935 #ifdef NET_SKBUFF_DATA_USES_OFFSET 937 #ifdef NET_SKBUFF_DATA_USES_OFFSET
936 n->transport_header += off; 938 n->transport_header += off;
937 n->network_header += off; 939 n->network_header += off;
938 if (skb_mac_header_was_set(skb)) 940 if (skb_mac_header_was_set(skb))
939 n->mac_header += off; 941 n->mac_header += off;
940 #endif 942 #endif
941 943
942 return n; 944 return n;
943 } 945 }
944 EXPORT_SYMBOL(skb_copy_expand); 946 EXPORT_SYMBOL(skb_copy_expand);
945 947
946 /** 948 /**
947 * skb_pad - zero pad the tail of an skb 949 * skb_pad - zero pad the tail of an skb
948 * @skb: buffer to pad 950 * @skb: buffer to pad
949 * @pad: space to pad 951 * @pad: space to pad
950 * 952 *
951 * Ensure that a buffer is followed by a padding area that is zero 953 * Ensure that a buffer is followed by a padding area that is zero
952 * filled. Used by network drivers which may DMA or transfer data 954 * filled. Used by network drivers which may DMA or transfer data
953 * beyond the buffer end onto the wire. 955 * beyond the buffer end onto the wire.
954 * 956 *
955 * May return error in out of memory cases. The skb is freed on error. 957 * May return error in out of memory cases. The skb is freed on error.
956 */ 958 */
957 959
958 int skb_pad(struct sk_buff *skb, int pad) 960 int skb_pad(struct sk_buff *skb, int pad)
959 { 961 {
960 int err; 962 int err;
961 int ntail; 963 int ntail;
962 964
963 /* If the skbuff is non linear tailroom is always zero.. */ 965 /* If the skbuff is non linear tailroom is always zero.. */
964 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 966 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) {
965 memset(skb->data+skb->len, 0, pad); 967 memset(skb->data+skb->len, 0, pad);
966 return 0; 968 return 0;
967 } 969 }
968 970
969 ntail = skb->data_len + pad - (skb->end - skb->tail); 971 ntail = skb->data_len + pad - (skb->end - skb->tail);
970 if (likely(skb_cloned(skb) || ntail > 0)) { 972 if (likely(skb_cloned(skb) || ntail > 0)) {
971 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 973 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC);
972 if (unlikely(err)) 974 if (unlikely(err))
973 goto free_skb; 975 goto free_skb;
974 } 976 }
975 977
976 /* FIXME: The use of this function with non-linear skb's really needs 978 /* FIXME: The use of this function with non-linear skb's really needs
977 * to be audited. 979 * to be audited.
978 */ 980 */
979 err = skb_linearize(skb); 981 err = skb_linearize(skb);
980 if (unlikely(err)) 982 if (unlikely(err))
981 goto free_skb; 983 goto free_skb;
982 984
983 memset(skb->data + skb->len, 0, pad); 985 memset(skb->data + skb->len, 0, pad);
984 return 0; 986 return 0;
985 987
986 free_skb: 988 free_skb:
987 kfree_skb(skb); 989 kfree_skb(skb);
988 return err; 990 return err;
989 } 991 }
990 EXPORT_SYMBOL(skb_pad); 992 EXPORT_SYMBOL(skb_pad);
991 993
992 /** 994 /**
993 * skb_put - add data to a buffer 995 * skb_put - add data to a buffer
994 * @skb: buffer to use 996 * @skb: buffer to use
995 * @len: amount of data to add 997 * @len: amount of data to add
996 * 998 *
997 * This function extends the used data area of the buffer. If this would 999 * This function extends the used data area of the buffer. If this would
998 * exceed the total buffer size the kernel will panic. A pointer to the 1000 * exceed the total buffer size the kernel will panic. A pointer to the
999 * first byte of the extra data is returned. 1001 * first byte of the extra data is returned.
1000 */ 1002 */
1001 unsigned char *skb_put(struct sk_buff *skb, unsigned int len) 1003 unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
1002 { 1004 {
1003 unsigned char *tmp = skb_tail_pointer(skb); 1005 unsigned char *tmp = skb_tail_pointer(skb);
1004 SKB_LINEAR_ASSERT(skb); 1006 SKB_LINEAR_ASSERT(skb);
1005 skb->tail += len; 1007 skb->tail += len;
1006 skb->len += len; 1008 skb->len += len;
1007 if (unlikely(skb->tail > skb->end)) 1009 if (unlikely(skb->tail > skb->end))
1008 skb_over_panic(skb, len, __builtin_return_address(0)); 1010 skb_over_panic(skb, len, __builtin_return_address(0));
1009 return tmp; 1011 return tmp;
1010 } 1012 }
1011 EXPORT_SYMBOL(skb_put); 1013 EXPORT_SYMBOL(skb_put);
1012 1014
1013 /** 1015 /**
1014 * skb_push - add data to the start of a buffer 1016 * skb_push - add data to the start of a buffer
1015 * @skb: buffer to use 1017 * @skb: buffer to use
1016 * @len: amount of data to add 1018 * @len: amount of data to add
1017 * 1019 *
1018 * This function extends the used data area of the buffer at the buffer 1020 * This function extends the used data area of the buffer at the buffer
1019 * start. If this would exceed the total buffer headroom the kernel will 1021 * start. If this would exceed the total buffer headroom the kernel will
1020 * panic. A pointer to the first byte of the extra data is returned. 1022 * panic. A pointer to the first byte of the extra data is returned.
1021 */ 1023 */
1022 unsigned char *skb_push(struct sk_buff *skb, unsigned int len) 1024 unsigned char *skb_push(struct sk_buff *skb, unsigned int len)
1023 { 1025 {
1024 skb->data -= len; 1026 skb->data -= len;
1025 skb->len += len; 1027 skb->len += len;
1026 if (unlikely(skb->data<skb->head)) 1028 if (unlikely(skb->data<skb->head))
1027 skb_under_panic(skb, len, __builtin_return_address(0)); 1029 skb_under_panic(skb, len, __builtin_return_address(0));
1028 return skb->data; 1030 return skb->data;
1029 } 1031 }
1030 EXPORT_SYMBOL(skb_push); 1032 EXPORT_SYMBOL(skb_push);
1031 1033
1032 /** 1034 /**
1033 * skb_pull - remove data from the start of a buffer 1035 * skb_pull - remove data from the start of a buffer
1034 * @skb: buffer to use 1036 * @skb: buffer to use
1035 * @len: amount of data to remove 1037 * @len: amount of data to remove
1036 * 1038 *
1037 * This function removes data from the start of a buffer, returning 1039 * This function removes data from the start of a buffer, returning
1038 * the memory to the headroom. A pointer to the next data in the buffer 1040 * the memory to the headroom. A pointer to the next data in the buffer
1039 * is returned. Once the data has been pulled future pushes will overwrite 1041 * is returned. Once the data has been pulled future pushes will overwrite
1040 * the old data. 1042 * the old data.
1041 */ 1043 */
1042 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len) 1044 unsigned char *skb_pull(struct sk_buff *skb, unsigned int len)
1043 { 1045 {
1044 return skb_pull_inline(skb, len); 1046 return skb_pull_inline(skb, len);
1045 } 1047 }
1046 EXPORT_SYMBOL(skb_pull); 1048 EXPORT_SYMBOL(skb_pull);
1047 1049
1048 /** 1050 /**
1049 * skb_trim - remove end from a buffer 1051 * skb_trim - remove end from a buffer
1050 * @skb: buffer to alter 1052 * @skb: buffer to alter
1051 * @len: new length 1053 * @len: new length
1052 * 1054 *
1053 * Cut the length of a buffer down by removing data from the tail. If 1055 * Cut the length of a buffer down by removing data from the tail. If
1054 * the buffer is already under the length specified it is not modified. 1056 * the buffer is already under the length specified it is not modified.
1055 * The skb must be linear. 1057 * The skb must be linear.
1056 */ 1058 */
1057 void skb_trim(struct sk_buff *skb, unsigned int len) 1059 void skb_trim(struct sk_buff *skb, unsigned int len)
1058 { 1060 {
1059 if (skb->len > len) 1061 if (skb->len > len)
1060 __skb_trim(skb, len); 1062 __skb_trim(skb, len);
1061 } 1063 }
1062 EXPORT_SYMBOL(skb_trim); 1064 EXPORT_SYMBOL(skb_trim);
1063 1065
1064 /* Trims skb to length len. It can change skb pointers. 1066 /* Trims skb to length len. It can change skb pointers.
1065 */ 1067 */
1066 1068
1067 int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1069 int ___pskb_trim(struct sk_buff *skb, unsigned int len)
1068 { 1070 {
1069 struct sk_buff **fragp; 1071 struct sk_buff **fragp;
1070 struct sk_buff *frag; 1072 struct sk_buff *frag;
1071 int offset = skb_headlen(skb); 1073 int offset = skb_headlen(skb);
1072 int nfrags = skb_shinfo(skb)->nr_frags; 1074 int nfrags = skb_shinfo(skb)->nr_frags;
1073 int i; 1075 int i;
1074 int err; 1076 int err;
1075 1077
1076 if (skb_cloned(skb) && 1078 if (skb_cloned(skb) &&
1077 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1079 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC))))
1078 return err; 1080 return err;
1079 1081
1080 i = 0; 1082 i = 0;
1081 if (offset >= len) 1083 if (offset >= len)
1082 goto drop_pages; 1084 goto drop_pages;
1083 1085
1084 for (; i < nfrags; i++) { 1086 for (; i < nfrags; i++) {
1085 int end = offset + skb_shinfo(skb)->frags[i].size; 1087 int end = offset + skb_shinfo(skb)->frags[i].size;
1086 1088
1087 if (end < len) { 1089 if (end < len) {
1088 offset = end; 1090 offset = end;
1089 continue; 1091 continue;
1090 } 1092 }
1091 1093
1092 skb_shinfo(skb)->frags[i++].size = len - offset; 1094 skb_shinfo(skb)->frags[i++].size = len - offset;
1093 1095
1094 drop_pages: 1096 drop_pages:
1095 skb_shinfo(skb)->nr_frags = i; 1097 skb_shinfo(skb)->nr_frags = i;
1096 1098
1097 for (; i < nfrags; i++) 1099 for (; i < nfrags; i++)
1098 put_page(skb_shinfo(skb)->frags[i].page); 1100 put_page(skb_shinfo(skb)->frags[i].page);
1099 1101
1100 if (skb_has_frag_list(skb)) 1102 if (skb_has_frag_list(skb))
1101 skb_drop_fraglist(skb); 1103 skb_drop_fraglist(skb);
1102 goto done; 1104 goto done;
1103 } 1105 }
1104 1106
1105 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1107 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp);
1106 fragp = &frag->next) { 1108 fragp = &frag->next) {
1107 int end = offset + frag->len; 1109 int end = offset + frag->len;
1108 1110
1109 if (skb_shared(frag)) { 1111 if (skb_shared(frag)) {
1110 struct sk_buff *nfrag; 1112 struct sk_buff *nfrag;
1111 1113
1112 nfrag = skb_clone(frag, GFP_ATOMIC); 1114 nfrag = skb_clone(frag, GFP_ATOMIC);
1113 if (unlikely(!nfrag)) 1115 if (unlikely(!nfrag))
1114 return -ENOMEM; 1116 return -ENOMEM;
1115 1117
1116 nfrag->next = frag->next; 1118 nfrag->next = frag->next;
1117 kfree_skb(frag); 1119 kfree_skb(frag);
1118 frag = nfrag; 1120 frag = nfrag;
1119 *fragp = frag; 1121 *fragp = frag;
1120 } 1122 }
1121 1123
1122 if (end < len) { 1124 if (end < len) {
1123 offset = end; 1125 offset = end;
1124 continue; 1126 continue;
1125 } 1127 }
1126 1128
1127 if (end > len && 1129 if (end > len &&
1128 unlikely((err = pskb_trim(frag, len - offset)))) 1130 unlikely((err = pskb_trim(frag, len - offset))))
1129 return err; 1131 return err;
1130 1132
1131 if (frag->next) 1133 if (frag->next)
1132 skb_drop_list(&frag->next); 1134 skb_drop_list(&frag->next);
1133 break; 1135 break;
1134 } 1136 }
1135 1137
1136 done: 1138 done:
1137 if (len > skb_headlen(skb)) { 1139 if (len > skb_headlen(skb)) {
1138 skb->data_len -= skb->len - len; 1140 skb->data_len -= skb->len - len;
1139 skb->len = len; 1141 skb->len = len;
1140 } else { 1142 } else {
1141 skb->len = len; 1143 skb->len = len;
1142 skb->data_len = 0; 1144 skb->data_len = 0;
1143 skb_set_tail_pointer(skb, len); 1145 skb_set_tail_pointer(skb, len);
1144 } 1146 }
1145 1147
1146 return 0; 1148 return 0;
1147 } 1149 }
1148 EXPORT_SYMBOL(___pskb_trim); 1150 EXPORT_SYMBOL(___pskb_trim);
1149 1151
1150 /** 1152 /**
1151 * __pskb_pull_tail - advance tail of skb header 1153 * __pskb_pull_tail - advance tail of skb header
1152 * @skb: buffer to reallocate 1154 * @skb: buffer to reallocate
1153 * @delta: number of bytes to advance tail 1155 * @delta: number of bytes to advance tail
1154 * 1156 *
1155 * The function makes a sense only on a fragmented &sk_buff, 1157 * The function makes a sense only on a fragmented &sk_buff,
1156 * it expands header moving its tail forward and copying necessary 1158 * it expands header moving its tail forward and copying necessary
1157 * data from fragmented part. 1159 * data from fragmented part.
1158 * 1160 *
1159 * &sk_buff MUST have reference count of 1. 1161 * &sk_buff MUST have reference count of 1.
1160 * 1162 *
1161 * Returns %NULL (and &sk_buff does not change) if pull failed 1163 * Returns %NULL (and &sk_buff does not change) if pull failed
1162 * or value of new tail of skb in the case of success. 1164 * or value of new tail of skb in the case of success.
1163 * 1165 *
1164 * All the pointers pointing into skb header may change and must be 1166 * All the pointers pointing into skb header may change and must be
1165 * reloaded after call to this function. 1167 * reloaded after call to this function.
1166 */ 1168 */
1167 1169
1168 /* Moves tail of skb head forward, copying data from fragmented part, 1170 /* Moves tail of skb head forward, copying data from fragmented part,
1169 * when it is necessary. 1171 * when it is necessary.
1170 * 1. It may fail due to malloc failure. 1172 * 1. It may fail due to malloc failure.
1171 * 2. It may change skb pointers. 1173 * 2. It may change skb pointers.
1172 * 1174 *
1173 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1175 * It is pretty complicated. Luckily, it is called only in exceptional cases.
1174 */ 1176 */
1175 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) 1177 unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta)
1176 { 1178 {
1177 /* If skb has not enough free space at tail, get new one 1179 /* If skb has not enough free space at tail, get new one
1178 * plus 128 bytes for future expansions. If we have enough 1180 * plus 128 bytes for future expansions. If we have enough
1179 * room at tail, reallocate without expansion only if skb is cloned. 1181 * room at tail, reallocate without expansion only if skb is cloned.
1180 */ 1182 */
1181 int i, k, eat = (skb->tail + delta) - skb->end; 1183 int i, k, eat = (skb->tail + delta) - skb->end;
1182 1184
1183 if (eat > 0 || skb_cloned(skb)) { 1185 if (eat > 0 || skb_cloned(skb)) {
1184 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1186 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0,
1185 GFP_ATOMIC)) 1187 GFP_ATOMIC))
1186 return NULL; 1188 return NULL;
1187 } 1189 }
1188 1190
1189 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta)) 1191 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta))
1190 BUG(); 1192 BUG();
1191 1193
1192 /* Optimization: no fragments, no reasons to preestimate 1194 /* Optimization: no fragments, no reasons to preestimate
1193 * size of pulled pages. Superb. 1195 * size of pulled pages. Superb.
1194 */ 1196 */
1195 if (!skb_has_frag_list(skb)) 1197 if (!skb_has_frag_list(skb))
1196 goto pull_pages; 1198 goto pull_pages;
1197 1199
1198 /* Estimate size of pulled pages. */ 1200 /* Estimate size of pulled pages. */
1199 eat = delta; 1201 eat = delta;
1200 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1202 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1201 if (skb_shinfo(skb)->frags[i].size >= eat) 1203 if (skb_shinfo(skb)->frags[i].size >= eat)
1202 goto pull_pages; 1204 goto pull_pages;
1203 eat -= skb_shinfo(skb)->frags[i].size; 1205 eat -= skb_shinfo(skb)->frags[i].size;
1204 } 1206 }
1205 1207
1206 /* If we need update frag list, we are in troubles. 1208 /* If we need update frag list, we are in troubles.
1207 * Certainly, it possible to add an offset to skb data, 1209 * Certainly, it possible to add an offset to skb data,
1208 * but taking into account that pulling is expected to 1210 * but taking into account that pulling is expected to
1209 * be very rare operation, it is worth to fight against 1211 * be very rare operation, it is worth to fight against
1210 * further bloating skb head and crucify ourselves here instead. 1212 * further bloating skb head and crucify ourselves here instead.
1211 * Pure masohism, indeed. 8)8) 1213 * Pure masohism, indeed. 8)8)
1212 */ 1214 */
1213 if (eat) { 1215 if (eat) {
1214 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1216 struct sk_buff *list = skb_shinfo(skb)->frag_list;
1215 struct sk_buff *clone = NULL; 1217 struct sk_buff *clone = NULL;
1216 struct sk_buff *insp = NULL; 1218 struct sk_buff *insp = NULL;
1217 1219
1218 do { 1220 do {
1219 BUG_ON(!list); 1221 BUG_ON(!list);
1220 1222
1221 if (list->len <= eat) { 1223 if (list->len <= eat) {
1222 /* Eaten as whole. */ 1224 /* Eaten as whole. */
1223 eat -= list->len; 1225 eat -= list->len;
1224 list = list->next; 1226 list = list->next;
1225 insp = list; 1227 insp = list;
1226 } else { 1228 } else {
1227 /* Eaten partially. */ 1229 /* Eaten partially. */
1228 1230
1229 if (skb_shared(list)) { 1231 if (skb_shared(list)) {
1230 /* Sucks! We need to fork list. :-( */ 1232 /* Sucks! We need to fork list. :-( */
1231 clone = skb_clone(list, GFP_ATOMIC); 1233 clone = skb_clone(list, GFP_ATOMIC);
1232 if (!clone) 1234 if (!clone)
1233 return NULL; 1235 return NULL;
1234 insp = list->next; 1236 insp = list->next;
1235 list = clone; 1237 list = clone;
1236 } else { 1238 } else {
1237 /* This may be pulled without 1239 /* This may be pulled without
1238 * problems. */ 1240 * problems. */
1239 insp = list; 1241 insp = list;
1240 } 1242 }
1241 if (!pskb_pull(list, eat)) { 1243 if (!pskb_pull(list, eat)) {
1242 kfree_skb(clone); 1244 kfree_skb(clone);
1243 return NULL; 1245 return NULL;
1244 } 1246 }
1245 break; 1247 break;
1246 } 1248 }
1247 } while (eat); 1249 } while (eat);
1248 1250
1249 /* Free pulled out fragments. */ 1251 /* Free pulled out fragments. */
1250 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1252 while ((list = skb_shinfo(skb)->frag_list) != insp) {
1251 skb_shinfo(skb)->frag_list = list->next; 1253 skb_shinfo(skb)->frag_list = list->next;
1252 kfree_skb(list); 1254 kfree_skb(list);
1253 } 1255 }
1254 /* And insert new clone at head. */ 1256 /* And insert new clone at head. */
1255 if (clone) { 1257 if (clone) {
1256 clone->next = list; 1258 clone->next = list;
1257 skb_shinfo(skb)->frag_list = clone; 1259 skb_shinfo(skb)->frag_list = clone;
1258 } 1260 }
1259 } 1261 }
1260 /* Success! Now we may commit changes to skb data. */ 1262 /* Success! Now we may commit changes to skb data. */
1261 1263
1262 pull_pages: 1264 pull_pages:
1263 eat = delta; 1265 eat = delta;
1264 k = 0; 1266 k = 0;
1265 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1267 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1266 if (skb_shinfo(skb)->frags[i].size <= eat) { 1268 if (skb_shinfo(skb)->frags[i].size <= eat) {
1267 put_page(skb_shinfo(skb)->frags[i].page); 1269 put_page(skb_shinfo(skb)->frags[i].page);
1268 eat -= skb_shinfo(skb)->frags[i].size; 1270 eat -= skb_shinfo(skb)->frags[i].size;
1269 } else { 1271 } else {
1270 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 1272 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i];
1271 if (eat) { 1273 if (eat) {
1272 skb_shinfo(skb)->frags[k].page_offset += eat; 1274 skb_shinfo(skb)->frags[k].page_offset += eat;
1273 skb_shinfo(skb)->frags[k].size -= eat; 1275 skb_shinfo(skb)->frags[k].size -= eat;
1274 eat = 0; 1276 eat = 0;
1275 } 1277 }
1276 k++; 1278 k++;
1277 } 1279 }
1278 } 1280 }
1279 skb_shinfo(skb)->nr_frags = k; 1281 skb_shinfo(skb)->nr_frags = k;
1280 1282
1281 skb->tail += delta; 1283 skb->tail += delta;
1282 skb->data_len -= delta; 1284 skb->data_len -= delta;
1283 1285
1284 return skb_tail_pointer(skb); 1286 return skb_tail_pointer(skb);
1285 } 1287 }
1286 EXPORT_SYMBOL(__pskb_pull_tail); 1288 EXPORT_SYMBOL(__pskb_pull_tail);
1287 1289
1288 /* Copy some data bits from skb to kernel buffer. */ 1290 /* Copy some data bits from skb to kernel buffer. */
1289 1291
1290 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 1292 int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len)
1291 { 1293 {
1292 int start = skb_headlen(skb); 1294 int start = skb_headlen(skb);
1293 struct sk_buff *frag_iter; 1295 struct sk_buff *frag_iter;
1294 int i, copy; 1296 int i, copy;
1295 1297
1296 if (offset > (int)skb->len - len) 1298 if (offset > (int)skb->len - len)
1297 goto fault; 1299 goto fault;
1298 1300
1299 /* Copy header. */ 1301 /* Copy header. */
1300 if ((copy = start - offset) > 0) { 1302 if ((copy = start - offset) > 0) {
1301 if (copy > len) 1303 if (copy > len)
1302 copy = len; 1304 copy = len;
1303 skb_copy_from_linear_data_offset(skb, offset, to, copy); 1305 skb_copy_from_linear_data_offset(skb, offset, to, copy);
1304 if ((len -= copy) == 0) 1306 if ((len -= copy) == 0)
1305 return 0; 1307 return 0;
1306 offset += copy; 1308 offset += copy;
1307 to += copy; 1309 to += copy;
1308 } 1310 }
1309 1311
1310 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1312 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1311 int end; 1313 int end;
1312 1314
1313 WARN_ON(start > offset + len); 1315 WARN_ON(start > offset + len);
1314 1316
1315 end = start + skb_shinfo(skb)->frags[i].size; 1317 end = start + skb_shinfo(skb)->frags[i].size;
1316 if ((copy = end - offset) > 0) { 1318 if ((copy = end - offset) > 0) {
1317 u8 *vaddr; 1319 u8 *vaddr;
1318 1320
1319 if (copy > len) 1321 if (copy > len)
1320 copy = len; 1322 copy = len;
1321 1323
1322 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]); 1324 vaddr = kmap_skb_frag(&skb_shinfo(skb)->frags[i]);
1323 memcpy(to, 1325 memcpy(to,
1324 vaddr + skb_shinfo(skb)->frags[i].page_offset+ 1326 vaddr + skb_shinfo(skb)->frags[i].page_offset+
1325 offset - start, copy); 1327 offset - start, copy);
1326 kunmap_skb_frag(vaddr); 1328 kunmap_skb_frag(vaddr);
1327 1329
1328 if ((len -= copy) == 0) 1330 if ((len -= copy) == 0)
1329 return 0; 1331 return 0;
1330 offset += copy; 1332 offset += copy;
1331 to += copy; 1333 to += copy;
1332 } 1334 }
1333 start = end; 1335 start = end;
1334 } 1336 }
1335 1337
1336 skb_walk_frags(skb, frag_iter) { 1338 skb_walk_frags(skb, frag_iter) {
1337 int end; 1339 int end;
1338 1340
1339 WARN_ON(start > offset + len); 1341 WARN_ON(start > offset + len);
1340 1342
1341 end = start + frag_iter->len; 1343 end = start + frag_iter->len;
1342 if ((copy = end - offset) > 0) { 1344 if ((copy = end - offset) > 0) {
1343 if (copy > len) 1345 if (copy > len)
1344 copy = len; 1346 copy = len;
1345 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 1347 if (skb_copy_bits(frag_iter, offset - start, to, copy))
1346 goto fault; 1348 goto fault;
1347 if ((len -= copy) == 0) 1349 if ((len -= copy) == 0)
1348 return 0; 1350 return 0;
1349 offset += copy; 1351 offset += copy;
1350 to += copy; 1352 to += copy;
1351 } 1353 }
1352 start = end; 1354 start = end;
1353 } 1355 }
1354 if (!len) 1356 if (!len)
1355 return 0; 1357 return 0;
1356 1358
1357 fault: 1359 fault:
1358 return -EFAULT; 1360 return -EFAULT;
1359 } 1361 }
1360 EXPORT_SYMBOL(skb_copy_bits); 1362 EXPORT_SYMBOL(skb_copy_bits);
1361 1363
1362 /* 1364 /*
1363 * Callback from splice_to_pipe(), if we need to release some pages 1365 * Callback from splice_to_pipe(), if we need to release some pages
1364 * at the end of the spd in case we error'ed out in filling the pipe. 1366 * at the end of the spd in case we error'ed out in filling the pipe.
1365 */ 1367 */
1366 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 1368 static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
1367 { 1369 {
1368 put_page(spd->pages[i]); 1370 put_page(spd->pages[i]);
1369 } 1371 }
1370 1372
1371 static inline struct page *linear_to_page(struct page *page, unsigned int *len, 1373 static inline struct page *linear_to_page(struct page *page, unsigned int *len,
1372 unsigned int *offset, 1374 unsigned int *offset,
1373 struct sk_buff *skb, struct sock *sk) 1375 struct sk_buff *skb, struct sock *sk)
1374 { 1376 {
1375 struct page *p = sk->sk_sndmsg_page; 1377 struct page *p = sk->sk_sndmsg_page;
1376 unsigned int off; 1378 unsigned int off;
1377 1379
1378 if (!p) { 1380 if (!p) {
1379 new_page: 1381 new_page:
1380 p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0); 1382 p = sk->sk_sndmsg_page = alloc_pages(sk->sk_allocation, 0);
1381 if (!p) 1383 if (!p)
1382 return NULL; 1384 return NULL;
1383 1385
1384 off = sk->sk_sndmsg_off = 0; 1386 off = sk->sk_sndmsg_off = 0;
1385 /* hold one ref to this page until it's full */ 1387 /* hold one ref to this page until it's full */
1386 } else { 1388 } else {
1387 unsigned int mlen; 1389 unsigned int mlen;
1388 1390
1389 off = sk->sk_sndmsg_off; 1391 off = sk->sk_sndmsg_off;
1390 mlen = PAGE_SIZE - off; 1392 mlen = PAGE_SIZE - off;
1391 if (mlen < 64 && mlen < *len) { 1393 if (mlen < 64 && mlen < *len) {
1392 put_page(p); 1394 put_page(p);
1393 goto new_page; 1395 goto new_page;
1394 } 1396 }
1395 1397
1396 *len = min_t(unsigned int, *len, mlen); 1398 *len = min_t(unsigned int, *len, mlen);
1397 } 1399 }
1398 1400
1399 memcpy(page_address(p) + off, page_address(page) + *offset, *len); 1401 memcpy(page_address(p) + off, page_address(page) + *offset, *len);
1400 sk->sk_sndmsg_off += *len; 1402 sk->sk_sndmsg_off += *len;
1401 *offset = off; 1403 *offset = off;
1402 get_page(p); 1404 get_page(p);
1403 1405
1404 return p; 1406 return p;
1405 } 1407 }
1406 1408
1407 /* 1409 /*
1408 * Fill page/offset/length into spd, if it can hold more pages. 1410 * Fill page/offset/length into spd, if it can hold more pages.
1409 */ 1411 */
1410 static inline int spd_fill_page(struct splice_pipe_desc *spd, 1412 static inline int spd_fill_page(struct splice_pipe_desc *spd,
1411 struct pipe_inode_info *pipe, struct page *page, 1413 struct pipe_inode_info *pipe, struct page *page,
1412 unsigned int *len, unsigned int offset, 1414 unsigned int *len, unsigned int offset,
1413 struct sk_buff *skb, int linear, 1415 struct sk_buff *skb, int linear,
1414 struct sock *sk) 1416 struct sock *sk)
1415 { 1417 {
1416 if (unlikely(spd->nr_pages == pipe->buffers)) 1418 if (unlikely(spd->nr_pages == pipe->buffers))
1417 return 1; 1419 return 1;
1418 1420
1419 if (linear) { 1421 if (linear) {
1420 page = linear_to_page(page, len, &offset, skb, sk); 1422 page = linear_to_page(page, len, &offset, skb, sk);
1421 if (!page) 1423 if (!page)
1422 return 1; 1424 return 1;
1423 } else 1425 } else
1424 get_page(page); 1426 get_page(page);
1425 1427
1426 spd->pages[spd->nr_pages] = page; 1428 spd->pages[spd->nr_pages] = page;
1427 spd->partial[spd->nr_pages].len = *len; 1429 spd->partial[spd->nr_pages].len = *len;
1428 spd->partial[spd->nr_pages].offset = offset; 1430 spd->partial[spd->nr_pages].offset = offset;
1429 spd->nr_pages++; 1431 spd->nr_pages++;
1430 1432
1431 return 0; 1433 return 0;
1432 } 1434 }
1433 1435
1434 static inline void __segment_seek(struct page **page, unsigned int *poff, 1436 static inline void __segment_seek(struct page **page, unsigned int *poff,
1435 unsigned int *plen, unsigned int off) 1437 unsigned int *plen, unsigned int off)
1436 { 1438 {
1437 unsigned long n; 1439 unsigned long n;
1438 1440
1439 *poff += off; 1441 *poff += off;
1440 n = *poff / PAGE_SIZE; 1442 n = *poff / PAGE_SIZE;
1441 if (n) 1443 if (n)
1442 *page = nth_page(*page, n); 1444 *page = nth_page(*page, n);
1443 1445
1444 *poff = *poff % PAGE_SIZE; 1446 *poff = *poff % PAGE_SIZE;
1445 *plen -= off; 1447 *plen -= off;
1446 } 1448 }
1447 1449
1448 static inline int __splice_segment(struct page *page, unsigned int poff, 1450 static inline int __splice_segment(struct page *page, unsigned int poff,
1449 unsigned int plen, unsigned int *off, 1451 unsigned int plen, unsigned int *off,
1450 unsigned int *len, struct sk_buff *skb, 1452 unsigned int *len, struct sk_buff *skb,
1451 struct splice_pipe_desc *spd, int linear, 1453 struct splice_pipe_desc *spd, int linear,
1452 struct sock *sk, 1454 struct sock *sk,
1453 struct pipe_inode_info *pipe) 1455 struct pipe_inode_info *pipe)
1454 { 1456 {
1455 if (!*len) 1457 if (!*len)
1456 return 1; 1458 return 1;
1457 1459
1458 /* skip this segment if already processed */ 1460 /* skip this segment if already processed */
1459 if (*off >= plen) { 1461 if (*off >= plen) {
1460 *off -= plen; 1462 *off -= plen;
1461 return 0; 1463 return 0;
1462 } 1464 }
1463 1465
1464 /* ignore any bits we already processed */ 1466 /* ignore any bits we already processed */
1465 if (*off) { 1467 if (*off) {
1466 __segment_seek(&page, &poff, &plen, *off); 1468 __segment_seek(&page, &poff, &plen, *off);
1467 *off = 0; 1469 *off = 0;
1468 } 1470 }
1469 1471
1470 do { 1472 do {
1471 unsigned int flen = min(*len, plen); 1473 unsigned int flen = min(*len, plen);
1472 1474
1473 /* the linear region may spread across several pages */ 1475 /* the linear region may spread across several pages */
1474 flen = min_t(unsigned int, flen, PAGE_SIZE - poff); 1476 flen = min_t(unsigned int, flen, PAGE_SIZE - poff);
1475 1477
1476 if (spd_fill_page(spd, pipe, page, &flen, poff, skb, linear, sk)) 1478 if (spd_fill_page(spd, pipe, page, &flen, poff, skb, linear, sk))
1477 return 1; 1479 return 1;
1478 1480
1479 __segment_seek(&page, &poff, &plen, flen); 1481 __segment_seek(&page, &poff, &plen, flen);
1480 *len -= flen; 1482 *len -= flen;
1481 1483
1482 } while (*len && plen); 1484 } while (*len && plen);
1483 1485
1484 return 0; 1486 return 0;
1485 } 1487 }
1486 1488
1487 /* 1489 /*
1488 * Map linear and fragment data from the skb to spd. It reports failure if the 1490 * Map linear and fragment data from the skb to spd. It reports failure if the
1489 * pipe is full or if we already spliced the requested length. 1491 * pipe is full or if we already spliced the requested length.
1490 */ 1492 */
1491 static int __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 1493 static int __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe,
1492 unsigned int *offset, unsigned int *len, 1494 unsigned int *offset, unsigned int *len,
1493 struct splice_pipe_desc *spd, struct sock *sk) 1495 struct splice_pipe_desc *spd, struct sock *sk)
1494 { 1496 {
1495 int seg; 1497 int seg;
1496 1498
1497 /* 1499 /*
1498 * map the linear part 1500 * map the linear part
1499 */ 1501 */
1500 if (__splice_segment(virt_to_page(skb->data), 1502 if (__splice_segment(virt_to_page(skb->data),
1501 (unsigned long) skb->data & (PAGE_SIZE - 1), 1503 (unsigned long) skb->data & (PAGE_SIZE - 1),
1502 skb_headlen(skb), 1504 skb_headlen(skb),
1503 offset, len, skb, spd, 1, sk, pipe)) 1505 offset, len, skb, spd, 1, sk, pipe))
1504 return 1; 1506 return 1;
1505 1507
1506 /* 1508 /*
1507 * then map the fragments 1509 * then map the fragments
1508 */ 1510 */
1509 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 1511 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) {
1510 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 1512 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg];
1511 1513
1512 if (__splice_segment(f->page, f->page_offset, f->size, 1514 if (__splice_segment(f->page, f->page_offset, f->size,
1513 offset, len, skb, spd, 0, sk, pipe)) 1515 offset, len, skb, spd, 0, sk, pipe))
1514 return 1; 1516 return 1;
1515 } 1517 }
1516 1518
1517 return 0; 1519 return 0;
1518 } 1520 }
1519 1521
1520 /* 1522 /*
1521 * Map data from the skb to a pipe. Should handle both the linear part, 1523 * Map data from the skb to a pipe. Should handle both the linear part,
1522 * the fragments, and the frag list. It does NOT handle frag lists within 1524 * the fragments, and the frag list. It does NOT handle frag lists within
1523 * the frag list, if such a thing exists. We'd probably need to recurse to 1525 * the frag list, if such a thing exists. We'd probably need to recurse to
1524 * handle that cleanly. 1526 * handle that cleanly.
1525 */ 1527 */
1526 int skb_splice_bits(struct sk_buff *skb, unsigned int offset, 1528 int skb_splice_bits(struct sk_buff *skb, unsigned int offset,
1527 struct pipe_inode_info *pipe, unsigned int tlen, 1529 struct pipe_inode_info *pipe, unsigned int tlen,
1528 unsigned int flags) 1530 unsigned int flags)
1529 { 1531 {
1530 struct partial_page partial[PIPE_DEF_BUFFERS]; 1532 struct partial_page partial[PIPE_DEF_BUFFERS];
1531 struct page *pages[PIPE_DEF_BUFFERS]; 1533 struct page *pages[PIPE_DEF_BUFFERS];
1532 struct splice_pipe_desc spd = { 1534 struct splice_pipe_desc spd = {
1533 .pages = pages, 1535 .pages = pages,
1534 .partial = partial, 1536 .partial = partial,
1535 .flags = flags, 1537 .flags = flags,
1536 .ops = &sock_pipe_buf_ops, 1538 .ops = &sock_pipe_buf_ops,
1537 .spd_release = sock_spd_release, 1539 .spd_release = sock_spd_release,
1538 }; 1540 };
1539 struct sk_buff *frag_iter; 1541 struct sk_buff *frag_iter;
1540 struct sock *sk = skb->sk; 1542 struct sock *sk = skb->sk;
1541 int ret = 0; 1543 int ret = 0;
1542 1544
1543 if (splice_grow_spd(pipe, &spd)) 1545 if (splice_grow_spd(pipe, &spd))
1544 return -ENOMEM; 1546 return -ENOMEM;
1545 1547
1546 /* 1548 /*
1547 * __skb_splice_bits() only fails if the output has no room left, 1549 * __skb_splice_bits() only fails if the output has no room left,
1548 * so no point in going over the frag_list for the error case. 1550 * so no point in going over the frag_list for the error case.
1549 */ 1551 */
1550 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk)) 1552 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk))
1551 goto done; 1553 goto done;
1552 else if (!tlen) 1554 else if (!tlen)
1553 goto done; 1555 goto done;
1554 1556
1555 /* 1557 /*
1556 * now see if we have a frag_list to map 1558 * now see if we have a frag_list to map
1557 */ 1559 */
1558 skb_walk_frags(skb, frag_iter) { 1560 skb_walk_frags(skb, frag_iter) {
1559 if (!tlen) 1561 if (!tlen)
1560 break; 1562 break;
1561 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk)) 1563 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk))
1562 break; 1564 break;
1563 } 1565 }
1564 1566
1565 done: 1567 done:
1566 if (spd.nr_pages) { 1568 if (spd.nr_pages) {
1567 /* 1569 /*
1568 * Drop the socket lock, otherwise we have reverse 1570 * Drop the socket lock, otherwise we have reverse
1569 * locking dependencies between sk_lock and i_mutex 1571 * locking dependencies between sk_lock and i_mutex
1570 * here as compared to sendfile(). We enter here 1572 * here as compared to sendfile(). We enter here
1571 * with the socket lock held, and splice_to_pipe() will 1573 * with the socket lock held, and splice_to_pipe() will
1572 * grab the pipe inode lock. For sendfile() emulation, 1574 * grab the pipe inode lock. For sendfile() emulation,
1573 * we call into ->sendpage() with the i_mutex lock held 1575 * we call into ->sendpage() with the i_mutex lock held
1574 * and networking will grab the socket lock. 1576 * and networking will grab the socket lock.
1575 */ 1577 */
1576 release_sock(sk); 1578 release_sock(sk);
1577 ret = splice_to_pipe(pipe, &spd); 1579 ret = splice_to_pipe(pipe, &spd);
1578 lock_sock(sk); 1580 lock_sock(sk);
1579 } 1581 }
1580 1582
1581 splice_shrink_spd(pipe, &spd); 1583 splice_shrink_spd(pipe, &spd);
1582 return ret; 1584 return ret;
1583 } 1585 }
1584 1586
1585 /** 1587 /**
1586 * skb_store_bits - store bits from kernel buffer to skb 1588 * skb_store_bits - store bits from kernel buffer to skb
1587 * @skb: destination buffer 1589 * @skb: destination buffer
1588 * @offset: offset in destination 1590 * @offset: offset in destination
1589 * @from: source buffer 1591 * @from: source buffer
1590 * @len: number of bytes to copy 1592 * @len: number of bytes to copy
1591 * 1593 *
1592 * Copy the specified number of bytes from the source buffer to the 1594 * Copy the specified number of bytes from the source buffer to the
1593 * destination skb. This function handles all the messy bits of 1595 * destination skb. This function handles all the messy bits of
1594 * traversing fragment lists and such. 1596 * traversing fragment lists and such.
1595 */ 1597 */
1596 1598
1597 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 1599 int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len)
1598 { 1600 {
1599 int start = skb_headlen(skb); 1601 int start = skb_headlen(skb);
1600 struct sk_buff *frag_iter; 1602 struct sk_buff *frag_iter;
1601 int i, copy; 1603 int i, copy;
1602 1604
1603 if (offset > (int)skb->len - len) 1605 if (offset > (int)skb->len - len)
1604 goto fault; 1606 goto fault;
1605 1607
1606 if ((copy = start - offset) > 0) { 1608 if ((copy = start - offset) > 0) {
1607 if (copy > len) 1609 if (copy > len)
1608 copy = len; 1610 copy = len;
1609 skb_copy_to_linear_data_offset(skb, offset, from, copy); 1611 skb_copy_to_linear_data_offset(skb, offset, from, copy);
1610 if ((len -= copy) == 0) 1612 if ((len -= copy) == 0)
1611 return 0; 1613 return 0;
1612 offset += copy; 1614 offset += copy;
1613 from += copy; 1615 from += copy;
1614 } 1616 }
1615 1617
1616 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1618 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1617 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1619 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1618 int end; 1620 int end;
1619 1621
1620 WARN_ON(start > offset + len); 1622 WARN_ON(start > offset + len);
1621 1623
1622 end = start + frag->size; 1624 end = start + frag->size;
1623 if ((copy = end - offset) > 0) { 1625 if ((copy = end - offset) > 0) {
1624 u8 *vaddr; 1626 u8 *vaddr;
1625 1627
1626 if (copy > len) 1628 if (copy > len)
1627 copy = len; 1629 copy = len;
1628 1630
1629 vaddr = kmap_skb_frag(frag); 1631 vaddr = kmap_skb_frag(frag);
1630 memcpy(vaddr + frag->page_offset + offset - start, 1632 memcpy(vaddr + frag->page_offset + offset - start,
1631 from, copy); 1633 from, copy);
1632 kunmap_skb_frag(vaddr); 1634 kunmap_skb_frag(vaddr);
1633 1635
1634 if ((len -= copy) == 0) 1636 if ((len -= copy) == 0)
1635 return 0; 1637 return 0;
1636 offset += copy; 1638 offset += copy;
1637 from += copy; 1639 from += copy;
1638 } 1640 }
1639 start = end; 1641 start = end;
1640 } 1642 }
1641 1643
1642 skb_walk_frags(skb, frag_iter) { 1644 skb_walk_frags(skb, frag_iter) {
1643 int end; 1645 int end;
1644 1646
1645 WARN_ON(start > offset + len); 1647 WARN_ON(start > offset + len);
1646 1648
1647 end = start + frag_iter->len; 1649 end = start + frag_iter->len;
1648 if ((copy = end - offset) > 0) { 1650 if ((copy = end - offset) > 0) {
1649 if (copy > len) 1651 if (copy > len)
1650 copy = len; 1652 copy = len;
1651 if (skb_store_bits(frag_iter, offset - start, 1653 if (skb_store_bits(frag_iter, offset - start,
1652 from, copy)) 1654 from, copy))
1653 goto fault; 1655 goto fault;
1654 if ((len -= copy) == 0) 1656 if ((len -= copy) == 0)
1655 return 0; 1657 return 0;
1656 offset += copy; 1658 offset += copy;
1657 from += copy; 1659 from += copy;
1658 } 1660 }
1659 start = end; 1661 start = end;
1660 } 1662 }
1661 if (!len) 1663 if (!len)
1662 return 0; 1664 return 0;
1663 1665
1664 fault: 1666 fault:
1665 return -EFAULT; 1667 return -EFAULT;
1666 } 1668 }
1667 EXPORT_SYMBOL(skb_store_bits); 1669 EXPORT_SYMBOL(skb_store_bits);
1668 1670
1669 /* Checksum skb data. */ 1671 /* Checksum skb data. */
1670 1672
1671 __wsum skb_checksum(const struct sk_buff *skb, int offset, 1673 __wsum skb_checksum(const struct sk_buff *skb, int offset,
1672 int len, __wsum csum) 1674 int len, __wsum csum)
1673 { 1675 {
1674 int start = skb_headlen(skb); 1676 int start = skb_headlen(skb);
1675 int i, copy = start - offset; 1677 int i, copy = start - offset;
1676 struct sk_buff *frag_iter; 1678 struct sk_buff *frag_iter;
1677 int pos = 0; 1679 int pos = 0;
1678 1680
1679 /* Checksum header. */ 1681 /* Checksum header. */
1680 if (copy > 0) { 1682 if (copy > 0) {
1681 if (copy > len) 1683 if (copy > len)
1682 copy = len; 1684 copy = len;
1683 csum = csum_partial(skb->data + offset, copy, csum); 1685 csum = csum_partial(skb->data + offset, copy, csum);
1684 if ((len -= copy) == 0) 1686 if ((len -= copy) == 0)
1685 return csum; 1687 return csum;
1686 offset += copy; 1688 offset += copy;
1687 pos = copy; 1689 pos = copy;
1688 } 1690 }
1689 1691
1690 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1692 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1691 int end; 1693 int end;
1692 1694
1693 WARN_ON(start > offset + len); 1695 WARN_ON(start > offset + len);
1694 1696
1695 end = start + skb_shinfo(skb)->frags[i].size; 1697 end = start + skb_shinfo(skb)->frags[i].size;
1696 if ((copy = end - offset) > 0) { 1698 if ((copy = end - offset) > 0) {
1697 __wsum csum2; 1699 __wsum csum2;
1698 u8 *vaddr; 1700 u8 *vaddr;
1699 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1701 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1700 1702
1701 if (copy > len) 1703 if (copy > len)
1702 copy = len; 1704 copy = len;
1703 vaddr = kmap_skb_frag(frag); 1705 vaddr = kmap_skb_frag(frag);
1704 csum2 = csum_partial(vaddr + frag->page_offset + 1706 csum2 = csum_partial(vaddr + frag->page_offset +
1705 offset - start, copy, 0); 1707 offset - start, copy, 0);
1706 kunmap_skb_frag(vaddr); 1708 kunmap_skb_frag(vaddr);
1707 csum = csum_block_add(csum, csum2, pos); 1709 csum = csum_block_add(csum, csum2, pos);
1708 if (!(len -= copy)) 1710 if (!(len -= copy))
1709 return csum; 1711 return csum;
1710 offset += copy; 1712 offset += copy;
1711 pos += copy; 1713 pos += copy;
1712 } 1714 }
1713 start = end; 1715 start = end;
1714 } 1716 }
1715 1717
1716 skb_walk_frags(skb, frag_iter) { 1718 skb_walk_frags(skb, frag_iter) {
1717 int end; 1719 int end;
1718 1720
1719 WARN_ON(start > offset + len); 1721 WARN_ON(start > offset + len);
1720 1722
1721 end = start + frag_iter->len; 1723 end = start + frag_iter->len;
1722 if ((copy = end - offset) > 0) { 1724 if ((copy = end - offset) > 0) {
1723 __wsum csum2; 1725 __wsum csum2;
1724 if (copy > len) 1726 if (copy > len)
1725 copy = len; 1727 copy = len;
1726 csum2 = skb_checksum(frag_iter, offset - start, 1728 csum2 = skb_checksum(frag_iter, offset - start,
1727 copy, 0); 1729 copy, 0);
1728 csum = csum_block_add(csum, csum2, pos); 1730 csum = csum_block_add(csum, csum2, pos);
1729 if ((len -= copy) == 0) 1731 if ((len -= copy) == 0)
1730 return csum; 1732 return csum;
1731 offset += copy; 1733 offset += copy;
1732 pos += copy; 1734 pos += copy;
1733 } 1735 }
1734 start = end; 1736 start = end;
1735 } 1737 }
1736 BUG_ON(len); 1738 BUG_ON(len);
1737 1739
1738 return csum; 1740 return csum;
1739 } 1741 }
1740 EXPORT_SYMBOL(skb_checksum); 1742 EXPORT_SYMBOL(skb_checksum);
1741 1743
1742 /* Both of above in one bottle. */ 1744 /* Both of above in one bottle. */
1743 1745
1744 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 1746 __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset,
1745 u8 *to, int len, __wsum csum) 1747 u8 *to, int len, __wsum csum)
1746 { 1748 {
1747 int start = skb_headlen(skb); 1749 int start = skb_headlen(skb);
1748 int i, copy = start - offset; 1750 int i, copy = start - offset;
1749 struct sk_buff *frag_iter; 1751 struct sk_buff *frag_iter;
1750 int pos = 0; 1752 int pos = 0;
1751 1753
1752 /* Copy header. */ 1754 /* Copy header. */
1753 if (copy > 0) { 1755 if (copy > 0) {
1754 if (copy > len) 1756 if (copy > len)
1755 copy = len; 1757 copy = len;
1756 csum = csum_partial_copy_nocheck(skb->data + offset, to, 1758 csum = csum_partial_copy_nocheck(skb->data + offset, to,
1757 copy, csum); 1759 copy, csum);
1758 if ((len -= copy) == 0) 1760 if ((len -= copy) == 0)
1759 return csum; 1761 return csum;
1760 offset += copy; 1762 offset += copy;
1761 to += copy; 1763 to += copy;
1762 pos = copy; 1764 pos = copy;
1763 } 1765 }
1764 1766
1765 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1767 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1766 int end; 1768 int end;
1767 1769
1768 WARN_ON(start > offset + len); 1770 WARN_ON(start > offset + len);
1769 1771
1770 end = start + skb_shinfo(skb)->frags[i].size; 1772 end = start + skb_shinfo(skb)->frags[i].size;
1771 if ((copy = end - offset) > 0) { 1773 if ((copy = end - offset) > 0) {
1772 __wsum csum2; 1774 __wsum csum2;
1773 u8 *vaddr; 1775 u8 *vaddr;
1774 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1776 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1775 1777
1776 if (copy > len) 1778 if (copy > len)
1777 copy = len; 1779 copy = len;
1778 vaddr = kmap_skb_frag(frag); 1780 vaddr = kmap_skb_frag(frag);
1779 csum2 = csum_partial_copy_nocheck(vaddr + 1781 csum2 = csum_partial_copy_nocheck(vaddr +
1780 frag->page_offset + 1782 frag->page_offset +
1781 offset - start, to, 1783 offset - start, to,
1782 copy, 0); 1784 copy, 0);
1783 kunmap_skb_frag(vaddr); 1785 kunmap_skb_frag(vaddr);
1784 csum = csum_block_add(csum, csum2, pos); 1786 csum = csum_block_add(csum, csum2, pos);
1785 if (!(len -= copy)) 1787 if (!(len -= copy))
1786 return csum; 1788 return csum;
1787 offset += copy; 1789 offset += copy;
1788 to += copy; 1790 to += copy;
1789 pos += copy; 1791 pos += copy;
1790 } 1792 }
1791 start = end; 1793 start = end;
1792 } 1794 }
1793 1795
1794 skb_walk_frags(skb, frag_iter) { 1796 skb_walk_frags(skb, frag_iter) {
1795 __wsum csum2; 1797 __wsum csum2;
1796 int end; 1798 int end;
1797 1799
1798 WARN_ON(start > offset + len); 1800 WARN_ON(start > offset + len);
1799 1801
1800 end = start + frag_iter->len; 1802 end = start + frag_iter->len;
1801 if ((copy = end - offset) > 0) { 1803 if ((copy = end - offset) > 0) {
1802 if (copy > len) 1804 if (copy > len)
1803 copy = len; 1805 copy = len;
1804 csum2 = skb_copy_and_csum_bits(frag_iter, 1806 csum2 = skb_copy_and_csum_bits(frag_iter,
1805 offset - start, 1807 offset - start,
1806 to, copy, 0); 1808 to, copy, 0);
1807 csum = csum_block_add(csum, csum2, pos); 1809 csum = csum_block_add(csum, csum2, pos);
1808 if ((len -= copy) == 0) 1810 if ((len -= copy) == 0)
1809 return csum; 1811 return csum;
1810 offset += copy; 1812 offset += copy;
1811 to += copy; 1813 to += copy;
1812 pos += copy; 1814 pos += copy;
1813 } 1815 }
1814 start = end; 1816 start = end;
1815 } 1817 }
1816 BUG_ON(len); 1818 BUG_ON(len);
1817 return csum; 1819 return csum;
1818 } 1820 }
1819 EXPORT_SYMBOL(skb_copy_and_csum_bits); 1821 EXPORT_SYMBOL(skb_copy_and_csum_bits);
1820 1822
1821 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 1823 void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to)
1822 { 1824 {
1823 __wsum csum; 1825 __wsum csum;
1824 long csstart; 1826 long csstart;
1825 1827
1826 if (skb->ip_summed == CHECKSUM_PARTIAL) 1828 if (skb->ip_summed == CHECKSUM_PARTIAL)
1827 csstart = skb_checksum_start_offset(skb); 1829 csstart = skb_checksum_start_offset(skb);
1828 else 1830 else
1829 csstart = skb_headlen(skb); 1831 csstart = skb_headlen(skb);
1830 1832
1831 BUG_ON(csstart > skb_headlen(skb)); 1833 BUG_ON(csstart > skb_headlen(skb));
1832 1834
1833 skb_copy_from_linear_data(skb, to, csstart); 1835 skb_copy_from_linear_data(skb, to, csstart);
1834 1836
1835 csum = 0; 1837 csum = 0;
1836 if (csstart != skb->len) 1838 if (csstart != skb->len)
1837 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 1839 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart,
1838 skb->len - csstart, 0); 1840 skb->len - csstart, 0);
1839 1841
1840 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1842 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1841 long csstuff = csstart + skb->csum_offset; 1843 long csstuff = csstart + skb->csum_offset;
1842 1844
1843 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 1845 *((__sum16 *)(to + csstuff)) = csum_fold(csum);
1844 } 1846 }
1845 } 1847 }
1846 EXPORT_SYMBOL(skb_copy_and_csum_dev); 1848 EXPORT_SYMBOL(skb_copy_and_csum_dev);
1847 1849
1848 /** 1850 /**
1849 * skb_dequeue - remove from the head of the queue 1851 * skb_dequeue - remove from the head of the queue
1850 * @list: list to dequeue from 1852 * @list: list to dequeue from
1851 * 1853 *
1852 * Remove the head of the list. The list lock is taken so the function 1854 * Remove the head of the list. The list lock is taken so the function
1853 * may be used safely with other locking list functions. The head item is 1855 * may be used safely with other locking list functions. The head item is
1854 * returned or %NULL if the list is empty. 1856 * returned or %NULL if the list is empty.
1855 */ 1857 */
1856 1858
1857 struct sk_buff *skb_dequeue(struct sk_buff_head *list) 1859 struct sk_buff *skb_dequeue(struct sk_buff_head *list)
1858 { 1860 {
1859 unsigned long flags; 1861 unsigned long flags;
1860 struct sk_buff *result; 1862 struct sk_buff *result;
1861 1863
1862 spin_lock_irqsave(&list->lock, flags); 1864 spin_lock_irqsave(&list->lock, flags);
1863 result = __skb_dequeue(list); 1865 result = __skb_dequeue(list);
1864 spin_unlock_irqrestore(&list->lock, flags); 1866 spin_unlock_irqrestore(&list->lock, flags);
1865 return result; 1867 return result;
1866 } 1868 }
1867 EXPORT_SYMBOL(skb_dequeue); 1869 EXPORT_SYMBOL(skb_dequeue);
1868 1870
1869 /** 1871 /**
1870 * skb_dequeue_tail - remove from the tail of the queue 1872 * skb_dequeue_tail - remove from the tail of the queue
1871 * @list: list to dequeue from 1873 * @list: list to dequeue from
1872 * 1874 *
1873 * Remove the tail of the list. The list lock is taken so the function 1875 * Remove the tail of the list. The list lock is taken so the function
1874 * may be used safely with other locking list functions. The tail item is 1876 * may be used safely with other locking list functions. The tail item is
1875 * returned or %NULL if the list is empty. 1877 * returned or %NULL if the list is empty.
1876 */ 1878 */
1877 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 1879 struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list)
1878 { 1880 {
1879 unsigned long flags; 1881 unsigned long flags;
1880 struct sk_buff *result; 1882 struct sk_buff *result;
1881 1883
1882 spin_lock_irqsave(&list->lock, flags); 1884 spin_lock_irqsave(&list->lock, flags);
1883 result = __skb_dequeue_tail(list); 1885 result = __skb_dequeue_tail(list);
1884 spin_unlock_irqrestore(&list->lock, flags); 1886 spin_unlock_irqrestore(&list->lock, flags);
1885 return result; 1887 return result;
1886 } 1888 }
1887 EXPORT_SYMBOL(skb_dequeue_tail); 1889 EXPORT_SYMBOL(skb_dequeue_tail);
1888 1890
1889 /** 1891 /**
1890 * skb_queue_purge - empty a list 1892 * skb_queue_purge - empty a list
1891 * @list: list to empty 1893 * @list: list to empty
1892 * 1894 *
1893 * Delete all buffers on an &sk_buff list. Each buffer is removed from 1895 * Delete all buffers on an &sk_buff list. Each buffer is removed from
1894 * the list and one reference dropped. This function takes the list 1896 * the list and one reference dropped. This function takes the list
1895 * lock and is atomic with respect to other list locking functions. 1897 * lock and is atomic with respect to other list locking functions.
1896 */ 1898 */
1897 void skb_queue_purge(struct sk_buff_head *list) 1899 void skb_queue_purge(struct sk_buff_head *list)
1898 { 1900 {
1899 struct sk_buff *skb; 1901 struct sk_buff *skb;
1900 while ((skb = skb_dequeue(list)) != NULL) 1902 while ((skb = skb_dequeue(list)) != NULL)
1901 kfree_skb(skb); 1903 kfree_skb(skb);
1902 } 1904 }
1903 EXPORT_SYMBOL(skb_queue_purge); 1905 EXPORT_SYMBOL(skb_queue_purge);
1904 1906
1905 /** 1907 /**
1906 * skb_queue_head - queue a buffer at the list head 1908 * skb_queue_head - queue a buffer at the list head
1907 * @list: list to use 1909 * @list: list to use
1908 * @newsk: buffer to queue 1910 * @newsk: buffer to queue
1909 * 1911 *
1910 * Queue a buffer at the start of the list. This function takes the 1912 * Queue a buffer at the start of the list. This function takes the
1911 * list lock and can be used safely with other locking &sk_buff functions 1913 * list lock and can be used safely with other locking &sk_buff functions
1912 * safely. 1914 * safely.
1913 * 1915 *
1914 * A buffer cannot be placed on two lists at the same time. 1916 * A buffer cannot be placed on two lists at the same time.
1915 */ 1917 */
1916 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 1918 void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk)
1917 { 1919 {
1918 unsigned long flags; 1920 unsigned long flags;
1919 1921
1920 spin_lock_irqsave(&list->lock, flags); 1922 spin_lock_irqsave(&list->lock, flags);
1921 __skb_queue_head(list, newsk); 1923 __skb_queue_head(list, newsk);
1922 spin_unlock_irqrestore(&list->lock, flags); 1924 spin_unlock_irqrestore(&list->lock, flags);
1923 } 1925 }
1924 EXPORT_SYMBOL(skb_queue_head); 1926 EXPORT_SYMBOL(skb_queue_head);
1925 1927
1926 /** 1928 /**
1927 * skb_queue_tail - queue a buffer at the list tail 1929 * skb_queue_tail - queue a buffer at the list tail
1928 * @list: list to use 1930 * @list: list to use
1929 * @newsk: buffer to queue 1931 * @newsk: buffer to queue
1930 * 1932 *
1931 * Queue a buffer at the tail of the list. This function takes the 1933 * Queue a buffer at the tail of the list. This function takes the
1932 * list lock and can be used safely with other locking &sk_buff functions 1934 * list lock and can be used safely with other locking &sk_buff functions
1933 * safely. 1935 * safely.
1934 * 1936 *
1935 * A buffer cannot be placed on two lists at the same time. 1937 * A buffer cannot be placed on two lists at the same time.
1936 */ 1938 */
1937 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 1939 void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk)
1938 { 1940 {
1939 unsigned long flags; 1941 unsigned long flags;
1940 1942
1941 spin_lock_irqsave(&list->lock, flags); 1943 spin_lock_irqsave(&list->lock, flags);
1942 __skb_queue_tail(list, newsk); 1944 __skb_queue_tail(list, newsk);
1943 spin_unlock_irqrestore(&list->lock, flags); 1945 spin_unlock_irqrestore(&list->lock, flags);
1944 } 1946 }
1945 EXPORT_SYMBOL(skb_queue_tail); 1947 EXPORT_SYMBOL(skb_queue_tail);
1946 1948
1947 /** 1949 /**
1948 * skb_unlink - remove a buffer from a list 1950 * skb_unlink - remove a buffer from a list
1949 * @skb: buffer to remove 1951 * @skb: buffer to remove
1950 * @list: list to use 1952 * @list: list to use
1951 * 1953 *
1952 * Remove a packet from a list. The list locks are taken and this 1954 * Remove a packet from a list. The list locks are taken and this
1953 * function is atomic with respect to other list locked calls 1955 * function is atomic with respect to other list locked calls
1954 * 1956 *
1955 * You must know what list the SKB is on. 1957 * You must know what list the SKB is on.
1956 */ 1958 */
1957 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 1959 void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list)
1958 { 1960 {
1959 unsigned long flags; 1961 unsigned long flags;
1960 1962
1961 spin_lock_irqsave(&list->lock, flags); 1963 spin_lock_irqsave(&list->lock, flags);
1962 __skb_unlink(skb, list); 1964 __skb_unlink(skb, list);
1963 spin_unlock_irqrestore(&list->lock, flags); 1965 spin_unlock_irqrestore(&list->lock, flags);
1964 } 1966 }
1965 EXPORT_SYMBOL(skb_unlink); 1967 EXPORT_SYMBOL(skb_unlink);
1966 1968
1967 /** 1969 /**
1968 * skb_append - append a buffer 1970 * skb_append - append a buffer
1969 * @old: buffer to insert after 1971 * @old: buffer to insert after
1970 * @newsk: buffer to insert 1972 * @newsk: buffer to insert
1971 * @list: list to use 1973 * @list: list to use
1972 * 1974 *
1973 * Place a packet after a given packet in a list. The list locks are taken 1975 * Place a packet after a given packet in a list. The list locks are taken
1974 * and this function is atomic with respect to other list locked calls. 1976 * and this function is atomic with respect to other list locked calls.
1975 * A buffer cannot be placed on two lists at the same time. 1977 * A buffer cannot be placed on two lists at the same time.
1976 */ 1978 */
1977 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 1979 void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
1978 { 1980 {
1979 unsigned long flags; 1981 unsigned long flags;
1980 1982
1981 spin_lock_irqsave(&list->lock, flags); 1983 spin_lock_irqsave(&list->lock, flags);
1982 __skb_queue_after(list, old, newsk); 1984 __skb_queue_after(list, old, newsk);
1983 spin_unlock_irqrestore(&list->lock, flags); 1985 spin_unlock_irqrestore(&list->lock, flags);
1984 } 1986 }
1985 EXPORT_SYMBOL(skb_append); 1987 EXPORT_SYMBOL(skb_append);
1986 1988
1987 /** 1989 /**
1988 * skb_insert - insert a buffer 1990 * skb_insert - insert a buffer
1989 * @old: buffer to insert before 1991 * @old: buffer to insert before
1990 * @newsk: buffer to insert 1992 * @newsk: buffer to insert
1991 * @list: list to use 1993 * @list: list to use
1992 * 1994 *
1993 * Place a packet before a given packet in a list. The list locks are 1995 * Place a packet before a given packet in a list. The list locks are
1994 * taken and this function is atomic with respect to other list locked 1996 * taken and this function is atomic with respect to other list locked
1995 * calls. 1997 * calls.
1996 * 1998 *
1997 * A buffer cannot be placed on two lists at the same time. 1999 * A buffer cannot be placed on two lists at the same time.
1998 */ 2000 */
1999 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2001 void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list)
2000 { 2002 {
2001 unsigned long flags; 2003 unsigned long flags;
2002 2004
2003 spin_lock_irqsave(&list->lock, flags); 2005 spin_lock_irqsave(&list->lock, flags);
2004 __skb_insert(newsk, old->prev, old, list); 2006 __skb_insert(newsk, old->prev, old, list);
2005 spin_unlock_irqrestore(&list->lock, flags); 2007 spin_unlock_irqrestore(&list->lock, flags);
2006 } 2008 }
2007 EXPORT_SYMBOL(skb_insert); 2009 EXPORT_SYMBOL(skb_insert);
2008 2010
2009 static inline void skb_split_inside_header(struct sk_buff *skb, 2011 static inline void skb_split_inside_header(struct sk_buff *skb,
2010 struct sk_buff* skb1, 2012 struct sk_buff* skb1,
2011 const u32 len, const int pos) 2013 const u32 len, const int pos)
2012 { 2014 {
2013 int i; 2015 int i;
2014 2016
2015 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 2017 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len),
2016 pos - len); 2018 pos - len);
2017 /* And move data appendix as is. */ 2019 /* And move data appendix as is. */
2018 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2020 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
2019 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 2021 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i];
2020 2022
2021 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 2023 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags;
2022 skb_shinfo(skb)->nr_frags = 0; 2024 skb_shinfo(skb)->nr_frags = 0;
2023 skb1->data_len = skb->data_len; 2025 skb1->data_len = skb->data_len;
2024 skb1->len += skb1->data_len; 2026 skb1->len += skb1->data_len;
2025 skb->data_len = 0; 2027 skb->data_len = 0;
2026 skb->len = len; 2028 skb->len = len;
2027 skb_set_tail_pointer(skb, len); 2029 skb_set_tail_pointer(skb, len);
2028 } 2030 }
2029 2031
2030 static inline void skb_split_no_header(struct sk_buff *skb, 2032 static inline void skb_split_no_header(struct sk_buff *skb,
2031 struct sk_buff* skb1, 2033 struct sk_buff* skb1,
2032 const u32 len, int pos) 2034 const u32 len, int pos)
2033 { 2035 {
2034 int i, k = 0; 2036 int i, k = 0;
2035 const int nfrags = skb_shinfo(skb)->nr_frags; 2037 const int nfrags = skb_shinfo(skb)->nr_frags;
2036 2038
2037 skb_shinfo(skb)->nr_frags = 0; 2039 skb_shinfo(skb)->nr_frags = 0;
2038 skb1->len = skb1->data_len = skb->len - len; 2040 skb1->len = skb1->data_len = skb->len - len;
2039 skb->len = len; 2041 skb->len = len;
2040 skb->data_len = len - pos; 2042 skb->data_len = len - pos;
2041 2043
2042 for (i = 0; i < nfrags; i++) { 2044 for (i = 0; i < nfrags; i++) {
2043 int size = skb_shinfo(skb)->frags[i].size; 2045 int size = skb_shinfo(skb)->frags[i].size;
2044 2046
2045 if (pos + size > len) { 2047 if (pos + size > len) {
2046 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 2048 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i];
2047 2049
2048 if (pos < len) { 2050 if (pos < len) {
2049 /* Split frag. 2051 /* Split frag.
2050 * We have two variants in this case: 2052 * We have two variants in this case:
2051 * 1. Move all the frag to the second 2053 * 1. Move all the frag to the second
2052 * part, if it is possible. F.e. 2054 * part, if it is possible. F.e.
2053 * this approach is mandatory for TUX, 2055 * this approach is mandatory for TUX,
2054 * where splitting is expensive. 2056 * where splitting is expensive.
2055 * 2. Split is accurately. We make this. 2057 * 2. Split is accurately. We make this.
2056 */ 2058 */
2057 get_page(skb_shinfo(skb)->frags[i].page); 2059 get_page(skb_shinfo(skb)->frags[i].page);
2058 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 2060 skb_shinfo(skb1)->frags[0].page_offset += len - pos;
2059 skb_shinfo(skb1)->frags[0].size -= len - pos; 2061 skb_shinfo(skb1)->frags[0].size -= len - pos;
2060 skb_shinfo(skb)->frags[i].size = len - pos; 2062 skb_shinfo(skb)->frags[i].size = len - pos;
2061 skb_shinfo(skb)->nr_frags++; 2063 skb_shinfo(skb)->nr_frags++;
2062 } 2064 }
2063 k++; 2065 k++;
2064 } else 2066 } else
2065 skb_shinfo(skb)->nr_frags++; 2067 skb_shinfo(skb)->nr_frags++;
2066 pos += size; 2068 pos += size;
2067 } 2069 }
2068 skb_shinfo(skb1)->nr_frags = k; 2070 skb_shinfo(skb1)->nr_frags = k;
2069 } 2071 }
2070 2072
2071 /** 2073 /**
2072 * skb_split - Split fragmented skb to two parts at length len. 2074 * skb_split - Split fragmented skb to two parts at length len.
2073 * @skb: the buffer to split 2075 * @skb: the buffer to split
2074 * @skb1: the buffer to receive the second part 2076 * @skb1: the buffer to receive the second part
2075 * @len: new length for skb 2077 * @len: new length for skb
2076 */ 2078 */
2077 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 2079 void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len)
2078 { 2080 {
2079 int pos = skb_headlen(skb); 2081 int pos = skb_headlen(skb);
2080 2082
2081 if (len < pos) /* Split line is inside header. */ 2083 if (len < pos) /* Split line is inside header. */
2082 skb_split_inside_header(skb, skb1, len, pos); 2084 skb_split_inside_header(skb, skb1, len, pos);
2083 else /* Second chunk has no header, nothing to copy. */ 2085 else /* Second chunk has no header, nothing to copy. */
2084 skb_split_no_header(skb, skb1, len, pos); 2086 skb_split_no_header(skb, skb1, len, pos);
2085 } 2087 }
2086 EXPORT_SYMBOL(skb_split); 2088 EXPORT_SYMBOL(skb_split);
2087 2089
2088 /* Shifting from/to a cloned skb is a no-go. 2090 /* Shifting from/to a cloned skb is a no-go.
2089 * 2091 *
2090 * Caller cannot keep skb_shinfo related pointers past calling here! 2092 * Caller cannot keep skb_shinfo related pointers past calling here!
2091 */ 2093 */
2092 static int skb_prepare_for_shift(struct sk_buff *skb) 2094 static int skb_prepare_for_shift(struct sk_buff *skb)
2093 { 2095 {
2094 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2096 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2095 } 2097 }
2096 2098
2097 /** 2099 /**
2098 * skb_shift - Shifts paged data partially from skb to another 2100 * skb_shift - Shifts paged data partially from skb to another
2099 * @tgt: buffer into which tail data gets added 2101 * @tgt: buffer into which tail data gets added
2100 * @skb: buffer from which the paged data comes from 2102 * @skb: buffer from which the paged data comes from
2101 * @shiftlen: shift up to this many bytes 2103 * @shiftlen: shift up to this many bytes
2102 * 2104 *
2103 * Attempts to shift up to shiftlen worth of bytes, which may be less than 2105 * Attempts to shift up to shiftlen worth of bytes, which may be less than
2104 * the length of the skb, from tgt to skb. Returns number bytes shifted. 2106 * the length of the skb, from tgt to skb. Returns number bytes shifted.
2105 * It's up to caller to free skb if everything was shifted. 2107 * It's up to caller to free skb if everything was shifted.
2106 * 2108 *
2107 * If @tgt runs out of frags, the whole operation is aborted. 2109 * If @tgt runs out of frags, the whole operation is aborted.
2108 * 2110 *
2109 * Skb cannot include anything else but paged data while tgt is allowed 2111 * Skb cannot include anything else but paged data while tgt is allowed
2110 * to have non-paged data as well. 2112 * to have non-paged data as well.
2111 * 2113 *
2112 * TODO: full sized shift could be optimized but that would need 2114 * TODO: full sized shift could be optimized but that would need
2113 * specialized skb free'er to handle frags without up-to-date nr_frags. 2115 * specialized skb free'er to handle frags without up-to-date nr_frags.
2114 */ 2116 */
2115 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 2117 int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen)
2116 { 2118 {
2117 int from, to, merge, todo; 2119 int from, to, merge, todo;
2118 struct skb_frag_struct *fragfrom, *fragto; 2120 struct skb_frag_struct *fragfrom, *fragto;
2119 2121
2120 BUG_ON(shiftlen > skb->len); 2122 BUG_ON(shiftlen > skb->len);
2121 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */ 2123 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */
2122 2124
2123 todo = shiftlen; 2125 todo = shiftlen;
2124 from = 0; 2126 from = 0;
2125 to = skb_shinfo(tgt)->nr_frags; 2127 to = skb_shinfo(tgt)->nr_frags;
2126 fragfrom = &skb_shinfo(skb)->frags[from]; 2128 fragfrom = &skb_shinfo(skb)->frags[from];
2127 2129
2128 /* Actual merge is delayed until the point when we know we can 2130 /* Actual merge is delayed until the point when we know we can
2129 * commit all, so that we don't have to undo partial changes 2131 * commit all, so that we don't have to undo partial changes
2130 */ 2132 */
2131 if (!to || 2133 if (!to ||
2132 !skb_can_coalesce(tgt, to, fragfrom->page, fragfrom->page_offset)) { 2134 !skb_can_coalesce(tgt, to, fragfrom->page, fragfrom->page_offset)) {
2133 merge = -1; 2135 merge = -1;
2134 } else { 2136 } else {
2135 merge = to - 1; 2137 merge = to - 1;
2136 2138
2137 todo -= fragfrom->size; 2139 todo -= fragfrom->size;
2138 if (todo < 0) { 2140 if (todo < 0) {
2139 if (skb_prepare_for_shift(skb) || 2141 if (skb_prepare_for_shift(skb) ||
2140 skb_prepare_for_shift(tgt)) 2142 skb_prepare_for_shift(tgt))
2141 return 0; 2143 return 0;
2142 2144
2143 /* All previous frag pointers might be stale! */ 2145 /* All previous frag pointers might be stale! */
2144 fragfrom = &skb_shinfo(skb)->frags[from]; 2146 fragfrom = &skb_shinfo(skb)->frags[from];
2145 fragto = &skb_shinfo(tgt)->frags[merge]; 2147 fragto = &skb_shinfo(tgt)->frags[merge];
2146 2148
2147 fragto->size += shiftlen; 2149 fragto->size += shiftlen;
2148 fragfrom->size -= shiftlen; 2150 fragfrom->size -= shiftlen;
2149 fragfrom->page_offset += shiftlen; 2151 fragfrom->page_offset += shiftlen;
2150 2152
2151 goto onlymerged; 2153 goto onlymerged;
2152 } 2154 }
2153 2155
2154 from++; 2156 from++;
2155 } 2157 }
2156 2158
2157 /* Skip full, not-fitting skb to avoid expensive operations */ 2159 /* Skip full, not-fitting skb to avoid expensive operations */
2158 if ((shiftlen == skb->len) && 2160 if ((shiftlen == skb->len) &&
2159 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 2161 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to))
2160 return 0; 2162 return 0;
2161 2163
2162 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 2164 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt))
2163 return 0; 2165 return 0;
2164 2166
2165 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 2167 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
2166 if (to == MAX_SKB_FRAGS) 2168 if (to == MAX_SKB_FRAGS)
2167 return 0; 2169 return 0;
2168 2170
2169 fragfrom = &skb_shinfo(skb)->frags[from]; 2171 fragfrom = &skb_shinfo(skb)->frags[from];
2170 fragto = &skb_shinfo(tgt)->frags[to]; 2172 fragto = &skb_shinfo(tgt)->frags[to];
2171 2173
2172 if (todo >= fragfrom->size) { 2174 if (todo >= fragfrom->size) {
2173 *fragto = *fragfrom; 2175 *fragto = *fragfrom;
2174 todo -= fragfrom->size; 2176 todo -= fragfrom->size;
2175 from++; 2177 from++;
2176 to++; 2178 to++;
2177 2179
2178 } else { 2180 } else {
2179 get_page(fragfrom->page); 2181 get_page(fragfrom->page);
2180 fragto->page = fragfrom->page; 2182 fragto->page = fragfrom->page;
2181 fragto->page_offset = fragfrom->page_offset; 2183 fragto->page_offset = fragfrom->page_offset;
2182 fragto->size = todo; 2184 fragto->size = todo;
2183 2185
2184 fragfrom->page_offset += todo; 2186 fragfrom->page_offset += todo;
2185 fragfrom->size -= todo; 2187 fragfrom->size -= todo;
2186 todo = 0; 2188 todo = 0;
2187 2189
2188 to++; 2190 to++;
2189 break; 2191 break;
2190 } 2192 }
2191 } 2193 }
2192 2194
2193 /* Ready to "commit" this state change to tgt */ 2195 /* Ready to "commit" this state change to tgt */
2194 skb_shinfo(tgt)->nr_frags = to; 2196 skb_shinfo(tgt)->nr_frags = to;
2195 2197
2196 if (merge >= 0) { 2198 if (merge >= 0) {
2197 fragfrom = &skb_shinfo(skb)->frags[0]; 2199 fragfrom = &skb_shinfo(skb)->frags[0];
2198 fragto = &skb_shinfo(tgt)->frags[merge]; 2200 fragto = &skb_shinfo(tgt)->frags[merge];
2199 2201
2200 fragto->size += fragfrom->size; 2202 fragto->size += fragfrom->size;
2201 put_page(fragfrom->page); 2203 put_page(fragfrom->page);
2202 } 2204 }
2203 2205
2204 /* Reposition in the original skb */ 2206 /* Reposition in the original skb */
2205 to = 0; 2207 to = 0;
2206 while (from < skb_shinfo(skb)->nr_frags) 2208 while (from < skb_shinfo(skb)->nr_frags)
2207 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 2209 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
2208 skb_shinfo(skb)->nr_frags = to; 2210 skb_shinfo(skb)->nr_frags = to;
2209 2211
2210 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 2212 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
2211 2213
2212 onlymerged: 2214 onlymerged:
2213 /* Most likely the tgt won't ever need its checksum anymore, skb on 2215 /* Most likely the tgt won't ever need its checksum anymore, skb on
2214 * the other hand might need it if it needs to be resent 2216 * the other hand might need it if it needs to be resent
2215 */ 2217 */
2216 tgt->ip_summed = CHECKSUM_PARTIAL; 2218 tgt->ip_summed = CHECKSUM_PARTIAL;
2217 skb->ip_summed = CHECKSUM_PARTIAL; 2219 skb->ip_summed = CHECKSUM_PARTIAL;
2218 2220
2219 /* Yak, is it really working this way? Some helper please? */ 2221 /* Yak, is it really working this way? Some helper please? */
2220 skb->len -= shiftlen; 2222 skb->len -= shiftlen;
2221 skb->data_len -= shiftlen; 2223 skb->data_len -= shiftlen;
2222 skb->truesize -= shiftlen; 2224 skb->truesize -= shiftlen;
2223 tgt->len += shiftlen; 2225 tgt->len += shiftlen;
2224 tgt->data_len += shiftlen; 2226 tgt->data_len += shiftlen;
2225 tgt->truesize += shiftlen; 2227 tgt->truesize += shiftlen;
2226 2228
2227 return shiftlen; 2229 return shiftlen;
2228 } 2230 }
2229 2231
2230 /** 2232 /**
2231 * skb_prepare_seq_read - Prepare a sequential read of skb data 2233 * skb_prepare_seq_read - Prepare a sequential read of skb data
2232 * @skb: the buffer to read 2234 * @skb: the buffer to read
2233 * @from: lower offset of data to be read 2235 * @from: lower offset of data to be read
2234 * @to: upper offset of data to be read 2236 * @to: upper offset of data to be read
2235 * @st: state variable 2237 * @st: state variable
2236 * 2238 *
2237 * Initializes the specified state variable. Must be called before 2239 * Initializes the specified state variable. Must be called before
2238 * invoking skb_seq_read() for the first time. 2240 * invoking skb_seq_read() for the first time.
2239 */ 2241 */
2240 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 2242 void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from,
2241 unsigned int to, struct skb_seq_state *st) 2243 unsigned int to, struct skb_seq_state *st)
2242 { 2244 {
2243 st->lower_offset = from; 2245 st->lower_offset = from;
2244 st->upper_offset = to; 2246 st->upper_offset = to;
2245 st->root_skb = st->cur_skb = skb; 2247 st->root_skb = st->cur_skb = skb;
2246 st->frag_idx = st->stepped_offset = 0; 2248 st->frag_idx = st->stepped_offset = 0;
2247 st->frag_data = NULL; 2249 st->frag_data = NULL;
2248 } 2250 }
2249 EXPORT_SYMBOL(skb_prepare_seq_read); 2251 EXPORT_SYMBOL(skb_prepare_seq_read);
2250 2252
2251 /** 2253 /**
2252 * skb_seq_read - Sequentially read skb data 2254 * skb_seq_read - Sequentially read skb data
2253 * @consumed: number of bytes consumed by the caller so far 2255 * @consumed: number of bytes consumed by the caller so far
2254 * @data: destination pointer for data to be returned 2256 * @data: destination pointer for data to be returned
2255 * @st: state variable 2257 * @st: state variable
2256 * 2258 *
2257 * Reads a block of skb data at &consumed relative to the 2259 * Reads a block of skb data at &consumed relative to the
2258 * lower offset specified to skb_prepare_seq_read(). Assigns 2260 * lower offset specified to skb_prepare_seq_read(). Assigns
2259 * the head of the data block to &data and returns the length 2261 * the head of the data block to &data and returns the length
2260 * of the block or 0 if the end of the skb data or the upper 2262 * of the block or 0 if the end of the skb data or the upper
2261 * offset has been reached. 2263 * offset has been reached.
2262 * 2264 *
2263 * The caller is not required to consume all of the data 2265 * The caller is not required to consume all of the data
2264 * returned, i.e. &consumed is typically set to the number 2266 * returned, i.e. &consumed is typically set to the number
2265 * of bytes already consumed and the next call to 2267 * of bytes already consumed and the next call to
2266 * skb_seq_read() will return the remaining part of the block. 2268 * skb_seq_read() will return the remaining part of the block.
2267 * 2269 *
2268 * Note 1: The size of each block of data returned can be arbitary, 2270 * Note 1: The size of each block of data returned can be arbitary,
2269 * this limitation is the cost for zerocopy seqeuental 2271 * this limitation is the cost for zerocopy seqeuental
2270 * reads of potentially non linear data. 2272 * reads of potentially non linear data.
2271 * 2273 *
2272 * Note 2: Fragment lists within fragments are not implemented 2274 * Note 2: Fragment lists within fragments are not implemented
2273 * at the moment, state->root_skb could be replaced with 2275 * at the moment, state->root_skb could be replaced with
2274 * a stack for this purpose. 2276 * a stack for this purpose.
2275 */ 2277 */
2276 unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 2278 unsigned int skb_seq_read(unsigned int consumed, const u8 **data,
2277 struct skb_seq_state *st) 2279 struct skb_seq_state *st)
2278 { 2280 {
2279 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 2281 unsigned int block_limit, abs_offset = consumed + st->lower_offset;
2280 skb_frag_t *frag; 2282 skb_frag_t *frag;
2281 2283
2282 if (unlikely(abs_offset >= st->upper_offset)) 2284 if (unlikely(abs_offset >= st->upper_offset))
2283 return 0; 2285 return 0;
2284 2286
2285 next_skb: 2287 next_skb:
2286 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 2288 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset;
2287 2289
2288 if (abs_offset < block_limit && !st->frag_data) { 2290 if (abs_offset < block_limit && !st->frag_data) {
2289 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 2291 *data = st->cur_skb->data + (abs_offset - st->stepped_offset);
2290 return block_limit - abs_offset; 2292 return block_limit - abs_offset;
2291 } 2293 }
2292 2294
2293 if (st->frag_idx == 0 && !st->frag_data) 2295 if (st->frag_idx == 0 && !st->frag_data)
2294 st->stepped_offset += skb_headlen(st->cur_skb); 2296 st->stepped_offset += skb_headlen(st->cur_skb);
2295 2297
2296 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 2298 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) {
2297 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 2299 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx];
2298 block_limit = frag->size + st->stepped_offset; 2300 block_limit = frag->size + st->stepped_offset;
2299 2301
2300 if (abs_offset < block_limit) { 2302 if (abs_offset < block_limit) {
2301 if (!st->frag_data) 2303 if (!st->frag_data)
2302 st->frag_data = kmap_skb_frag(frag); 2304 st->frag_data = kmap_skb_frag(frag);
2303 2305
2304 *data = (u8 *) st->frag_data + frag->page_offset + 2306 *data = (u8 *) st->frag_data + frag->page_offset +
2305 (abs_offset - st->stepped_offset); 2307 (abs_offset - st->stepped_offset);
2306 2308
2307 return block_limit - abs_offset; 2309 return block_limit - abs_offset;
2308 } 2310 }
2309 2311
2310 if (st->frag_data) { 2312 if (st->frag_data) {
2311 kunmap_skb_frag(st->frag_data); 2313 kunmap_skb_frag(st->frag_data);
2312 st->frag_data = NULL; 2314 st->frag_data = NULL;
2313 } 2315 }
2314 2316
2315 st->frag_idx++; 2317 st->frag_idx++;
2316 st->stepped_offset += frag->size; 2318 st->stepped_offset += frag->size;
2317 } 2319 }
2318 2320
2319 if (st->frag_data) { 2321 if (st->frag_data) {
2320 kunmap_skb_frag(st->frag_data); 2322 kunmap_skb_frag(st->frag_data);
2321 st->frag_data = NULL; 2323 st->frag_data = NULL;
2322 } 2324 }
2323 2325
2324 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 2326 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) {
2325 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 2327 st->cur_skb = skb_shinfo(st->root_skb)->frag_list;
2326 st->frag_idx = 0; 2328 st->frag_idx = 0;
2327 goto next_skb; 2329 goto next_skb;
2328 } else if (st->cur_skb->next) { 2330 } else if (st->cur_skb->next) {
2329 st->cur_skb = st->cur_skb->next; 2331 st->cur_skb = st->cur_skb->next;
2330 st->frag_idx = 0; 2332 st->frag_idx = 0;
2331 goto next_skb; 2333 goto next_skb;
2332 } 2334 }
2333 2335
2334 return 0; 2336 return 0;
2335 } 2337 }
2336 EXPORT_SYMBOL(skb_seq_read); 2338 EXPORT_SYMBOL(skb_seq_read);
2337 2339
2338 /** 2340 /**
2339 * skb_abort_seq_read - Abort a sequential read of skb data 2341 * skb_abort_seq_read - Abort a sequential read of skb data
2340 * @st: state variable 2342 * @st: state variable
2341 * 2343 *
2342 * Must be called if skb_seq_read() was not called until it 2344 * Must be called if skb_seq_read() was not called until it
2343 * returned 0. 2345 * returned 0.
2344 */ 2346 */
2345 void skb_abort_seq_read(struct skb_seq_state *st) 2347 void skb_abort_seq_read(struct skb_seq_state *st)
2346 { 2348 {
2347 if (st->frag_data) 2349 if (st->frag_data)
2348 kunmap_skb_frag(st->frag_data); 2350 kunmap_skb_frag(st->frag_data);
2349 } 2351 }
2350 EXPORT_SYMBOL(skb_abort_seq_read); 2352 EXPORT_SYMBOL(skb_abort_seq_read);
2351 2353
2352 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 2354 #define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb))
2353 2355
2354 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 2356 static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text,
2355 struct ts_config *conf, 2357 struct ts_config *conf,
2356 struct ts_state *state) 2358 struct ts_state *state)
2357 { 2359 {
2358 return skb_seq_read(offset, text, TS_SKB_CB(state)); 2360 return skb_seq_read(offset, text, TS_SKB_CB(state));
2359 } 2361 }
2360 2362
2361 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 2363 static void skb_ts_finish(struct ts_config *conf, struct ts_state *state)
2362 { 2364 {
2363 skb_abort_seq_read(TS_SKB_CB(state)); 2365 skb_abort_seq_read(TS_SKB_CB(state));
2364 } 2366 }
2365 2367
2366 /** 2368 /**
2367 * skb_find_text - Find a text pattern in skb data 2369 * skb_find_text - Find a text pattern in skb data
2368 * @skb: the buffer to look in 2370 * @skb: the buffer to look in
2369 * @from: search offset 2371 * @from: search offset
2370 * @to: search limit 2372 * @to: search limit
2371 * @config: textsearch configuration 2373 * @config: textsearch configuration
2372 * @state: uninitialized textsearch state variable 2374 * @state: uninitialized textsearch state variable
2373 * 2375 *
2374 * Finds a pattern in the skb data according to the specified 2376 * Finds a pattern in the skb data according to the specified
2375 * textsearch configuration. Use textsearch_next() to retrieve 2377 * textsearch configuration. Use textsearch_next() to retrieve
2376 * subsequent occurrences of the pattern. Returns the offset 2378 * subsequent occurrences of the pattern. Returns the offset
2377 * to the first occurrence or UINT_MAX if no match was found. 2379 * to the first occurrence or UINT_MAX if no match was found.
2378 */ 2380 */
2379 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 2381 unsigned int skb_find_text(struct sk_buff *skb, unsigned int from,
2380 unsigned int to, struct ts_config *config, 2382 unsigned int to, struct ts_config *config,
2381 struct ts_state *state) 2383 struct ts_state *state)
2382 { 2384 {
2383 unsigned int ret; 2385 unsigned int ret;
2384 2386
2385 config->get_next_block = skb_ts_get_next_block; 2387 config->get_next_block = skb_ts_get_next_block;
2386 config->finish = skb_ts_finish; 2388 config->finish = skb_ts_finish;
2387 2389
2388 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state)); 2390 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state));
2389 2391
2390 ret = textsearch_find(config, state); 2392 ret = textsearch_find(config, state);
2391 return (ret <= to - from ? ret : UINT_MAX); 2393 return (ret <= to - from ? ret : UINT_MAX);
2392 } 2394 }
2393 EXPORT_SYMBOL(skb_find_text); 2395 EXPORT_SYMBOL(skb_find_text);
2394 2396
2395 /** 2397 /**
2396 * skb_append_datato_frags: - append the user data to a skb 2398 * skb_append_datato_frags: - append the user data to a skb
2397 * @sk: sock structure 2399 * @sk: sock structure
2398 * @skb: skb structure to be appened with user data. 2400 * @skb: skb structure to be appened with user data.
2399 * @getfrag: call back function to be used for getting the user data 2401 * @getfrag: call back function to be used for getting the user data
2400 * @from: pointer to user message iov 2402 * @from: pointer to user message iov
2401 * @length: length of the iov message 2403 * @length: length of the iov message
2402 * 2404 *
2403 * Description: This procedure append the user data in the fragment part 2405 * Description: This procedure append the user data in the fragment part
2404 * of the skb if any page alloc fails user this procedure returns -ENOMEM 2406 * of the skb if any page alloc fails user this procedure returns -ENOMEM
2405 */ 2407 */
2406 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 2408 int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb,
2407 int (*getfrag)(void *from, char *to, int offset, 2409 int (*getfrag)(void *from, char *to, int offset,
2408 int len, int odd, struct sk_buff *skb), 2410 int len, int odd, struct sk_buff *skb),
2409 void *from, int length) 2411 void *from, int length)
2410 { 2412 {
2411 int frg_cnt = 0; 2413 int frg_cnt = 0;
2412 skb_frag_t *frag = NULL; 2414 skb_frag_t *frag = NULL;
2413 struct page *page = NULL; 2415 struct page *page = NULL;
2414 int copy, left; 2416 int copy, left;
2415 int offset = 0; 2417 int offset = 0;
2416 int ret; 2418 int ret;
2417 2419
2418 do { 2420 do {
2419 /* Return error if we don't have space for new frag */ 2421 /* Return error if we don't have space for new frag */
2420 frg_cnt = skb_shinfo(skb)->nr_frags; 2422 frg_cnt = skb_shinfo(skb)->nr_frags;
2421 if (frg_cnt >= MAX_SKB_FRAGS) 2423 if (frg_cnt >= MAX_SKB_FRAGS)
2422 return -EFAULT; 2424 return -EFAULT;
2423 2425
2424 /* allocate a new page for next frag */ 2426 /* allocate a new page for next frag */
2425 page = alloc_pages(sk->sk_allocation, 0); 2427 page = alloc_pages(sk->sk_allocation, 0);
2426 2428
2427 /* If alloc_page fails just return failure and caller will 2429 /* If alloc_page fails just return failure and caller will
2428 * free previous allocated pages by doing kfree_skb() 2430 * free previous allocated pages by doing kfree_skb()
2429 */ 2431 */
2430 if (page == NULL) 2432 if (page == NULL)
2431 return -ENOMEM; 2433 return -ENOMEM;
2432 2434
2433 /* initialize the next frag */ 2435 /* initialize the next frag */
2434 sk->sk_sndmsg_page = page; 2436 sk->sk_sndmsg_page = page;
2435 sk->sk_sndmsg_off = 0; 2437 sk->sk_sndmsg_off = 0;
2436 skb_fill_page_desc(skb, frg_cnt, page, 0, 0); 2438 skb_fill_page_desc(skb, frg_cnt, page, 0, 0);
2437 skb->truesize += PAGE_SIZE; 2439 skb->truesize += PAGE_SIZE;
2438 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc); 2440 atomic_add(PAGE_SIZE, &sk->sk_wmem_alloc);
2439 2441
2440 /* get the new initialized frag */ 2442 /* get the new initialized frag */
2441 frg_cnt = skb_shinfo(skb)->nr_frags; 2443 frg_cnt = skb_shinfo(skb)->nr_frags;
2442 frag = &skb_shinfo(skb)->frags[frg_cnt - 1]; 2444 frag = &skb_shinfo(skb)->frags[frg_cnt - 1];
2443 2445
2444 /* copy the user data to page */ 2446 /* copy the user data to page */
2445 left = PAGE_SIZE - frag->page_offset; 2447 left = PAGE_SIZE - frag->page_offset;
2446 copy = (length > left)? left : length; 2448 copy = (length > left)? left : length;
2447 2449
2448 ret = getfrag(from, (page_address(frag->page) + 2450 ret = getfrag(from, (page_address(frag->page) +
2449 frag->page_offset + frag->size), 2451 frag->page_offset + frag->size),
2450 offset, copy, 0, skb); 2452 offset, copy, 0, skb);
2451 if (ret < 0) 2453 if (ret < 0)
2452 return -EFAULT; 2454 return -EFAULT;
2453 2455
2454 /* copy was successful so update the size parameters */ 2456 /* copy was successful so update the size parameters */
2455 sk->sk_sndmsg_off += copy; 2457 sk->sk_sndmsg_off += copy;
2456 frag->size += copy; 2458 frag->size += copy;
2457 skb->len += copy; 2459 skb->len += copy;
2458 skb->data_len += copy; 2460 skb->data_len += copy;
2459 offset += copy; 2461 offset += copy;
2460 length -= copy; 2462 length -= copy;
2461 2463
2462 } while (length > 0); 2464 } while (length > 0);
2463 2465
2464 return 0; 2466 return 0;
2465 } 2467 }
2466 EXPORT_SYMBOL(skb_append_datato_frags); 2468 EXPORT_SYMBOL(skb_append_datato_frags);
2467 2469
2468 /** 2470 /**
2469 * skb_pull_rcsum - pull skb and update receive checksum 2471 * skb_pull_rcsum - pull skb and update receive checksum
2470 * @skb: buffer to update 2472 * @skb: buffer to update
2471 * @len: length of data pulled 2473 * @len: length of data pulled
2472 * 2474 *
2473 * This function performs an skb_pull on the packet and updates 2475 * This function performs an skb_pull on the packet and updates
2474 * the CHECKSUM_COMPLETE checksum. It should be used on 2476 * the CHECKSUM_COMPLETE checksum. It should be used on
2475 * receive path processing instead of skb_pull unless you know 2477 * receive path processing instead of skb_pull unless you know
2476 * that the checksum difference is zero (e.g., a valid IP header) 2478 * that the checksum difference is zero (e.g., a valid IP header)
2477 * or you are setting ip_summed to CHECKSUM_NONE. 2479 * or you are setting ip_summed to CHECKSUM_NONE.
2478 */ 2480 */
2479 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 2481 unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len)
2480 { 2482 {
2481 BUG_ON(len > skb->len); 2483 BUG_ON(len > skb->len);
2482 skb->len -= len; 2484 skb->len -= len;
2483 BUG_ON(skb->len < skb->data_len); 2485 BUG_ON(skb->len < skb->data_len);
2484 skb_postpull_rcsum(skb, skb->data, len); 2486 skb_postpull_rcsum(skb, skb->data, len);
2485 return skb->data += len; 2487 return skb->data += len;
2486 } 2488 }
2487 EXPORT_SYMBOL_GPL(skb_pull_rcsum); 2489 EXPORT_SYMBOL_GPL(skb_pull_rcsum);
2488 2490
2489 /** 2491 /**
2490 * skb_segment - Perform protocol segmentation on skb. 2492 * skb_segment - Perform protocol segmentation on skb.
2491 * @skb: buffer to segment 2493 * @skb: buffer to segment
2492 * @features: features for the output path (see dev->features) 2494 * @features: features for the output path (see dev->features)
2493 * 2495 *
2494 * This function performs segmentation on the given skb. It returns 2496 * This function performs segmentation on the given skb. It returns
2495 * a pointer to the first in a list of new skbs for the segments. 2497 * a pointer to the first in a list of new skbs for the segments.
2496 * In case of error it returns ERR_PTR(err). 2498 * In case of error it returns ERR_PTR(err).
2497 */ 2499 */
2498 struct sk_buff *skb_segment(struct sk_buff *skb, int features) 2500 struct sk_buff *skb_segment(struct sk_buff *skb, int features)
2499 { 2501 {
2500 struct sk_buff *segs = NULL; 2502 struct sk_buff *segs = NULL;
2501 struct sk_buff *tail = NULL; 2503 struct sk_buff *tail = NULL;
2502 struct sk_buff *fskb = skb_shinfo(skb)->frag_list; 2504 struct sk_buff *fskb = skb_shinfo(skb)->frag_list;
2503 unsigned int mss = skb_shinfo(skb)->gso_size; 2505 unsigned int mss = skb_shinfo(skb)->gso_size;
2504 unsigned int doffset = skb->data - skb_mac_header(skb); 2506 unsigned int doffset = skb->data - skb_mac_header(skb);
2505 unsigned int offset = doffset; 2507 unsigned int offset = doffset;
2506 unsigned int headroom; 2508 unsigned int headroom;
2507 unsigned int len; 2509 unsigned int len;
2508 int sg = features & NETIF_F_SG; 2510 int sg = features & NETIF_F_SG;
2509 int nfrags = skb_shinfo(skb)->nr_frags; 2511 int nfrags = skb_shinfo(skb)->nr_frags;
2510 int err = -ENOMEM; 2512 int err = -ENOMEM;
2511 int i = 0; 2513 int i = 0;
2512 int pos; 2514 int pos;
2513 2515
2514 __skb_push(skb, doffset); 2516 __skb_push(skb, doffset);
2515 headroom = skb_headroom(skb); 2517 headroom = skb_headroom(skb);
2516 pos = skb_headlen(skb); 2518 pos = skb_headlen(skb);
2517 2519
2518 do { 2520 do {
2519 struct sk_buff *nskb; 2521 struct sk_buff *nskb;
2520 skb_frag_t *frag; 2522 skb_frag_t *frag;
2521 int hsize; 2523 int hsize;
2522 int size; 2524 int size;
2523 2525
2524 len = skb->len - offset; 2526 len = skb->len - offset;
2525 if (len > mss) 2527 if (len > mss)
2526 len = mss; 2528 len = mss;
2527 2529
2528 hsize = skb_headlen(skb) - offset; 2530 hsize = skb_headlen(skb) - offset;
2529 if (hsize < 0) 2531 if (hsize < 0)
2530 hsize = 0; 2532 hsize = 0;
2531 if (hsize > len || !sg) 2533 if (hsize > len || !sg)
2532 hsize = len; 2534 hsize = len;
2533 2535
2534 if (!hsize && i >= nfrags) { 2536 if (!hsize && i >= nfrags) {
2535 BUG_ON(fskb->len != len); 2537 BUG_ON(fskb->len != len);
2536 2538
2537 pos += len; 2539 pos += len;
2538 nskb = skb_clone(fskb, GFP_ATOMIC); 2540 nskb = skb_clone(fskb, GFP_ATOMIC);
2539 fskb = fskb->next; 2541 fskb = fskb->next;
2540 2542
2541 if (unlikely(!nskb)) 2543 if (unlikely(!nskb))
2542 goto err; 2544 goto err;
2543 2545
2544 hsize = skb_end_pointer(nskb) - nskb->head; 2546 hsize = skb_end_pointer(nskb) - nskb->head;
2545 if (skb_cow_head(nskb, doffset + headroom)) { 2547 if (skb_cow_head(nskb, doffset + headroom)) {
2546 kfree_skb(nskb); 2548 kfree_skb(nskb);
2547 goto err; 2549 goto err;
2548 } 2550 }
2549 2551
2550 nskb->truesize += skb_end_pointer(nskb) - nskb->head - 2552 nskb->truesize += skb_end_pointer(nskb) - nskb->head -
2551 hsize; 2553 hsize;
2552 skb_release_head_state(nskb); 2554 skb_release_head_state(nskb);
2553 __skb_push(nskb, doffset); 2555 __skb_push(nskb, doffset);
2554 } else { 2556 } else {
2555 nskb = alloc_skb(hsize + doffset + headroom, 2557 nskb = alloc_skb(hsize + doffset + headroom,
2556 GFP_ATOMIC); 2558 GFP_ATOMIC);
2557 2559
2558 if (unlikely(!nskb)) 2560 if (unlikely(!nskb))
2559 goto err; 2561 goto err;
2560 2562
2561 skb_reserve(nskb, headroom); 2563 skb_reserve(nskb, headroom);
2562 __skb_put(nskb, doffset); 2564 __skb_put(nskb, doffset);
2563 } 2565 }
2564 2566
2565 if (segs) 2567 if (segs)
2566 tail->next = nskb; 2568 tail->next = nskb;
2567 else 2569 else
2568 segs = nskb; 2570 segs = nskb;
2569 tail = nskb; 2571 tail = nskb;
2570 2572
2571 __copy_skb_header(nskb, skb); 2573 __copy_skb_header(nskb, skb);
2572 nskb->mac_len = skb->mac_len; 2574 nskb->mac_len = skb->mac_len;
2573 2575
2574 /* nskb and skb might have different headroom */ 2576 /* nskb and skb might have different headroom */
2575 if (nskb->ip_summed == CHECKSUM_PARTIAL) 2577 if (nskb->ip_summed == CHECKSUM_PARTIAL)
2576 nskb->csum_start += skb_headroom(nskb) - headroom; 2578 nskb->csum_start += skb_headroom(nskb) - headroom;
2577 2579
2578 skb_reset_mac_header(nskb); 2580 skb_reset_mac_header(nskb);
2579 skb_set_network_header(nskb, skb->mac_len); 2581 skb_set_network_header(nskb, skb->mac_len);
2580 nskb->transport_header = (nskb->network_header + 2582 nskb->transport_header = (nskb->network_header +
2581 skb_network_header_len(skb)); 2583 skb_network_header_len(skb));
2582 skb_copy_from_linear_data(skb, nskb->data, doffset); 2584 skb_copy_from_linear_data(skb, nskb->data, doffset);
2583 2585
2584 if (fskb != skb_shinfo(skb)->frag_list) 2586 if (fskb != skb_shinfo(skb)->frag_list)
2585 continue; 2587 continue;
2586 2588
2587 if (!sg) { 2589 if (!sg) {
2588 nskb->ip_summed = CHECKSUM_NONE; 2590 nskb->ip_summed = CHECKSUM_NONE;
2589 nskb->csum = skb_copy_and_csum_bits(skb, offset, 2591 nskb->csum = skb_copy_and_csum_bits(skb, offset,
2590 skb_put(nskb, len), 2592 skb_put(nskb, len),
2591 len, 0); 2593 len, 0);
2592 continue; 2594 continue;
2593 } 2595 }
2594 2596
2595 frag = skb_shinfo(nskb)->frags; 2597 frag = skb_shinfo(nskb)->frags;
2596 2598
2597 skb_copy_from_linear_data_offset(skb, offset, 2599 skb_copy_from_linear_data_offset(skb, offset,
2598 skb_put(nskb, hsize), hsize); 2600 skb_put(nskb, hsize), hsize);
2599 2601
2600 while (pos < offset + len && i < nfrags) { 2602 while (pos < offset + len && i < nfrags) {
2601 *frag = skb_shinfo(skb)->frags[i]; 2603 *frag = skb_shinfo(skb)->frags[i];
2602 get_page(frag->page); 2604 get_page(frag->page);
2603 size = frag->size; 2605 size = frag->size;
2604 2606
2605 if (pos < offset) { 2607 if (pos < offset) {
2606 frag->page_offset += offset - pos; 2608 frag->page_offset += offset - pos;
2607 frag->size -= offset - pos; 2609 frag->size -= offset - pos;
2608 } 2610 }
2609 2611
2610 skb_shinfo(nskb)->nr_frags++; 2612 skb_shinfo(nskb)->nr_frags++;
2611 2613
2612 if (pos + size <= offset + len) { 2614 if (pos + size <= offset + len) {
2613 i++; 2615 i++;
2614 pos += size; 2616 pos += size;
2615 } else { 2617 } else {
2616 frag->size -= pos + size - (offset + len); 2618 frag->size -= pos + size - (offset + len);
2617 goto skip_fraglist; 2619 goto skip_fraglist;
2618 } 2620 }
2619 2621
2620 frag++; 2622 frag++;
2621 } 2623 }
2622 2624
2623 if (pos < offset + len) { 2625 if (pos < offset + len) {
2624 struct sk_buff *fskb2 = fskb; 2626 struct sk_buff *fskb2 = fskb;
2625 2627
2626 BUG_ON(pos + fskb->len != offset + len); 2628 BUG_ON(pos + fskb->len != offset + len);
2627 2629
2628 pos += fskb->len; 2630 pos += fskb->len;
2629 fskb = fskb->next; 2631 fskb = fskb->next;
2630 2632
2631 if (fskb2->next) { 2633 if (fskb2->next) {
2632 fskb2 = skb_clone(fskb2, GFP_ATOMIC); 2634 fskb2 = skb_clone(fskb2, GFP_ATOMIC);
2633 if (!fskb2) 2635 if (!fskb2)
2634 goto err; 2636 goto err;
2635 } else 2637 } else
2636 skb_get(fskb2); 2638 skb_get(fskb2);
2637 2639
2638 SKB_FRAG_ASSERT(nskb); 2640 SKB_FRAG_ASSERT(nskb);
2639 skb_shinfo(nskb)->frag_list = fskb2; 2641 skb_shinfo(nskb)->frag_list = fskb2;
2640 } 2642 }
2641 2643
2642 skip_fraglist: 2644 skip_fraglist:
2643 nskb->data_len = len - hsize; 2645 nskb->data_len = len - hsize;
2644 nskb->len += nskb->data_len; 2646 nskb->len += nskb->data_len;
2645 nskb->truesize += nskb->data_len; 2647 nskb->truesize += nskb->data_len;
2646 } while ((offset += len) < skb->len); 2648 } while ((offset += len) < skb->len);
2647 2649
2648 return segs; 2650 return segs;
2649 2651
2650 err: 2652 err:
2651 while ((skb = segs)) { 2653 while ((skb = segs)) {
2652 segs = skb->next; 2654 segs = skb->next;
2653 kfree_skb(skb); 2655 kfree_skb(skb);
2654 } 2656 }
2655 return ERR_PTR(err); 2657 return ERR_PTR(err);
2656 } 2658 }
2657 EXPORT_SYMBOL_GPL(skb_segment); 2659 EXPORT_SYMBOL_GPL(skb_segment);
2658 2660
2659 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 2661 int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
2660 { 2662 {
2661 struct sk_buff *p = *head; 2663 struct sk_buff *p = *head;
2662 struct sk_buff *nskb; 2664 struct sk_buff *nskb;
2663 struct skb_shared_info *skbinfo = skb_shinfo(skb); 2665 struct skb_shared_info *skbinfo = skb_shinfo(skb);
2664 struct skb_shared_info *pinfo = skb_shinfo(p); 2666 struct skb_shared_info *pinfo = skb_shinfo(p);
2665 unsigned int headroom; 2667 unsigned int headroom;
2666 unsigned int len = skb_gro_len(skb); 2668 unsigned int len = skb_gro_len(skb);
2667 unsigned int offset = skb_gro_offset(skb); 2669 unsigned int offset = skb_gro_offset(skb);
2668 unsigned int headlen = skb_headlen(skb); 2670 unsigned int headlen = skb_headlen(skb);
2669 2671
2670 if (p->len + len >= 65536) 2672 if (p->len + len >= 65536)
2671 return -E2BIG; 2673 return -E2BIG;
2672 2674
2673 if (pinfo->frag_list) 2675 if (pinfo->frag_list)
2674 goto merge; 2676 goto merge;
2675 else if (headlen <= offset) { 2677 else if (headlen <= offset) {
2676 skb_frag_t *frag; 2678 skb_frag_t *frag;
2677 skb_frag_t *frag2; 2679 skb_frag_t *frag2;
2678 int i = skbinfo->nr_frags; 2680 int i = skbinfo->nr_frags;
2679 int nr_frags = pinfo->nr_frags + i; 2681 int nr_frags = pinfo->nr_frags + i;
2680 2682
2681 offset -= headlen; 2683 offset -= headlen;
2682 2684
2683 if (nr_frags > MAX_SKB_FRAGS) 2685 if (nr_frags > MAX_SKB_FRAGS)
2684 return -E2BIG; 2686 return -E2BIG;
2685 2687
2686 pinfo->nr_frags = nr_frags; 2688 pinfo->nr_frags = nr_frags;
2687 skbinfo->nr_frags = 0; 2689 skbinfo->nr_frags = 0;
2688 2690
2689 frag = pinfo->frags + nr_frags; 2691 frag = pinfo->frags + nr_frags;
2690 frag2 = skbinfo->frags + i; 2692 frag2 = skbinfo->frags + i;
2691 do { 2693 do {
2692 *--frag = *--frag2; 2694 *--frag = *--frag2;
2693 } while (--i); 2695 } while (--i);
2694 2696
2695 frag->page_offset += offset; 2697 frag->page_offset += offset;
2696 frag->size -= offset; 2698 frag->size -= offset;
2697 2699
2698 skb->truesize -= skb->data_len; 2700 skb->truesize -= skb->data_len;
2699 skb->len -= skb->data_len; 2701 skb->len -= skb->data_len;
2700 skb->data_len = 0; 2702 skb->data_len = 0;
2701 2703
2702 NAPI_GRO_CB(skb)->free = 1; 2704 NAPI_GRO_CB(skb)->free = 1;
2703 goto done; 2705 goto done;
2704 } else if (skb_gro_len(p) != pinfo->gso_size) 2706 } else if (skb_gro_len(p) != pinfo->gso_size)
2705 return -E2BIG; 2707 return -E2BIG;
2706 2708
2707 headroom = skb_headroom(p); 2709 headroom = skb_headroom(p);
2708 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC); 2710 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC);
2709 if (unlikely(!nskb)) 2711 if (unlikely(!nskb))
2710 return -ENOMEM; 2712 return -ENOMEM;
2711 2713
2712 __copy_skb_header(nskb, p); 2714 __copy_skb_header(nskb, p);
2713 nskb->mac_len = p->mac_len; 2715 nskb->mac_len = p->mac_len;
2714 2716
2715 skb_reserve(nskb, headroom); 2717 skb_reserve(nskb, headroom);
2716 __skb_put(nskb, skb_gro_offset(p)); 2718 __skb_put(nskb, skb_gro_offset(p));
2717 2719
2718 skb_set_mac_header(nskb, skb_mac_header(p) - p->data); 2720 skb_set_mac_header(nskb, skb_mac_header(p) - p->data);
2719 skb_set_network_header(nskb, skb_network_offset(p)); 2721 skb_set_network_header(nskb, skb_network_offset(p));
2720 skb_set_transport_header(nskb, skb_transport_offset(p)); 2722 skb_set_transport_header(nskb, skb_transport_offset(p));
2721 2723
2722 __skb_pull(p, skb_gro_offset(p)); 2724 __skb_pull(p, skb_gro_offset(p));
2723 memcpy(skb_mac_header(nskb), skb_mac_header(p), 2725 memcpy(skb_mac_header(nskb), skb_mac_header(p),
2724 p->data - skb_mac_header(p)); 2726 p->data - skb_mac_header(p));
2725 2727
2726 *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p); 2728 *NAPI_GRO_CB(nskb) = *NAPI_GRO_CB(p);
2727 skb_shinfo(nskb)->frag_list = p; 2729 skb_shinfo(nskb)->frag_list = p;
2728 skb_shinfo(nskb)->gso_size = pinfo->gso_size; 2730 skb_shinfo(nskb)->gso_size = pinfo->gso_size;
2729 pinfo->gso_size = 0; 2731 pinfo->gso_size = 0;
2730 skb_header_release(p); 2732 skb_header_release(p);
2731 nskb->prev = p; 2733 nskb->prev = p;
2732 2734
2733 nskb->data_len += p->len; 2735 nskb->data_len += p->len;
2734 nskb->truesize += p->len; 2736 nskb->truesize += p->len;
2735 nskb->len += p->len; 2737 nskb->len += p->len;
2736 2738
2737 *head = nskb; 2739 *head = nskb;
2738 nskb->next = p->next; 2740 nskb->next = p->next;
2739 p->next = NULL; 2741 p->next = NULL;
2740 2742
2741 p = nskb; 2743 p = nskb;
2742 2744
2743 merge: 2745 merge:
2744 if (offset > headlen) { 2746 if (offset > headlen) {
2745 skbinfo->frags[0].page_offset += offset - headlen; 2747 skbinfo->frags[0].page_offset += offset - headlen;
2746 skbinfo->frags[0].size -= offset - headlen; 2748 skbinfo->frags[0].size -= offset - headlen;
2747 offset = headlen; 2749 offset = headlen;
2748 } 2750 }
2749 2751
2750 __skb_pull(skb, offset); 2752 __skb_pull(skb, offset);
2751 2753
2752 p->prev->next = skb; 2754 p->prev->next = skb;
2753 p->prev = skb; 2755 p->prev = skb;
2754 skb_header_release(skb); 2756 skb_header_release(skb);
2755 2757
2756 done: 2758 done:
2757 NAPI_GRO_CB(p)->count++; 2759 NAPI_GRO_CB(p)->count++;
2758 p->data_len += len; 2760 p->data_len += len;
2759 p->truesize += len; 2761 p->truesize += len;
2760 p->len += len; 2762 p->len += len;
2761 2763
2762 NAPI_GRO_CB(skb)->same_flow = 1; 2764 NAPI_GRO_CB(skb)->same_flow = 1;
2763 return 0; 2765 return 0;
2764 } 2766 }
2765 EXPORT_SYMBOL_GPL(skb_gro_receive); 2767 EXPORT_SYMBOL_GPL(skb_gro_receive);
2766 2768
2767 void __init skb_init(void) 2769 void __init skb_init(void)
2768 { 2770 {
2769 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 2771 skbuff_head_cache = kmem_cache_create("skbuff_head_cache",
2770 sizeof(struct sk_buff), 2772 sizeof(struct sk_buff),
2771 0, 2773 0,
2772 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 2774 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2773 NULL); 2775 NULL);
2774 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 2776 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
2775 (2*sizeof(struct sk_buff)) + 2777 (2*sizeof(struct sk_buff)) +
2776 sizeof(atomic_t), 2778 sizeof(atomic_t),
2777 0, 2779 0,
2778 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 2780 SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2779 NULL); 2781 NULL);
2780 } 2782 }
2781 2783
2782 /** 2784 /**
2783 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 2785 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer
2784 * @skb: Socket buffer containing the buffers to be mapped 2786 * @skb: Socket buffer containing the buffers to be mapped
2785 * @sg: The scatter-gather list to map into 2787 * @sg: The scatter-gather list to map into
2786 * @offset: The offset into the buffer's contents to start mapping 2788 * @offset: The offset into the buffer's contents to start mapping
2787 * @len: Length of buffer space to be mapped 2789 * @len: Length of buffer space to be mapped
2788 * 2790 *
2789 * Fill the specified scatter-gather list with mappings/pointers into a 2791 * Fill the specified scatter-gather list with mappings/pointers into a
2790 * region of the buffer space attached to a socket buffer. 2792 * region of the buffer space attached to a socket buffer.
2791 */ 2793 */
2792 static int 2794 static int
2793 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 2795 __skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
2794 { 2796 {
2795 int start = skb_headlen(skb); 2797 int start = skb_headlen(skb);
2796 int i, copy = start - offset; 2798 int i, copy = start - offset;
2797 struct sk_buff *frag_iter; 2799 struct sk_buff *frag_iter;
2798 int elt = 0; 2800 int elt = 0;
2799 2801
2800 if (copy > 0) { 2802 if (copy > 0) {
2801 if (copy > len) 2803 if (copy > len)
2802 copy = len; 2804 copy = len;
2803 sg_set_buf(sg, skb->data + offset, copy); 2805 sg_set_buf(sg, skb->data + offset, copy);
2804 elt++; 2806 elt++;
2805 if ((len -= copy) == 0) 2807 if ((len -= copy) == 0)
2806 return elt; 2808 return elt;
2807 offset += copy; 2809 offset += copy;
2808 } 2810 }
2809 2811
2810 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2812 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2811 int end; 2813 int end;
2812 2814
2813 WARN_ON(start > offset + len); 2815 WARN_ON(start > offset + len);
2814 2816
2815 end = start + skb_shinfo(skb)->frags[i].size; 2817 end = start + skb_shinfo(skb)->frags[i].size;
2816 if ((copy = end - offset) > 0) { 2818 if ((copy = end - offset) > 0) {
2817 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2819 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2818 2820
2819 if (copy > len) 2821 if (copy > len)
2820 copy = len; 2822 copy = len;
2821 sg_set_page(&sg[elt], frag->page, copy, 2823 sg_set_page(&sg[elt], frag->page, copy,
2822 frag->page_offset+offset-start); 2824 frag->page_offset+offset-start);
2823 elt++; 2825 elt++;
2824 if (!(len -= copy)) 2826 if (!(len -= copy))
2825 return elt; 2827 return elt;
2826 offset += copy; 2828 offset += copy;
2827 } 2829 }
2828 start = end; 2830 start = end;
2829 } 2831 }
2830 2832
2831 skb_walk_frags(skb, frag_iter) { 2833 skb_walk_frags(skb, frag_iter) {
2832 int end; 2834 int end;
2833 2835
2834 WARN_ON(start > offset + len); 2836 WARN_ON(start > offset + len);
2835 2837
2836 end = start + frag_iter->len; 2838 end = start + frag_iter->len;
2837 if ((copy = end - offset) > 0) { 2839 if ((copy = end - offset) > 0) {
2838 if (copy > len) 2840 if (copy > len)
2839 copy = len; 2841 copy = len;
2840 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start, 2842 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start,
2841 copy); 2843 copy);
2842 if ((len -= copy) == 0) 2844 if ((len -= copy) == 0)
2843 return elt; 2845 return elt;
2844 offset += copy; 2846 offset += copy;
2845 } 2847 }
2846 start = end; 2848 start = end;
2847 } 2849 }
2848 BUG_ON(len); 2850 BUG_ON(len);
2849 return elt; 2851 return elt;
2850 } 2852 }
2851 2853
2852 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 2854 int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len)
2853 { 2855 {
2854 int nsg = __skb_to_sgvec(skb, sg, offset, len); 2856 int nsg = __skb_to_sgvec(skb, sg, offset, len);
2855 2857
2856 sg_mark_end(&sg[nsg - 1]); 2858 sg_mark_end(&sg[nsg - 1]);
2857 2859
2858 return nsg; 2860 return nsg;
2859 } 2861 }
2860 EXPORT_SYMBOL_GPL(skb_to_sgvec); 2862 EXPORT_SYMBOL_GPL(skb_to_sgvec);
2861 2863
2862 /** 2864 /**
2863 * skb_cow_data - Check that a socket buffer's data buffers are writable 2865 * skb_cow_data - Check that a socket buffer's data buffers are writable
2864 * @skb: The socket buffer to check. 2866 * @skb: The socket buffer to check.
2865 * @tailbits: Amount of trailing space to be added 2867 * @tailbits: Amount of trailing space to be added
2866 * @trailer: Returned pointer to the skb where the @tailbits space begins 2868 * @trailer: Returned pointer to the skb where the @tailbits space begins
2867 * 2869 *
2868 * Make sure that the data buffers attached to a socket buffer are 2870 * Make sure that the data buffers attached to a socket buffer are
2869 * writable. If they are not, private copies are made of the data buffers 2871 * writable. If they are not, private copies are made of the data buffers
2870 * and the socket buffer is set to use these instead. 2872 * and the socket buffer is set to use these instead.
2871 * 2873 *
2872 * If @tailbits is given, make sure that there is space to write @tailbits 2874 * If @tailbits is given, make sure that there is space to write @tailbits
2873 * bytes of data beyond current end of socket buffer. @trailer will be 2875 * bytes of data beyond current end of socket buffer. @trailer will be
2874 * set to point to the skb in which this space begins. 2876 * set to point to the skb in which this space begins.
2875 * 2877 *
2876 * The number of scatterlist elements required to completely map the 2878 * The number of scatterlist elements required to completely map the
2877 * COW'd and extended socket buffer will be returned. 2879 * COW'd and extended socket buffer will be returned.
2878 */ 2880 */
2879 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 2881 int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer)
2880 { 2882 {
2881 int copyflag; 2883 int copyflag;
2882 int elt; 2884 int elt;
2883 struct sk_buff *skb1, **skb_p; 2885 struct sk_buff *skb1, **skb_p;
2884 2886
2885 /* If skb is cloned or its head is paged, reallocate 2887 /* If skb is cloned or its head is paged, reallocate
2886 * head pulling out all the pages (pages are considered not writable 2888 * head pulling out all the pages (pages are considered not writable
2887 * at the moment even if they are anonymous). 2889 * at the moment even if they are anonymous).
2888 */ 2890 */
2889 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 2891 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) &&
2890 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 2892 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL)
2891 return -ENOMEM; 2893 return -ENOMEM;
2892 2894
2893 /* Easy case. Most of packets will go this way. */ 2895 /* Easy case. Most of packets will go this way. */
2894 if (!skb_has_frag_list(skb)) { 2896 if (!skb_has_frag_list(skb)) {
2895 /* A little of trouble, not enough of space for trailer. 2897 /* A little of trouble, not enough of space for trailer.
2896 * This should not happen, when stack is tuned to generate 2898 * This should not happen, when stack is tuned to generate
2897 * good frames. OK, on miss we reallocate and reserve even more 2899 * good frames. OK, on miss we reallocate and reserve even more
2898 * space, 128 bytes is fair. */ 2900 * space, 128 bytes is fair. */
2899 2901
2900 if (skb_tailroom(skb) < tailbits && 2902 if (skb_tailroom(skb) < tailbits &&
2901 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 2903 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC))
2902 return -ENOMEM; 2904 return -ENOMEM;
2903 2905
2904 /* Voila! */ 2906 /* Voila! */
2905 *trailer = skb; 2907 *trailer = skb;
2906 return 1; 2908 return 1;
2907 } 2909 }
2908 2910
2909 /* Misery. We are in troubles, going to mincer fragments... */ 2911 /* Misery. We are in troubles, going to mincer fragments... */
2910 2912
2911 elt = 1; 2913 elt = 1;
2912 skb_p = &skb_shinfo(skb)->frag_list; 2914 skb_p = &skb_shinfo(skb)->frag_list;
2913 copyflag = 0; 2915 copyflag = 0;
2914 2916
2915 while ((skb1 = *skb_p) != NULL) { 2917 while ((skb1 = *skb_p) != NULL) {
2916 int ntail = 0; 2918 int ntail = 0;
2917 2919
2918 /* The fragment is partially pulled by someone, 2920 /* The fragment is partially pulled by someone,
2919 * this can happen on input. Copy it and everything 2921 * this can happen on input. Copy it and everything
2920 * after it. */ 2922 * after it. */
2921 2923
2922 if (skb_shared(skb1)) 2924 if (skb_shared(skb1))
2923 copyflag = 1; 2925 copyflag = 1;
2924 2926
2925 /* If the skb is the last, worry about trailer. */ 2927 /* If the skb is the last, worry about trailer. */
2926 2928
2927 if (skb1->next == NULL && tailbits) { 2929 if (skb1->next == NULL && tailbits) {
2928 if (skb_shinfo(skb1)->nr_frags || 2930 if (skb_shinfo(skb1)->nr_frags ||
2929 skb_has_frag_list(skb1) || 2931 skb_has_frag_list(skb1) ||
2930 skb_tailroom(skb1) < tailbits) 2932 skb_tailroom(skb1) < tailbits)
2931 ntail = tailbits + 128; 2933 ntail = tailbits + 128;
2932 } 2934 }
2933 2935
2934 if (copyflag || 2936 if (copyflag ||
2935 skb_cloned(skb1) || 2937 skb_cloned(skb1) ||
2936 ntail || 2938 ntail ||
2937 skb_shinfo(skb1)->nr_frags || 2939 skb_shinfo(skb1)->nr_frags ||
2938 skb_has_frag_list(skb1)) { 2940 skb_has_frag_list(skb1)) {
2939 struct sk_buff *skb2; 2941 struct sk_buff *skb2;
2940 2942
2941 /* Fuck, we are miserable poor guys... */ 2943 /* Fuck, we are miserable poor guys... */
2942 if (ntail == 0) 2944 if (ntail == 0)
2943 skb2 = skb_copy(skb1, GFP_ATOMIC); 2945 skb2 = skb_copy(skb1, GFP_ATOMIC);
2944 else 2946 else
2945 skb2 = skb_copy_expand(skb1, 2947 skb2 = skb_copy_expand(skb1,
2946 skb_headroom(skb1), 2948 skb_headroom(skb1),
2947 ntail, 2949 ntail,
2948 GFP_ATOMIC); 2950 GFP_ATOMIC);
2949 if (unlikely(skb2 == NULL)) 2951 if (unlikely(skb2 == NULL))
2950 return -ENOMEM; 2952 return -ENOMEM;
2951 2953
2952 if (skb1->sk) 2954 if (skb1->sk)
2953 skb_set_owner_w(skb2, skb1->sk); 2955 skb_set_owner_w(skb2, skb1->sk);
2954 2956
2955 /* Looking around. Are we still alive? 2957 /* Looking around. Are we still alive?
2956 * OK, link new skb, drop old one */ 2958 * OK, link new skb, drop old one */
2957 2959
2958 skb2->next = skb1->next; 2960 skb2->next = skb1->next;
2959 *skb_p = skb2; 2961 *skb_p = skb2;
2960 kfree_skb(skb1); 2962 kfree_skb(skb1);
2961 skb1 = skb2; 2963 skb1 = skb2;
2962 } 2964 }
2963 elt++; 2965 elt++;
2964 *trailer = skb1; 2966 *trailer = skb1;
2965 skb_p = &skb1->next; 2967 skb_p = &skb1->next;
2966 } 2968 }
2967 2969
2968 return elt; 2970 return elt;
2969 } 2971 }
2970 EXPORT_SYMBOL_GPL(skb_cow_data); 2972 EXPORT_SYMBOL_GPL(skb_cow_data);
2971 2973
2972 static void sock_rmem_free(struct sk_buff *skb) 2974 static void sock_rmem_free(struct sk_buff *skb)
2973 { 2975 {
2974 struct sock *sk = skb->sk; 2976 struct sock *sk = skb->sk;
2975 2977
2976 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 2978 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
2977 } 2979 }
2978 2980
2979 /* 2981 /*
2980 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 2982 * Note: We dont mem charge error packets (no sk_forward_alloc changes)
2981 */ 2983 */
2982 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 2984 int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
2983 { 2985 {
2984 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 2986 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
2985 (unsigned)sk->sk_rcvbuf) 2987 (unsigned)sk->sk_rcvbuf)
2986 return -ENOMEM; 2988 return -ENOMEM;
2987 2989
2988 skb_orphan(skb); 2990 skb_orphan(skb);
2989 skb->sk = sk; 2991 skb->sk = sk;
2990 skb->destructor = sock_rmem_free; 2992 skb->destructor = sock_rmem_free;
2991 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 2993 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2992 2994
2993 skb_queue_tail(&sk->sk_error_queue, skb); 2995 skb_queue_tail(&sk->sk_error_queue, skb);
2994 if (!sock_flag(sk, SOCK_DEAD)) 2996 if (!sock_flag(sk, SOCK_DEAD))
2995 sk->sk_data_ready(sk, skb->len); 2997 sk->sk_data_ready(sk, skb->len);
2996 return 0; 2998 return 0;
2997 } 2999 }
2998 EXPORT_SYMBOL(sock_queue_err_skb); 3000 EXPORT_SYMBOL(sock_queue_err_skb);
2999 3001
3000 void skb_tstamp_tx(struct sk_buff *orig_skb, 3002 void skb_tstamp_tx(struct sk_buff *orig_skb,
3001 struct skb_shared_hwtstamps *hwtstamps) 3003 struct skb_shared_hwtstamps *hwtstamps)
3002 { 3004 {
3003 struct sock *sk = orig_skb->sk; 3005 struct sock *sk = orig_skb->sk;
3004 struct sock_exterr_skb *serr; 3006 struct sock_exterr_skb *serr;
3005 struct sk_buff *skb; 3007 struct sk_buff *skb;
3006 int err; 3008 int err;
3007 3009
3008 if (!sk) 3010 if (!sk)
3009 return; 3011 return;
3010 3012
3011 skb = skb_clone(orig_skb, GFP_ATOMIC); 3013 skb = skb_clone(orig_skb, GFP_ATOMIC);
3012 if (!skb) 3014 if (!skb)
3013 return; 3015 return;
3014 3016
3015 if (hwtstamps) { 3017 if (hwtstamps) {
3016 *skb_hwtstamps(skb) = 3018 *skb_hwtstamps(skb) =
3017 *hwtstamps; 3019 *hwtstamps;
3018 } else { 3020 } else {
3019 /* 3021 /*
3020 * no hardware time stamps available, 3022 * no hardware time stamps available,
3021 * so keep the shared tx_flags and only 3023 * so keep the shared tx_flags and only
3022 * store software time stamp 3024 * store software time stamp
3023 */ 3025 */
3024 skb->tstamp = ktime_get_real(); 3026 skb->tstamp = ktime_get_real();
3025 } 3027 }
3026 3028
3027 serr = SKB_EXT_ERR(skb); 3029 serr = SKB_EXT_ERR(skb);
3028 memset(serr, 0, sizeof(*serr)); 3030 memset(serr, 0, sizeof(*serr));
3029 serr->ee.ee_errno = ENOMSG; 3031 serr->ee.ee_errno = ENOMSG;
3030 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 3032 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
3031 3033
3032 err = sock_queue_err_skb(sk, skb); 3034 err = sock_queue_err_skb(sk, skb);
3033 3035
3034 if (err) 3036 if (err)
3035 kfree_skb(skb); 3037 kfree_skb(skb);
3036 } 3038 }
3037 EXPORT_SYMBOL_GPL(skb_tstamp_tx); 3039 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
3038 3040
3039 3041
3040 /** 3042 /**
3041 * skb_partial_csum_set - set up and verify partial csum values for packet 3043 * skb_partial_csum_set - set up and verify partial csum values for packet
3042 * @skb: the skb to set 3044 * @skb: the skb to set
3043 * @start: the number of bytes after skb->data to start checksumming. 3045 * @start: the number of bytes after skb->data to start checksumming.
3044 * @off: the offset from start to place the checksum. 3046 * @off: the offset from start to place the checksum.
3045 * 3047 *
3046 * For untrusted partially-checksummed packets, we need to make sure the values 3048 * For untrusted partially-checksummed packets, we need to make sure the values
3047 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 3049 * for skb->csum_start and skb->csum_offset are valid so we don't oops.
3048 * 3050 *
3049 * This function checks and sets those values and skb->ip_summed: if this 3051 * This function checks and sets those values and skb->ip_summed: if this
3050 * returns false you should drop the packet. 3052 * returns false you should drop the packet.
3051 */ 3053 */
3052 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 3054 bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off)
3053 { 3055 {
3054 if (unlikely(start > skb_headlen(skb)) || 3056 if (unlikely(start > skb_headlen(skb)) ||
3055 unlikely((int)start + off > skb_headlen(skb) - 2)) { 3057 unlikely((int)start + off > skb_headlen(skb) - 2)) {
3056 if (net_ratelimit()) 3058 if (net_ratelimit())
3057 printk(KERN_WARNING 3059 printk(KERN_WARNING
3058 "bad partial csum: csum=%u/%u len=%u\n", 3060 "bad partial csum: csum=%u/%u len=%u\n",
3059 start, off, skb_headlen(skb)); 3061 start, off, skb_headlen(skb));
3060 return false; 3062 return false;
3061 } 3063 }
3062 skb->ip_summed = CHECKSUM_PARTIAL; 3064 skb->ip_summed = CHECKSUM_PARTIAL;
3063 skb->csum_start = skb_headroom(skb) + start; 3065 skb->csum_start = skb_headroom(skb) + start;
3064 skb->csum_offset = off; 3066 skb->csum_offset = off;
3065 return true; 3067 return true;
3066 } 3068 }
3067 EXPORT_SYMBOL_GPL(skb_partial_csum_set); 3069 EXPORT_SYMBOL_GPL(skb_partial_csum_set);
3068 3070
3069 void __skb_warn_lro_forwarding(const struct sk_buff *skb) 3071 void __skb_warn_lro_forwarding(const struct sk_buff *skb)
3070 { 3072 {
3071 if (net_ratelimit()) 3073 if (net_ratelimit())
3072 pr_warning("%s: received packets cannot be forwarded" 3074 pr_warning("%s: received packets cannot be forwarded"
3073 " while LRO is enabled\n", skb->dev->name); 3075 " while LRO is enabled\n", skb->dev->name);
3074 } 3076 }
3075 EXPORT_SYMBOL(__skb_warn_lro_forwarding); 3077 EXPORT_SYMBOL(__skb_warn_lro_forwarding);
3076 3078
net/ipv6/netfilter/nf_defrag_ipv6_hooks.c
Diff comments   View file @ 464143c
1 /* (C) 1999-2001 Paul `Rusty' Russell 1 /* (C) 1999-2001 Paul `Rusty' Russell
2 * (C) 2002-2004 Netfilter Core Team <coreteam@netfilter.org> 2 * (C) 2002-2004 Netfilter Core Team <coreteam@netfilter.org>
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify 4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as 5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation. 6 * published by the Free Software Foundation.
7 */ 7 */
8 8
9 #include <linux/types.h> 9 #include <linux/types.h>
10 #include <linux/ipv6.h> 10 #include <linux/ipv6.h>
11 #include <linux/in6.h> 11 #include <linux/in6.h>
12 #include <linux/netfilter.h> 12 #include <linux/netfilter.h>
13 #include <linux/module.h> 13 #include <linux/module.h>
14 #include <linux/skbuff.h> 14 #include <linux/skbuff.h>
15 #include <linux/icmp.h> 15 #include <linux/icmp.h>
16 #include <linux/sysctl.h> 16 #include <linux/sysctl.h>
17 #include <net/ipv6.h> 17 #include <net/ipv6.h>
18 #include <net/inet_frag.h> 18 #include <net/inet_frag.h>
19 19
20 #include <linux/netfilter_ipv6.h> 20 #include <linux/netfilter_ipv6.h>
21 #include <linux/netfilter_bridge.h> 21 #include <linux/netfilter_bridge.h>
22 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
22 #include <net/netfilter/nf_conntrack.h> 23 #include <net/netfilter/nf_conntrack.h>
23 #include <net/netfilter/nf_conntrack_helper.h> 24 #include <net/netfilter/nf_conntrack_helper.h>
24 #include <net/netfilter/nf_conntrack_l4proto.h> 25 #include <net/netfilter/nf_conntrack_l4proto.h>
25 #include <net/netfilter/nf_conntrack_l3proto.h> 26 #include <net/netfilter/nf_conntrack_l3proto.h>
26 #include <net/netfilter/nf_conntrack_core.h> 27 #include <net/netfilter/nf_conntrack_core.h>
27 #include <net/netfilter/nf_conntrack_zones.h>
28 #include <net/netfilter/ipv6/nf_conntrack_ipv6.h> 28 #include <net/netfilter/ipv6/nf_conntrack_ipv6.h>
29 #endif
30 #include <net/netfilter/nf_conntrack_zones.h>
29 #include <net/netfilter/ipv6/nf_defrag_ipv6.h> 31 #include <net/netfilter/ipv6/nf_defrag_ipv6.h>
30 32
31 static enum ip6_defrag_users nf_ct6_defrag_user(unsigned int hooknum, 33 static enum ip6_defrag_users nf_ct6_defrag_user(unsigned int hooknum,
32 struct sk_buff *skb) 34 struct sk_buff *skb)
33 { 35 {
34 u16 zone = NF_CT_DEFAULT_ZONE; 36 u16 zone = NF_CT_DEFAULT_ZONE;
35 37
38 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
36 if (skb->nfct) 39 if (skb->nfct)
37 zone = nf_ct_zone((struct nf_conn *)skb->nfct); 40 zone = nf_ct_zone((struct nf_conn *)skb->nfct);
41 #endif
38 42
39 #ifdef CONFIG_BRIDGE_NETFILTER 43 #ifdef CONFIG_BRIDGE_NETFILTER
40 if (skb->nf_bridge && 44 if (skb->nf_bridge &&
41 skb->nf_bridge->mask & BRNF_NF_BRIDGE_PREROUTING) 45 skb->nf_bridge->mask & BRNF_NF_BRIDGE_PREROUTING)
42 return IP6_DEFRAG_CONNTRACK_BRIDGE_IN + zone; 46 return IP6_DEFRAG_CONNTRACK_BRIDGE_IN + zone;
43 #endif 47 #endif
44 if (hooknum == NF_INET_PRE_ROUTING) 48 if (hooknum == NF_INET_PRE_ROUTING)
45 return IP6_DEFRAG_CONNTRACK_IN + zone; 49 return IP6_DEFRAG_CONNTRACK_IN + zone;
46 else 50 else
47 return IP6_DEFRAG_CONNTRACK_OUT + zone; 51 return IP6_DEFRAG_CONNTRACK_OUT + zone;
48 52
49 } 53 }
50 54
51 static unsigned int ipv6_defrag(unsigned int hooknum, 55 static unsigned int ipv6_defrag(unsigned int hooknum,
52 struct sk_buff *skb, 56 struct sk_buff *skb,
53 const struct net_device *in, 57 const struct net_device *in,
54 const struct net_device *out, 58 const struct net_device *out,
55 int (*okfn)(struct sk_buff *)) 59 int (*okfn)(struct sk_buff *))
56 { 60 {
57 struct sk_buff *reasm; 61 struct sk_buff *reasm;
58 62
63 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
59 /* Previously seen (loopback)? */ 64 /* Previously seen (loopback)? */
60 if (skb->nfct && !nf_ct_is_template((struct nf_conn *)skb->nfct)) 65 if (skb->nfct && !nf_ct_is_template((struct nf_conn *)skb->nfct))
61 return NF_ACCEPT; 66 return NF_ACCEPT;
67 #endif
62 68
63 reasm = nf_ct_frag6_gather(skb, nf_ct6_defrag_user(hooknum, skb)); 69 reasm = nf_ct_frag6_gather(skb, nf_ct6_defrag_user(hooknum, skb));
64 /* queued */ 70 /* queued */
65 if (reasm == NULL) 71 if (reasm == NULL)
66 return NF_STOLEN; 72 return NF_STOLEN;
67 73
68 /* error occured or not fragmented */ 74 /* error occured or not fragmented */
69 if (reasm == skb) 75 if (reasm == skb)
70 return NF_ACCEPT; 76 return NF_ACCEPT;
71 77
72 nf_ct_frag6_output(hooknum, reasm, (struct net_device *)in, 78 nf_ct_frag6_output(hooknum, reasm, (struct net_device *)in,
73 (struct net_device *)out, okfn); 79 (struct net_device *)out, okfn);
74 80
75 return NF_STOLEN; 81 return NF_STOLEN;
76 } 82 }
77 83
78 static struct nf_hook_ops ipv6_defrag_ops[] = { 84 static struct nf_hook_ops ipv6_defrag_ops[] = {
79 { 85 {
80 .hook = ipv6_defrag, 86 .hook = ipv6_defrag,
81 .owner = THIS_MODULE, 87 .owner = THIS_MODULE,
82 .pf = NFPROTO_IPV6, 88 .pf = NFPROTO_IPV6,
83 .hooknum = NF_INET_PRE_ROUTING, 89 .hooknum = NF_INET_PRE_ROUTING,
84 .priority = NF_IP6_PRI_CONNTRACK_DEFRAG, 90 .priority = NF_IP6_PRI_CONNTRACK_DEFRAG,
85 }, 91 },
86 { 92 {
87 .hook = ipv6_defrag, 93 .hook = ipv6_defrag,
88 .owner = THIS_MODULE, 94 .owner = THIS_MODULE,
89 .pf = NFPROTO_IPV6, 95 .pf = NFPROTO_IPV6,
90 .hooknum = NF_INET_LOCAL_OUT, 96 .hooknum = NF_INET_LOCAL_OUT,
91 .priority = NF_IP6_PRI_CONNTRACK_DEFRAG, 97 .priority = NF_IP6_PRI_CONNTRACK_DEFRAG,
92 }, 98 },
93 }; 99 };
94 100
95 static int __init nf_defrag_init(void) 101 static int __init nf_defrag_init(void)
96 { 102 {
97 int ret = 0; 103 int ret = 0;
98 104
99 ret = nf_ct_frag6_init(); 105 ret = nf_ct_frag6_init();
100 if (ret < 0) { 106 if (ret < 0) {
101 pr_err("nf_defrag_ipv6: can't initialize frag6.\n"); 107 pr_err("nf_defrag_ipv6: can't initialize frag6.\n");
102 return ret; 108 return ret;
103 } 109 }
104 ret = nf_register_hooks(ipv6_defrag_ops, ARRAY_SIZE(ipv6_defrag_ops)); 110 ret = nf_register_hooks(ipv6_defrag_ops, ARRAY_SIZE(ipv6_defrag_ops));
105 if (ret < 0) { 111 if (ret < 0) {
106 pr_err("nf_defrag_ipv6: can't register hooks\n"); 112 pr_err("nf_defrag_ipv6: can't register hooks\n");
107 goto cleanup_frag6; 113 goto cleanup_frag6;
108 } 114 }
109 return ret; 115 return ret;
110 116
111 cleanup_frag6: 117 cleanup_frag6:
112 nf_ct_frag6_cleanup(); 118 nf_ct_frag6_cleanup();
113 return ret; 119 return ret;
114 120
115 } 121 }
116 122
117 static void __exit nf_defrag_fini(void) 123 static void __exit nf_defrag_fini(void)
118 { 124 {
119 nf_unregister_hooks(ipv6_defrag_ops, ARRAY_SIZE(ipv6_defrag_ops)); 125 nf_unregister_hooks(ipv6_defrag_ops, ARRAY_SIZE(ipv6_defrag_ops));
120 nf_ct_frag6_cleanup(); 126 nf_ct_frag6_cleanup();
121 } 127 }
122 128
123 void nf_defrag_ipv6_enable(void) 129 void nf_defrag_ipv6_enable(void)
124 { 130 {
125 } 131 }
126 EXPORT_SYMBOL_GPL(nf_defrag_ipv6_enable); 132 EXPORT_SYMBOL_GPL(nf_defrag_ipv6_enable);
127 133
128 module_init(nf_defrag_init); 134 module_init(nf_defrag_init);
129 module_exit(nf_defrag_fini); 135 module_exit(nf_defrag_fini);
130 136
131 MODULE_LICENSE("GPL"); 137 MODULE_LICENSE("GPL");