Blame view
net/core/datagram.c
20.5 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
1da177e4c Linux-2.6.12-rc2 |
2 3 4 5 6 7 8 9 10 11 12 |
/* * SUCS NET3: * * Generic datagram handling routines. These are generic for all * protocols. Possibly a generic IP version on top of these would * make sense. Not tonight however 8-). * This is used because UDP, RAW, PACKET, DDP, IPX, AX.25 and * NetROM layer all have identical poll code and mostly * identical recvmsg() code. So we share it here. The poll was * shared before but buried in udp.c so I moved it. * |
113aa838e net: Rationalise ... |
13 |
* Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>. (datagram_poll() from old |
1da177e4c Linux-2.6.12-rc2 |
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
* udp.c code) * * Fixes: * Alan Cox : NULL return from skb_peek_copy() * understood * Alan Cox : Rewrote skb_read_datagram to avoid the * skb_peek_copy stuff. * Alan Cox : Added support for SOCK_SEQPACKET. * IPX can no longer use the SO_TYPE hack * but AX.25 now works right, and SPX is * feasible. * Alan Cox : Fixed write poll of non IP protocol * crash. * Florian La Roche: Changed for my new skbuff handling. * Darryl Miles : Fixed non-blocking SOCK_SEQPACKET. * Linus Torvalds : BSD semantic fixes. * Alan Cox : Datagram iovec handling * Darryl Miles : Fixed non-blocking SOCK_STREAM. * Alan Cox : POSIXisms * Pete Wyckoff : Unconnected accept() fix. * */ #include <linux/module.h> #include <linux/types.h> #include <linux/kernel.h> |
7c0f6ba68 Replace <asm/uacc... |
40 |
#include <linux/uaccess.h> |
1da177e4c Linux-2.6.12-rc2 |
41 42 43 44 45 |
#include <linux/mm.h> #include <linux/interrupt.h> #include <linux/errno.h> #include <linux/sched.h> #include <linux/inet.h> |
1da177e4c Linux-2.6.12-rc2 |
46 47 48 49 |
#include <linux/netdevice.h> #include <linux/rtnetlink.h> #include <linux/poll.h> #include <linux/highmem.h> |
3305b80c2 [IP]: Simplify an... |
50 |
#include <linux/spinlock.h> |
5a0e3ad6a include cleanup: ... |
51 |
#include <linux/slab.h> |
0433547aa net: use release_... |
52 |
#include <linux/pagemap.h> |
a8f820aa4 inet: Add skb_cop... |
53 |
#include <linux/uio.h> |
29f3490ba net: use indirect... |
54 |
#include <linux/indirect_call_wrapper.h> |
1da177e4c Linux-2.6.12-rc2 |
55 56 57 |
#include <net/protocol.h> #include <linux/skbuff.h> |
1da177e4c Linux-2.6.12-rc2 |
58 |
|
c752f0739 [TCP]: Move the t... |
59 60 61 |
#include <net/checksum.h> #include <net/sock.h> #include <net/tcp_states.h> |
e9b3cc1b3 net: skb ftracer ... |
62 |
#include <trace/events/skb.h> |
076bb0c82 net: rename inclu... |
63 |
#include <net/busy_poll.h> |
1da177e4c Linux-2.6.12-rc2 |
64 |
|
7b7ed885a net/core: Allow t... |
65 |
#include "datagram.h" |
1da177e4c Linux-2.6.12-rc2 |
66 67 68 69 70 71 72 |
/* * Is a socket 'connection oriented' ? */ static inline int connection_based(struct sock *sk) { return sk->sk_type == SOCK_SEQPACKET || sk->sk_type == SOCK_STREAM; } |
ac6424b98 sched/wait: Renam... |
73 |
static int receiver_wake_function(wait_queue_entry_t *wait, unsigned int mode, int sync, |
bf368e4e7 net: Avoid extra ... |
74 75 |
void *key) { |
bf368e4e7 net: Avoid extra ... |
76 77 78 |
/* * Avoid a wakeup if event not interesting for us */ |
a9a08845e vfs: do bulk POLL... |
79 |
if (key && !(key_to_poll(key) & (EPOLLIN | EPOLLERR))) |
bf368e4e7 net: Avoid extra ... |
80 81 82 |
return 0; return autoremove_wake_function(wait, mode, sync, key); } |
1da177e4c Linux-2.6.12-rc2 |
83 |
/* |
39cc86130 unix/dgram: fix p... |
84 |
* Wait for the last received packet to be different from skb |
1da177e4c Linux-2.6.12-rc2 |
85 |
*/ |
b50b0580d net: add queue ar... |
86 87 |
int __skb_wait_for_more_packets(struct sock *sk, struct sk_buff_head *queue, int *err, long *timeo_p, |
ea3793ee2 core: enable more... |
88 |
const struct sk_buff *skb) |
1da177e4c Linux-2.6.12-rc2 |
89 90 |
{ int error; |
bf368e4e7 net: Avoid extra ... |
91 |
DEFINE_WAIT_FUNC(wait, receiver_wake_function); |
1da177e4c Linux-2.6.12-rc2 |
92 |
|
aa3951451 net: sk_sleep() h... |
93 |
prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); |
1da177e4c Linux-2.6.12-rc2 |
94 95 96 97 98 |
/* Socket errors? */ error = sock_error(sk); if (error) goto out_err; |
b50b0580d net: add queue ar... |
99 |
if (READ_ONCE(queue->prev) != skb) |
1da177e4c Linux-2.6.12-rc2 |
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 |
goto out; /* Socket shut down? */ if (sk->sk_shutdown & RCV_SHUTDOWN) goto out_noerr; /* Sequenced packets can come disconnected. * If so we report the problem */ error = -ENOTCONN; if (connection_based(sk) && !(sk->sk_state == TCP_ESTABLISHED || sk->sk_state == TCP_LISTEN)) goto out_err; /* handle signals */ if (signal_pending(current)) goto interrupted; error = 0; *timeo_p = schedule_timeout(*timeo_p); out: |
aa3951451 net: sk_sleep() h... |
121 |
finish_wait(sk_sleep(sk), &wait); |
1da177e4c Linux-2.6.12-rc2 |
122 123 124 125 126 127 128 129 130 131 132 |
return error; interrupted: error = sock_intr_errno(*timeo_p); out_err: *err = error; goto out; out_noerr: *err = 0; error = 1; goto out; } |
ea3793ee2 core: enable more... |
133 |
EXPORT_SYMBOL(__skb_wait_for_more_packets); |
1da177e4c Linux-2.6.12-rc2 |
134 |
|
a0a2a6602 net: Fix skb_set_... |
135 |
static struct sk_buff *skb_set_peeked(struct sk_buff *skb) |
738ac1ebb net: Clone skb be... |
136 137 138 139 |
{ struct sk_buff *nskb; if (skb->peeked) |
a0a2a6602 net: Fix skb_set_... |
140 |
return skb; |
738ac1ebb net: Clone skb be... |
141 142 143 144 145 146 147 |
/* We have to unshare an skb before modifying it. */ if (!skb_shared(skb)) goto done; nskb = skb_clone(skb, GFP_ATOMIC); if (!nskb) |
a0a2a6602 net: Fix skb_set_... |
148 |
return ERR_PTR(-ENOMEM); |
738ac1ebb net: Clone skb be... |
149 150 151 152 153 154 155 156 157 158 159 |
skb->prev->next = nskb; skb->next->prev = nskb; nskb->prev = skb->prev; nskb->next = skb->next; consume_skb(skb); skb = nskb; done: skb->peeked = 1; |
a0a2a6602 net: Fix skb_set_... |
160 |
return skb; |
738ac1ebb net: Clone skb be... |
161 |
} |
65101aeca net/sock: factor ... |
162 163 164 |
struct sk_buff *__skb_try_recv_from_queue(struct sock *sk, struct sk_buff_head *queue, unsigned int flags, |
fd69c399c datagram: remove ... |
165 |
int *off, int *err, |
65101aeca net/sock: factor ... |
166 167 |
struct sk_buff **last) { |
a0917e0bc datagram: When pe... |
168 |
bool peek_at_off = false; |
65101aeca net/sock: factor ... |
169 |
struct sk_buff *skb; |
a0917e0bc datagram: When pe... |
170 171 172 173 174 175 |
int _off = 0; if (unlikely(flags & MSG_PEEK && *off >= 0)) { peek_at_off = true; _off = *off; } |
65101aeca net/sock: factor ... |
176 177 178 179 |
*last = queue->prev; skb_queue_walk(queue, skb) { if (flags & MSG_PEEK) { |
a0917e0bc datagram: When pe... |
180 181 |
if (peek_at_off && _off >= skb->len && (_off || skb->peeked)) { |
de321ed38 net: fix __skb_tr... |
182 |
_off -= skb->len; |
65101aeca net/sock: factor ... |
183 184 185 186 |
continue; } if (!skb->len) { skb = skb_set_peeked(skb); |
98e4fcff3 datagram: Remove ... |
187 |
if (IS_ERR(skb)) { |
65101aeca net/sock: factor ... |
188 |
*err = PTR_ERR(skb); |
de321ed38 net: fix __skb_tr... |
189 |
return NULL; |
65101aeca net/sock: factor ... |
190 191 |
} } |
633547973 net: convert sk_b... |
192 |
refcount_inc(&skb->users); |
65101aeca net/sock: factor ... |
193 194 |
} else { __skb_unlink(skb, queue); |
65101aeca net/sock: factor ... |
195 |
} |
de321ed38 net: fix __skb_tr... |
196 |
*off = _off; |
65101aeca net/sock: factor ... |
197 198 199 200 |
return skb; } return NULL; } |
1da177e4c Linux-2.6.12-rc2 |
201 |
/** |
ea3793ee2 core: enable more... |
202 |
* __skb_try_recv_datagram - Receive a datagram skbuff |
4dc3b16ba [PATCH] DocBook: ... |
203 |
* @sk: socket |
b50b0580d net: add queue ar... |
204 |
* @queue: socket queue from which to receive |
d3f6cd9e6 datagram: fix ker... |
205 |
* @flags: MSG\_ flags |
3f518bf74 datagram: Add off... |
206 207 |
* @off: an offset in bytes to peek skb from. Returns an offset * within an skb where data actually starts |
4dc3b16ba [PATCH] DocBook: ... |
208 |
* @err: error code returned |
ea3793ee2 core: enable more... |
209 210 |
* @last: set to last peeked message to inform the wait function * what to look for when peeking |
1da177e4c Linux-2.6.12-rc2 |
211 212 213 214 215 216 217 |
* * Get a datagram skbuff, understands the peeking, nonblocking wakeups * and possible races. This replaces identical code in packet, raw and * udp, as well as the IPX AX.25 and Appletalk. It also finally fixes * the long standing peek and read race for datagram sockets. If you * alter this routine remember it must be re-entrant. * |
ea3793ee2 core: enable more... |
218 219 |
* This function will lock the socket if a skb is returned, so * the caller needs to unlock the socket in that case (usually by |
d651983dd net: fix some ide... |
220 |
* calling skb_free_datagram). Returns NULL with @err set to |
ea3793ee2 core: enable more... |
221 222 |
* -EAGAIN if no data was available or to some other value if an * error was detected. |
1da177e4c Linux-2.6.12-rc2 |
223 224 225 226 227 228 229 230 231 232 233 234 235 |
* * * It does not lock socket since today. This function is * * free of race conditions. This measure should/can improve * * significantly datagram socket latencies at high loads, * * when data copying to user space takes lots of time. * * (BTW I've just killed the last cli() in IP/IPv6/core/netlink/packet * * 8) Great win.) * * --ANK (980729) * * The order of the tests when we find no data waiting are specified * quite explicitly by POSIX 1003.1g, don't change them without having * the standard around please. */ |
b50b0580d net: add queue ar... |
236 237 |
struct sk_buff *__skb_try_recv_datagram(struct sock *sk, struct sk_buff_head *queue, |
e427cad6e net: datagram: dr... |
238 |
unsigned int flags, int *off, int *err, |
ea3793ee2 core: enable more... |
239 |
struct sk_buff **last) |
1da177e4c Linux-2.6.12-rc2 |
240 |
{ |
ea3793ee2 core: enable more... |
241 |
struct sk_buff *skb; |
738ac1ebb net: Clone skb be... |
242 |
unsigned long cpu_flags; |
1da177e4c Linux-2.6.12-rc2 |
243 244 245 246 247 248 249 |
/* * Caller is allowed not to check sk->sk_err before skb_recv_datagram() */ int error = sock_error(sk); if (error) goto no_packet; |
1da177e4c Linux-2.6.12-rc2 |
250 251 252 253 254 |
do { /* Again only user level code calls this function, so nothing * interrupt level will suddenly eat the receive_queue. * * Look at current nfs client by the way... |
8917a3c0b Fix a typo in dat... |
255 |
* However, this function was correct in any case. 8) |
1da177e4c Linux-2.6.12-rc2 |
256 |
*/ |
4934b0329 datagram: Factor ... |
257 |
spin_lock_irqsave(&queue->lock, cpu_flags); |
e427cad6e net: datagram: dr... |
258 259 |
skb = __skb_try_recv_from_queue(sk, queue, flags, off, &error, last); |
3f518bf74 datagram: Add off... |
260 |
spin_unlock_irqrestore(&queue->lock, cpu_flags); |
de321ed38 net: fix __skb_tr... |
261 262 |
if (error) goto no_packet; |
65101aeca net/sock: factor ... |
263 264 |
if (skb) return skb; |
2b5cd0dfa net: Change retur... |
265 266 267 268 269 |
if (!sk_can_busy_loop(sk)) break; sk_busy_loop(sk, flags & MSG_DONTWAIT); |
b50b0580d net: add queue ar... |
270 |
} while (READ_ONCE(queue->prev) != *last); |
1da177e4c Linux-2.6.12-rc2 |
271 |
|
ea3793ee2 core: enable more... |
272 |
error = -EAGAIN; |
a5b50476f udp: add low late... |
273 |
|
ea3793ee2 core: enable more... |
274 275 276 277 278 |
no_packet: *err = error; return NULL; } EXPORT_SYMBOL(__skb_try_recv_datagram); |
1da177e4c Linux-2.6.12-rc2 |
279 |
|
b50b0580d net: add queue ar... |
280 281 |
struct sk_buff *__skb_recv_datagram(struct sock *sk, struct sk_buff_head *sk_queue, |
e427cad6e net: datagram: dr... |
282 |
unsigned int flags, int *off, int *err) |
ea3793ee2 core: enable more... |
283 284 285 |
{ struct sk_buff *skb, *last; long timeo; |
1da177e4c Linux-2.6.12-rc2 |
286 |
|
ea3793ee2 core: enable more... |
287 288 289 |
timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); do { |
e427cad6e net: datagram: dr... |
290 291 |
skb = __skb_try_recv_datagram(sk, sk_queue, flags, off, err, &last); |
ea3793ee2 core: enable more... |
292 293 |
if (skb) return skb; |
760a43224 net: Fix inverted... |
294 |
if (*err != -EAGAIN) |
ea3793ee2 core: enable more... |
295 296 |
break; } while (timeo && |
b50b0580d net: add queue ar... |
297 298 |
!__skb_wait_for_more_packets(sk, sk_queue, err, &timeo, last)); |
1da177e4c Linux-2.6.12-rc2 |
299 |
|
1da177e4c Linux-2.6.12-rc2 |
300 301 |
return NULL; } |
a59322be0 [UDP]: Only incre... |
302 |
EXPORT_SYMBOL(__skb_recv_datagram); |
95c961747 net: cleanup unsi... |
303 |
struct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, |
a59322be0 [UDP]: Only incre... |
304 305 |
int noblock, int *err) { |
fd69c399c datagram: remove ... |
306 |
int off = 0; |
a59322be0 [UDP]: Only incre... |
307 |
|
b50b0580d net: add queue ar... |
308 309 |
return __skb_recv_datagram(sk, &sk->sk_receive_queue, flags | (noblock ? MSG_DONTWAIT : 0), |
e427cad6e net: datagram: dr... |
310 |
&off, err); |
a59322be0 [UDP]: Only incre... |
311 |
} |
9e34a5b51 net/core: EXPORT_... |
312 |
EXPORT_SYMBOL(skb_recv_datagram); |
1da177e4c Linux-2.6.12-rc2 |
313 314 315 |
void skb_free_datagram(struct sock *sk, struct sk_buff *skb) { |
ead2ceb0e Network Drop Moni... |
316 |
consume_skb(skb); |
270acefaf net: sk_free_data... |
317 |
sk_mem_reclaim_partial(sk); |
1da177e4c Linux-2.6.12-rc2 |
318 |
} |
9d410c796 net: fix sk_forwa... |
319 |
EXPORT_SYMBOL(skb_free_datagram); |
627d2d6b5 udp: enable MSG_P... |
320 |
void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len) |
9d410c796 net: fix sk_forwa... |
321 |
{ |
8a74ad60a net: fix lock_soc... |
322 |
bool slow; |
3889a803e net: factor out a... |
323 |
if (!skb_unref(skb)) { |
627d2d6b5 udp: enable MSG_P... |
324 |
sk_peek_offset_bwd(sk, len); |
93bb64eac net: skb_free_dat... |
325 |
return; |
627d2d6b5 udp: enable MSG_P... |
326 |
} |
93bb64eac net: skb_free_dat... |
327 |
|
8a74ad60a net: fix lock_soc... |
328 |
slow = lock_sock_fast(sk); |
627d2d6b5 udp: enable MSG_P... |
329 |
sk_peek_offset_bwd(sk, len); |
4b0b72f7d net: speedup udp ... |
330 331 |
skb_orphan(skb); sk_mem_reclaim_partial(sk); |
8a74ad60a net: fix lock_soc... |
332 |
unlock_sock_fast(sk, slow); |
4b0b72f7d net: speedup udp ... |
333 |
|
93bb64eac net: skb_free_dat... |
334 335 |
/* skb is now orphaned, can be freed outside of locked section */ __kfree_skb(skb); |
9d410c796 net: fix sk_forwa... |
336 |
} |
627d2d6b5 udp: enable MSG_P... |
337 |
EXPORT_SYMBOL(__skb_free_datagram_locked); |
1da177e4c Linux-2.6.12-rc2 |
338 |
|
65101aeca net/sock: factor ... |
339 340 |
int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue, struct sk_buff *skb, unsigned int flags, |
69629464e udp: properly cop... |
341 342 |
void (*destructor)(struct sock *sk, struct sk_buff *skb)) |
f8c3bf00d net/socket: facto... |
343 344 345 346 347 |
{ int err = 0; if (flags & MSG_PEEK) { err = -ENOENT; |
65101aeca net/sock: factor ... |
348 |
spin_lock_bh(&sk_queue->lock); |
fd6055a80 udp: on peeking b... |
349 |
if (skb->next) { |
65101aeca net/sock: factor ... |
350 |
__skb_unlink(skb, sk_queue); |
633547973 net: convert sk_b... |
351 |
refcount_dec(&skb->users); |
69629464e udp: properly cop... |
352 353 |
if (destructor) destructor(sk, skb); |
f8c3bf00d net/socket: facto... |
354 355 |
err = 0; } |
65101aeca net/sock: factor ... |
356 |
spin_unlock_bh(&sk_queue->lock); |
f8c3bf00d net/socket: facto... |
357 358 359 360 361 362 |
} atomic_inc(&sk->sk_drops); return err; } EXPORT_SYMBOL(__sk_queue_drop_skb); |
1da177e4c Linux-2.6.12-rc2 |
363 |
/** |
3305b80c2 [IP]: Simplify an... |
364 365 366 |
* skb_kill_datagram - Free a datagram skbuff forcibly * @sk: socket * @skb: datagram skbuff |
d3f6cd9e6 datagram: fix ker... |
367 |
* @flags: MSG\_ flags |
3305b80c2 [IP]: Simplify an... |
368 369 370 371 372 373 374 375 376 377 378 379 |
* * This function frees a datagram skbuff that was received by * skb_recv_datagram. The flags argument must match the one * used for skb_recv_datagram. * * If the MSG_PEEK flag is set, and the packet is still on the * receive queue of the socket, it will be taken off the queue * before it is freed. * * This function currently only disables BH when acquiring the * sk_receive_queue lock. Therefore it must not be used in a * context where that lock is acquired in an IRQ context. |
27ab25686 [UDP]: Avoid repe... |
380 381 |
* * It returns 0 if the packet was removed by us. |
3305b80c2 [IP]: Simplify an... |
382 |
*/ |
27ab25686 [UDP]: Avoid repe... |
383 |
int skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags) |
3305b80c2 [IP]: Simplify an... |
384 |
{ |
65101aeca net/sock: factor ... |
385 386 |
int err = __sk_queue_drop_skb(sk, &sk->sk_receive_queue, skb, flags, NULL); |
3305b80c2 [IP]: Simplify an... |
387 |
|
61de71c67 Network Drop Moni... |
388 389 |
kfree_skb(skb); sk_mem_reclaim_partial(sk); |
27ab25686 [UDP]: Avoid repe... |
390 |
return err; |
3305b80c2 [IP]: Simplify an... |
391 |
} |
3305b80c2 [IP]: Simplify an... |
392 |
EXPORT_SYMBOL(skb_kill_datagram); |
29f3490ba net: use indirect... |
393 394 395 396 |
INDIRECT_CALLABLE_DECLARE(static size_t simple_copy_to_iter(const void *addr, size_t bytes, void *data __always_unused, struct iov_iter *i)); |
56dc6d635 datagram: Make __... |
397 398 399 400 |
static int __skb_datagram_iter(const struct sk_buff *skb, int offset, struct iov_iter *to, int len, bool fault_short, size_t (*cb)(const void *, size_t, void *, struct iov_iter *), void *data) |
a8f820aa4 inet: Add skb_cop... |
401 402 |
{ int start = skb_headlen(skb); |
327868212 make skb_copy_dat... |
403 |
int i, copy = start - offset, start_off = offset, n; |
a8f820aa4 inet: Add skb_cop... |
404 |
struct sk_buff *frag_iter; |
a8f820aa4 inet: Add skb_cop... |
405 406 407 408 |
/* Copy header. */ if (copy > 0) { if (copy > len) copy = len; |
29f3490ba net: use indirect... |
409 410 |
n = INDIRECT_CALL_1(cb, simple_copy_to_iter, skb->data + offset, copy, data, to); |
327868212 make skb_copy_dat... |
411 412 |
offset += n; if (n != copy) |
a8f820aa4 inet: Add skb_cop... |
413 414 415 |
goto short_copy; if ((len -= copy) == 0) return 0; |
a8f820aa4 inet: Add skb_cop... |
416 417 418 419 420 421 422 423 424 425 426 |
} /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; WARN_ON(start > offset + len); end = start + skb_frag_size(frag); if ((copy = end - offset) > 0) { |
0fc07791b datagram: open-co... |
427 428 |
struct page *page = skb_frag_page(frag); u8 *vaddr = kmap(page); |
a8f820aa4 inet: Add skb_cop... |
429 430 |
if (copy > len) copy = len; |
29f3490ba net: use indirect... |
431 432 433 |
n = INDIRECT_CALL_1(cb, simple_copy_to_iter, vaddr + skb_frag_off(frag) + offset - start, copy, data, to); |
0fc07791b datagram: open-co... |
434 |
kunmap(page); |
327868212 make skb_copy_dat... |
435 436 |
offset += n; if (n != copy) |
a8f820aa4 inet: Add skb_cop... |
437 438 439 |
goto short_copy; if (!(len -= copy)) return 0; |
a8f820aa4 inet: Add skb_cop... |
440 441 442 443 444 445 446 447 448 449 450 451 452 |
} start = end; } skb_walk_frags(skb, frag_iter) { int end; WARN_ON(start > offset + len); end = start + frag_iter->len; if ((copy = end - offset) > 0) { if (copy > len) copy = len; |
950fcaecd datagram: consoli... |
453 |
if (__skb_datagram_iter(frag_iter, offset - start, |
65d69e250 datagram: introdu... |
454 |
to, copy, fault_short, cb, data)) |
a8f820aa4 inet: Add skb_cop... |
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 |
goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; } if (!len) return 0; /* This is not really a user copy fault, but rather someone * gave us a bogus length on the skb. We should probably * print a warning here as it may indicate a kernel bug. */ fault: |
327868212 make skb_copy_dat... |
471 |
iov_iter_revert(to, offset - start_off); |
a8f820aa4 inet: Add skb_cop... |
472 473 474 |
return -EFAULT; short_copy: |
950fcaecd datagram: consoli... |
475 |
if (fault_short || iov_iter_count(to)) |
a8f820aa4 inet: Add skb_cop... |
476 477 478 479 |
goto fault; return 0; } |
950fcaecd datagram: consoli... |
480 |
|
65d69e250 datagram: introdu... |
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 |
/** * skb_copy_and_hash_datagram_iter - Copy datagram to an iovec iterator * and update a hash. * @skb: buffer to copy * @offset: offset in the buffer to start copying from * @to: iovec iterator to copy to * @len: amount of data to copy from buffer to iovec * @hash: hash request to update */ int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset, struct iov_iter *to, int len, struct ahash_request *hash) { return __skb_datagram_iter(skb, offset, to, len, true, hash_and_copy_to_iter, hash); } EXPORT_SYMBOL(skb_copy_and_hash_datagram_iter); |
950fcaecd datagram: consoli... |
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 |
static size_t simple_copy_to_iter(const void *addr, size_t bytes, void *data __always_unused, struct iov_iter *i) { return copy_to_iter(addr, bytes, i); } /** * skb_copy_datagram_iter - Copy a datagram to an iovec iterator. * @skb: buffer to copy * @offset: offset in the buffer to start copying from * @to: iovec iterator to copy to * @len: amount of data to copy from buffer to iovec */ int skb_copy_datagram_iter(const struct sk_buff *skb, int offset, struct iov_iter *to, int len) { trace_skb_copy_datagram_iovec(skb, len); return __skb_datagram_iter(skb, offset, to, len, false, simple_copy_to_iter, NULL); } |
a8f820aa4 inet: Add skb_cop... |
518 519 520 |
EXPORT_SYMBOL(skb_copy_datagram_iter); /** |
8feb2fb2b switch AF_PACKET ... |
521 |
* skb_copy_datagram_from_iter - Copy a datagram from an iov_iter. |
db543c1f9 net: skb_copy_dat... |
522 523 |
* @skb: buffer to copy * @offset: offset in the buffer to start copying to |
8feb2fb2b switch AF_PACKET ... |
524 |
* @from: the copy source |
db543c1f9 net: skb_copy_dat... |
525 526 527 |
* @len: amount of data to copy to buffer from iovec * * Returns 0 or -EFAULT. |
db543c1f9 net: skb_copy_dat... |
528 |
*/ |
3a654f975 new helpers: skb_... |
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 |
int skb_copy_datagram_from_iter(struct sk_buff *skb, int offset, struct iov_iter *from, int len) { int start = skb_headlen(skb); int i, copy = start - offset; struct sk_buff *frag_iter; /* Copy header. */ if (copy > 0) { if (copy > len) copy = len; if (copy_from_iter(skb->data + offset, copy, from) != copy) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } /* Copy paged appendix. Hmm... why does this look so complicated? */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; WARN_ON(start > offset + len); end = start + skb_frag_size(frag); if ((copy = end - offset) > 0) { size_t copied; if (copy > len) copy = len; copied = copy_page_from_iter(skb_frag_page(frag), |
b54c9d5bd net: Use skb_frag... |
562 |
skb_frag_off(frag) + offset - start, |
3a654f975 new helpers: skb_... |
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 |
copy, from); if (copied != copy) goto fault; if (!(len -= copy)) return 0; offset += copy; } start = end; } skb_walk_frags(skb, frag_iter) { int end; WARN_ON(start > offset + len); end = start + frag_iter->len; if ((copy = end - offset) > 0) { if (copy > len) copy = len; if (skb_copy_datagram_from_iter(frag_iter, offset - start, from, copy)) goto fault; if ((len -= copy) == 0) return 0; offset += copy; } start = end; } if (!len) return 0; fault: return -EFAULT; } EXPORT_SYMBOL(skb_copy_datagram_from_iter); |
52267790e sock: add MSG_ZER... |
600 601 |
int __zerocopy_sg_from_iter(struct sock *sk, struct sk_buff *skb, struct iov_iter *from, size_t length) |
3a654f975 new helpers: skb_... |
602 |
{ |
52267790e sock: add MSG_ZER... |
603 |
int frag = skb_shinfo(skb)->nr_frags; |
3a654f975 new helpers: skb_... |
604 |
|
52267790e sock: add MSG_ZER... |
605 |
while (length && iov_iter_count(from)) { |
3a654f975 new helpers: skb_... |
606 |
struct page *pages[MAX_SKB_FRAGS]; |
394fcd8a8 net: zerocopy: co... |
607 |
struct page *last_head = NULL; |
3a654f975 new helpers: skb_... |
608 609 610 |
size_t start; ssize_t copied; unsigned long truesize; |
394fcd8a8 net: zerocopy: co... |
611 |
int refs, n = 0; |
3a654f975 new helpers: skb_... |
612 613 614 |
if (frag == MAX_SKB_FRAGS) return -EMSGSIZE; |
52267790e sock: add MSG_ZER... |
615 |
copied = iov_iter_get_pages(from, pages, length, |
3a654f975 new helpers: skb_... |
616 617 618 619 620 |
MAX_SKB_FRAGS - frag, &start); if (copied < 0) return -EFAULT; iov_iter_advance(from, copied); |
52267790e sock: add MSG_ZER... |
621 |
length -= copied; |
3a654f975 new helpers: skb_... |
622 623 624 625 626 |
truesize = PAGE_ALIGN(copied + start); skb->data_len += copied; skb->len += copied; skb->truesize += truesize; |
52267790e sock: add MSG_ZER... |
627 |
if (sk && sk->sk_type == SOCK_STREAM) { |
ab4e846a8 tcp: annotate sk-... |
628 |
sk_wmem_queued_add(sk, truesize); |
52267790e sock: add MSG_ZER... |
629 630 631 632 |
sk_mem_charge(sk, truesize); } else { refcount_add(truesize, &skb->sk->sk_wmem_alloc); } |
394fcd8a8 net: zerocopy: co... |
633 |
for (refs = 0; copied != 0; start = 0) { |
3a654f975 new helpers: skb_... |
634 |
int size = min_t(int, copied, PAGE_SIZE - start); |
394fcd8a8 net: zerocopy: co... |
635 636 637 |
struct page *head = compound_head(pages[n]); start += (pages[n] - head) << PAGE_SHIFT; |
3a654f975 new helpers: skb_... |
638 639 |
copied -= size; n++; |
394fcd8a8 net: zerocopy: co... |
640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 |
if (frag) { skb_frag_t *last = &skb_shinfo(skb)->frags[frag - 1]; if (head == skb_frag_page(last) && start == skb_frag_off(last) + skb_frag_size(last)) { skb_frag_size_add(last, size); /* We combined this page, we need to release * a reference. Since compound pages refcount * is shared among many pages, batch the refcount * adjustments to limit false sharing. */ last_head = head; refs++; continue; } } if (refs) { page_ref_sub(last_head, refs); refs = 0; } skb_fill_page_desc(skb, frag++, head, start, size); |
3a654f975 new helpers: skb_... |
661 |
} |
394fcd8a8 net: zerocopy: co... |
662 663 |
if (refs) page_ref_sub(last_head, refs); |
3a654f975 new helpers: skb_... |
664 665 666 |
} return 0; } |
52267790e sock: add MSG_ZER... |
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 |
EXPORT_SYMBOL(__zerocopy_sg_from_iter); /** * zerocopy_sg_from_iter - Build a zerocopy datagram from an iov_iter * @skb: buffer to copy * @from: the source to copy from * * The function will first copy up to headlen, and then pin the userspace * pages and build frags through them. * * Returns 0, -EFAULT or -EMSGSIZE. */ int zerocopy_sg_from_iter(struct sk_buff *skb, struct iov_iter *from) { int copy = min_t(int, skb_headlen(skb), iov_iter_count(from)); /* copy up to skb headlen */ if (skb_copy_datagram_from_iter(skb, 0, from, copy)) return -EFAULT; return __zerocopy_sg_from_iter(NULL, skb, from, ~0U); } |
3a654f975 new helpers: skb_... |
689 |
EXPORT_SYMBOL(zerocopy_sg_from_iter); |
950fcaecd datagram: consoli... |
690 691 692 693 694 695 696 697 698 |
/** * skb_copy_and_csum_datagram_iter - Copy datagram to an iovec iterator * and update a checksum. * @skb: buffer to copy * @offset: offset in the buffer to start copying from * @to: iovec iterator to copy to * @len: amount of data to copy from buffer to iovec * @csump: checksum pointer */ |
1da177e4c Linux-2.6.12-rc2 |
699 |
static int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, |
e5a4b0bb8 switch memcpy_to_... |
700 |
struct iov_iter *to, int len, |
5084205fa [NET]: Annotate c... |
701 |
__wsum *csump) |
1da177e4c Linux-2.6.12-rc2 |
702 |
{ |
950fcaecd datagram: consoli... |
703 704 |
return __skb_datagram_iter(skb, offset, to, len, true, csum_and_copy_to_iter, csump); |
1da177e4c Linux-2.6.12-rc2 |
705 706 707 |
} /** |
e5a4b0bb8 switch memcpy_to_... |
708 |
* skb_copy_and_csum_datagram_msg - Copy and checksum skb to user iovec. |
4dc3b16ba [PATCH] DocBook: ... |
709 710 |
* @skb: skbuff * @hlen: hardware length |
e5a4b0bb8 switch memcpy_to_... |
711 |
* @msg: destination |
4ec93edb1 [NET] CORE: Fix w... |
712 |
* |
1da177e4c Linux-2.6.12-rc2 |
713 714 715 716 |
* Caller _must_ check that skb will fit to this iovec. * * Returns: 0 - success. * -EINVAL - checksum failure. |
e5a4b0bb8 switch memcpy_to_... |
717 |
* -EFAULT - fault during copy. |
1da177e4c Linux-2.6.12-rc2 |
718 |
*/ |
e5a4b0bb8 switch memcpy_to_... |
719 720 |
int skb_copy_and_csum_datagram_msg(struct sk_buff *skb, int hlen, struct msghdr *msg) |
1da177e4c Linux-2.6.12-rc2 |
721 |
{ |
d3bc23e7e [NET]: Annotate c... |
722 |
__wsum csum; |
1da177e4c Linux-2.6.12-rc2 |
723 |
int chunk = skb->len - hlen; |
ef8aef55c [NET]: Do not der... |
724 725 |
if (!chunk) return 0; |
01e97e651 new helper: msg_d... |
726 |
if (msg_data_left(msg) < chunk) { |
fb286bb29 [NET]: Detect har... |
727 |
if (__skb_checksum_complete(skb)) |
a6a599324 iov_iter: don't r... |
728 |
return -EINVAL; |
e5a4b0bb8 switch memcpy_to_... |
729 |
if (skb_copy_datagram_msg(skb, hlen, msg, chunk)) |
1da177e4c Linux-2.6.12-rc2 |
730 731 732 |
goto fault; } else { csum = csum_partial(skb->data, hlen, skb->csum); |
e5a4b0bb8 switch memcpy_to_... |
733 |
if (skb_copy_and_csum_datagram(skb, hlen, &msg->msg_iter, |
1da177e4c Linux-2.6.12-rc2 |
734 735 |
chunk, &csum)) goto fault; |
a6a599324 iov_iter: don't r... |
736 737 738 739 740 |
if (csum_fold(csum)) { iov_iter_revert(&msg->msg_iter, chunk); return -EINVAL; } |
db4f1be3c net: udp: fix han... |
741 742 |
if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && !skb->csum_complete_sw) |
7fe50ac83 net: dump more us... |
743 |
netdev_rx_csum_fault(NULL, skb); |
1da177e4c Linux-2.6.12-rc2 |
744 745 |
} return 0; |
1da177e4c Linux-2.6.12-rc2 |
746 747 748 |
fault: return -EFAULT; } |
e5a4b0bb8 switch memcpy_to_... |
749 |
EXPORT_SYMBOL(skb_copy_and_csum_datagram_msg); |
1da177e4c Linux-2.6.12-rc2 |
750 751 752 |
/** * datagram_poll - generic datagram poll |
a11e1d432 Revert changes to... |
753 |
* @file: file struct |
4dc3b16ba [PATCH] DocBook: ... |
754 |
* @sock: socket |
a11e1d432 Revert changes to... |
755 |
* @wait: poll table |
1da177e4c Linux-2.6.12-rc2 |
756 757 758 759 760 |
* * Datagram poll: Again totally generic. This also handles * sequenced packet sockets providing the socket receive queue * is only ever holding data ready to receive. * |
d3f6cd9e6 datagram: fix ker... |
761 |
* Note: when you *don't* use this routine for this protocol, |
1da177e4c Linux-2.6.12-rc2 |
762 763 764 |
* and you use a different write policy from sock_writeable() * then please supply your own write_space callback. */ |
a11e1d432 Revert changes to... |
765 766 |
__poll_t datagram_poll(struct file *file, struct socket *sock, poll_table *wait) |
1da177e4c Linux-2.6.12-rc2 |
767 768 |
{ struct sock *sk = sock->sk; |
a11e1d432 Revert changes to... |
769 |
__poll_t mask; |
89ab066d4 Revert "net: simp... |
770 |
sock_poll_wait(file, sock, wait); |
a11e1d432 Revert changes to... |
771 |
mask = 0; |
1da177e4c Linux-2.6.12-rc2 |
772 773 |
/* exceptional events? */ |
3ef7cf57c net: use skb_queu... |
774 |
if (sk->sk_err || !skb_queue_empty_lockless(&sk->sk_error_queue)) |
a9a08845e vfs: do bulk POLL... |
775 776 |
mask |= EPOLLERR | (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? EPOLLPRI : 0); |
7d4c04fc1 net: add option t... |
777 |
|
f348d70a3 [PATCH] POLLRDHUP... |
778 |
if (sk->sk_shutdown & RCV_SHUTDOWN) |
a9a08845e vfs: do bulk POLL... |
779 |
mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM; |
1da177e4c Linux-2.6.12-rc2 |
780 |
if (sk->sk_shutdown == SHUTDOWN_MASK) |
a9a08845e vfs: do bulk POLL... |
781 |
mask |= EPOLLHUP; |
1da177e4c Linux-2.6.12-rc2 |
782 783 |
/* readable? */ |
3ef7cf57c net: use skb_queu... |
784 |
if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) |
a9a08845e vfs: do bulk POLL... |
785 |
mask |= EPOLLIN | EPOLLRDNORM; |
1da177e4c Linux-2.6.12-rc2 |
786 787 788 789 |
/* Connection-based need to check for termination and startup */ if (connection_based(sk)) { if (sk->sk_state == TCP_CLOSE) |
a9a08845e vfs: do bulk POLL... |
790 |
mask |= EPOLLHUP; |
1da177e4c Linux-2.6.12-rc2 |
791 792 793 794 795 796 797 |
/* connection hasn't started yet? */ if (sk->sk_state == TCP_SYN_SENT) return mask; } /* writable? */ if (sock_writeable(sk)) |
a9a08845e vfs: do bulk POLL... |
798 |
mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND; |
1da177e4c Linux-2.6.12-rc2 |
799 |
else |
9cd3e072b net: rename SOCK_... |
800 |
sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); |
1da177e4c Linux-2.6.12-rc2 |
801 802 803 |
return mask; } |
a11e1d432 Revert changes to... |
804 |
EXPORT_SYMBOL(datagram_poll); |