Blame view

net/core/sock.c 88.5 KB
2874c5fd2   Thomas Gleixner   treewide: Replace...
1
  // SPDX-License-Identifier: GPL-2.0-or-later
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2
3
4
5
6
7
8
9
  /*
   * INET		An implementation of the TCP/IP protocol suite for the LINUX
   *		operating system.  INET is implemented using the  BSD Socket
   *		interface as the means of communication with the user level.
   *
   *		Generic socket support routines. Memory allocators, socket lock/release
   *		handler for protocols to use and generic option handler.
   *
02c30a84e   Jesper Juhl   [PATCH] update Ro...
10
   * Authors:	Ross Biro
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
   *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
   *		Florian La Roche, <flla@stud.uni-sb.de>
   *		Alan Cox, <A.Cox@swansea.ac.uk>
   *
   * Fixes:
   *		Alan Cox	: 	Numerous verify_area() problems
   *		Alan Cox	:	Connecting on a connecting socket
   *					now returns an error for tcp.
   *		Alan Cox	:	sock->protocol is set correctly.
   *					and is not sometimes left as 0.
   *		Alan Cox	:	connect handles icmp errors on a
   *					connect properly. Unfortunately there
   *					is a restart syscall nasty there. I
   *					can't match BSD without hacking the C
   *					library. Ideas urgently sought!
   *		Alan Cox	:	Disallow bind() to addresses that are
   *					not ours - especially broadcast ones!!
   *		Alan Cox	:	Socket 1024 _IS_ ok for users. (fencepost)
   *		Alan Cox	:	sock_wfree/sock_rfree don't destroy sockets,
   *					instead they leave that for the DESTROY timer.
   *		Alan Cox	:	Clean up error flag in accept
   *		Alan Cox	:	TCP ack handling is buggy, the DESTROY timer
   *					was buggy. Put a remove_sock() in the handler
   *					for memory when we hit 0. Also altered the timer
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
35
   *					code. The ACK stuff can wait and needs major
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
   *					TCP layer surgery.
   *		Alan Cox	:	Fixed TCP ack bug, removed remove sock
   *					and fixed timer/inet_bh race.
   *		Alan Cox	:	Added zapped flag for TCP
   *		Alan Cox	:	Move kfree_skb into skbuff.c and tidied up surplus code
   *		Alan Cox	:	for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
   *		Alan Cox	:	kfree_s calls now are kfree_skbmem so we can track skb resources
   *		Alan Cox	:	Supports socket option broadcast now as does udp. Packet and raw need fixing.
   *		Alan Cox	:	Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
   *		Rick Sladkey	:	Relaxed UDP rules for matching packets.
   *		C.E.Hawkins	:	IFF_PROMISC/SIOCGHWADDR support
   *	Pauline Middelink	:	identd support
   *		Alan Cox	:	Fixed connect() taking signals I think.
   *		Alan Cox	:	SO_LINGER supported
   *		Alan Cox	:	Error reporting fixes
   *		Anonymous	:	inet_create tidied up (sk->reuse setting)
   *		Alan Cox	:	inet sockets don't set sk->type!
   *		Alan Cox	:	Split socket option code
   *		Alan Cox	:	Callbacks
   *		Alan Cox	:	Nagle flag for Charles & Johannes stuff
   *		Alex		:	Removed restriction on inet fioctl
   *		Alan Cox	:	Splitting INET from NET core
   *		Alan Cox	:	Fixed bogus SO_TYPE handling in getsockopt()
   *		Adam Caldwell	:	Missing return in SO_DONTROUTE/SO_DEBUG code
   *		Alan Cox	:	Split IP from generic code
   *		Alan Cox	:	New kfree_skbmem()
   *		Alan Cox	:	Make SO_DEBUG superuser only.
   *		Alan Cox	:	Allow anyone to clear SO_DEBUG
   *					(compatibility fix)
   *		Alan Cox	:	Added optimistic memory grabbing for AF_UNIX throughput.
   *		Alan Cox	:	Allocator for a socket is settable.
   *		Alan Cox	:	SO_ERROR includes soft errors.
   *		Alan Cox	:	Allow NULL arguments on some SO_ opts
   *		Alan Cox	: 	Generic socket allocation to make hooks
   *					easier (suggested by Craig Metz).
   *		Michael Pall	:	SO_ERROR returns positive errno again
   *              Steve Whitehouse:       Added default destructor to free
   *                                      protocol private data.
   *              Steve Whitehouse:       Added various other default routines
   *                                      common to several socket families.
   *              Chris Evans     :       Call suser() check last on F_SETOWN
   *		Jay Schulist	:	Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
   *		Andi Kleen	:	Add sock_kmalloc()/sock_kfree_s()
   *		Andi Kleen	:	Fix write_space callback
   *		Chris Evans	:	Security fixes - signedness again
   *		Arnaldo C. Melo :       cleanups, use skb_queue_purge
   *
   * To Fix:
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
84
   */
e005d193d   Joe Perches   net: core: Use pr...
85
  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
80b14dee2   Richard Cochran   net: Add a new so...
86
  #include <asm/unaligned.h>
4fc268d24   Randy Dunlap   [PATCH] capable/c...
87
  #include <linux/capability.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
88
  #include <linux/errno.h>
cb820f8e4   Richard Cochran   net: Provide a ge...
89
  #include <linux/errqueue.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
90
91
92
93
  #include <linux/types.h>
  #include <linux/socket.h>
  #include <linux/in.h>
  #include <linux/kernel.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
94
95
96
97
  #include <linux/module.h>
  #include <linux/proc_fs.h>
  #include <linux/seq_file.h>
  #include <linux/sched.h>
f10830487   Vlastimil Babka   treewide: convert...
98
  #include <linux/sched/mm.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
99
100
101
102
103
104
105
106
107
108
  #include <linux/timer.h>
  #include <linux/string.h>
  #include <linux/sockios.h>
  #include <linux/net.h>
  #include <linux/mm.h>
  #include <linux/slab.h>
  #include <linux/interrupt.h>
  #include <linux/poll.h>
  #include <linux/tcp.h>
  #include <linux/init.h>
a1f8e7f7f   Al Viro   [PATCH] severing ...
109
  #include <linux/highmem.h>
3f551f943   Eric W. Biederman   sock: Introduce c...
110
  #include <linux/user_namespace.h>
c5905afb0   Ingo Molnar   static keys: Intr...
111
  #include <linux/static_key.h>
3969eb385   David S. Miller   net: Fix build wi...
112
  #include <linux/memcontrol.h>
8c1ae10d7   David S. Miller   net: Add missing ...
113
  #include <linux/prefetch.h>
a6c0d0934   Christoph Hellwig   net: explicitly i...
114
  #include <linux/compat.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
115

7c0f6ba68   Linus Torvalds   Replace <asm/uacc...
116
  #include <linux/uaccess.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
117
118
119
120
  
  #include <linux/netdevice.h>
  #include <net/protocol.h>
  #include <linux/skbuff.h>
457c4cbc5   Eric W. Biederman   [NET]: Make /proc...
121
  #include <net/net_namespace.h>
2e6599cb8   Arnaldo Carvalho de Melo   [NET] Generalise ...
122
  #include <net/request_sock.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
123
  #include <net/sock.h>
20d494735   Patrick Ohly   net: socket infra...
124
  #include <linux/net_tstamp.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
125
126
  #include <net/xfrm.h>
  #include <linux/ipsec.h>
f84517253   Herbert Xu   cls_cgroup: Store...
127
  #include <net/cls_cgroup.h>
5bc1421e3   Neil Horman   net: add network ...
128
  #include <net/netprio_cgroup.h>
eb4cb0085   Craig Gallek   sock_diag: define...
129
  #include <linux/sock_diag.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
130
131
  
  #include <linux/filter.h>
538950a1b   Craig Gallek   soreuseport: sets...
132
  #include <net/sock_reuseport.h>
6ac99e8f2   Martin KaFai Lau   bpf: Introduce bp...
133
  #include <net/bpf_sk_storage.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
134

3847ce32a   Satoru Moriya   core: add tracepo...
135
  #include <trace/events/sock.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
136
  #include <net/tcp.h>
076bb0c82   Eliezer Tamir   net: rename inclu...
137
  #include <net/busy_poll.h>
060212928   Eliezer Tamir   net: add low late...
138

36b77a520   Glauber Costa   net: fix sleeping...
139
  static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b37   Glauber Costa   tcp memory pressu...
140
  static LIST_HEAD(proto_list);
648845ab7   Tonghao Zhang   sock: Move the so...
141
  static void sock_inuse_add(struct net *net, int val);
a3b299da8   Eric W. Biederman   net: Add variants...
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
  /**
   * sk_ns_capable - General socket capability test
   * @sk: Socket to use a capability on or through
   * @user_ns: The user namespace of the capability to use
   * @cap: The capability to use
   *
   * Test to see if the opener of the socket had when the socket was
   * created and the current process has the capability @cap in the user
   * namespace @user_ns.
   */
  bool sk_ns_capable(const struct sock *sk,
  		   struct user_namespace *user_ns, int cap)
  {
  	return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
  		ns_capable(user_ns, cap);
  }
  EXPORT_SYMBOL(sk_ns_capable);
  
  /**
   * sk_capable - Socket global capability test
   * @sk: Socket to use a capability on or through
e793c0f70   Masanari Iida   net: treewide: Fi...
163
   * @cap: The global capability to use
a3b299da8   Eric W. Biederman   net: Add variants...
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
   *
   * Test to see if the opener of the socket had when the socket was
   * created and the current process has the capability @cap in all user
   * namespaces.
   */
  bool sk_capable(const struct sock *sk, int cap)
  {
  	return sk_ns_capable(sk, &init_user_ns, cap);
  }
  EXPORT_SYMBOL(sk_capable);
  
  /**
   * sk_net_capable - Network namespace socket capability test
   * @sk: Socket to use a capability on or through
   * @cap: The capability to use
   *
e793c0f70   Masanari Iida   net: treewide: Fi...
180
   * Test to see if the opener of the socket had when the socket was created
a3b299da8   Eric W. Biederman   net: Add variants...
181
182
183
184
185
186
187
188
   * and the current process has the capability @cap over the network namespace
   * the socket is a member of.
   */
  bool sk_net_capable(const struct sock *sk, int cap)
  {
  	return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
  }
  EXPORT_SYMBOL(sk_net_capable);
da21f24dd   Ingo Molnar   [PATCH] lockdep: ...
189
190
  /*
   * Each address family might have different locking rules, so we have
cdfbabfb2   David Howells   net: Work around ...
191
192
   * one slock key per address family and separate keys for internal and
   * userspace sockets.
da21f24dd   Ingo Molnar   [PATCH] lockdep: ...
193
   */
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
194
  static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb2   David Howells   net: Work around ...
195
  static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
196
  static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb2   David Howells   net: Work around ...
197
  static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
198

a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
199
200
201
202
203
  /*
   * Make lock validator output more readable. (we pre-construct these
   * strings build-time, so that runtime initialization of socket
   * locks is fast):
   */
cdfbabfb2   David Howells   net: Work around ...
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  
  #define _sock_locks(x)						  \
    x "AF_UNSPEC",	x "AF_UNIX"     ,	x "AF_INET"     , \
    x "AF_AX25"  ,	x "AF_IPX"      ,	x "AF_APPLETALK", \
    x "AF_NETROM",	x "AF_BRIDGE"   ,	x "AF_ATMPVC"   , \
    x "AF_X25"   ,	x "AF_INET6"    ,	x "AF_ROSE"     , \
    x "AF_DECnet",	x "AF_NETBEUI"  ,	x "AF_SECURITY" , \
    x "AF_KEY"   ,	x "AF_NETLINK"  ,	x "AF_PACKET"   , \
    x "AF_ASH"   ,	x "AF_ECONET"   ,	x "AF_ATMSVC"   , \
    x "AF_RDS"   ,	x "AF_SNA"      ,	x "AF_IRDA"     , \
    x "AF_PPPOX" ,	x "AF_WANPIPE"  ,	x "AF_LLC"      , \
    x "27"       ,	x "28"          ,	x "AF_CAN"      , \
    x "AF_TIPC"  ,	x "AF_BLUETOOTH",	x "IUCV"        , \
    x "AF_RXRPC" ,	x "AF_ISDN"     ,	x "AF_PHONET"   , \
    x "AF_IEEE802154",	x "AF_CAIF"	,	x "AF_ALG"      , \
    x "AF_NFC"   ,	x "AF_VSOCK"    ,	x "AF_KCM"      , \
68e8b849b   Björn Töpel   net: initial AF_X...
220
221
    x "AF_QIPCRTR",	x "AF_SMC"	,	x "AF_XDP"	, \
    x "AF_MAX"
cdfbabfb2   David Howells   net: Work around ...
222

36cbd3dcc   Jan Engelhardt   net: mark read-on...
223
  static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb2   David Howells   net: Work around ...
224
  	_sock_locks("sk_lock-")
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
225
  };
36cbd3dcc   Jan Engelhardt   net: mark read-on...
226
  static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb2   David Howells   net: Work around ...
227
  	_sock_locks("slock-")
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
228
  };
36cbd3dcc   Jan Engelhardt   net: mark read-on...
229
  static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb2   David Howells   net: Work around ...
230
231
232
233
234
235
236
237
238
239
240
  	_sock_locks("clock-")
  };
  
  static const char *const af_family_kern_key_strings[AF_MAX+1] = {
  	_sock_locks("k-sk_lock-")
  };
  static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
  	_sock_locks("k-slock-")
  };
  static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
  	_sock_locks("k-clock-")
443aef0ed   Peter Zijlstra   lockdep: fixup sk...
241
  };
581319c58   Paolo Abeni   net/socket: use p...
242
  static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
6b431d50d   Matthieu Baerts   net/socket: remov...
243
  	_sock_locks("rlock-")
581319c58   Paolo Abeni   net/socket: use p...
244
245
  };
  static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
6b431d50d   Matthieu Baerts   net/socket: remov...
246
  	_sock_locks("wlock-")
581319c58   Paolo Abeni   net/socket: use p...
247
248
  };
  static const char *const af_family_elock_key_strings[AF_MAX+1] = {
6b431d50d   Matthieu Baerts   net/socket: remov...
249
  	_sock_locks("elock-")
581319c58   Paolo Abeni   net/socket: use p...
250
  };
da21f24dd   Ingo Molnar   [PATCH] lockdep: ...
251
252
  
  /*
581319c58   Paolo Abeni   net/socket: use p...
253
   * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24dd   Ingo Molnar   [PATCH] lockdep: ...
254
255
256
   * so split the lock classes by using a per-AF key:
   */
  static struct lock_class_key af_callback_keys[AF_MAX];
581319c58   Paolo Abeni   net/socket: use p...
257
258
259
  static struct lock_class_key af_rlock_keys[AF_MAX];
  static struct lock_class_key af_wlock_keys[AF_MAX];
  static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb2   David Howells   net: Work around ...
260
  static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24dd   Ingo Molnar   [PATCH] lockdep: ...
261

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
262
  /* Run time adjustable parameters. */
ab32ea5d8   Brian Haley   [NET/IPV4/IPV6]: ...
263
  __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a2   Hans Schillstrom   net: export sysct...
264
  EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d8   Brian Haley   [NET/IPV4/IPV6]: ...
265
  __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a2   Hans Schillstrom   net: export sysct...
266
  EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d8   Brian Haley   [NET/IPV4/IPV6]: ...
267
268
  __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
  __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
269

25985edce   Lucas De Marchi   Fix common misspe...
270
  /* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d8   Brian Haley   [NET/IPV4/IPV6]: ...
271
  int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c2   Eric Dumazet   net: net/core/soc...
272
  EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
273

b245be1f4   Willem de Bruijn   net-timestamp: no...
274
  int sysctl_tstamp_allow_data __read_mostly = 1;
a7950ae82   Davidlohr Bueso   net/sock: Update ...
275
276
  DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
  EXPORT_SYMBOL_GPL(memalloc_socks_key);
c93bdd0e0   Mel Gorman   netvm: allow skb ...
277

7cb024049   Mel Gorman   netvm: allow the ...
278
279
280
281
282
283
284
285
286
287
288
289
  /**
   * sk_set_memalloc - sets %SOCK_MEMALLOC
   * @sk: socket to set it on
   *
   * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
   * It's the responsibility of the admin to adjust min_free_kbytes
   * to meet the requirements
   */
  void sk_set_memalloc(struct sock *sk)
  {
  	sock_set_flag(sk, SOCK_MEMALLOC);
  	sk->sk_allocation |= __GFP_MEMALLOC;
a7950ae82   Davidlohr Bueso   net/sock: Update ...
290
  	static_branch_inc(&memalloc_socks_key);
7cb024049   Mel Gorman   netvm: allow the ...
291
292
293
294
295
296
297
  }
  EXPORT_SYMBOL_GPL(sk_set_memalloc);
  
  void sk_clear_memalloc(struct sock *sk)
  {
  	sock_reset_flag(sk, SOCK_MEMALLOC);
  	sk->sk_allocation &= ~__GFP_MEMALLOC;
a7950ae82   Davidlohr Bueso   net/sock: Update ...
298
  	static_branch_dec(&memalloc_socks_key);
c76562b67   Mel Gorman   netvm: prevent a ...
299
300
301
  
  	/*
  	 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d7536102   Mel Gorman   net, swap: Remove...
302
303
304
305
  	 * progress of swapping. SOCK_MEMALLOC may be cleared while
  	 * it has rmem allocations due to the last swapfile being deactivated
  	 * but there is a risk that the socket is unusable due to exceeding
  	 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b67   Mel Gorman   netvm: prevent a ...
306
  	 */
5d7536102   Mel Gorman   net, swap: Remove...
307
  	sk_mem_reclaim(sk);
7cb024049   Mel Gorman   netvm: allow the ...
308
309
  }
  EXPORT_SYMBOL_GPL(sk_clear_memalloc);
b4b9e3558   Mel Gorman   netvm: set PF_MEM...
310
311
312
  int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
  {
  	int ret;
f10830487   Vlastimil Babka   treewide: convert...
313
  	unsigned int noreclaim_flag;
b4b9e3558   Mel Gorman   netvm: set PF_MEM...
314
315
316
  
  	/* these should have been dropped before queueing */
  	BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
f10830487   Vlastimil Babka   treewide: convert...
317
  	noreclaim_flag = memalloc_noreclaim_save();
b4b9e3558   Mel Gorman   netvm: set PF_MEM...
318
  	ret = sk->sk_backlog_rcv(sk, skb);
f10830487   Vlastimil Babka   treewide: convert...
319
  	memalloc_noreclaim_restore(noreclaim_flag);
b4b9e3558   Mel Gorman   netvm: set PF_MEM...
320
321
322
323
  
  	return ret;
  }
  EXPORT_SYMBOL(__sk_backlog_rcv);
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
324
  static int sock_get_timeout(long timeo, void *optval, bool old_timeval)
fe0c72f3d   Arnd Bergmann   socket: move comp...
325
  {
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
326
  	struct __kernel_sock_timeval tv;
fe0c72f3d   Arnd Bergmann   socket: move comp...
327
328
329
330
331
332
333
334
  
  	if (timeo == MAX_SCHEDULE_TIMEOUT) {
  		tv.tv_sec = 0;
  		tv.tv_usec = 0;
  	} else {
  		tv.tv_sec = timeo / HZ;
  		tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
  	}
e6986423d   Arnd Bergmann   socket: fix compa...
335
  	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3d   Arnd Bergmann   socket: move comp...
336
337
338
339
  		struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
  		*(struct old_timeval32 *)optval = tv32;
  		return sizeof(tv32);
  	}
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
340
341
342
343
344
  	if (old_timeval) {
  		struct __kernel_old_timeval old_tv;
  		old_tv.tv_sec = tv.tv_sec;
  		old_tv.tv_usec = tv.tv_usec;
  		*(struct __kernel_old_timeval *)optval = old_tv;
28e72b26d   Vito Caputo   sock_get_timeout:...
345
  		return sizeof(old_tv);
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
346
  	}
28e72b26d   Vito Caputo   sock_get_timeout:...
347
348
  	*(struct __kernel_sock_timeval *)optval = tv;
  	return sizeof(tv);
fe0c72f3d   Arnd Bergmann   socket: move comp...
349
  }
c34645ac2   Christoph Hellwig   net: switch sock_...
350
351
  static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
  			    bool old_timeval)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
352
  {
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
353
  	struct __kernel_sock_timeval tv;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
354

e6986423d   Arnd Bergmann   socket: fix compa...
355
  	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3d   Arnd Bergmann   socket: move comp...
356
357
358
359
  		struct old_timeval32 tv32;
  
  		if (optlen < sizeof(tv32))
  			return -EINVAL;
c34645ac2   Christoph Hellwig   net: switch sock_...
360
  		if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
fe0c72f3d   Arnd Bergmann   socket: move comp...
361
362
363
  			return -EFAULT;
  		tv.tv_sec = tv32.tv_sec;
  		tv.tv_usec = tv32.tv_usec;
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
364
365
366
367
368
  	} else if (old_timeval) {
  		struct __kernel_old_timeval old_tv;
  
  		if (optlen < sizeof(old_tv))
  			return -EINVAL;
c34645ac2   Christoph Hellwig   net: switch sock_...
369
  		if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
370
371
372
  			return -EFAULT;
  		tv.tv_sec = old_tv.tv_sec;
  		tv.tv_usec = old_tv.tv_usec;
fe0c72f3d   Arnd Bergmann   socket: move comp...
373
374
375
  	} else {
  		if (optlen < sizeof(tv))
  			return -EINVAL;
c34645ac2   Christoph Hellwig   net: switch sock_...
376
  		if (copy_from_sockptr(&tv, optval, sizeof(tv)))
fe0c72f3d   Arnd Bergmann   socket: move comp...
377
378
  			return -EFAULT;
  	}
ba78073e6   Vasily Averin   [NET]: "wrong tim...
379
380
  	if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
  		return -EDOM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
381

ba78073e6   Vasily Averin   [NET]: "wrong tim...
382
  	if (tv.tv_sec < 0) {
6f11df835   Andrew Morton   [NET]: "wrong tim...
383
  		static int warned __read_mostly;
ba78073e6   Vasily Averin   [NET]: "wrong tim...
384
  		*timeo_p = 0;
50aab54f3   Ilpo Järvinen   net: Add missing ...
385
  		if (warned < 10 && net_ratelimit()) {
ba78073e6   Vasily Averin   [NET]: "wrong tim...
386
  			warned++;
e005d193d   Joe Perches   net: core: Use pr...
387
388
389
  			pr_info("%s: `%s' (pid %d) tries to set negative timeout
  ",
  				__func__, current->comm, task_pid_nr(current));
50aab54f3   Ilpo Järvinen   net: Add missing ...
390
  		}
ba78073e6   Vasily Averin   [NET]: "wrong tim...
391
392
  		return 0;
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
393
394
395
  	*timeo_p = MAX_SCHEDULE_TIMEOUT;
  	if (tv.tv_sec == 0 && tv.tv_usec == 0)
  		return 0;
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
396
397
  	if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
  		*timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
398
399
  	return 0;
  }
080a270f5   Hannes Frederic Sowa   sock: don't enabl...
400
401
402
403
404
405
406
407
408
409
  static bool sock_needs_netstamp(const struct sock *sk)
  {
  	switch (sk->sk_family) {
  	case AF_UNSPEC:
  	case AF_UNIX:
  		return false;
  	default:
  		return true;
  	}
  }
08e29af3a   Eric Dumazet   net: optimize soc...
410
  static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
411
  {
08e29af3a   Eric Dumazet   net: optimize soc...
412
413
  	if (sk->sk_flags & flags) {
  		sk->sk_flags &= ~flags;
080a270f5   Hannes Frederic Sowa   sock: don't enabl...
414
415
  		if (sock_needs_netstamp(sk) &&
  		    !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d494735   Patrick Ohly   net: socket infra...
416
  			net_disable_timestamp();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
417
418
  	}
  }
e6afc8ace   samanthakumar   udp: remove heade...
419
  int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
420
  {
3b885787e   Neil Horman   net: Generalize s...
421
422
  	unsigned long flags;
  	struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
423

0fd7bac6b   Eric Dumazet   net: relax rcvbuf...
424
  	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037c   Eric Dumazet   net: sk_drops con...
425
  		atomic_inc(&sk->sk_drops);
3847ce32a   Satoru Moriya   core: add tracepo...
426
  		trace_sock_rcvqueue_full(sk, skb);
766e9037c   Eric Dumazet   net: sk_drops con...
427
  		return -ENOMEM;
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
428
  	}
c76562b67   Mel Gorman   netvm: prevent a ...
429
  	if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037c   Eric Dumazet   net: sk_drops con...
430
431
  		atomic_inc(&sk->sk_drops);
  		return -ENOBUFS;
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
432
  	}
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
433
434
  	skb->dev = NULL;
  	skb_set_owner_r(skb, sk);
49ad9599d   David S. Miller   Revert "net: rele...
435

7fee226ad   Eric Dumazet   net: add a noref ...
436
437
438
439
  	/* we escape from rcu protected region, make sure we dont leak
  	 * a norefcounted dst
  	 */
  	skb_dst_force(skb);
3b885787e   Neil Horman   net: Generalize s...
440
  	spin_lock_irqsave(&list->lock, flags);
3bc3b96f3   Eyal Birger   net: add common a...
441
  	sock_skb_set_dropcount(sk, skb);
3b885787e   Neil Horman   net: Generalize s...
442
443
  	__skb_queue_tail(list, skb);
  	spin_unlock_irqrestore(&list->lock, flags);
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
444
445
  
  	if (!sock_flag(sk, SOCK_DEAD))
676d23690   David S. Miller   net: Fix use afte...
446
  		sk->sk_data_ready(sk);
766e9037c   Eric Dumazet   net: sk_drops con...
447
  	return 0;
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
448
  }
e6afc8ace   samanthakumar   udp: remove heade...
449
450
451
452
453
454
455
456
457
458
459
460
  EXPORT_SYMBOL(__sock_queue_rcv_skb);
  
  int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
  {
  	int err;
  
  	err = sk_filter(sk, skb);
  	if (err)
  		return err;
  
  	return __sock_queue_rcv_skb(sk, skb);
  }
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
461
  EXPORT_SYMBOL(sock_queue_rcv_skb);
4f0c40d94   Willem de Bruijn   dccp: limit sk_fi...
462
  int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb3   Eric Dumazet   dccp: do not rele...
463
  		     const int nested, unsigned int trim_cap, bool refcounted)
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
464
465
  {
  	int rc = NET_RX_SUCCESS;
4f0c40d94   Willem de Bruijn   dccp: limit sk_fi...
466
  	if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
467
468
469
  		goto discard_and_relse;
  
  	skb->dev = NULL;
274f482d3   Sorin Dumitru   sock: remove skb ...
470
  	if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f2   Eric Dumazet   net: sk_add_backl...
471
472
473
  		atomic_inc(&sk->sk_drops);
  		goto discard_and_relse;
  	}
58a5a7b95   Arnaldo Carvalho de Melo   [NET]: Conditiona...
474
475
476
477
  	if (nested)
  		bh_lock_sock_nested(sk);
  	else
  		bh_lock_sock(sk);
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
478
479
480
481
482
  	if (!sock_owned_by_user(sk)) {
  		/*
  		 * trylock + unlock semantics:
  		 */
  		mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
c57943a1c   Peter Zijlstra   net: wrap sk->sk_...
483
  		rc = sk_backlog_rcv(sk, skb);
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
484

5facae4f3   Qian Cai   locking/lockdep: ...
485
  		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
8265792bf   Eric Dumazet   net: silence KCSA...
486
  	} else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
8eae939f1   Zhu Yi   net: add limit fo...
487
488
489
490
  		bh_unlock_sock(sk);
  		atomic_inc(&sk->sk_drops);
  		goto discard_and_relse;
  	}
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
491
492
  	bh_unlock_sock(sk);
  out:
c3f24cfb3   Eric Dumazet   dccp: do not rele...
493
494
  	if (refcounted)
  		sock_put(sk);
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
495
496
497
498
499
  	return rc;
  discard_and_relse:
  	kfree_skb(skb);
  	goto out;
  }
4f0c40d94   Willem de Bruijn   dccp: limit sk_fi...
500
  EXPORT_SYMBOL(__sk_receive_skb);
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
501
502
503
  
  struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
  {
b6c6712a4   Eric Dumazet   net: sk_dst_cache...
504
  	struct dst_entry *dst = __sk_dst_get(sk);
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
505
506
  
  	if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
e022f0b4a   Krishna Kumar   net: Introduce sk...
507
  		sk_tx_queue_clear(sk);
9b8805a32   Julian Anastasov   sock: add sk_dst_...
508
  		sk->sk_dst_pending_confirm = 0;
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
509
  		RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50e   Denis Vlasenko   [NET]: deinline 2...
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
  		dst_release(dst);
  		return NULL;
  	}
  
  	return dst;
  }
  EXPORT_SYMBOL(__sk_dst_check);
  
  struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
  {
  	struct dst_entry *dst = sk_dst_get(sk);
  
  	if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
  		sk_dst_reset(sk);
  		dst_release(dst);
  		return NULL;
  	}
  
  	return dst;
  }
  EXPORT_SYMBOL(sk_dst_check);
7594888c7   Christoph Hellwig   net: add sock_bin...
531
  static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
4878809f7   David S. Miller   [NET]: Fix two is...
532
533
534
  {
  	int ret = -ENOPROTOOPT;
  #ifdef CONFIG_NETDEVICES
3b1e0a655   YOSHIFUJI Hideaki   [NET] NETNS: Omit...
535
  	struct net *net = sock_net(sk);
4878809f7   David S. Miller   [NET]: Fix two is...
536
537
538
  
  	/* Sorry... */
  	ret = -EPERM;
c427bfec1   Vincent Bernat   net: core: enable...
539
  	if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
4878809f7   David S. Miller   [NET]: Fix two is...
540
541
542
  		goto out;
  
  	ret = -EINVAL;
f5dd3d0c9   David Herrmann   net: introduce SO...
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
  	if (ifindex < 0)
  		goto out;
  
  	sk->sk_bound_dev_if = ifindex;
  	if (sk->sk_prot->rehash)
  		sk->sk_prot->rehash(sk);
  	sk_dst_reset(sk);
  
  	ret = 0;
  
  out:
  #endif
  
  	return ret;
  }
8ea204c2b   Ferenc Fejes   net: Make locking...
558
  int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
7594888c7   Christoph Hellwig   net: add sock_bin...
559
560
  {
  	int ret;
8ea204c2b   Ferenc Fejes   net: Make locking...
561
562
  	if (lock_sk)
  		lock_sock(sk);
7594888c7   Christoph Hellwig   net: add sock_bin...
563
  	ret = sock_bindtoindex_locked(sk, ifindex);
8ea204c2b   Ferenc Fejes   net: Make locking...
564
565
  	if (lock_sk)
  		release_sock(sk);
7594888c7   Christoph Hellwig   net: add sock_bin...
566
567
568
569
  
  	return ret;
  }
  EXPORT_SYMBOL(sock_bindtoindex);
5790642b4   Christoph Hellwig   net: switch sock_...
570
  static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
f5dd3d0c9   David Herrmann   net: introduce SO...
571
572
573
574
575
576
577
578
  {
  	int ret = -ENOPROTOOPT;
  #ifdef CONFIG_NETDEVICES
  	struct net *net = sock_net(sk);
  	char devname[IFNAMSIZ];
  	int index;
  
  	ret = -EINVAL;
4878809f7   David S. Miller   [NET]: Fix two is...
579
580
581
582
583
584
585
586
587
588
589
590
591
  	if (optlen < 0)
  		goto out;
  
  	/* Bind this socket to a particular device like "eth0",
  	 * as specified in the passed interface name. If the
  	 * name is "" or the option length is zero the socket
  	 * is not bound.
  	 */
  	if (optlen > IFNAMSIZ - 1)
  		optlen = IFNAMSIZ - 1;
  	memset(devname, 0, sizeof(devname));
  
  	ret = -EFAULT;
5790642b4   Christoph Hellwig   net: switch sock_...
592
  	if (copy_from_sockptr(devname, optval, optlen))
4878809f7   David S. Miller   [NET]: Fix two is...
593
  		goto out;
000ba2e43   David S. Miller   net: Fix build wa...
594
595
  	index = 0;
  	if (devname[0] != '\0') {
bf8e56bfc   Eric Dumazet   net: sock_bindtod...
596
  		struct net_device *dev;
4878809f7   David S. Miller   [NET]: Fix two is...
597

bf8e56bfc   Eric Dumazet   net: sock_bindtod...
598
599
600
601
602
  		rcu_read_lock();
  		dev = dev_get_by_name_rcu(net, devname);
  		if (dev)
  			index = dev->ifindex;
  		rcu_read_unlock();
4878809f7   David S. Miller   [NET]: Fix two is...
603
604
605
  		ret = -ENODEV;
  		if (!dev)
  			goto out;
4878809f7   David S. Miller   [NET]: Fix two is...
606
  	}
8ea204c2b   Ferenc Fejes   net: Make locking...
607
  	return sock_bindtoindex(sk, index, true);
4878809f7   David S. Miller   [NET]: Fix two is...
608
609
610
611
612
  out:
  #endif
  
  	return ret;
  }
c91f6df2d   Brian Haley   sockopt: Change g...
613
614
615
616
617
618
  static int sock_getbindtodevice(struct sock *sk, char __user *optval,
  				int __user *optlen, int len)
  {
  	int ret = -ENOPROTOOPT;
  #ifdef CONFIG_NETDEVICES
  	struct net *net = sock_net(sk);
c91f6df2d   Brian Haley   sockopt: Change g...
619
  	char devname[IFNAMSIZ];
c91f6df2d   Brian Haley   sockopt: Change g...
620
621
622
623
624
625
626
627
628
  
  	if (sk->sk_bound_dev_if == 0) {
  		len = 0;
  		goto zero;
  	}
  
  	ret = -EINVAL;
  	if (len < IFNAMSIZ)
  		goto out;
5dbe7c178   Nicolas Schichan   net: fix kernel d...
629
630
  	ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
  	if (ret)
c91f6df2d   Brian Haley   sockopt: Change g...
631
  		goto out;
c91f6df2d   Brian Haley   sockopt: Change g...
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
  
  	len = strlen(devname) + 1;
  
  	ret = -EFAULT;
  	if (copy_to_user(optval, devname, len))
  		goto out;
  
  zero:
  	ret = -EFAULT;
  	if (put_user(len, optlen))
  		goto out;
  
  	ret = 0;
  
  out:
  #endif
  
  	return ret;
  }
f60e5990d   hannes@stressinduktion.org   ipv6: protect skb...
651
652
653
654
655
656
657
658
659
660
661
662
663
664
  bool sk_mc_loop(struct sock *sk)
  {
  	if (dev_recursion_level())
  		return false;
  	if (!sk)
  		return true;
  	switch (sk->sk_family) {
  	case AF_INET:
  		return inet_sk(sk)->mc_loop;
  #if IS_ENABLED(CONFIG_IPV6)
  	case AF_INET6:
  		return inet6_sk(sk)->mc_loop;
  #endif
  	}
0ad6f6e76   Eric Dumazet   net: increment xm...
665
  	WARN_ON_ONCE(1);
f60e5990d   hannes@stressinduktion.org   ipv6: protect skb...
666
667
668
  	return true;
  }
  EXPORT_SYMBOL(sk_mc_loop);
b58f0e8f3   Christoph Hellwig   net: add sock_set...
669
670
671
672
673
674
675
  void sock_set_reuseaddr(struct sock *sk)
  {
  	lock_sock(sk);
  	sk->sk_reuse = SK_CAN_REUSE;
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_reuseaddr);
fe31a326a   Christoph Hellwig   net: add sock_set...
676
677
678
679
680
681
682
  void sock_set_reuseport(struct sock *sk)
  {
  	lock_sock(sk);
  	sk->sk_reuseport = true;
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_reuseport);
c433594c0   Christoph Hellwig   net: add sock_no_...
683
684
685
686
687
688
689
690
  void sock_no_linger(struct sock *sk)
  {
  	lock_sock(sk);
  	sk->sk_lingertime = 0;
  	sock_set_flag(sk, SOCK_LINGER);
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_no_linger);
6e4349674   Christoph Hellwig   net: add sock_set...
691
692
693
694
695
696
697
  void sock_set_priority(struct sock *sk, u32 priority)
  {
  	lock_sock(sk);
  	sk->sk_priority = priority;
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_priority);
76ee0785f   Christoph Hellwig   net: add sock_set...
698
699
700
701
702
703
704
705
706
707
  void sock_set_sndtimeo(struct sock *sk, s64 secs)
  {
  	lock_sock(sk);
  	if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
  		sk->sk_sndtimeo = secs * HZ;
  	else
  		sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_sndtimeo);
783da70e8   Christoph Hellwig   net: add sock_ena...
708
709
710
711
712
713
714
715
716
717
  static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
  {
  	if (val)  {
  		sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
  		sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns);
  		sock_set_flag(sk, SOCK_RCVTSTAMP);
  		sock_enable_timestamp(sk, SOCK_TIMESTAMP);
  	} else {
  		sock_reset_flag(sk, SOCK_RCVTSTAMP);
  		sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
783da70e8   Christoph Hellwig   net: add sock_ena...
718
719
720
721
722
723
724
725
726
727
  	}
  }
  
  void sock_enable_timestamps(struct sock *sk)
  {
  	lock_sock(sk);
  	__sock_set_timestamps(sk, true, false, true);
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_enable_timestamps);
ce3d9544c   Christoph Hellwig   net: add sock_set...
728
729
730
731
732
733
734
735
736
  void sock_set_keepalive(struct sock *sk)
  {
  	lock_sock(sk);
  	if (sk->sk_prot->keepalive)
  		sk->sk_prot->keepalive(sk, true);
  	sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_keepalive);
26cfabf9c   Christoph Hellwig   net: add sock_set...
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
  static void __sock_set_rcvbuf(struct sock *sk, int val)
  {
  	/* Ensure val * 2 fits into an int, to prevent max_t() from treating it
  	 * as a negative value.
  	 */
  	val = min_t(int, val, INT_MAX / 2);
  	sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
  
  	/* We double it on the way in to account for "struct sk_buff" etc.
  	 * overhead.   Applications assume that the SO_RCVBUF setting they make
  	 * will allow that much actual data to be received on that socket.
  	 *
  	 * Applications are unaware that "struct sk_buff" and other overheads
  	 * allocate from the receive buffer during socket buffer allocation.
  	 *
  	 * And after considering the possible alternatives, returning the value
  	 * we actually used in getsockopt is the most desirable behavior.
  	 */
  	WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
  }
  
  void sock_set_rcvbuf(struct sock *sk, int val)
  {
  	lock_sock(sk);
  	__sock_set_rcvbuf(sk, val);
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_rcvbuf);
84d1c6174   Alexander Aring   net: sock: add so...
765
766
767
768
769
770
771
  void sock_set_mark(struct sock *sk, u32 val)
  {
  	lock_sock(sk);
  	sk->sk_mark = val;
  	release_sock(sk);
  }
  EXPORT_SYMBOL(sock_set_mark);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
772
773
774
775
776
777
  /*
   *	This is meant for all protocols to use and covers goings on
   *	at the socket level. Everything here is generic.
   */
  
  int sock_setsockopt(struct socket *sock, int level, int optname,
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
778
  		    sockptr_t optval, unsigned int optlen)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
779
  {
80b14dee2   Richard Cochran   net: Add a new so...
780
  	struct sock_txtime sk_txtime;
2a91525c2   Eric Dumazet   net: net/core/soc...
781
  	struct sock *sk = sock->sk;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
782
783
784
785
  	int val;
  	int valbool;
  	struct linger ling;
  	int ret = 0;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
786

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
787
788
789
  	/*
  	 *	Options without arguments
  	 */
4878809f7   David S. Miller   [NET]: Fix two is...
790
  	if (optname == SO_BINDTODEVICE)
c91f6df2d   Brian Haley   sockopt: Change g...
791
  		return sock_setbindtodevice(sk, optval, optlen);
4878809f7   David S. Miller   [NET]: Fix two is...
792

e71a4783a   Stephen Hemminger   [NET] core: white...
793
794
  	if (optlen < sizeof(int))
  		return -EINVAL;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
795

c8c1bbb6e   Christoph Hellwig   net: switch sock_...
796
  	if (copy_from_sockptr(&val, optval, sizeof(val)))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
797
  		return -EFAULT;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
798

2a91525c2   Eric Dumazet   net: net/core/soc...
799
  	valbool = val ? 1 : 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
800
801
  
  	lock_sock(sk);
2a91525c2   Eric Dumazet   net: net/core/soc...
802
  	switch (optname) {
e71a4783a   Stephen Hemminger   [NET] core: white...
803
  	case SO_DEBUG:
2a91525c2   Eric Dumazet   net: net/core/soc...
804
  		if (val && !capable(CAP_NET_ADMIN))
e71a4783a   Stephen Hemminger   [NET] core: white...
805
  			ret = -EACCES;
2a91525c2   Eric Dumazet   net: net/core/soc...
806
  		else
c0ef877b2   Pavel Emelyanov   [NET]: Move sock_...
807
  			sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783a   Stephen Hemminger   [NET] core: white...
808
809
  		break;
  	case SO_REUSEADDR:
cdb8744d8   Bart Van Assche   Revert "net: do n...
810
  		sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783a   Stephen Hemminger   [NET] core: white...
811
  		break;
055dc21a1   Tom Herbert   soreuseport: infr...
812
813
814
  	case SO_REUSEPORT:
  		sk->sk_reuseport = valbool;
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
815
  	case SO_TYPE:
49c794e94   Jan Engelhardt   net: implement a ...
816
  	case SO_PROTOCOL:
0d6038ee7   Jan Engelhardt   net: implement a ...
817
  	case SO_DOMAIN:
e71a4783a   Stephen Hemminger   [NET] core: white...
818
819
820
821
  	case SO_ERROR:
  		ret = -ENOPROTOOPT;
  		break;
  	case SO_DONTROUTE:
c0ef877b2   Pavel Emelyanov   [NET]: Move sock_...
822
  		sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
0fbe82e62   yupeng   net: call sk_dst_...
823
  		sk_dst_reset(sk);
e71a4783a   Stephen Hemminger   [NET] core: white...
824
825
826
827
828
829
  		break;
  	case SO_BROADCAST:
  		sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
  		break;
  	case SO_SNDBUF:
  		/* Don't error on this BSD doesn't and if you think
829819301   Eric Dumazet   net: cleanups in ...
830
831
832
833
834
  		 * about it this is right. Otherwise apps have to
  		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
  		 * are treated in BSD as hints
  		 */
  		val = min_t(u32, val, sysctl_wmem_max);
b0573dea1   Patrick McHardy   [NET]: Introduce ...
835
  set_sndbuf:
4057765f2   Guillaume Nault   sock: consistent ...
836
837
838
839
  		/* Ensure val * 2 fits into an int, to prevent max_t()
  		 * from treating it as a negative value.
  		 */
  		val = min_t(int, val, INT_MAX / 2);
e71a4783a   Stephen Hemminger   [NET] core: white...
840
  		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
e292f05e0   Eric Dumazet   tcp: annotate sk-...
841
842
  		WRITE_ONCE(sk->sk_sndbuf,
  			   max_t(int, val * 2, SOCK_MIN_SNDBUF));
829819301   Eric Dumazet   net: cleanups in ...
843
  		/* Wake up sending tasks if we upped the value. */
e71a4783a   Stephen Hemminger   [NET] core: white...
844
845
  		sk->sk_write_space(sk);
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
846

e71a4783a   Stephen Hemminger   [NET] core: white...
847
848
849
850
851
  	case SO_SNDBUFFORCE:
  		if (!capable(CAP_NET_ADMIN)) {
  			ret = -EPERM;
  			break;
  		}
4057765f2   Guillaume Nault   sock: consistent ...
852
853
854
855
856
857
  
  		/* No negative values (to prevent underflow, as val will be
  		 * multiplied by 2).
  		 */
  		if (val < 0)
  			val = 0;
e71a4783a   Stephen Hemminger   [NET] core: white...
858
  		goto set_sndbuf;
b0573dea1   Patrick McHardy   [NET]: Introduce ...
859

e71a4783a   Stephen Hemminger   [NET] core: white...
860
861
  	case SO_RCVBUF:
  		/* Don't error on this BSD doesn't and if you think
829819301   Eric Dumazet   net: cleanups in ...
862
863
864
865
  		 * about it this is right. Otherwise apps have to
  		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
  		 * are treated in BSD as hints
  		 */
26cfabf9c   Christoph Hellwig   net: add sock_set...
866
  		__sock_set_rcvbuf(sk, min_t(u32, val, sysctl_rmem_max));
e71a4783a   Stephen Hemminger   [NET] core: white...
867
868
869
870
871
  		break;
  
  	case SO_RCVBUFFORCE:
  		if (!capable(CAP_NET_ADMIN)) {
  			ret = -EPERM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
872
  			break;
e71a4783a   Stephen Hemminger   [NET] core: white...
873
  		}
4057765f2   Guillaume Nault   sock: consistent ...
874
875
876
877
  
  		/* No negative values (to prevent underflow, as val will be
  		 * multiplied by 2).
  		 */
26cfabf9c   Christoph Hellwig   net: add sock_set...
878
879
  		__sock_set_rcvbuf(sk, max(val, 0));
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
880

e71a4783a   Stephen Hemminger   [NET] core: white...
881
  	case SO_KEEPALIVE:
4b9d07a44   Ursula Braun   net: introduce ke...
882
883
  		if (sk->sk_prot->keepalive)
  			sk->sk_prot->keepalive(sk, valbool);
e71a4783a   Stephen Hemminger   [NET] core: white...
884
885
886
887
888
889
890
891
  		sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
  		break;
  
  	case SO_OOBINLINE:
  		sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
  		break;
  
  	case SO_NO_CHECK:
28448b804   Tom Herbert   net: Split sk_no_...
892
  		sk->sk_no_check_tx = valbool;
e71a4783a   Stephen Hemminger   [NET] core: white...
893
894
895
  		break;
  
  	case SO_PRIORITY:
5e1fccc0b   Eric W. Biederman   net: Allow userns...
896
897
  		if ((val >= 0 && val <= 6) ||
  		    ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783a   Stephen Hemminger   [NET] core: white...
898
899
900
901
902
903
904
905
  			sk->sk_priority = val;
  		else
  			ret = -EPERM;
  		break;
  
  	case SO_LINGER:
  		if (optlen < sizeof(ling)) {
  			ret = -EINVAL;	/* 1003.1g */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
906
  			break;
e71a4783a   Stephen Hemminger   [NET] core: white...
907
  		}
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
908
  		if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
e71a4783a   Stephen Hemminger   [NET] core: white...
909
  			ret = -EFAULT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
910
  			break;
e71a4783a   Stephen Hemminger   [NET] core: white...
911
912
913
914
  		}
  		if (!ling.l_onoff)
  			sock_reset_flag(sk, SOCK_LINGER);
  		else {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
915
  #if (BITS_PER_LONG == 32)
e71a4783a   Stephen Hemminger   [NET] core: white...
916
917
  			if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
  				sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
918
  			else
e71a4783a   Stephen Hemminger   [NET] core: white...
919
920
921
922
923
924
925
  #endif
  				sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
  			sock_set_flag(sk, SOCK_LINGER);
  		}
  		break;
  
  	case SO_BSDCOMPAT:
e71a4783a   Stephen Hemminger   [NET] core: white...
926
927
928
929
930
931
932
933
  		break;
  
  	case SO_PASSCRED:
  		if (valbool)
  			set_bit(SOCK_PASSCRED, &sock->flags);
  		else
  			clear_bit(SOCK_PASSCRED, &sock->flags);
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
934
  	case SO_TIMESTAMP_OLD:
783da70e8   Christoph Hellwig   net: add sock_ena...
935
936
  		__sock_set_timestamps(sk, valbool, false, false);
  		break;
887feae36   Deepa Dinamani   socket: Add SO_TI...
937
  	case SO_TIMESTAMP_NEW:
783da70e8   Christoph Hellwig   net: add sock_ena...
938
939
  		__sock_set_timestamps(sk, valbool, true, false);
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
940
  	case SO_TIMESTAMPNS_OLD:
783da70e8   Christoph Hellwig   net: add sock_ena...
941
942
  		__sock_set_timestamps(sk, valbool, false, true);
  		break;
887feae36   Deepa Dinamani   socket: Add SO_TI...
943
  	case SO_TIMESTAMPNS_NEW:
783da70e8   Christoph Hellwig   net: add sock_ena...
944
  		__sock_set_timestamps(sk, valbool, true, true);
e71a4783a   Stephen Hemminger   [NET] core: white...
945
  		break;
9718475e6   Deepa Dinamani   socket: Add SO_TI...
946
  	case SO_TIMESTAMPING_NEW:
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
947
  	case SO_TIMESTAMPING_OLD:
20d494735   Patrick Ohly   net: socket infra...
948
  		if (val & ~SOF_TIMESTAMPING_MASK) {
f249fb783   Rémi Denis-Courmont   Fix error return ...
949
  			ret = -EINVAL;
20d494735   Patrick Ohly   net: socket infra...
950
951
  			break;
  		}
b245be1f4   Willem de Bruijn   net-timestamp: no...
952

09c2d251b   Willem de Bruijn   net-timestamp: ad...
953
  		if (val & SOF_TIMESTAMPING_OPT_ID &&
4ed2d765d   Willem de Bruijn   net-timestamp: TC...
954
  		    !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
ac5cc9779   WANG Cong   net: check both t...
955
956
  			if (sk->sk_protocol == IPPROTO_TCP &&
  			    sk->sk_type == SOCK_STREAM) {
6db8b963a   Soheil Hassas Yeganeh   tcp: accept SOF_T...
957
958
  				if ((1 << sk->sk_state) &
  				    (TCPF_CLOSE | TCPF_LISTEN)) {
4ed2d765d   Willem de Bruijn   net-timestamp: TC...
959
960
961
962
963
964
965
966
  					ret = -EINVAL;
  					break;
  				}
  				sk->sk_tskey = tcp_sk(sk)->snd_una;
  			} else {
  				sk->sk_tskey = 0;
  			}
  		}
1c885808e   Francis Yan   tcp: SOF_TIMESTAM...
967
968
969
970
971
972
  
  		if (val & SOF_TIMESTAMPING_OPT_STATS &&
  		    !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
  			ret = -EINVAL;
  			break;
  		}
b9f40e21e   Willem de Bruijn   net-timestamp: mo...
973
  		sk->sk_tsflags = val;
59e611a56   Christian Eggers   socket: fix optio...
974
  		sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW);
20d494735   Patrick Ohly   net: socket infra...
975
976
977
  		if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
  			sock_enable_timestamp(sk,
  					      SOCK_TIMESTAMPING_RX_SOFTWARE);
59e611a56   Christian Eggers   socket: fix optio...
978
  		else
20d494735   Patrick Ohly   net: socket infra...
979
  			sock_disable_timestamp(sk,
08e29af3a   Eric Dumazet   net: optimize soc...
980
  					       (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
20d494735   Patrick Ohly   net: socket infra...
981
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
982
983
984
  	case SO_RCVLOWAT:
  		if (val < 0)
  			val = INT_MAX;
d1361840f   Eric Dumazet   tcp: fix SO_RCVLO...
985
986
987
  		if (sock->ops->set_rcvlowat)
  			ret = sock->ops->set_rcvlowat(sk, val);
  		else
eac66402d   Eric Dumazet   net: annotate sk-...
988
  			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
e71a4783a   Stephen Hemminger   [NET] core: white...
989
  		break;
45bdc6615   Deepa Dinamani   socket: Rename SO...
990
  	case SO_RCVTIMEO_OLD:
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
991
  	case SO_RCVTIMEO_NEW:
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
992
  		ret = sock_set_timeout(&sk->sk_rcvtimeo, optval,
c34645ac2   Christoph Hellwig   net: switch sock_...
993
  				       optlen, optname == SO_RCVTIMEO_OLD);
e71a4783a   Stephen Hemminger   [NET] core: white...
994
  		break;
45bdc6615   Deepa Dinamani   socket: Rename SO...
995
  	case SO_SNDTIMEO_OLD:
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
996
  	case SO_SNDTIMEO_NEW:
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
997
  		ret = sock_set_timeout(&sk->sk_sndtimeo, optval,
c34645ac2   Christoph Hellwig   net: switch sock_...
998
  				       optlen, optname == SO_SNDTIMEO_OLD);
e71a4783a   Stephen Hemminger   [NET] core: white...
999
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1000

4d295e546   Christoph Hellwig   net: simplify cBP...
1001
1002
  	case SO_ATTACH_FILTER: {
  		struct sock_fprog fprog;
e71a4783a   Stephen Hemminger   [NET] core: white...
1003

c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1004
  		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e546   Christoph Hellwig   net: simplify cBP...
1005
  		if (!ret)
e71a4783a   Stephen Hemminger   [NET] core: white...
1006
  			ret = sk_attach_filter(&fprog, sk);
e71a4783a   Stephen Hemminger   [NET] core: white...
1007
  		break;
4d295e546   Christoph Hellwig   net: simplify cBP...
1008
  	}
89aa07583   Alexei Starovoitov   net: sock: allow ...
1009
1010
1011
1012
1013
1014
  	case SO_ATTACH_BPF:
  		ret = -EINVAL;
  		if (optlen == sizeof(u32)) {
  			u32 ufd;
  
  			ret = -EFAULT;
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1015
  			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
89aa07583   Alexei Starovoitov   net: sock: allow ...
1016
1017
1018
1019
1020
  				break;
  
  			ret = sk_attach_bpf(ufd, sk);
  		}
  		break;
4d295e546   Christoph Hellwig   net: simplify cBP...
1021
1022
  	case SO_ATTACH_REUSEPORT_CBPF: {
  		struct sock_fprog fprog;
538950a1b   Craig Gallek   soreuseport: sets...
1023

c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1024
  		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e546   Christoph Hellwig   net: simplify cBP...
1025
  		if (!ret)
538950a1b   Craig Gallek   soreuseport: sets...
1026
  			ret = sk_reuseport_attach_filter(&fprog, sk);
538950a1b   Craig Gallek   soreuseport: sets...
1027
  		break;
4d295e546   Christoph Hellwig   net: simplify cBP...
1028
  	}
538950a1b   Craig Gallek   soreuseport: sets...
1029
1030
1031
1032
1033
1034
  	case SO_ATTACH_REUSEPORT_EBPF:
  		ret = -EINVAL;
  		if (optlen == sizeof(u32)) {
  			u32 ufd;
  
  			ret = -EFAULT;
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1035
  			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
538950a1b   Craig Gallek   soreuseport: sets...
1036
1037
1038
1039
1040
  				break;
  
  			ret = sk_reuseport_attach_bpf(ufd, sk);
  		}
  		break;
99f3a064b   Martin KaFai Lau   bpf: net: Add SO_...
1041
1042
1043
  	case SO_DETACH_REUSEPORT_BPF:
  		ret = reuseport_detach_prog(sk);
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1044
  	case SO_DETACH_FILTER:
55b333253   Pavel Emelyanov   [NET]: Introduce ...
1045
  		ret = sk_detach_filter(sk);
e71a4783a   Stephen Hemminger   [NET] core: white...
1046
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1047

d59577b6f   Vincent Bernat   sk-filter: Add ab...
1048
1049
1050
1051
1052
1053
  	case SO_LOCK_FILTER:
  		if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
  			ret = -EPERM;
  		else
  			sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1054
1055
1056
1057
1058
1059
  	case SO_PASSSEC:
  		if (valbool)
  			set_bit(SOCK_PASSSEC, &sock->flags);
  		else
  			clear_bit(SOCK_PASSSEC, &sock->flags);
  		break;
4a19ec580   Laszlo Attila Toth   [NET]: Introducin...
1060
  	case SO_MARK:
50254256f   David Barmann   sock: Reset dst w...
1061
  		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
4a19ec580   Laszlo Attila Toth   [NET]: Introducin...
1062
  			ret = -EPERM;
50254256f   David Barmann   sock: Reset dst w...
1063
  		} else if (val != sk->sk_mark) {
4a19ec580   Laszlo Attila Toth   [NET]: Introducin...
1064
  			sk->sk_mark = val;
50254256f   David Barmann   sock: Reset dst w...
1065
1066
  			sk_dst_reset(sk);
  		}
4a19ec580   Laszlo Attila Toth   [NET]: Introducin...
1067
  		break;
877ce7c1b   Catherine Zhang   [AF_UNIX]: Datagr...
1068

3b885787e   Neil Horman   net: Generalize s...
1069
  	case SO_RXQ_OVFL:
8083f0fc9   Johannes Berg   net: use sock_val...
1070
  		sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787e   Neil Horman   net: Generalize s...
1071
  		break;
6e3e939f3   Johannes Berg   net: add wireless...
1072
1073
1074
1075
  
  	case SO_WIFI_STATUS:
  		sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
  		break;
ef64a54f6   Pavel Emelyanov   sock: Introduce t...
1076
1077
  	case SO_PEEK_OFF:
  		if (sock->ops->set_peek_off)
12663bfc9   Sasha Levin   net: unix: allow ...
1078
  			ret = sock->ops->set_peek_off(sk, val);
ef64a54f6   Pavel Emelyanov   sock: Introduce t...
1079
1080
1081
  		else
  			ret = -EOPNOTSUPP;
  		break;
3bdc0eba0   Ben Greear   net: Add framewor...
1082
1083
1084
1085
  
  	case SO_NOFCS:
  		sock_valbool_flag(sk, SOCK_NOFCS, valbool);
  		break;
7d4c04fc1   Keller, Jacob E   net: add option t...
1086
1087
1088
  	case SO_SELECT_ERR_QUEUE:
  		sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
  		break;
e0d1095ae   Cong Wang   net: rename CONFI...
1089
  #ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc517   Eliezer Tamir   net: rename busy ...
1090
  	case SO_BUSY_POLL:
dafcc4380   Eliezer Tamir   net: add socket o...
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
  		/* allow unprivileged users to decrease the value */
  		if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
  			ret = -EPERM;
  		else {
  			if (val < 0)
  				ret = -EINVAL;
  			else
  				sk->sk_ll_usec = val;
  		}
  		break;
7fd3253a7   Björn Töpel   net: Introduce pr...
1101
1102
1103
1104
1105
1106
  	case SO_PREFER_BUSY_POLL:
  		if (valbool && !capable(CAP_NET_ADMIN))
  			ret = -EPERM;
  		else
  			WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
  		break;
7c951cafc   Björn Töpel   net: Add SO_BUSY_...
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
  	case SO_BUSY_POLL_BUDGET:
  		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !capable(CAP_NET_ADMIN)) {
  			ret = -EPERM;
  		} else {
  			if (val < 0 || val > U16_MAX)
  				ret = -EINVAL;
  			else
  				WRITE_ONCE(sk->sk_busy_poll_budget, val);
  		}
  		break;
dafcc4380   Eliezer Tamir   net: add socket o...
1117
  #endif
62748f32d   Eric Dumazet   net: introduce SO...
1118
1119
  
  	case SO_MAX_PACING_RATE:
6bdef102d   Eric Dumazet   net: support 64bi...
1120
  		{
700465fd3   Ke Li   net: Properly typ...
1121
  		unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val;
6bdef102d   Eric Dumazet   net: support 64bi...
1122
1123
1124
  
  		if (sizeof(ulval) != sizeof(val) &&
  		    optlen >= sizeof(ulval) &&
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1125
  		    copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
6bdef102d   Eric Dumazet   net: support 64bi...
1126
1127
1128
1129
  			ret = -EFAULT;
  			break;
  		}
  		if (ulval != ~0UL)
218af599f   Eric Dumazet   tcp: internal imp...
1130
1131
1132
  			cmpxchg(&sk->sk_pacing_status,
  				SK_PACING_NONE,
  				SK_PACING_NEEDED);
6bdef102d   Eric Dumazet   net: support 64bi...
1133
1134
  		sk->sk_max_pacing_rate = ulval;
  		sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
62748f32d   Eric Dumazet   net: introduce SO...
1135
  		break;
6bdef102d   Eric Dumazet   net: support 64bi...
1136
  		}
70da268b5   Eric Dumazet   net: SO_INCOMING_...
1137
  	case SO_INCOMING_CPU:
7170a9777   Eric Dumazet   net: annotate acc...
1138
  		WRITE_ONCE(sk->sk_incoming_cpu, val);
70da268b5   Eric Dumazet   net: SO_INCOMING_...
1139
  		break;
a87cb3e48   Tom Herbert   net: Facility to ...
1140
1141
1142
1143
  	case SO_CNX_ADVICE:
  		if (val == 1)
  			dst_negative_advice(sk);
  		break;
76851d121   Willem de Bruijn   sock: add SOCK_ZE...
1144
1145
  
  	case SO_ZEROCOPY:
28190752c   Sowmini Varadhan   sock: permit SO_Z...
1146
  		if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
b5947e5d1   Willem de Bruijn   udp: msg_zerocopy
1147
1148
1149
1150
  			if (!((sk->sk_type == SOCK_STREAM &&
  			       sk->sk_protocol == IPPROTO_TCP) ||
  			      (sk->sk_type == SOCK_DGRAM &&
  			       sk->sk_protocol == IPPROTO_UDP)))
28190752c   Sowmini Varadhan   sock: permit SO_Z...
1151
  				ret = -ENOTSUPP;
28190752c   Sowmini Varadhan   sock: permit SO_Z...
1152
  		} else if (sk->sk_family != PF_RDS) {
76851d121   Willem de Bruijn   sock: add SOCK_ZE...
1153
  			ret = -ENOTSUPP;
28190752c   Sowmini Varadhan   sock: permit SO_Z...
1154
1155
1156
1157
1158
1159
  		}
  		if (!ret) {
  			if (val < 0 || val > 1)
  				ret = -EINVAL;
  			else
  				sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
28190752c   Sowmini Varadhan   sock: permit SO_Z...
1160
  		}
334e64131   Jesus Sanchez-Palencia   sock: Fix SO_ZERO...
1161
  		break;
80b14dee2   Richard Cochran   net: Add a new so...
1162
  	case SO_TXTIME:
790709f24   Eric Dumazet   net: relax SO_TXT...
1163
  		if (optlen != sizeof(struct sock_txtime)) {
80b14dee2   Richard Cochran   net: Add a new so...
1164
  			ret = -EINVAL;
790709f24   Eric Dumazet   net: relax SO_TXT...
1165
  			break;
c8c1bbb6e   Christoph Hellwig   net: switch sock_...
1166
  		} else if (copy_from_sockptr(&sk_txtime, optval,
80b14dee2   Richard Cochran   net: Add a new so...
1167
1168
  			   sizeof(struct sock_txtime))) {
  			ret = -EFAULT;
790709f24   Eric Dumazet   net: relax SO_TXT...
1169
  			break;
80b14dee2   Richard Cochran   net: Add a new so...
1170
1171
  		} else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
  			ret = -EINVAL;
790709f24   Eric Dumazet   net: relax SO_TXT...
1172
1173
1174
1175
1176
1177
1178
1179
1180
  			break;
  		}
  		/* CLOCK_MONOTONIC is only used by sch_fq, and this packet
  		 * scheduler has enough safe guards.
  		 */
  		if (sk_txtime.clockid != CLOCK_MONOTONIC &&
  		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
  			ret = -EPERM;
  			break;
80b14dee2   Richard Cochran   net: Add a new so...
1181
  		}
790709f24   Eric Dumazet   net: relax SO_TXT...
1182
1183
1184
1185
1186
1187
  		sock_valbool_flag(sk, SOCK_TXTIME, true);
  		sk->sk_clockid = sk_txtime.clockid;
  		sk->sk_txtime_deadline_mode =
  			!!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
  		sk->sk_txtime_report_errors =
  			!!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
80b14dee2   Richard Cochran   net: Add a new so...
1188
  		break;
f5dd3d0c9   David Herrmann   net: introduce SO...
1189
  	case SO_BINDTOIFINDEX:
7594888c7   Christoph Hellwig   net: add sock_bin...
1190
  		ret = sock_bindtoindex_locked(sk, val);
f5dd3d0c9   David Herrmann   net: introduce SO...
1191
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1192
1193
1194
  	default:
  		ret = -ENOPROTOOPT;
  		break;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1195
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1196
1197
1198
  	release_sock(sk);
  	return ret;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
1199
  EXPORT_SYMBOL(sock_setsockopt);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1200

8f09898bf   stephen hemminger   socket: cleanups
1201
1202
  static void cred_to_ucred(struct pid *pid, const struct cred *cred,
  			  struct ucred *ucred)
3f551f943   Eric W. Biederman   sock: Introduce c...
1203
1204
1205
1206
1207
  {
  	ucred->pid = pid_vnr(pid);
  	ucred->uid = ucred->gid = -1;
  	if (cred) {
  		struct user_namespace *current_ns = current_user_ns();
b2e4f544f   Eric W. Biederman   userns: Convert n...
1208
1209
  		ucred->uid = from_kuid_munged(current_ns, cred->euid);
  		ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f943   Eric W. Biederman   sock: Introduce c...
1210
1211
  	}
  }
28b5ba2aa   David Herrmann   net: introduce SO...
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
  static int groups_to_user(gid_t __user *dst, const struct group_info *src)
  {
  	struct user_namespace *user_ns = current_user_ns();
  	int i;
  
  	for (i = 0; i < src->ngroups; i++)
  		if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
  			return -EFAULT;
  
  	return 0;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1223
1224
1225
1226
  int sock_getsockopt(struct socket *sock, int level, int optname,
  		    char __user *optval, int __user *optlen)
  {
  	struct sock *sk = sock->sk;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1227

e71a4783a   Stephen Hemminger   [NET] core: white...
1228
  	union {
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1229
  		int val;
5daab9db7   Chenbo Feng   New getsockopt op...
1230
  		u64 val64;
677f136c6   Eric Dumazet   net: support 64bi...
1231
  		unsigned long ulval;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1232
  		struct linger ling;
fe0c72f3d   Arnd Bergmann   socket: move comp...
1233
1234
  		struct old_timeval32 tm32;
  		struct __kernel_old_timeval tm;
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
1235
  		struct  __kernel_sock_timeval stm;
80b14dee2   Richard Cochran   net: Add a new so...
1236
  		struct sock_txtime txtime;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1237
  	} v;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1238

4d0392be2   H Hartley Sweeten   net/core/sock.c: ...
1239
  	int lv = sizeof(int);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1240
  	int len;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1241

e71a4783a   Stephen Hemminger   [NET] core: white...
1242
  	if (get_user(len, optlen))
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1243
  		return -EFAULT;
e71a4783a   Stephen Hemminger   [NET] core: white...
1244
  	if (len < 0)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1245
  		return -EINVAL;
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1246

50fee1dec   Eugene Teo   net: amend the fi...
1247
  	memset(&v, 0, sizeof(v));
df0bca049   Clément Lecigne   net: 4 bytes kern...
1248

2a91525c2   Eric Dumazet   net: net/core/soc...
1249
  	switch (optname) {
e71a4783a   Stephen Hemminger   [NET] core: white...
1250
1251
1252
1253
1254
1255
1256
1257
1258
  	case SO_DEBUG:
  		v.val = sock_flag(sk, SOCK_DBG);
  		break;
  
  	case SO_DONTROUTE:
  		v.val = sock_flag(sk, SOCK_LOCALROUTE);
  		break;
  
  	case SO_BROADCAST:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1259
  		v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783a   Stephen Hemminger   [NET] core: white...
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
  		break;
  
  	case SO_SNDBUF:
  		v.val = sk->sk_sndbuf;
  		break;
  
  	case SO_RCVBUF:
  		v.val = sk->sk_rcvbuf;
  		break;
  
  	case SO_REUSEADDR:
  		v.val = sk->sk_reuse;
  		break;
055dc21a1   Tom Herbert   soreuseport: infr...
1273
1274
1275
  	case SO_REUSEPORT:
  		v.val = sk->sk_reuseport;
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1276
  	case SO_KEEPALIVE:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1277
  		v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783a   Stephen Hemminger   [NET] core: white...
1278
1279
1280
1281
1282
  		break;
  
  	case SO_TYPE:
  		v.val = sk->sk_type;
  		break;
49c794e94   Jan Engelhardt   net: implement a ...
1283
1284
1285
  	case SO_PROTOCOL:
  		v.val = sk->sk_protocol;
  		break;
0d6038ee7   Jan Engelhardt   net: implement a ...
1286
1287
1288
  	case SO_DOMAIN:
  		v.val = sk->sk_family;
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1289
1290
  	case SO_ERROR:
  		v.val = -sock_error(sk);
2a91525c2   Eric Dumazet   net: net/core/soc...
1291
  		if (v.val == 0)
e71a4783a   Stephen Hemminger   [NET] core: white...
1292
1293
1294
1295
  			v.val = xchg(&sk->sk_err_soft, 0);
  		break;
  
  	case SO_OOBINLINE:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1296
  		v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783a   Stephen Hemminger   [NET] core: white...
1297
1298
1299
  		break;
  
  	case SO_NO_CHECK:
28448b804   Tom Herbert   net: Split sk_no_...
1300
  		v.val = sk->sk_no_check_tx;
e71a4783a   Stephen Hemminger   [NET] core: white...
1301
1302
1303
1304
1305
1306
1307
1308
  		break;
  
  	case SO_PRIORITY:
  		v.val = sk->sk_priority;
  		break;
  
  	case SO_LINGER:
  		lv		= sizeof(v.ling);
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1309
  		v.ling.l_onoff	= sock_flag(sk, SOCK_LINGER);
e71a4783a   Stephen Hemminger   [NET] core: white...
1310
1311
1312
1313
  		v.ling.l_linger	= sk->sk_lingertime / HZ;
  		break;
  
  	case SO_BSDCOMPAT:
e71a4783a   Stephen Hemminger   [NET] core: white...
1314
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
1315
  	case SO_TIMESTAMP_OLD:
92f37fd2e   Eric Dumazet   [NET]: Adding SO_...
1316
  		v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
887feae36   Deepa Dinamani   socket: Add SO_TI...
1317
  				!sock_flag(sk, SOCK_TSTAMP_NEW) &&
92f37fd2e   Eric Dumazet   [NET]: Adding SO_...
1318
1319
  				!sock_flag(sk, SOCK_RCVTSTAMPNS);
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
1320
  	case SO_TIMESTAMPNS_OLD:
887feae36   Deepa Dinamani   socket: Add SO_TI...
1321
1322
1323
1324
1325
1326
1327
1328
1329
  		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
  		break;
  
  	case SO_TIMESTAMP_NEW:
  		v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
  		break;
  
  	case SO_TIMESTAMPNS_NEW:
  		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
e71a4783a   Stephen Hemminger   [NET] core: white...
1330
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
1331
  	case SO_TIMESTAMPING_OLD:
b9f40e21e   Willem de Bruijn   net-timestamp: mo...
1332
  		v.val = sk->sk_tsflags;
20d494735   Patrick Ohly   net: socket infra...
1333
  		break;
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
1334
1335
1336
  	case SO_RCVTIMEO_OLD:
  	case SO_RCVTIMEO_NEW:
  		lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
e71a4783a   Stephen Hemminger   [NET] core: white...
1337
  		break;
a9beb86ae   Deepa Dinamani   sock: Add SO_RCVT...
1338
1339
1340
  	case SO_SNDTIMEO_OLD:
  	case SO_SNDTIMEO_NEW:
  		lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
e71a4783a   Stephen Hemminger   [NET] core: white...
1341
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1342

e71a4783a   Stephen Hemminger   [NET] core: white...
1343
1344
1345
  	case SO_RCVLOWAT:
  		v.val = sk->sk_rcvlowat;
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1346

e71a4783a   Stephen Hemminger   [NET] core: white...
1347
  	case SO_SNDLOWAT:
2a91525c2   Eric Dumazet   net: net/core/soc...
1348
  		v.val = 1;
e71a4783a   Stephen Hemminger   [NET] core: white...
1349
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1350

e71a4783a   Stephen Hemminger   [NET] core: white...
1351
  	case SO_PASSCRED:
829819301   Eric Dumazet   net: cleanups in ...
1352
  		v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783a   Stephen Hemminger   [NET] core: white...
1353
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1354

e71a4783a   Stephen Hemminger   [NET] core: white...
1355
  	case SO_PEERCRED:
109f6e39f   Eric W. Biederman   af_unix: Allow SO...
1356
1357
1358
1359
1360
1361
  	{
  		struct ucred peercred;
  		if (len > sizeof(peercred))
  			len = sizeof(peercred);
  		cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
  		if (copy_to_user(optval, &peercred, len))
e71a4783a   Stephen Hemminger   [NET] core: white...
1362
1363
  			return -EFAULT;
  		goto lenout;
109f6e39f   Eric W. Biederman   af_unix: Allow SO...
1364
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1365

28b5ba2aa   David Herrmann   net: introduce SO...
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
  	case SO_PEERGROUPS:
  	{
  		int ret, n;
  
  		if (!sk->sk_peer_cred)
  			return -ENODATA;
  
  		n = sk->sk_peer_cred->group_info->ngroups;
  		if (len < n * sizeof(gid_t)) {
  			len = n * sizeof(gid_t);
  			return put_user(len, optlen) ? -EFAULT : -ERANGE;
  		}
  		len = n * sizeof(gid_t);
  
  		ret = groups_to_user((gid_t __user *)optval,
  				     sk->sk_peer_cred->group_info);
  		if (ret)
  			return ret;
  		goto lenout;
  	}
e71a4783a   Stephen Hemminger   [NET] core: white...
1386
1387
1388
  	case SO_PEERNAME:
  	{
  		char address[128];
9b2c45d47   Denys Vlasenko   net: make getname...
1389
1390
  		lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
  		if (lv < 0)
e71a4783a   Stephen Hemminger   [NET] core: white...
1391
1392
1393
1394
1395
1396
1397
  			return -ENOTCONN;
  		if (lv < len)
  			return -EINVAL;
  		if (copy_to_user(optval, address, len))
  			return -EFAULT;
  		goto lenout;
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1398

e71a4783a   Stephen Hemminger   [NET] core: white...
1399
1400
1401
1402
1403
1404
  	/* Dubious BSD thing... Probably nobody even uses it, but
  	 * the UNIX standard wants it for whatever reason... -DaveM
  	 */
  	case SO_ACCEPTCONN:
  		v.val = sk->sk_state == TCP_LISTEN;
  		break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1405

e71a4783a   Stephen Hemminger   [NET] core: white...
1406
  	case SO_PASSSEC:
829819301   Eric Dumazet   net: cleanups in ...
1407
  		v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783a   Stephen Hemminger   [NET] core: white...
1408
  		break;
877ce7c1b   Catherine Zhang   [AF_UNIX]: Datagr...
1409

e71a4783a   Stephen Hemminger   [NET] core: white...
1410
1411
  	case SO_PEERSEC:
  		return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1412

4a19ec580   Laszlo Attila Toth   [NET]: Introducin...
1413
1414
1415
  	case SO_MARK:
  		v.val = sk->sk_mark;
  		break;
3b885787e   Neil Horman   net: Generalize s...
1416
  	case SO_RXQ_OVFL:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1417
  		v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787e   Neil Horman   net: Generalize s...
1418
  		break;
6e3e939f3   Johannes Berg   net: add wireless...
1419
  	case SO_WIFI_STATUS:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1420
  		v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f3   Johannes Berg   net: add wireless...
1421
  		break;
ef64a54f6   Pavel Emelyanov   sock: Introduce t...
1422
1423
1424
1425
1426
1427
  	case SO_PEEK_OFF:
  		if (!sock->ops->set_peek_off)
  			return -EOPNOTSUPP;
  
  		v.val = sk->sk_peek_off;
  		break;
bc2f79968   David S. Miller   net: Add missing ...
1428
  	case SO_NOFCS:
1b23a5dfc   Eric Dumazet   net: sock_flag() ...
1429
  		v.val = sock_flag(sk, SOCK_NOFCS);
bc2f79968   David S. Miller   net: Add missing ...
1430
  		break;
c91f6df2d   Brian Haley   sockopt: Change g...
1431

f7b86bfe8   Pavel Emelyanov   sockopt: Make SO_...
1432
  	case SO_BINDTODEVICE:
c91f6df2d   Brian Haley   sockopt: Change g...
1433
  		return sock_getbindtodevice(sk, optval, optlen, len);
a8fc92778   Pavel Emelyanov   sk-filter: Add ab...
1434
1435
1436
1437
1438
1439
  	case SO_GET_FILTER:
  		len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
  		if (len < 0)
  			return len;
  
  		goto lenout;
c91f6df2d   Brian Haley   sockopt: Change g...
1440

d59577b6f   Vincent Bernat   sk-filter: Add ab...
1441
1442
1443
  	case SO_LOCK_FILTER:
  		v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
  		break;
ea02f9411   Michal Sekletar   net: introduce SO...
1444
1445
1446
  	case SO_BPF_EXTENSIONS:
  		v.val = bpf_tell_extensions();
  		break;
7d4c04fc1   Keller, Jacob E   net: add option t...
1447
1448
1449
  	case SO_SELECT_ERR_QUEUE:
  		v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
  		break;
e0d1095ae   Cong Wang   net: rename CONFI...
1450
  #ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc517   Eliezer Tamir   net: rename busy ...
1451
  	case SO_BUSY_POLL:
dafcc4380   Eliezer Tamir   net: add socket o...
1452
1453
  		v.val = sk->sk_ll_usec;
  		break;
7fd3253a7   Björn Töpel   net: Introduce pr...
1454
1455
1456
  	case SO_PREFER_BUSY_POLL:
  		v.val = READ_ONCE(sk->sk_prefer_busy_poll);
  		break;
dafcc4380   Eliezer Tamir   net: add socket o...
1457
  #endif
62748f32d   Eric Dumazet   net: introduce SO...
1458
  	case SO_MAX_PACING_RATE:
677f136c6   Eric Dumazet   net: support 64bi...
1459
1460
1461
1462
1463
1464
1465
  		if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
  			lv = sizeof(v.ulval);
  			v.ulval = sk->sk_max_pacing_rate;
  		} else {
  			/* 32bit version */
  			v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
  		}
62748f32d   Eric Dumazet   net: introduce SO...
1466
  		break;
2c8c56e15   Eric Dumazet   net: introduce SO...
1467
  	case SO_INCOMING_CPU:
7170a9777   Eric Dumazet   net: annotate acc...
1468
  		v.val = READ_ONCE(sk->sk_incoming_cpu);
2c8c56e15   Eric Dumazet   net: introduce SO...
1469
  		break;
a2d133b1d   Josh Hunt   sock: introduce S...
1470
1471
1472
  	case SO_MEMINFO:
  	{
  		u32 meminfo[SK_MEMINFO_VARS];
a2d133b1d   Josh Hunt   sock: introduce S...
1473
1474
1475
1476
1477
1478
1479
1480
  		sk_get_meminfo(sk, meminfo);
  
  		len = min_t(unsigned int, len, sizeof(meminfo));
  		if (copy_to_user(optval, &meminfo, len))
  			return -EFAULT;
  
  		goto lenout;
  	}
6d4339028   Sridhar Samudrala   net: Introduce SO...
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
  
  #ifdef CONFIG_NET_RX_BUSY_POLL
  	case SO_INCOMING_NAPI_ID:
  		v.val = READ_ONCE(sk->sk_napi_id);
  
  		/* aggregate non-NAPI IDs down to 0 */
  		if (v.val < MIN_NAPI_ID)
  			v.val = 0;
  
  		break;
  #endif
5daab9db7   Chenbo Feng   New getsockopt op...
1492
1493
1494
1495
1496
1497
  	case SO_COOKIE:
  		lv = sizeof(u64);
  		if (len < lv)
  			return -EINVAL;
  		v.val64 = sock_gen_cookie(sk);
  		break;
76851d121   Willem de Bruijn   sock: add SOCK_ZE...
1498
1499
1500
  	case SO_ZEROCOPY:
  		v.val = sock_flag(sk, SOCK_ZEROCOPY);
  		break;
80b14dee2   Richard Cochran   net: Add a new so...
1501
1502
1503
1504
1505
  	case SO_TXTIME:
  		lv = sizeof(v.txtime);
  		v.txtime.clockid = sk->sk_clockid;
  		v.txtime.flags |= sk->sk_txtime_deadline_mode ?
  				  SOF_TXTIME_DEADLINE_MODE : 0;
4b15c7075   Jesus Sanchez-Palencia   net/sched: Make e...
1506
1507
  		v.txtime.flags |= sk->sk_txtime_report_errors ?
  				  SOF_TXTIME_REPORT_ERRORS : 0;
80b14dee2   Richard Cochran   net: Add a new so...
1508
  		break;
f5dd3d0c9   David Herrmann   net: introduce SO...
1509
1510
1511
  	case SO_BINDTOIFINDEX:
  		v.val = sk->sk_bound_dev_if;
  		break;
e71a4783a   Stephen Hemminger   [NET] core: white...
1512
  	default:
443b5991a   YOSHIFUJI Hideaki/吉藤英明   net: Move the com...
1513
1514
1515
  		/* We implement the SO_SNDLOWAT etc to not be settable
  		 * (1003.1g 7).
  		 */
e71a4783a   Stephen Hemminger   [NET] core: white...
1516
  		return -ENOPROTOOPT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1517
  	}
e71a4783a   Stephen Hemminger   [NET] core: white...
1518

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1519
1520
1521
1522
1523
  	if (len > lv)
  		len = lv;
  	if (copy_to_user(optval, &v, len))
  		return -EFAULT;
  lenout:
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1524
1525
1526
  	if (put_user(len, optlen))
  		return -EFAULT;
  	return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1527
  }
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
1528
1529
1530
1531
1532
  /*
   * Initialize an sk_lock.
   *
   * (We also register the sk_lock with the lock validator.)
   */
b6f99a211   Dave Jones   [NET]: fix up mis...
1533
  static inline void sock_lock_init(struct sock *sk)
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
1534
  {
cdfbabfb2   David Howells   net: Work around ...
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
  	if (sk->sk_kern_sock)
  		sock_lock_init_class_and_name(
  			sk,
  			af_family_kern_slock_key_strings[sk->sk_family],
  			af_family_kern_slock_keys + sk->sk_family,
  			af_family_kern_key_strings[sk->sk_family],
  			af_family_kern_keys + sk->sk_family);
  	else
  		sock_lock_init_class_and_name(
  			sk,
ed07536ed   Peter Zijlstra   [PATCH] lockdep: ...
1545
1546
1547
1548
  			af_family_slock_key_strings[sk->sk_family],
  			af_family_slock_keys + sk->sk_family,
  			af_family_key_strings[sk->sk_family],
  			af_family_keys + sk->sk_family);
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
1549
  }
4dc6dc716   Eric Dumazet   net: sock_copy() ...
1550
1551
1552
  /*
   * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
   * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba4   Eric Dumazet   net: optimize INE...
1553
   * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc716   Eric Dumazet   net: sock_copy() ...
1554
   */
f1a6c4da1   Pavel Emelyanov   [NET]: Move the s...
1555
1556
  static void sock_copy(struct sock *nsk, const struct sock *osk)
  {
b8e202d1d   Jakub Sitnicki   net, sk_msg: Anno...
1557
  	const struct proto *prot = READ_ONCE(osk->sk_prot);
f1a6c4da1   Pavel Emelyanov   [NET]: Move the s...
1558
1559
1560
  #ifdef CONFIG_SECURITY_NETWORK
  	void *sptr = nsk->sk_security;
  #endif
68835aba4   Eric Dumazet   net: optimize INE...
1561
1562
1563
  	memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
  
  	memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
b8e202d1d   Jakub Sitnicki   net, sk_msg: Anno...
1564
  	       prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
68835aba4   Eric Dumazet   net: optimize INE...
1565

f1a6c4da1   Pavel Emelyanov   [NET]: Move the s...
1566
1567
1568
1569
1570
  #ifdef CONFIG_SECURITY_NETWORK
  	nsk->sk_security = sptr;
  	security_sk_clone(osk, nsk);
  #endif
  }
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1571
1572
  static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
  		int family)
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1573
1574
1575
1576
1577
  {
  	struct sock *sk;
  	struct kmem_cache *slab;
  
  	slab = prot->slab;
e912b1142   Eric Dumazet   net: sk_prot_allo...
1578
1579
1580
1581
  	if (slab != NULL) {
  		sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
  		if (!sk)
  			return sk;
6471384af   Alexander Potapenko   mm: security: int...
1582
  		if (want_init_on_alloc(priority))
ba2489b0e   Eric Dumazet   net: remove clear...
1583
  			sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d9   Octavian Purdila   net: fix nulls li...
1584
  	} else
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1585
  		sk = kmalloc(prot->obj_size, priority);
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1586
1587
1588
1589
1590
1591
  	if (sk != NULL) {
  		if (security_sk_alloc(sk, family, priority))
  			goto out_free;
  
  		if (!try_module_get(prot->owner))
  			goto out_free_sec;
e022f0b4a   Krishna Kumar   net: Introduce sk...
1592
  		sk_tx_queue_clear(sk);
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1593
  	}
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1594
  	return sk;
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1595
1596
1597
1598
1599
1600
1601
1602
1603
  
  out_free_sec:
  	security_sk_free(sk);
  out_free:
  	if (slab != NULL)
  		kmem_cache_free(slab, sk);
  	else
  		kfree(sk);
  	return NULL;
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1604
1605
1606
1607
1608
  }
  
  static void sk_prot_free(struct proto *prot, struct sock *sk)
  {
  	struct kmem_cache *slab;
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1609
  	struct module *owner;
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1610

2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1611
  	owner = prot->owner;
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1612
  	slab = prot->slab;
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1613

bd1060a1d   Tejun Heo   sock, cgroup: add...
1614
  	cgroup_sk_free(&sk->sk_cgrp_data);
2d7580738   Johannes Weiner   mm: memcontrol: c...
1615
  	mem_cgroup_sk_free(sk);
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1616
  	security_sk_free(sk);
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1617
1618
1619
1620
  	if (slab != NULL)
  		kmem_cache_free(slab, sk);
  	else
  		kfree(sk);
2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1621
  	module_put(owner);
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1622
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1623
1624
  /**
   *	sk_alloc - All socket objects are allocated here
c4ea43c55   Randy Dunlap   net core: fix ker...
1625
   *	@net: the applicable net namespace
4dc3b16ba   Pavel Pisa   [PATCH] DocBook: ...
1626
1627
1628
   *	@family: protocol family
   *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
   *	@prot: struct proto associated with this new sock instance
11aa9c28b   Eric W. Biederman   net: Pass kern fr...
1629
   *	@kern: is this to be a kernel socket?
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1630
   */
1b8d7ae42   Eric W. Biederman   [NET]: Make socke...
1631
  struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28b   Eric W. Biederman   net: Pass kern fr...
1632
  		      struct proto *prot, int kern)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1633
  {
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1634
  	struct sock *sk;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1635

154adbc84   Pavel Emelyanov   [NET]: Remove bog...
1636
  	sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1637
  	if (sk) {
154adbc84   Pavel Emelyanov   [NET]: Remove bog...
1638
1639
1640
1641
1642
1643
  		sk->sk_family = family;
  		/*
  		 * See comment in struct sock definition to understand
  		 * why we need sk_prot_creator -acme
  		 */
  		sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb2   David Howells   net: Work around ...
1644
  		sk->sk_kern_sock = kern;
154adbc84   Pavel Emelyanov   [NET]: Remove bog...
1645
  		sock_lock_init(sk);
26abe1437   Eric W. Biederman   net: Modify sk_al...
1646
  		sk->sk_net_refcnt = kern ? 0 : 1;
648845ab7   Tonghao Zhang   sock: Move the so...
1647
  		if (likely(sk->sk_net_refcnt)) {
26abe1437   Eric W. Biederman   net: Modify sk_al...
1648
  			get_net(net);
648845ab7   Tonghao Zhang   sock: Move the so...
1649
1650
  			sock_inuse_add(net, 1);
  		}
26abe1437   Eric W. Biederman   net: Modify sk_al...
1651
  		sock_net_set(sk, net);
14afee4b6   Reshetova, Elena   net: convert sock...
1652
  		refcount_set(&sk->sk_wmem_alloc, 1);
f84517253   Herbert Xu   cls_cgroup: Store...
1653

2d7580738   Johannes Weiner   mm: memcontrol: c...
1654
  		mem_cgroup_sk_alloc(sk);
d979a39d7   Johannes Weiner   cgroup: duplicate...
1655
  		cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fec   Tejun Heo   net: wrap sock->s...
1656
1657
  		sock_update_classid(&sk->sk_cgrp_data);
  		sock_update_netprioidx(&sk->sk_cgrp_data);
41b14fb87   Tariq Toukan   net: Do not clear...
1658
  		sk_tx_queue_clear(sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1659
  	}
a79af59ef   Frank Filz   [NET]: Fix module...
1660

2e4afe7b3   Pavel Emelyanov   [NET]: Move some ...
1661
  	return sk;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1662
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
1663
  EXPORT_SYMBOL(sk_alloc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1664

a4298e452   Eric Dumazet   net: add SOCK_RCU...
1665
1666
1667
1668
  /* Sockets having SOCK_RCU_FREE will call this function after one RCU
   * grace period. This is the case for UDP sockets and TCP listeners.
   */
  static void __sk_destruct(struct rcu_head *head)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1669
  {
a4298e452   Eric Dumazet   net: add SOCK_RCU...
1670
  	struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1671
  	struct sk_filter *filter;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1672
1673
1674
  
  	if (sk->sk_destruct)
  		sk->sk_destruct(sk);
a898def29   Paul E. McKenney   net: Add checking...
1675
  	filter = rcu_dereference_check(sk->sk_filter,
14afee4b6   Reshetova, Elena   net: convert sock...
1676
  				       refcount_read(&sk->sk_wmem_alloc) == 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1677
  	if (filter) {
309dd5fc8   Pavel Emelyanov   [NET]: Move the f...
1678
  		sk_filter_uncharge(sk, filter);
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
1679
  		RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1680
  	}
08e29af3a   Eric Dumazet   net: optimize soc...
1681
  	sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1682

6ac99e8f2   Martin KaFai Lau   bpf: Introduce bp...
1683
1684
1685
  #ifdef CONFIG_BPF_SYSCALL
  	bpf_sk_storage_free(sk);
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1686
  	if (atomic_read(&sk->sk_omem_alloc))
e005d193d   Joe Perches   net: core: Use pr...
1687
1688
1689
  		pr_debug("%s: optmem leakage (%d bytes) detected
  ",
  			 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1690

22a0e18ea   Eric Dumazet   net: properly rel...
1691
1692
1693
1694
  	if (sk->sk_frag.page) {
  		put_page(sk->sk_frag.page);
  		sk->sk_frag.page = NULL;
  	}
109f6e39f   Eric W. Biederman   af_unix: Allow SO...
1695
1696
1697
  	if (sk->sk_peer_cred)
  		put_cred(sk->sk_peer_cred);
  	put_pid(sk->sk_peer_pid);
26abe1437   Eric W. Biederman   net: Modify sk_al...
1698
1699
  	if (likely(sk->sk_net_refcnt))
  		put_net(sock_net(sk));
c308c1b20   Pavel Emelyanov   [NET]: Cleanup th...
1700
  	sk_prot_free(sk->sk_prot_creator, sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1701
  }
2b85a34e9   Eric Dumazet   net: No more expe...
1702

a4298e452   Eric Dumazet   net: add SOCK_RCU...
1703
1704
  void sk_destruct(struct sock *sk)
  {
8c7138b33   Martin KaFai Lau   net: Unpublish sk...
1705
1706
1707
1708
1709
1710
1711
1712
  	bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
  
  	if (rcu_access_pointer(sk->sk_reuseport_cb)) {
  		reuseport_detach_sock(sk);
  		use_call_rcu = true;
  	}
  
  	if (use_call_rcu)
a4298e452   Eric Dumazet   net: add SOCK_RCU...
1713
1714
1715
1716
  		call_rcu(&sk->sk_rcu, __sk_destruct);
  	else
  		__sk_destruct(&sk->sk_rcu);
  }
eb4cb0085   Craig Gallek   sock_diag: define...
1717
1718
  static void __sk_free(struct sock *sk)
  {
648845ab7   Tonghao Zhang   sock: Move the so...
1719
1720
  	if (likely(sk->sk_net_refcnt))
  		sock_inuse_add(sock_net(sk), -1);
9709020c8   Eric Dumazet   sock_diag: fix us...
1721
  	if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
eb4cb0085   Craig Gallek   sock_diag: define...
1722
1723
1724
1725
  		sock_diag_broadcast_destroy(sk);
  	else
  		sk_destruct(sk);
  }
2b85a34e9   Eric Dumazet   net: No more expe...
1726
1727
1728
  void sk_free(struct sock *sk)
  {
  	/*
25985edce   Lucas De Marchi   Fix common misspe...
1729
  	 * We subtract one from sk_wmem_alloc and can know if
2b85a34e9   Eric Dumazet   net: No more expe...
1730
1731
1732
  	 * some packets are still in some tx queue.
  	 * If not null, sock_wfree() will call __sk_free(sk) later
  	 */
14afee4b6   Reshetova, Elena   net: convert sock...
1733
  	if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2b85a34e9   Eric Dumazet   net: No more expe...
1734
1735
  		__sk_free(sk);
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
1736
  EXPORT_SYMBOL(sk_free);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1737

581319c58   Paolo Abeni   net/socket: use p...
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
  static void sk_init_common(struct sock *sk)
  {
  	skb_queue_head_init(&sk->sk_receive_queue);
  	skb_queue_head_init(&sk->sk_write_queue);
  	skb_queue_head_init(&sk->sk_error_queue);
  
  	rwlock_init(&sk->sk_callback_lock);
  	lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
  			af_rlock_keys + sk->sk_family,
  			af_family_rlock_key_strings[sk->sk_family]);
  	lockdep_set_class_and_name(&sk->sk_write_queue.lock,
  			af_wlock_keys + sk->sk_family,
  			af_family_wlock_key_strings[sk->sk_family]);
  	lockdep_set_class_and_name(&sk->sk_error_queue.lock,
  			af_elock_keys + sk->sk_family,
  			af_family_elock_key_strings[sk->sk_family]);
  	lockdep_set_class_and_name(&sk->sk_callback_lock,
  			af_callback_keys + sk->sk_family,
  			af_family_clock_key_strings[sk->sk_family]);
  }
e56c57d0d   Eric Dumazet   net: rename sk_cl...
1758
1759
1760
1761
1762
1763
1764
1765
  /**
   *	sk_clone_lock - clone a socket, and lock its clone
   *	@sk: the socket to clone
   *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
   *
   *	Caller must unlock socket even in error path (bh_unlock_sock(newsk))
   */
  struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1766
  {
b8e202d1d   Jakub Sitnicki   net, sk_msg: Anno...
1767
  	struct proto *prot = READ_ONCE(sk->sk_prot);
8fd1d178a   Pavel Emelyanov   [NET]: Make the s...
1768
  	struct sock *newsk;
278571bac   Alexei Starovoitov   net: filter: simp...
1769
  	bool is_charged = true;
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1770

b8e202d1d   Jakub Sitnicki   net, sk_msg: Anno...
1771
  	newsk = sk_prot_alloc(prot, priority, sk->sk_family);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1772
1773
  	if (newsk != NULL) {
  		struct sk_filter *filter;
892c141e6   Venkat Yekkirala   [MLSXFRM]: Add se...
1774
  		sock_copy(newsk, sk);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1775

b8e202d1d   Jakub Sitnicki   net, sk_msg: Anno...
1776
  		newsk->sk_prot_creator = prot;
9d538fa60   Christoph Paasch   net: Set sk_prot_...
1777

87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1778
  		/* SANITY */
8a6817369   Sowmini Varadhan   net: sk_clone_loc...
1779
1780
  		if (likely(newsk->sk_net_refcnt))
  			get_net(sock_net(newsk));
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1781
1782
1783
  		sk_node_init(&newsk->sk_node);
  		sock_lock_init(newsk);
  		bh_lock_sock(newsk);
fa438ccfd   Eric Dumazet   [NET]: Keep sk_ba...
1784
  		newsk->sk_backlog.head	= newsk->sk_backlog.tail = NULL;
8eae939f1   Zhu Yi   net: add limit fo...
1785
  		newsk->sk_backlog.len = 0;
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1786
1787
  
  		atomic_set(&newsk->sk_rmem_alloc, 0);
2b85a34e9   Eric Dumazet   net: No more expe...
1788
1789
1790
  		/*
  		 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
  		 */
14afee4b6   Reshetova, Elena   net: convert sock...
1791
  		refcount_set(&newsk->sk_wmem_alloc, 1);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1792
  		atomic_set(&newsk->sk_omem_alloc, 0);
581319c58   Paolo Abeni   net/socket: use p...
1793
  		sk_init_common(newsk);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1794
1795
  
  		newsk->sk_dst_cache	= NULL;
9b8805a32   Julian Anastasov   sock: add sk_dst_...
1796
  		newsk->sk_dst_pending_confirm = 0;
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1797
1798
  		newsk->sk_wmem_queued	= 0;
  		newsk->sk_forward_alloc = 0;
9caad8641   Eric Dumazet   tcp: increment sk...
1799
  		atomic_set(&newsk->sk_drops, 0);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1800
  		newsk->sk_send_head	= NULL;
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1801
  		newsk->sk_userlocks	= sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
52267790e   Willem de Bruijn   sock: add MSG_ZER...
1802
  		atomic_set(&newsk->sk_zckey, 0);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1803
1804
  
  		sock_reset_flag(newsk, SOCK_DONE);
d752a4986   Shakeel Butt   net: memcg: late ...
1805
1806
1807
  
  		/* sk->sk_memcg will be populated at accept() time */
  		newsk->sk_memcg = NULL;
ad0f75e5f   Cong Wang   cgroup: fix cgrou...
1808
  		cgroup_sk_clone(&newsk->sk_cgrp_data);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1809

eefca20eb   Eric Dumazet   socket, bpf: fix ...
1810
1811
  		rcu_read_lock();
  		filter = rcu_dereference(sk->sk_filter);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1812
  		if (filter != NULL)
278571bac   Alexei Starovoitov   net: filter: simp...
1813
1814
1815
1816
1817
  			/* though it's an empty new sock, the charging may fail
  			 * if sysctl_optmem_max was changed between creation of
  			 * original socket and cloning
  			 */
  			is_charged = sk_filter_charge(newsk, filter);
eefca20eb   Eric Dumazet   socket, bpf: fix ...
1818
1819
  		RCU_INIT_POINTER(newsk->sk_filter, filter);
  		rcu_read_unlock();
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1820

d188ba86d   Eric Dumazet   xfrm: add rcu pro...
1821
  		if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
a97e50cc4   Daniel Borkmann   socket, bpf: fix ...
1822
1823
1824
1825
1826
1827
  			/* We need to make sure that we don't uncharge the new
  			 * socket if we couldn't charge it in the first place
  			 * as otherwise we uncharge the parent's filter.
  			 */
  			if (!is_charged)
  				RCU_INIT_POINTER(newsk->sk_filter, NULL);
94352d450   Arnaldo Carvalho de Melo   net: Introduce sk...
1828
  			sk_free_unlock_clone(newsk);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1829
1830
1831
  			newsk = NULL;
  			goto out;
  		}
fa4634976   Craig Gallek   soreuseport: Prep...
1832
  		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
8f51dfc73   Stanislav Fomichev   bpf: support clon...
1833
1834
1835
1836
1837
1838
  
  		if (bpf_sk_storage_clone(sk, newsk)) {
  			sk_free_unlock_clone(newsk);
  			newsk = NULL;
  			goto out;
  		}
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1839

f1ff5ce2c   Jakub Sitnicki   net, sk_msg: Clea...
1840
1841
1842
1843
  		/* Clear sk_user_data if parent had the pointer tagged
  		 * as not suitable for copying when cloning.
  		 */
  		if (sk_user_data_is_nocopy(newsk))
7a1ca9726   Jakub Sitnicki   net, sk_msg: Don'...
1844
  			newsk->sk_user_data = NULL;
f1ff5ce2c   Jakub Sitnicki   net, sk_msg: Clea...
1845

87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1846
  		newsk->sk_err	   = 0;
e551c32d5   Eric Dumazet   net: clear sk_err...
1847
  		newsk->sk_err_soft = 0;
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1848
  		newsk->sk_priority = 0;
2c8c56e15   Eric Dumazet   net: introduce SO...
1849
  		newsk->sk_incoming_cpu = raw_smp_processor_id();
648845ab7   Tonghao Zhang   sock: Move the so...
1850
1851
  		if (likely(newsk->sk_net_refcnt))
  			sock_inuse_add(sock_net(newsk), 1);
d979a39d7   Johannes Weiner   cgroup: duplicate...
1852

4dc6dc716   Eric Dumazet   net: sock_copy() ...
1853
1854
  		/*
  		 * Before updating sk_refcnt, we must commit prior changes to memory
2cdb54c93   Mauro Carvalho Chehab   docs: RCU: Conver...
1855
  		 * (Documentation/RCU/rculist_nulls.rst for details)
4dc6dc716   Eric Dumazet   net: sock_copy() ...
1856
1857
  		 */
  		smp_wmb();
41c6d650f   Reshetova, Elena   net: convert sock...
1858
  		refcount_set(&newsk->sk_refcnt, 2);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
  
  		/*
  		 * Increment the counter in the same struct proto as the master
  		 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
  		 * is the same as sk->sk_prot->socks, as this field was copied
  		 * with memcpy).
  		 *
  		 * This _changes_ the previous behaviour, where
  		 * tcp_create_openreq_child always was incrementing the
  		 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
  		 * to be taken into account in all callers. -acme
  		 */
  		sk_refcnt_debug_inc(newsk);
972692e0d   David S. Miller   net: Add sk_set_s...
1872
  		sk_set_socket(newsk, NULL);
41b14fb87   Tariq Toukan   net: Do not clear...
1873
  		sk_tx_queue_clear(newsk);
c2f26e8f8   Li RongQing   net: Use RCU_INIT...
1874
  		RCU_INIT_POINTER(newsk->sk_wq, NULL);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1875
1876
  
  		if (newsk->sk_prot->sockets_allocated)
180d8cd94   Glauber Costa   foundations of pe...
1877
  			sk_sockets_allocated_inc(newsk);
704da560c   Octavian Purdila   tcp: update the n...
1878

080a270f5   Hannes Frederic Sowa   sock: don't enabl...
1879
1880
  		if (sock_needs_netstamp(sk) &&
  		    newsk->sk_flags & SK_FLAGS_TIMESTAMP)
704da560c   Octavian Purdila   tcp: update the n...
1881
  			net_enable_timestamp();
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1882
1883
1884
1885
  	}
  out:
  	return newsk;
  }
e56c57d0d   Eric Dumazet   net: rename sk_cl...
1886
  EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb9   Arnaldo Carvalho de Melo   [SOCK]: Introduce...
1887

94352d450   Arnaldo Carvalho de Melo   net: Introduce sk...
1888
1889
1890
1891
1892
1893
1894
1895
1896
  void sk_free_unlock_clone(struct sock *sk)
  {
  	/* It is still raw copy of parent, so invalidate
  	 * destructor and make plain sk_free() */
  	sk->sk_destruct = NULL;
  	bh_unlock_sock(sk);
  	sk_free(sk);
  }
  EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
9958089a4   Andi Kleen   [NET]: Move sk_se...
1897
1898
  void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
  {
d6a4e26af   Eric Dumazet   tcp: tcp_tso_auto...
1899
  	u32 max_segs = 1;
6bd4f355d   Eric Dumazet   ipv6: kill sk_dst...
1900
  	sk_dst_set(sk, dst);
0a6b2a1dc   Eric Dumazet   tcp: switch to GS...
1901
  	sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps;
9958089a4   Andi Kleen   [NET]: Move sk_se...
1902
  	if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b991   Herbert Xu   [NET] gso: Fix GS...
1903
  		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
a465419b1   Eric Dumazet   net: Introduce sk...
1904
  	sk->sk_route_caps &= ~sk->sk_route_nocaps;
9958089a4   Andi Kleen   [NET]: Move sk_se...
1905
  	if (sk_can_gso(sk)) {
f70f250a7   Steffen Klassert   net: Allow IPsec ...
1906
  		if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
9958089a4   Andi Kleen   [NET]: Move sk_se...
1907
  			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a5   Peter P Waskiewicz Jr   [NET]: Add per-co...
1908
  		} else {
9958089a4   Andi Kleen   [NET]: Move sk_se...
1909
  			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
82cc1a7a5   Peter P Waskiewicz Jr   [NET]: Add per-co...
1910
  			sk->sk_gso_max_size = dst->dev->gso_max_size;
d6a4e26af   Eric Dumazet   tcp: tcp_tso_auto...
1911
  			max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
82cc1a7a5   Peter P Waskiewicz Jr   [NET]: Add per-co...
1912
  		}
9958089a4   Andi Kleen   [NET]: Move sk_se...
1913
  	}
d6a4e26af   Eric Dumazet   tcp: tcp_tso_auto...
1914
  	sk->sk_gso_max_segs = max_segs;
9958089a4   Andi Kleen   [NET]: Move sk_se...
1915
1916
  }
  EXPORT_SYMBOL_GPL(sk_setup_caps);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1917
1918
1919
  /*
   *	Simple resource managers for sockets.
   */
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
1920
1921
  /*
   * Write buffer destructor automatically called from kfree_skb.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1922
1923
1924
1925
   */
  void sock_wfree(struct sk_buff *skb)
  {
  	struct sock *sk = skb->sk;
d99927f4d   Eric Dumazet   net: Fix sock_wfr...
1926
  	unsigned int len = skb->truesize;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1927

d99927f4d   Eric Dumazet   net: Fix sock_wfr...
1928
1929
1930
1931
1932
  	if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
  		/*
  		 * Keep a reference on sk_wmem_alloc, this will be released
  		 * after sk_write_space() call
  		 */
14afee4b6   Reshetova, Elena   net: convert sock...
1933
  		WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1934
  		sk->sk_write_space(sk);
d99927f4d   Eric Dumazet   net: Fix sock_wfr...
1935
1936
  		len = 1;
  	}
2b85a34e9   Eric Dumazet   net: No more expe...
1937
  	/*
d99927f4d   Eric Dumazet   net: Fix sock_wfr...
1938
1939
  	 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
  	 * could not do because of in-flight packets
2b85a34e9   Eric Dumazet   net: No more expe...
1940
  	 */
14afee4b6   Reshetova, Elena   net: convert sock...
1941
  	if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e9   Eric Dumazet   net: No more expe...
1942
  		__sk_free(sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1943
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
1944
  EXPORT_SYMBOL(sock_wfree);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1945

1d2077ac0   Eric Dumazet   net: add __sock_w...
1946
1947
1948
1949
1950
1951
  /* This variant of sock_wfree() is used by TCP,
   * since it sets SOCK_USE_WRITE_QUEUE.
   */
  void __sock_wfree(struct sk_buff *skb)
  {
  	struct sock *sk = skb->sk;
14afee4b6   Reshetova, Elena   net: convert sock...
1952
  	if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1d2077ac0   Eric Dumazet   net: add __sock_w...
1953
1954
  		__sk_free(sk);
  }
9e17f8a47   Eric Dumazet   net: make skb_set...
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
  void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
  {
  	skb_orphan(skb);
  	skb->sk = sk;
  #ifdef CONFIG_INET
  	if (unlikely(!sk_fullsock(sk))) {
  		skb->destructor = sock_edemux;
  		sock_hold(sk);
  		return;
  	}
  #endif
  	skb->destructor = sock_wfree;
  	skb_set_hash_from_sk(skb, sk);
  	/*
  	 * We used to take a refcount on sk, but following operation
  	 * is enough to guarantee sk_free() wont free this sock until
  	 * all in-flight packets are completed
  	 */
14afee4b6   Reshetova, Elena   net: convert sock...
1973
  	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
9e17f8a47   Eric Dumazet   net: make skb_set...
1974
1975
  }
  EXPORT_SYMBOL(skb_set_owner_w);
414776621   Jakub Kicinski   net/tls: prevent ...
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
  static bool can_skb_orphan_partial(const struct sk_buff *skb)
  {
  #ifdef CONFIG_TLS_DEVICE
  	/* Drivers depend on in-order delivery for crypto offload,
  	 * partial orphan breaks out-of-order-OK logic.
  	 */
  	if (skb->decrypted)
  		return false;
  #endif
  	return (skb->destructor == sock_wfree ||
  		(IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
  }
1d2077ac0   Eric Dumazet   net: add __sock_w...
1988
1989
1990
1991
  /* This helper is used by netem, as it can hold packets in its
   * delay queue. We want to allow the owner socket to send more
   * packets, as if they were already TX completed by a typical driver.
   * But we also want to keep skb->sk set because some packet schedulers
f6ba8d33c   Eric Dumazet   netem: fix skb_or...
1992
   * rely on it (sch_fq for example).
1d2077ac0   Eric Dumazet   net: add __sock_w...
1993
   */
f2f872f92   Eric Dumazet   netem: Introduce ...
1994
1995
  void skb_orphan_partial(struct sk_buff *skb)
  {
f6ba8d33c   Eric Dumazet   netem: fix skb_or...
1996
  	if (skb_is_tcp_pure_ack(skb))
1d2077ac0   Eric Dumazet   net: add __sock_w...
1997
  		return;
414776621   Jakub Kicinski   net/tls: prevent ...
1998
  	if (can_skb_orphan_partial(skb)) {
f6ba8d33c   Eric Dumazet   netem: fix skb_or...
1999
  		struct sock *sk = skb->sk;
41c6d650f   Reshetova, Elena   net: convert sock...
2000
  		if (refcount_inc_not_zero(&sk->sk_refcnt)) {
14afee4b6   Reshetova, Elena   net: convert sock...
2001
  			WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc));
f6ba8d33c   Eric Dumazet   netem: fix skb_or...
2002
2003
  			skb->destructor = sock_efree;
  		}
f2f872f92   Eric Dumazet   netem: Introduce ...
2004
2005
2006
2007
2008
  	} else {
  		skb_orphan(skb);
  	}
  }
  EXPORT_SYMBOL(skb_orphan_partial);
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2009
2010
  /*
   * Read buffer destructor automatically called from kfree_skb.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2011
2012
2013
2014
   */
  void sock_rfree(struct sk_buff *skb)
  {
  	struct sock *sk = skb->sk;
d361fd599   Eric Dumazet   net: sock_free() ...
2015
  	unsigned int len = skb->truesize;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2016

d361fd599   Eric Dumazet   net: sock_free() ...
2017
2018
  	atomic_sub(len, &sk->sk_rmem_alloc);
  	sk_mem_uncharge(sk, len);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2019
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2020
  EXPORT_SYMBOL(sock_rfree);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2021

7768eed8b   Oliver Hartkopp   net: add comment ...
2022
2023
2024
2025
  /*
   * Buffer destructor for skbs that are not used directly in read or write
   * path, e.g. for error handler skbs. Automatically called from kfree_skb.
   */
62bccb8cd   Alexander Duyck   net-timestamp: Ma...
2026
2027
2028
2029
2030
  void sock_efree(struct sk_buff *skb)
  {
  	sock_put(skb->sk);
  }
  EXPORT_SYMBOL(sock_efree);
cf7fbe660   Joe Stringer   bpf: Add socket a...
2031
2032
2033
2034
2035
2036
  /* Buffer destructor for prefetch/receive path where reference count may
   * not be held, e.g. for listen sockets.
   */
  #ifdef CONFIG_INET
  void sock_pfree(struct sk_buff *skb)
  {
7ae215d23   Joe Stringer   bpf: Don't refcou...
2037
2038
  	if (sk_is_refcounted(skb->sk))
  		sock_gen_put(skb->sk);
cf7fbe660   Joe Stringer   bpf: Add socket a...
2039
2040
2041
  }
  EXPORT_SYMBOL(sock_pfree);
  #endif /* CONFIG_INET */
976d02015   Eric W. Biederman   userns: Convert s...
2042
  kuid_t sock_i_uid(struct sock *sk)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2043
  {
976d02015   Eric W. Biederman   userns: Convert s...
2044
  	kuid_t uid;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2045

f064af1e5   Eric Dumazet   net: fix a lockde...
2046
  	read_lock_bh(&sk->sk_callback_lock);
976d02015   Eric W. Biederman   userns: Convert s...
2047
  	uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e5   Eric Dumazet   net: fix a lockde...
2048
  	read_unlock_bh(&sk->sk_callback_lock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2049
2050
  	return uid;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2051
  EXPORT_SYMBOL(sock_i_uid);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2052
2053
2054
2055
  
  unsigned long sock_i_ino(struct sock *sk)
  {
  	unsigned long ino;
f064af1e5   Eric Dumazet   net: fix a lockde...
2056
  	read_lock_bh(&sk->sk_callback_lock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2057
  	ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e5   Eric Dumazet   net: fix a lockde...
2058
  	read_unlock_bh(&sk->sk_callback_lock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2059
2060
  	return ino;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2061
  EXPORT_SYMBOL(sock_i_ino);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2062
2063
2064
2065
  
  /*
   * Allocate a skb from the socket's send buffer.
   */
86a76caf8   Victor Fusco   [NET]: Fix sparse...
2066
  struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66fb   Al Viro   [PATCH] gfp flags...
2067
  			     gfp_t priority)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2068
  {
e292f05e0   Eric Dumazet   tcp: annotate sk-...
2069
2070
  	if (force ||
  	    refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2a91525c2   Eric Dumazet   net: net/core/soc...
2071
  		struct sk_buff *skb = alloc_skb(size, priority);
e292f05e0   Eric Dumazet   tcp: annotate sk-...
2072

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2073
2074
2075
2076
2077
2078
2079
  		if (skb) {
  			skb_set_owner_w(skb, sk);
  			return skb;
  		}
  	}
  	return NULL;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2080
  EXPORT_SYMBOL(sock_wmalloc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2081

98ba0bd55   Willem de Bruijn   sock: allocate sk...
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
  static void sock_ofree(struct sk_buff *skb)
  {
  	struct sock *sk = skb->sk;
  
  	atomic_sub(skb->truesize, &sk->sk_omem_alloc);
  }
  
  struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
  			     gfp_t priority)
  {
  	struct sk_buff *skb;
  
  	/* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
  	if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
  	    sysctl_optmem_max)
  		return NULL;
  
  	skb = alloc_skb(size, priority);
  	if (!skb)
  		return NULL;
  
  	atomic_add(skb->truesize, &sk->sk_omem_alloc);
  	skb->sk = sk;
  	skb->destructor = sock_ofree;
  	return skb;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2108
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2109
   * Allocate a memory block from the socket's option memory buffer.
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2110
   */
dd0fc66fb   Al Viro   [PATCH] gfp flags...
2111
  void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2112
  {
95c961747   Eric Dumazet   net: cleanup unsi...
2113
  	if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2114
2115
2116
  	    atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
  		void *mem;
  		/* First do the add, to avoid the race if kmalloc
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2117
  		 * might sleep.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2118
2119
2120
2121
2122
2123
2124
2125
2126
  		 */
  		atomic_add(size, &sk->sk_omem_alloc);
  		mem = kmalloc(size, priority);
  		if (mem)
  			return mem;
  		atomic_sub(size, &sk->sk_omem_alloc);
  	}
  	return NULL;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2127
  EXPORT_SYMBOL(sock_kmalloc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2128

79e886599   Daniel Borkmann   crypto: algif - a...
2129
2130
2131
  /* Free an option memory block. Note, we actually want the inline
   * here as this allows gcc to detect the nullify and fold away the
   * condition entirely.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2132
   */
79e886599   Daniel Borkmann   crypto: algif - a...
2133
2134
  static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
  				  const bool nullify)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2135
  {
e53da5fbf   David S. Miller   net: Trap attempt...
2136
2137
  	if (WARN_ON_ONCE(!mem))
  		return;
79e886599   Daniel Borkmann   crypto: algif - a...
2138
  	if (nullify)
453431a54   Waiman Long   mm, treewide: ren...
2139
  		kfree_sensitive(mem);
79e886599   Daniel Borkmann   crypto: algif - a...
2140
2141
  	else
  		kfree(mem);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2142
2143
  	atomic_sub(size, &sk->sk_omem_alloc);
  }
79e886599   Daniel Borkmann   crypto: algif - a...
2144
2145
2146
2147
2148
  
  void sock_kfree_s(struct sock *sk, void *mem, int size)
  {
  	__sock_kfree_s(sk, mem, size, false);
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2149
  EXPORT_SYMBOL(sock_kfree_s);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2150

79e886599   Daniel Borkmann   crypto: algif - a...
2151
2152
2153
2154
2155
  void sock_kzfree_s(struct sock *sk, void *mem, int size)
  {
  	__sock_kfree_s(sk, mem, size, true);
  }
  EXPORT_SYMBOL(sock_kzfree_s);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2156
2157
2158
  /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
     I think, these locks should be removed for datagram sockets.
   */
2a91525c2   Eric Dumazet   net: net/core/soc...
2159
  static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2160
2161
  {
  	DEFINE_WAIT(wait);
9cd3e072b   Eric Dumazet   net: rename SOCK_...
2162
  	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2163
2164
2165
2166
2167
2168
  	for (;;) {
  		if (!timeo)
  			break;
  		if (signal_pending(current))
  			break;
  		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa3951451   Eric Dumazet   net: sk_sleep() h...
2169
  		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
e292f05e0   Eric Dumazet   tcp: annotate sk-...
2170
  		if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2171
2172
2173
2174
2175
2176
2177
  			break;
  		if (sk->sk_shutdown & SEND_SHUTDOWN)
  			break;
  		if (sk->sk_err)
  			break;
  		timeo = schedule_timeout(timeo);
  	}
aa3951451   Eric Dumazet   net: sk_sleep() h...
2178
  	finish_wait(sk_sleep(sk), &wait);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2179
2180
2181
2182
2183
2184
2185
  	return timeo;
  }
  
  
  /*
   *	Generic send/receive buffer handlers
   */
4cc7f68d6   Herbert Xu   net: Reexport soc...
2186
2187
  struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
  				     unsigned long data_len, int noblock,
28d642710   Eric Dumazet   net: attempt high...
2188
  				     int *errcode, int max_page_order)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2189
  {
2e4e44107   Eric Dumazet   net: add alloc_sk...
2190
  	struct sk_buff *skb;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2191
2192
  	long timeo;
  	int err;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2193
  	timeo = sock_sndtimeo(sk, noblock);
2e4e44107   Eric Dumazet   net: add alloc_sk...
2194
  	for (;;) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2195
2196
2197
2198
2199
2200
2201
  		err = sock_error(sk);
  		if (err != 0)
  			goto failure;
  
  		err = -EPIPE;
  		if (sk->sk_shutdown & SEND_SHUTDOWN)
  			goto failure;
e292f05e0   Eric Dumazet   tcp: annotate sk-...
2202
  		if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2e4e44107   Eric Dumazet   net: add alloc_sk...
2203
  			break;
28d642710   Eric Dumazet   net: attempt high...
2204

9cd3e072b   Eric Dumazet   net: rename SOCK_...
2205
  		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e44107   Eric Dumazet   net: add alloc_sk...
2206
2207
2208
  		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  		err = -EAGAIN;
  		if (!timeo)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2209
  			goto failure;
2e4e44107   Eric Dumazet   net: add alloc_sk...
2210
2211
2212
  		if (signal_pending(current))
  			goto interrupted;
  		timeo = sock_wait_for_wmem(sk, timeo);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2213
  	}
2e4e44107   Eric Dumazet   net: add alloc_sk...
2214
2215
2216
2217
  	skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
  				   errcode, sk->sk_allocation);
  	if (skb)
  		skb_set_owner_w(skb, sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2218
2219
2220
2221
2222
2223
2224
2225
  	return skb;
  
  interrupted:
  	err = sock_intr_errno(timeo);
  failure:
  	*errcode = err;
  	return NULL;
  }
4cc7f68d6   Herbert Xu   net: Reexport soc...
2226
  EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2227

4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2228
  struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2229
2230
  				    int noblock, int *errcode)
  {
28d642710   Eric Dumazet   net: attempt high...
2231
  	return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2232
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2233
  EXPORT_SYMBOL(sock_alloc_send_skb);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2234

39771b127   Willem de Bruijn   sock: break up so...
2235
2236
2237
  int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
  		     struct sockcm_cookie *sockc)
  {
3dd17e63f   Soheil Hassas Yeganeh   sock: accept SO_T...
2238
  	u32 tsflags;
39771b127   Willem de Bruijn   sock: break up so...
2239
2240
2241
2242
2243
2244
2245
2246
  	switch (cmsg->cmsg_type) {
  	case SO_MARK:
  		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
  			return -EPERM;
  		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
  			return -EINVAL;
  		sockc->mark = *(u32 *)CMSG_DATA(cmsg);
  		break;
7f1bc6e95   Deepa Dinamani   sockopt: Rename S...
2247
  	case SO_TIMESTAMPING_OLD:
3dd17e63f   Soheil Hassas Yeganeh   sock: accept SO_T...
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
  		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
  			return -EINVAL;
  
  		tsflags = *(u32 *)CMSG_DATA(cmsg);
  		if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
  			return -EINVAL;
  
  		sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
  		sockc->tsflags |= tsflags;
  		break;
80b14dee2   Richard Cochran   net: Add a new so...
2258
2259
2260
2261
2262
2263
2264
  	case SCM_TXTIME:
  		if (!sock_flag(sk, SOCK_TXTIME))
  			return -EINVAL;
  		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
  			return -EINVAL;
  		sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
  		break;
779f1edec   Soheil Hassas Yeganeh   sock: ignore SCM_...
2265
2266
2267
2268
  	/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
  	case SCM_RIGHTS:
  	case SCM_CREDENTIALS:
  		break;
39771b127   Willem de Bruijn   sock: break up so...
2269
2270
2271
2272
2273
2274
  	default:
  		return -EINVAL;
  	}
  	return 0;
  }
  EXPORT_SYMBOL(__sock_cmsg_send);
f28ea365c   Edward Jee   sock: support per...
2275
2276
2277
2278
  int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
  		   struct sockcm_cookie *sockc)
  {
  	struct cmsghdr *cmsg;
39771b127   Willem de Bruijn   sock: break up so...
2279
  	int ret;
f28ea365c   Edward Jee   sock: support per...
2280
2281
2282
2283
2284
2285
  
  	for_each_cmsghdr(cmsg, msg) {
  		if (!CMSG_OK(msg, cmsg))
  			return -EINVAL;
  		if (cmsg->cmsg_level != SOL_SOCKET)
  			continue;
39771b127   Willem de Bruijn   sock: break up so...
2286
2287
2288
  		ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
  		if (ret)
  			return ret;
f28ea365c   Edward Jee   sock: support per...
2289
2290
2291
2292
  	}
  	return 0;
  }
  EXPORT_SYMBOL(sock_cmsg_send);
060447511   Eric Dumazet   tcp: add TCPMemor...
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
  static void sk_enter_memory_pressure(struct sock *sk)
  {
  	if (!sk->sk_prot->enter_memory_pressure)
  		return;
  
  	sk->sk_prot->enter_memory_pressure(sk);
  }
  
  static void sk_leave_memory_pressure(struct sock *sk)
  {
  	if (sk->sk_prot->leave_memory_pressure) {
  		sk->sk_prot->leave_memory_pressure(sk);
  	} else {
  		unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
503978aca   Eric Dumazet   net: avoid possib...
2307
2308
  		if (memory_pressure && READ_ONCE(*memory_pressure))
  			WRITE_ONCE(*memory_pressure, 0);
060447511   Eric Dumazet   tcp: add TCPMemor...
2309
2310
  	}
  }
5640f7685   Eric Dumazet   net: use a per ta...
2311
  #define SKB_FRAG_PAGE_ORDER	get_order(32768)
ce27ec606   Eric Dumazet   net: add high_ord...
2312
  DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
5640f7685   Eric Dumazet   net: use a per ta...
2313

400dfd3ae   Eric Dumazet   net: refactor sk_...
2314
2315
2316
2317
  /**
   * skb_page_frag_refill - check that a page_frag contains enough room
   * @sz: minimum size of the fragment we want to get
   * @pfrag: pointer to page_frag
82d5e2b8b   Eric Dumazet   net: fix skb_page...
2318
   * @gfp: priority for memory allocation
400dfd3ae   Eric Dumazet   net: refactor sk_...
2319
2320
2321
2322
2323
   *
   * Note: While this allocator tries to use high order pages, there is
   * no guarantee that allocations succeed. Therefore, @sz MUST be
   * less or equal than PAGE_SIZE.
   */
d9b2938aa   Eric Dumazet   net: attempt a si...
2324
  bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f7685   Eric Dumazet   net: use a per ta...
2325
  {
5640f7685   Eric Dumazet   net: use a per ta...
2326
  	if (pfrag->page) {
fe896d187   Joonsoo Kim   mm: introduce pag...
2327
  		if (page_ref_count(pfrag->page) == 1) {
5640f7685   Eric Dumazet   net: use a per ta...
2328
2329
2330
  			pfrag->offset = 0;
  			return true;
  		}
400dfd3ae   Eric Dumazet   net: refactor sk_...
2331
  		if (pfrag->offset + sz <= pfrag->size)
5640f7685   Eric Dumazet   net: use a per ta...
2332
2333
2334
  			return true;
  		put_page(pfrag->page);
  	}
d9b2938aa   Eric Dumazet   net: attempt a si...
2335
  	pfrag->offset = 0;
ce27ec606   Eric Dumazet   net: add high_ord...
2336
2337
  	if (SKB_FRAG_PAGE_ORDER &&
  	    !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
d0164adc8   Mel Gorman   mm, page_alloc: d...
2338
2339
2340
2341
  		/* Avoid direct reclaim but allow kswapd to wake */
  		pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
  					  __GFP_COMP | __GFP_NOWARN |
  					  __GFP_NORETRY,
d9b2938aa   Eric Dumazet   net: attempt a si...
2342
  					  SKB_FRAG_PAGE_ORDER);
5640f7685   Eric Dumazet   net: use a per ta...
2343
  		if (likely(pfrag->page)) {
d9b2938aa   Eric Dumazet   net: attempt a si...
2344
  			pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f7685   Eric Dumazet   net: use a per ta...
2345
2346
  			return true;
  		}
d9b2938aa   Eric Dumazet   net: attempt a si...
2347
2348
2349
2350
2351
2352
  	}
  	pfrag->page = alloc_page(gfp);
  	if (likely(pfrag->page)) {
  		pfrag->size = PAGE_SIZE;
  		return true;
  	}
400dfd3ae   Eric Dumazet   net: refactor sk_...
2353
2354
2355
2356
2357
2358
2359
2360
  	return false;
  }
  EXPORT_SYMBOL(skb_page_frag_refill);
  
  bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
  {
  	if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
  		return true;
5640f7685   Eric Dumazet   net: use a per ta...
2361
2362
2363
2364
2365
  	sk_enter_memory_pressure(sk);
  	sk_stream_moderate_sndbuf(sk);
  	return false;
  }
  EXPORT_SYMBOL(sk_page_frag_refill);
ad80b0fc6   Paolo Abeni   mptcp: open code ...
2366
  void __lock_sock(struct sock *sk)
f39234d60   Namhyung Kim   net/core: add loc...
2367
2368
  	__releases(&sk->sk_lock.slock)
  	__acquires(&sk->sk_lock.slock)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2369
2370
  {
  	DEFINE_WAIT(wait);
e71a4783a   Stephen Hemminger   [NET] core: white...
2371
  	for (;;) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2372
2373
2374
2375
2376
  		prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
  					TASK_UNINTERRUPTIBLE);
  		spin_unlock_bh(&sk->sk_lock.slock);
  		schedule();
  		spin_lock_bh(&sk->sk_lock.slock);
e71a4783a   Stephen Hemminger   [NET] core: white...
2377
  		if (!sock_owned_by_user(sk))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2378
2379
2380
2381
  			break;
  	}
  	finish_wait(&sk->sk_lock.wq, &wait);
  }
8873c064d   Eric Dumazet   tcp: do not relea...
2382
  void __release_sock(struct sock *sk)
f39234d60   Namhyung Kim   net/core: add loc...
2383
2384
  	__releases(&sk->sk_lock.slock)
  	__acquires(&sk->sk_lock.slock)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2385
  {
5413d1bab   Eric Dumazet   net: do not block...
2386
  	struct sk_buff *skb, *next;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2387

5413d1bab   Eric Dumazet   net: do not block...
2388
  	while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2389
  		sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2390

5413d1bab   Eric Dumazet   net: do not block...
2391
  		spin_unlock_bh(&sk->sk_lock.slock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2392

5413d1bab   Eric Dumazet   net: do not block...
2393
2394
  		do {
  			next = skb->next;
e4cbb02a1   Eric Dumazet   net: add a prefet...
2395
  			prefetch(next);
7fee226ad   Eric Dumazet   net: add a noref ...
2396
  			WARN_ON_ONCE(skb_dst_is_noref(skb));
a8305bff6   David S. Miller   net: Add and use ...
2397
  			skb_mark_not_on_list(skb);
c57943a1c   Peter Zijlstra   net: wrap sk->sk_...
2398
  			sk_backlog_rcv(sk, skb);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2399

5413d1bab   Eric Dumazet   net: do not block...
2400
  			cond_resched();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2401
2402
2403
  
  			skb = next;
  		} while (skb != NULL);
5413d1bab   Eric Dumazet   net: do not block...
2404
2405
  		spin_lock_bh(&sk->sk_lock.slock);
  	}
8eae939f1   Zhu Yi   net: add limit fo...
2406
2407
2408
2409
2410
2411
  
  	/*
  	 * Doing the zeroing here guarantee we can not loop forever
  	 * while a wild producer attempts to flood us.
  	 */
  	sk->sk_backlog.len = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2412
  }
d41a69f1d   Eric Dumazet   tcp: make tcp_sen...
2413
2414
2415
2416
2417
2418
  void __sk_flush_backlog(struct sock *sk)
  {
  	spin_lock_bh(&sk->sk_lock.slock);
  	__release_sock(sk);
  	spin_unlock_bh(&sk->sk_lock.slock);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2419
2420
  /**
   * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16ba   Pavel Pisa   [PATCH] DocBook: ...
2421
2422
   * @sk:    sock to wait on
   * @timeo: for how long
dfbafc995   Sabrina Dubroca   tcp: fix recv wit...
2423
   * @skb:   last skb seen on sk_receive_queue
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2424
2425
2426
2427
2428
2429
   *
   * Now socket state including sk->sk_err is changed only under lock,
   * hence we may omit checks after joining wait queue.
   * We check receive queue before schedule() only as optimization;
   * it is very likely that release_sock() added new data.
   */
dfbafc995   Sabrina Dubroca   tcp: fix recv wit...
2430
  int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2431
  {
d9dc8b0f8   WANG Cong   net: fix sleeping...
2432
  	DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2433
  	int rc;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2434

d9dc8b0f8   WANG Cong   net: fix sleeping...
2435
  	add_wait_queue(sk_sleep(sk), &wait);
9cd3e072b   Eric Dumazet   net: rename SOCK_...
2436
  	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f8   WANG Cong   net: fix sleeping...
2437
  	rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072b   Eric Dumazet   net: rename SOCK_...
2438
  	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f8   WANG Cong   net: fix sleeping...
2439
  	remove_wait_queue(sk_sleep(sk), &wait);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2440
2441
  	return rc;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2442
  EXPORT_SYMBOL(sk_wait_data);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2443
  /**
f8c3bf00d   Paolo Abeni   net/socket: facto...
2444
   *	__sk_mem_raise_allocated - increase memory_allocated
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2445
2446
   *	@sk: socket
   *	@size: memory size to allocate
f8c3bf00d   Paolo Abeni   net/socket: facto...
2447
   *	@amt: pages to allocate
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2448
2449
   *	@kind: allocation type
   *
f8c3bf00d   Paolo Abeni   net/socket: facto...
2450
   *	Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2451
   */
f8c3bf00d   Paolo Abeni   net/socket: facto...
2452
  int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2453
2454
  {
  	struct proto *prot = sk->sk_prot;
f8c3bf00d   Paolo Abeni   net/socket: facto...
2455
  	long allocated = sk_memory_allocated_add(sk, amt);
d6f19938e   Yafang Shao   net: expose sk wm...
2456
  	bool charged = true;
e805605c7   Johannes Weiner   net: tcp_memcontr...
2457

baac50bbc   Johannes Weiner   net: tcp_memcontr...
2458
  	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
d6f19938e   Yafang Shao   net: expose sk wm...
2459
  	    !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt)))
e805605c7   Johannes Weiner   net: tcp_memcontr...
2460
  		goto suppress_allocation;
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2461
2462
  
  	/* Under limit. */
e805605c7   Johannes Weiner   net: tcp_memcontr...
2463
  	if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd94   Glauber Costa   foundations of pe...
2464
  		sk_leave_memory_pressure(sk);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2465
2466
  		return 1;
  	}
e805605c7   Johannes Weiner   net: tcp_memcontr...
2467
2468
  	/* Under pressure. */
  	if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd94   Glauber Costa   foundations of pe...
2469
  		sk_enter_memory_pressure(sk);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2470

e805605c7   Johannes Weiner   net: tcp_memcontr...
2471
2472
  	/* Over hard limit. */
  	if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2473
2474
2475
2476
  		goto suppress_allocation;
  
  	/* guarantee minimum buffer size under pressure */
  	if (kind == SK_MEM_RECV) {
a3dcaf17e   Eric Dumazet   net: allow per ne...
2477
  		if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2478
  			return 1;
180d8cd94   Glauber Costa   foundations of pe...
2479

3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2480
  	} else { /* SK_MEM_SEND */
a3dcaf17e   Eric Dumazet   net: allow per ne...
2481
  		int wmem0 = sk_get_wmem0(sk, prot);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2482
  		if (sk->sk_type == SOCK_STREAM) {
a3dcaf17e   Eric Dumazet   net: allow per ne...
2483
  			if (sk->sk_wmem_queued < wmem0)
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2484
  				return 1;
a3dcaf17e   Eric Dumazet   net: allow per ne...
2485
  		} else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2486
  				return 1;
a3dcaf17e   Eric Dumazet   net: allow per ne...
2487
  		}
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2488
  	}
180d8cd94   Glauber Costa   foundations of pe...
2489
  	if (sk_has_memory_pressure(sk)) {
5bf325a53   Eric Dumazet   net: fix possible...
2490
  		u64 alloc;
1748376b6   Eric Dumazet   net: Use a percpu...
2491

180d8cd94   Glauber Costa   foundations of pe...
2492
  		if (!sk_under_memory_pressure(sk))
1748376b6   Eric Dumazet   net: Use a percpu...
2493
  			return 1;
180d8cd94   Glauber Costa   foundations of pe...
2494
2495
  		alloc = sk_sockets_allocated_read_positive(sk);
  		if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
  		    sk_mem_pages(sk->sk_wmem_queued +
  				 atomic_read(&sk->sk_rmem_alloc) +
  				 sk->sk_forward_alloc))
  			return 1;
  	}
  
  suppress_allocation:
  
  	if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
  		sk_stream_moderate_sndbuf(sk);
  
  		/* Fail only if socket is _under_ its sndbuf.
  		 * In this case we cannot block, so that we have to fail.
  		 */
  		if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
  			return 1;
  	}
d6f19938e   Yafang Shao   net: expose sk wm...
2513
2514
  	if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
  		trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3847ce32a   Satoru Moriya   core: add tracepo...
2515

0e90b31f4   Glauber Costa   net: introduce re...
2516
  	sk_memory_allocated_sub(sk, amt);
180d8cd94   Glauber Costa   foundations of pe...
2517

baac50bbc   Johannes Weiner   net: tcp_memcontr...
2518
2519
  	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
  		mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c7   Johannes Weiner   net: tcp_memcontr...
2520

3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2521
2522
  	return 0;
  }
f8c3bf00d   Paolo Abeni   net/socket: facto...
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
  EXPORT_SYMBOL(__sk_mem_raise_allocated);
  
  /**
   *	__sk_mem_schedule - increase sk_forward_alloc and memory_allocated
   *	@sk: socket
   *	@size: memory size to allocate
   *	@kind: allocation type
   *
   *	If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
   *	rmem allocation. This function assumes that protocols which have
   *	memory_pressure use sk_wmem_queued as write buffer accounting.
   */
  int __sk_mem_schedule(struct sock *sk, int size, int kind)
  {
  	int ret, amt = sk_mem_pages(size);
  
  	sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
  	ret = __sk_mem_raise_allocated(sk, size, amt, kind);
  	if (!ret)
  		sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
  	return ret;
  }
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2545
2546
2547
  EXPORT_SYMBOL(__sk_mem_schedule);
  
  /**
f8c3bf00d   Paolo Abeni   net/socket: facto...
2548
   *	__sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2549
   *	@sk: socket
f8c3bf00d   Paolo Abeni   net/socket: facto...
2550
2551
2552
   *	@amount: number of quanta
   *
   *	Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2553
   */
f8c3bf00d   Paolo Abeni   net/socket: facto...
2554
  void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2555
  {
1a24e04e4   Eric Dumazet   net: fix sk_mem_r...
2556
  	sk_memory_allocated_sub(sk, amount);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2557

baac50bbc   Johannes Weiner   net: tcp_memcontr...
2558
2559
  	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
  		mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c7   Johannes Weiner   net: tcp_memcontr...
2560

180d8cd94   Glauber Costa   foundations of pe...
2561
2562
2563
  	if (sk_under_memory_pressure(sk) &&
  	    (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
  		sk_leave_memory_pressure(sk);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2564
  }
f8c3bf00d   Paolo Abeni   net/socket: facto...
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
  EXPORT_SYMBOL(__sk_mem_reduce_allocated);
  
  /**
   *	__sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
   *	@sk: socket
   *	@amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
   */
  void __sk_mem_reclaim(struct sock *sk, int amount)
  {
  	amount >>= SK_MEM_QUANTUM_SHIFT;
  	sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
  	__sk_mem_reduce_allocated(sk, amount);
  }
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2578
  EXPORT_SYMBOL(__sk_mem_reclaim);
627d2d6b5   samanthakumar   udp: enable MSG_P...
2579
2580
  int sk_set_peek_off(struct sock *sk, int val)
  {
627d2d6b5   samanthakumar   udp: enable MSG_P...
2581
2582
2583
2584
  	sk->sk_peek_off = val;
  	return 0;
  }
  EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be6   Hideo Aoki   [NET] CORE: Intro...
2585

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
  /*
   * Set of default routines for initialising struct proto_ops when
   * the protocol does not support a particular function. In certain
   * cases where it makes no sense for a protocol to have a "do nothing"
   * function, some default processing is provided.
   */
  
  int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2597
  EXPORT_SYMBOL(sock_no_bind);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2598

4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2599
  int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2600
2601
2602
2603
  		    int len, int flags)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2604
  EXPORT_SYMBOL(sock_no_connect);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2605
2606
2607
2608
2609
  
  int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2610
  EXPORT_SYMBOL(sock_no_socketpair);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2611

cdfbabfb2   David Howells   net: Work around ...
2612
2613
  int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
  		   bool kern)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2614
2615
2616
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2617
  EXPORT_SYMBOL(sock_no_accept);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2618

4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2619
  int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
9b2c45d47   Denys Vlasenko   net: make getname...
2620
  		    int peer)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2621
2622
2623
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2624
  EXPORT_SYMBOL(sock_no_getname);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2625

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2626
2627
2628
2629
  int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2630
  EXPORT_SYMBOL(sock_no_ioctl);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2631
2632
2633
2634
2635
  
  int sock_no_listen(struct socket *sock, int backlog)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2636
  EXPORT_SYMBOL(sock_no_listen);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2637
2638
2639
2640
2641
  
  int sock_no_shutdown(struct socket *sock, int how)
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2642
  EXPORT_SYMBOL(sock_no_shutdown);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2643

1b7841404   Ying Xue   net: Remove iocb ...
2644
  int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2645
2646
2647
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2648
  EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2649

306b13eb3   Tom Herbert   proto_ops: Add lo...
2650
2651
2652
2653
2654
  int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
  {
  	return -EOPNOTSUPP;
  }
  EXPORT_SYMBOL(sock_no_sendmsg_locked);
1b7841404   Ying Xue   net: Remove iocb ...
2655
2656
  int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
  		    int flags)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2657
2658
2659
  {
  	return -EOPNOTSUPP;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2660
  EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2661
2662
2663
2664
2665
2666
  
  int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
  {
  	/* Mirror missing mmap method error code */
  	return -ENODEV;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2667
  EXPORT_SYMBOL(sock_no_mmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2668

d9539752d   Kees Cook   net/compat: Add m...
2669
2670
2671
2672
2673
2674
2675
  /*
   * When a file is received (via SCM_RIGHTS, etc), we must bump the
   * various sock-based usage counts.
   */
  void __receive_sock(struct file *file)
  {
  	struct socket *sock;
d9539752d   Kees Cook   net/compat: Add m...
2676

dba4a9256   Florent Revest   net: Remove the e...
2677
  	sock = sock_from_file(file);
d9539752d   Kees Cook   net/compat: Add m...
2678
2679
2680
2681
2682
  	if (sock) {
  		sock_update_netprioidx(&sock->sk->sk_cgrp_data);
  		sock_update_classid(&sock->sk->sk_cgrp_data);
  	}
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
  ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
  {
  	ssize_t res;
  	struct msghdr msg = {.msg_flags = flags};
  	struct kvec iov;
  	char *kaddr = kmap(page);
  	iov.iov_base = kaddr + offset;
  	iov.iov_len = size;
  	res = kernel_sendmsg(sock, &msg, &iov, 1, size);
  	kunmap(page);
  	return res;
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2695
  EXPORT_SYMBOL(sock_no_sendpage);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2696

306b13eb3   Tom Herbert   proto_ops: Add lo...
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
  ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
  				int offset, size_t size, int flags)
  {
  	ssize_t res;
  	struct msghdr msg = {.msg_flags = flags};
  	struct kvec iov;
  	char *kaddr = kmap(page);
  
  	iov.iov_base = kaddr + offset;
  	iov.iov_len = size;
  	res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
  	kunmap(page);
  	return res;
  }
  EXPORT_SYMBOL(sock_no_sendpage_locked);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2712
2713
2714
2715
2716
2717
  /*
   *	Default Socket Callbacks
   */
  
  static void sock_def_wakeup(struct sock *sk)
  {
438154823   Eric Dumazet   net: sock_def_rea...
2718
2719
2720
2721
  	struct socket_wq *wq;
  
  	rcu_read_lock();
  	wq = rcu_dereference(sk->sk_wq);
1ce0bf50a   Herbert Xu   net: Generalise w...
2722
  	if (skwq_has_sleeper(wq))
438154823   Eric Dumazet   net: sock_def_rea...
2723
2724
  		wake_up_interruptible_all(&wq->wait);
  	rcu_read_unlock();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2725
2726
2727
2728
  }
  
  static void sock_def_error_report(struct sock *sk)
  {
438154823   Eric Dumazet   net: sock_def_rea...
2729
2730
2731
2732
  	struct socket_wq *wq;
  
  	rcu_read_lock();
  	wq = rcu_dereference(sk->sk_wq);
1ce0bf50a   Herbert Xu   net: Generalise w...
2733
  	if (skwq_has_sleeper(wq))
a9a08845e   Linus Torvalds   vfs: do bulk POLL...
2734
  		wake_up_interruptible_poll(&wq->wait, EPOLLERR);
8d8ad9d7c   Pavel Emelyanov   [NET]: Name magic...
2735
  	sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
438154823   Eric Dumazet   net: sock_def_rea...
2736
  	rcu_read_unlock();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2737
  }
43a825afc   Björn Töpel   xsk, net: Make so...
2738
  void sock_def_readable(struct sock *sk)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2739
  {
438154823   Eric Dumazet   net: sock_def_rea...
2740
2741
2742
2743
  	struct socket_wq *wq;
  
  	rcu_read_lock();
  	wq = rcu_dereference(sk->sk_wq);
1ce0bf50a   Herbert Xu   net: Generalise w...
2744
  	if (skwq_has_sleeper(wq))
a9a08845e   Linus Torvalds   vfs: do bulk POLL...
2745
2746
  		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
  						EPOLLRDNORM | EPOLLRDBAND);
8d8ad9d7c   Pavel Emelyanov   [NET]: Name magic...
2747
  	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
438154823   Eric Dumazet   net: sock_def_rea...
2748
  	rcu_read_unlock();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2749
2750
2751
2752
  }
  
  static void sock_def_write_space(struct sock *sk)
  {
438154823   Eric Dumazet   net: sock_def_rea...
2753
2754
2755
  	struct socket_wq *wq;
  
  	rcu_read_lock();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2756
2757
2758
2759
  
  	/* Do not wake up a writer until he can make "significant"
  	 * progress.  --DaveM
  	 */
e292f05e0   Eric Dumazet   tcp: annotate sk-...
2760
  	if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) {
438154823   Eric Dumazet   net: sock_def_rea...
2761
  		wq = rcu_dereference(sk->sk_wq);
1ce0bf50a   Herbert Xu   net: Generalise w...
2762
  		if (skwq_has_sleeper(wq))
a9a08845e   Linus Torvalds   vfs: do bulk POLL...
2763
2764
  			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
  						EPOLLWRNORM | EPOLLWRBAND);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2765
2766
2767
  
  		/* Should agree with poll, otherwise some programs break */
  		if (sock_writeable(sk))
8d8ad9d7c   Pavel Emelyanov   [NET]: Name magic...
2768
  			sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2769
  	}
438154823   Eric Dumazet   net: sock_def_rea...
2770
  	rcu_read_unlock();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2771
2772
2773
2774
  }
  
  static void sock_def_destruct(struct sock *sk)
  {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2775
2776
2777
2778
2779
2780
  }
  
  void sk_send_sigurg(struct sock *sk)
  {
  	if (sk->sk_socket && sk->sk_socket->file)
  		if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7c   Pavel Emelyanov   [NET]: Name magic...
2781
  			sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2782
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2783
  EXPORT_SYMBOL(sk_send_sigurg);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2784
2785
2786
2787
2788
2789
2790
  
  void sk_reset_timer(struct sock *sk, struct timer_list* timer,
  		    unsigned long expires)
  {
  	if (!mod_timer(timer, expires))
  		sock_hold(sk);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2791
2792
2793
2794
  EXPORT_SYMBOL(sk_reset_timer);
  
  void sk_stop_timer(struct sock *sk, struct timer_list* timer)
  {
25cc4ae91   Ying Xue   net: remove redun...
2795
  	if (del_timer(timer))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2796
2797
  		__sock_put(sk);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2798
  EXPORT_SYMBOL(sk_stop_timer);
08b81d873   Geliang Tang   mptcp: add sk_sto...
2799
2800
2801
2802
2803
2804
  void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer)
  {
  	if (del_timer_sync(timer))
  		__sock_put(sk);
  }
  EXPORT_SYMBOL(sk_stop_timer_sync);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2805
2806
  void sock_init_data(struct socket *sock, struct sock *sk)
  {
581319c58   Paolo Abeni   net/socket: use p...
2807
  	sk_init_common(sk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2808
  	sk->sk_send_head	=	NULL;
99767f278   Kees Cook   net/core: Convert...
2809
  	timer_setup(&sk->sk_timer, NULL, 0);
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2810

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2811
2812
2813
2814
  	sk->sk_allocation	=	GFP_KERNEL;
  	sk->sk_rcvbuf		=	sysctl_rmem_default;
  	sk->sk_sndbuf		=	sysctl_wmem_default;
  	sk->sk_state		=	TCP_CLOSE;
972692e0d   David S. Miller   net: Add sk_set_s...
2815
  	sk_set_socket(sk, sock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2816
2817
  
  	sock_set_flag(sk, SOCK_ZAPPED);
e71a4783a   Stephen Hemminger   [NET] core: white...
2818
  	if (sock) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2819
  		sk->sk_type	=	sock->type;
333f7909a   Al Viro   coallocate socket...
2820
  		RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2821
  		sock->sk	=	sk;
86741ec25   Lorenzo Colitti   net: core: Add a ...
2822
2823
  		sk->sk_uid	=	SOCK_INODE(sock)->i_uid;
  	} else {
c2f26e8f8   Li RongQing   net: Use RCU_INIT...
2824
  		RCU_INIT_POINTER(sk->sk_wq, NULL);
86741ec25   Lorenzo Colitti   net: core: Add a ...
2825
2826
  		sk->sk_uid	=	make_kuid(sock_net(sk)->user_ns, 0);
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2827

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2828
  	rwlock_init(&sk->sk_callback_lock);
cdfbabfb2   David Howells   net: Work around ...
2829
2830
2831
2832
2833
2834
2835
2836
  	if (sk->sk_kern_sock)
  		lockdep_set_class_and_name(
  			&sk->sk_callback_lock,
  			af_kern_callback_keys + sk->sk_family,
  			af_family_kern_clock_key_strings[sk->sk_family]);
  	else
  		lockdep_set_class_and_name(
  			&sk->sk_callback_lock,
443aef0ed   Peter Zijlstra   lockdep: fixup sk...
2837
2838
  			af_callback_keys + sk->sk_family,
  			af_family_clock_key_strings[sk->sk_family]);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2839
2840
2841
2842
2843
2844
  
  	sk->sk_state_change	=	sock_def_wakeup;
  	sk->sk_data_ready	=	sock_def_readable;
  	sk->sk_write_space	=	sock_def_write_space;
  	sk->sk_error_report	=	sock_def_error_report;
  	sk->sk_destruct		=	sock_def_destruct;
5640f7685   Eric Dumazet   net: use a per ta...
2845
2846
  	sk->sk_frag.page	=	NULL;
  	sk->sk_frag.offset	=	0;
ef64a54f6   Pavel Emelyanov   sock: Introduce t...
2847
  	sk->sk_peek_off		=	-1;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2848

109f6e39f   Eric W. Biederman   af_unix: Allow SO...
2849
2850
  	sk->sk_peer_pid 	=	NULL;
  	sk->sk_peer_cred	=	NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2851
2852
2853
2854
  	sk->sk_write_pending	=	0;
  	sk->sk_rcvlowat		=	1;
  	sk->sk_rcvtimeo		=	MAX_SCHEDULE_TIMEOUT;
  	sk->sk_sndtimeo		=	MAX_SCHEDULE_TIMEOUT;
6c7c98bad   Paolo Abeni   sock: avoid dirty...
2855
  	sk->sk_stamp = SK_DEFAULT_STAMP;
3a0ed3e96   Deepa Dinamani   sock: Make sock->...
2856
2857
2858
  #if BITS_PER_LONG==32
  	seqlock_init(&sk->sk_stamp_seq);
  #endif
52267790e   Willem de Bruijn   sock: add MSG_ZER...
2859
  	atomic_set(&sk->sk_zckey, 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2860

e0d1095ae   Cong Wang   net: rename CONFI...
2861
  #ifdef CONFIG_NET_RX_BUSY_POLL
060212928   Eliezer Tamir   net: add low late...
2862
  	sk->sk_napi_id		=	0;
64b0dc517   Eliezer Tamir   net: rename busy ...
2863
  	sk->sk_ll_usec		=	sysctl_net_busy_read;
060212928   Eliezer Tamir   net: add low late...
2864
  #endif
76a9ebe81   Eric Dumazet   net: extend sk_pa...
2865
2866
  	sk->sk_max_pacing_rate = ~0UL;
  	sk->sk_pacing_rate = ~0UL;
7c68fa2bd   Eric Dumazet   net: annotate loc...
2867
  	WRITE_ONCE(sk->sk_pacing_shift, 10);
70da268b5   Eric Dumazet   net: SO_INCOMING_...
2868
  	sk->sk_incoming_cpu = -1;
c6345ce7d   Amritha Nambiar   net: Record recei...
2869
2870
  
  	sk_rx_queue_clear(sk);
4dc6dc716   Eric Dumazet   net: sock_copy() ...
2871
2872
  	/*
  	 * Before updating sk_refcnt, we must commit prior changes to memory
2cdb54c93   Mauro Carvalho Chehab   docs: RCU: Conver...
2873
  	 * (Documentation/RCU/rculist_nulls.rst for details)
4dc6dc716   Eric Dumazet   net: sock_copy() ...
2874
2875
  	 */
  	smp_wmb();
41c6d650f   Reshetova, Elena   net: convert sock...
2876
  	refcount_set(&sk->sk_refcnt, 1);
33c732c36   Wang Chen   [IPV4]: Add raw d...
2877
  	atomic_set(&sk->sk_drops, 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2878
  }
2a91525c2   Eric Dumazet   net: net/core/soc...
2879
  EXPORT_SYMBOL(sock_init_data);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2880

b5606c2d4   Harvey Harrison   remove final fast...
2881
  void lock_sock_nested(struct sock *sk, int subclass)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2882
2883
  {
  	might_sleep();
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
2884
  	spin_lock_bh(&sk->sk_lock.slock);
d2e9117c7   John Heffner   [NET]: Change typ...
2885
  	if (sk->sk_lock.owned)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2886
  		__lock_sock(sk);
d2e9117c7   John Heffner   [NET]: Change typ...
2887
  	sk->sk_lock.owned = 1;
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
2888
2889
2890
2891
  	spin_unlock(&sk->sk_lock.slock);
  	/*
  	 * The sk_lock has mutex_lock() semantics here:
  	 */
fcc70d5fd   Peter Zijlstra   [BLUETOOTH] lockd...
2892
  	mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
2893
  	local_bh_enable();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2894
  }
fcc70d5fd   Peter Zijlstra   [BLUETOOTH] lockd...
2895
  EXPORT_SYMBOL(lock_sock_nested);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2896

b5606c2d4   Harvey Harrison   remove final fast...
2897
  void release_sock(struct sock *sk)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2898
  {
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
2899
  	spin_lock_bh(&sk->sk_lock.slock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2900
2901
  	if (sk->sk_backlog.tail)
  		__release_sock(sk);
46d3ceabd   Eric Dumazet   tcp: TCP Small Qu...
2902

c3f9b0184   Eric Dumazet   tcp: tcp_release_...
2903
2904
2905
  	/* Warning : release_cb() might need to release sk ownership,
  	 * ie call sock_release_ownership(sk) before us.
  	 */
46d3ceabd   Eric Dumazet   tcp: TCP Small Qu...
2906
2907
  	if (sk->sk_prot->release_cb)
  		sk->sk_prot->release_cb(sk);
c3f9b0184   Eric Dumazet   tcp: tcp_release_...
2908
  	sock_release_ownership(sk);
a5b5bb9a0   Ingo Molnar   [PATCH] lockdep: ...
2909
2910
2911
  	if (waitqueue_active(&sk->sk_lock.wq))
  		wake_up(&sk->sk_lock.wq);
  	spin_unlock_bh(&sk->sk_lock.slock);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2912
2913
  }
  EXPORT_SYMBOL(release_sock);
8a74ad60a   Eric Dumazet   net: fix lock_soc...
2914
2915
2916
2917
2918
  /**
   * lock_sock_fast - fast version of lock_sock
   * @sk: socket
   *
   * This version should be used for very small section, where process wont block
d651983dd   Mauro Carvalho Chehab   net: fix some ide...
2919
2920
   * return false if fast path is taken:
   *
8a74ad60a   Eric Dumazet   net: fix lock_soc...
2921
   *   sk_lock.slock locked, owned = 0, BH disabled
d651983dd   Mauro Carvalho Chehab   net: fix some ide...
2922
2923
2924
   *
   * return true if slow path is taken:
   *
8a74ad60a   Eric Dumazet   net: fix lock_soc...
2925
2926
   *   sk_lock.slock unlocked, owned = 1, BH enabled
   */
12f4bd862   Paolo Abeni   net: add annotati...
2927
  bool lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock)
8a74ad60a   Eric Dumazet   net: fix lock_soc...
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
  {
  	might_sleep();
  	spin_lock_bh(&sk->sk_lock.slock);
  
  	if (!sk->sk_lock.owned)
  		/*
  		 * Note : We must disable BH
  		 */
  		return false;
  
  	__lock_sock(sk);
  	sk->sk_lock.owned = 1;
  	spin_unlock(&sk->sk_lock.slock);
  	/*
  	 * The sk_lock has mutex_lock() semantics here:
  	 */
  	mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
12f4bd862   Paolo Abeni   net: add annotati...
2945
  	__acquire(&sk->sk_lock.slock);
8a74ad60a   Eric Dumazet   net: fix lock_soc...
2946
2947
2948
2949
  	local_bh_enable();
  	return true;
  }
  EXPORT_SYMBOL(lock_sock_fast);
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2950
2951
  int sock_gettstamp(struct socket *sock, void __user *userstamp,
  		   bool timeval, bool time32)
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2952
  {
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2953
2954
  	struct sock *sk = sock->sk;
  	struct timespec64 ts;
9dae34978   Yafang Shao   net: avoid unnece...
2955
2956
  
  	sock_enable_timestamp(sk, SOCK_TIMESTAMP);
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2957
2958
  	ts = ktime_to_timespec64(sock_read_timestamp(sk));
  	if (ts.tv_sec == -1)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2959
  		return -ENOENT;
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2960
  	if (ts.tv_sec == 0) {
3a0ed3e96   Deepa Dinamani   sock: Make sock->...
2961
  		ktime_t kt = ktime_get_real();
f95f96a49   YueHaibing   sock: remove unne...
2962
  		sock_write_timestamp(sk, kt);
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2963
  		ts = ktime_to_timespec64(kt);
b7aa0bf70   Eric Dumazet   [NET]: convert ne...
2964
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2965

c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2966
2967
  	if (timeval)
  		ts.tv_nsec /= 1000;
9dae34978   Yafang Shao   net: avoid unnece...
2968

c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2969
2970
2971
2972
2973
2974
2975
2976
  #ifdef CONFIG_COMPAT_32BIT_TIME
  	if (time32)
  		return put_old_timespec32(&ts, userstamp);
  #endif
  #ifdef CONFIG_SPARC64
  	/* beware of padding in sparc64 timeval */
  	if (timeval && !in_compat_syscall()) {
  		struct __kernel_old_timeval __user tv = {
c98f4822e   Stephen Rothwell   net: fix sparc64 ...
2977
2978
  			.tv_sec = ts.tv_sec,
  			.tv_usec = ts.tv_nsec,
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2979
  		};
c98f4822e   Stephen Rothwell   net: fix sparc64 ...
2980
  		if (copy_to_user(userstamp, &tv, sizeof(tv)))
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2981
2982
  			return -EFAULT;
  		return 0;
ae40eb1ef   Eric Dumazet   [NET]: Introduce ...
2983
  	}
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2984
2985
  #endif
  	return put_timespec64(&ts, userstamp);
ae40eb1ef   Eric Dumazet   [NET]: Introduce ...
2986
  }
c7cbdbf29   Arnd Bergmann   net: rework SIOCG...
2987
  EXPORT_SYMBOL(sock_gettstamp);
ae40eb1ef   Eric Dumazet   [NET]: Introduce ...
2988

193d357d0   Alexey Dobriyan   net: spread "enum...
2989
  void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
4ec93edb1   YOSHIFUJI Hideaki   [NET] CORE: Fix w...
2990
  {
20d494735   Patrick Ohly   net: socket infra...
2991
  	if (!sock_flag(sk, flag)) {
08e29af3a   Eric Dumazet   net: optimize soc...
2992
  		unsigned long previous_flags = sk->sk_flags;
20d494735   Patrick Ohly   net: socket infra...
2993
2994
2995
2996
2997
2998
  		sock_set_flag(sk, flag);
  		/*
  		 * we just set one of the two flags which require net
  		 * time stamping, but time stamping might have been on
  		 * already because of the other one
  		 */
080a270f5   Hannes Frederic Sowa   sock: don't enabl...
2999
3000
  		if (sock_needs_netstamp(sk) &&
  		    !(previous_flags & SK_FLAGS_TIMESTAMP))
20d494735   Patrick Ohly   net: socket infra...
3001
  			net_enable_timestamp();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3002
3003
  	}
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3004

cb820f8e4   Richard Cochran   net: Provide a ge...
3005
3006
3007
3008
  int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
  		       int level, int type)
  {
  	struct sock_exterr_skb *serr;
364a9e932   Willem de Bruijn   sock: deduplicate...
3009
  	struct sk_buff *skb;
cb820f8e4   Richard Cochran   net: Provide a ge...
3010
3011
3012
  	int copied, err;
  
  	err = -EAGAIN;
364a9e932   Willem de Bruijn   sock: deduplicate...
3013
  	skb = sock_dequeue_err_skb(sk);
cb820f8e4   Richard Cochran   net: Provide a ge...
3014
3015
3016
3017
3018
3019
3020
3021
  	if (skb == NULL)
  		goto out;
  
  	copied = skb->len;
  	if (copied > len) {
  		msg->msg_flags |= MSG_TRUNC;
  		copied = len;
  	}
51f3d02b9   David S. Miller   net: Add and use ...
3022
  	err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e4   Richard Cochran   net: Provide a ge...
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
  	if (err)
  		goto out_free_skb;
  
  	sock_recv_timestamp(msg, sk, skb);
  
  	serr = SKB_EXT_ERR(skb);
  	put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
  
  	msg->msg_flags |= MSG_ERRQUEUE;
  	err = copied;
cb820f8e4   Richard Cochran   net: Provide a ge...
3033
3034
3035
3036
3037
3038
  out_free_skb:
  	kfree_skb(skb);
  out:
  	return err;
  }
  EXPORT_SYMBOL(sock_recv_errqueue);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
  /*
   *	Get a socket option on an socket.
   *
   *	FIX: POSIX 1003.1g is very ambiguous here. It states that
   *	asynchronous errors should be reported by getsockopt. We assume
   *	this means if you specify SO_ERROR (otherwise whats the point of it).
   */
  int sock_common_getsockopt(struct socket *sock, int level, int optname,
  			   char __user *optval, int __user *optlen)
  {
  	struct sock *sk = sock->sk;
  
  	return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3053
  EXPORT_SYMBOL(sock_common_getsockopt);
1b7841404   Ying Xue   net: Remove iocb ...
3054
3055
  int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
  			int flags)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3056
3057
3058
3059
  {
  	struct sock *sk = sock->sk;
  	int addr_len = 0;
  	int err;
1b7841404   Ying Xue   net: Remove iocb ...
3060
  	err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3061
3062
3063
3064
3065
  				   flags & ~MSG_DONTWAIT, &addr_len);
  	if (err >= 0)
  		msg->msg_namelen = addr_len;
  	return err;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3066
3067
3068
3069
3070
3071
  EXPORT_SYMBOL(sock_common_recvmsg);
  
  /*
   *	Set socket options on an inet socket.
   */
  int sock_common_setsockopt(struct socket *sock, int level, int optname,
a7b75c5a8   Christoph Hellwig   net: pass a sockp...
3072
  			   sockptr_t optval, unsigned int optlen)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3073
3074
3075
3076
3077
  {
  	struct sock *sk = sock->sk;
  
  	return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3078
3079
3080
3081
3082
3083
3084
3085
  EXPORT_SYMBOL(sock_common_setsockopt);
  
  void sk_common_release(struct sock *sk)
  {
  	if (sk->sk_prot->destroy)
  		sk->sk_prot->destroy(sk);
  
  	/*
645f08975   Miaohe Lin   net: Fix some com...
3086
  	 * Observation: when sk_common_release is called, processes have
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
  	 * no access to socket. But net still has.
  	 * Step one, detach it from networking:
  	 *
  	 * A. Remove from hash tables.
  	 */
  
  	sk->sk_prot->unhash(sk);
  
  	/*
  	 * In this point socket cannot receive new packets, but it is possible
  	 * that some packets are in flight because some CPU runs receiver and
  	 * did hash table lookup before we unhashed socket. They will achieve
  	 * receive queue and will be purged by socket destructor.
  	 *
  	 * Also we still have packets pending on receive queue and probably,
  	 * our own packets waiting in device queues. sock_destroy will drain
  	 * receive queue, but transmitted packets will delay socket destruction
  	 * until the last reference will be released.
  	 */
  
  	sock_orphan(sk);
  
  	xfrm_sk_free_policy(sk);
e6848976b   Arnaldo Carvalho de Melo   [NET]: Cleanup IN...
3110
  	sk_refcnt_debug_release(sk);
5640f7685   Eric Dumazet   net: use a per ta...
3111

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3112
3113
  	sock_put(sk);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3114
  EXPORT_SYMBOL(sk_common_release);
a2d133b1d   Josh Hunt   sock: introduce S...
3115
3116
3117
3118
3119
  void sk_get_meminfo(const struct sock *sk, u32 *mem)
  {
  	memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
  
  	mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
ebb3b78db   Eric Dumazet   tcp: annotate sk-...
3120
  	mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
a2d133b1d   Josh Hunt   sock: introduce S...
3121
  	mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
e292f05e0   Eric Dumazet   tcp: annotate sk-...
3122
  	mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
a2d133b1d   Josh Hunt   sock: introduce S...
3123
  	mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
ab4e846a8   Eric Dumazet   tcp: annotate sk-...
3124
  	mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
a2d133b1d   Josh Hunt   sock: introduce S...
3125
  	mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
70c265584   Eric Dumazet   net: silence KCSA...
3126
  	mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
a2d133b1d   Josh Hunt   sock: introduce S...
3127
3128
  	mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
  }
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3129
3130
  #ifdef CONFIG_PROC_FS
  #define PROTO_INUSE_NR	64	/* should be enough for the first time */
1338d466d   Pavel Emelyanov   [SOCK]: Introduce...
3131
3132
3133
  struct prot_inuse {
  	int val[PROTO_INUSE_NR];
  };
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3134
3135
  
  static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3136

70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3137
3138
  void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
  {
08fc7f814   Tonghao Zhang   sock: Change the ...
3139
  	__this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3140
3141
3142
3143
3144
3145
3146
3147
3148
  }
  EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
  
  int sock_prot_inuse_get(struct net *net, struct proto *prot)
  {
  	int cpu, idx = prot->inuse_idx;
  	int res = 0;
  
  	for_each_possible_cpu(cpu)
08fc7f814   Tonghao Zhang   sock: Change the ...
3149
  		res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3150
3151
3152
3153
  
  	return res >= 0 ? res : 0;
  }
  EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
648845ab7   Tonghao Zhang   sock: Move the so...
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
  static void sock_inuse_add(struct net *net, int val)
  {
  	this_cpu_add(*net->core.sock_inuse, val);
  }
  
  int sock_inuse_get(struct net *net)
  {
  	int cpu, res = 0;
  
  	for_each_possible_cpu(cpu)
  		res += *per_cpu_ptr(net->core.sock_inuse, cpu);
  
  	return res;
  }
  
  EXPORT_SYMBOL_GPL(sock_inuse_get);
2c8c1e729   Alexey Dobriyan   net: spread __net...
3170
  static int __net_init sock_inuse_init_net(struct net *net)
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3171
  {
08fc7f814   Tonghao Zhang   sock: Change the ...
3172
  	net->core.prot_inuse = alloc_percpu(struct prot_inuse);
648845ab7   Tonghao Zhang   sock: Move the so...
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
  	if (net->core.prot_inuse == NULL)
  		return -ENOMEM;
  
  	net->core.sock_inuse = alloc_percpu(int);
  	if (net->core.sock_inuse == NULL)
  		goto out;
  
  	return 0;
  
  out:
  	free_percpu(net->core.prot_inuse);
  	return -ENOMEM;
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3185
  }
2c8c1e729   Alexey Dobriyan   net: spread __net...
3186
  static void __net_exit sock_inuse_exit_net(struct net *net)
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3187
  {
08fc7f814   Tonghao Zhang   sock: Change the ...
3188
  	free_percpu(net->core.prot_inuse);
648845ab7   Tonghao Zhang   sock: Move the so...
3189
  	free_percpu(net->core.sock_inuse);
70ee11594   Pavel Emelyanov   [SOCK][NETNS]: Ad...
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
  }
  
  static struct pernet_operations net_inuse_ops = {
  	.init = sock_inuse_init_net,
  	.exit = sock_inuse_exit_net,
  };
  
  static __init int net_inuse_init(void)
  {
  	if (register_pernet_subsys(&net_inuse_ops))
  		panic("Cannot initialize net inuse counters");
  
  	return 0;
  }
  
  core_initcall(net_inuse_init);
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3206

b45ce3213   zhanglin   sock: fix potenti...
3207
  static int assign_proto_idx(struct proto *prot)
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3208
3209
3210
3211
  {
  	prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
  
  	if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193d   Joe Perches   net: core: Use pr...
3212
3213
  		pr_err("PROTO_INUSE_NR exhausted
  ");
b45ce3213   zhanglin   sock: fix potenti...
3214
  		return -ENOSPC;
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3215
3216
3217
  	}
  
  	set_bit(prot->inuse_idx, proto_inuse_idx);
b45ce3213   zhanglin   sock: fix potenti...
3218
  	return 0;
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3219
3220
3221
3222
3223
3224
3225
3226
  }
  
  static void release_proto_idx(struct proto *prot)
  {
  	if (prot->inuse_idx != PROTO_INUSE_NR - 1)
  		clear_bit(prot->inuse_idx, proto_inuse_idx);
  }
  #else
b45ce3213   zhanglin   sock: fix potenti...
3227
  static inline int assign_proto_idx(struct proto *prot)
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3228
  {
b45ce3213   zhanglin   sock: fix potenti...
3229
  	return 0;
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3230
3231
3232
3233
3234
  }
  
  static inline void release_proto_idx(struct proto *prot)
  {
  }
648845ab7   Tonghao Zhang   sock: Move the so...
3235
3236
3237
3238
  
  static void sock_inuse_add(struct net *net, int val)
  {
  }
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3239
  #endif
0f5907af3   Miaohe Lin   net: Fix potentia...
3240
3241
3242
3243
3244
3245
3246
3247
3248
  static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot)
  {
  	if (!twsk_prot)
  		return;
  	kfree(twsk_prot->twsk_slab_name);
  	twsk_prot->twsk_slab_name = NULL;
  	kmem_cache_destroy(twsk_prot->twsk_slab);
  	twsk_prot->twsk_slab = NULL;
  }
0159dfd3d   Eric Dumazet   net: add req_prot...
3249
3250
3251
3252
3253
3254
  static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
  {
  	if (!rsk_prot)
  		return;
  	kfree(rsk_prot->slab_name);
  	rsk_prot->slab_name = NULL;
adf78edac   Julia Lawall   net: core: drop n...
3255
3256
  	kmem_cache_destroy(rsk_prot->slab);
  	rsk_prot->slab = NULL;
0159dfd3d   Eric Dumazet   net: add req_prot...
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
  }
  
  static int req_prot_init(const struct proto *prot)
  {
  	struct request_sock_ops *rsk_prot = prot->rsk_prot;
  
  	if (!rsk_prot)
  		return 0;
  
  	rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
  					prot->name);
  	if (!rsk_prot->slab_name)
  		return -ENOMEM;
  
  	rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
  					   rsk_prot->obj_size, 0,
e699e2c6a   Shakeel Butt   net, mm: account ...
3273
3274
  					   SLAB_ACCOUNT | prot->slab_flags,
  					   NULL);
0159dfd3d   Eric Dumazet   net: add req_prot...
3275
3276
3277
3278
3279
3280
3281
3282
3283
  
  	if (!rsk_prot->slab) {
  		pr_crit("%s: Can't create request sock SLAB cache!
  ",
  			prot->name);
  		return -ENOMEM;
  	}
  	return 0;
  }
b733c007e   Pavel Emelyanov   [NET]: Clean prot...
3284
3285
  int proto_register(struct proto *prot, int alloc_slab)
  {
b45ce3213   zhanglin   sock: fix potenti...
3286
  	int ret = -ENOBUFS;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3287
  	if (alloc_slab) {
30c2c9f15   David Windsor   net: Define userc...
3288
3289
  		prot->slab = kmem_cache_create_usercopy(prot->name,
  					prot->obj_size, 0,
e699e2c6a   Shakeel Butt   net, mm: account ...
3290
3291
  					SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
  					prot->slab_flags,
289a4860d   Kees Cook   net: Restrict unw...
3292
  					prot->useroffset, prot->usersize,
271b72c7f   Eric Dumazet   udp: RCU handling...
3293
  					NULL);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3294
3295
  
  		if (prot->slab == NULL) {
e005d193d   Joe Perches   net: core: Use pr...
3296
3297
3298
  			pr_crit("%s: Can't create sock SLAB cache!
  ",
  				prot->name);
60e7663d4   Pavel Emelyanov   [SOCK]: Drop per-...
3299
  			goto out;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3300
  		}
2e6599cb8   Arnaldo Carvalho de Melo   [NET] Generalise ...
3301

0159dfd3d   Eric Dumazet   net: add req_prot...
3302
3303
  		if (req_prot_init(prot))
  			goto out_free_request_sock_slab;
8feaf0c0a   Arnaldo Carvalho de Melo   [INET]: Generalis...
3304

6d6ee43e0   Arnaldo Carvalho de Melo   [TWSK]: Introduce...
3305
  		if (prot->twsk_prot != NULL) {
faf234220   Alexey Dobriyan   net: use kasprint...
3306
  			prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
8feaf0c0a   Arnaldo Carvalho de Melo   [INET]: Generalis...
3307

7e56b5d69   Catalin Marinas   net: Fix memory l...
3308
  			if (prot->twsk_prot->twsk_slab_name == NULL)
8feaf0c0a   Arnaldo Carvalho de Melo   [INET]: Generalis...
3309
  				goto out_free_request_sock_slab;
6d6ee43e0   Arnaldo Carvalho de Melo   [TWSK]: Introduce...
3310
  			prot->twsk_prot->twsk_slab =
7e56b5d69   Catalin Marinas   net: Fix memory l...
3311
  				kmem_cache_create(prot->twsk_prot->twsk_slab_name,
6d6ee43e0   Arnaldo Carvalho de Melo   [TWSK]: Introduce...
3312
  						  prot->twsk_prot->twsk_obj_size,
3ab5aee7f   Eric Dumazet   net: Convert TCP ...
3313
  						  0,
e699e2c6a   Shakeel Butt   net, mm: account ...
3314
  						  SLAB_ACCOUNT |
52db70dca   Eric Dumazet   tcp: do not cache...
3315
  						  prot->slab_flags,
20c2df83d   Paul Mundt   mm: Remove slab d...
3316
  						  NULL);
6d6ee43e0   Arnaldo Carvalho de Melo   [TWSK]: Introduce...
3317
  			if (prot->twsk_prot->twsk_slab == NULL)
0f5907af3   Miaohe Lin   net: Fix potentia...
3318
  				goto out_free_timewait_sock_slab;
8feaf0c0a   Arnaldo Carvalho de Melo   [INET]: Generalis...
3319
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3320
  	}
36b77a520   Glauber Costa   net: fix sleeping...
3321
  	mutex_lock(&proto_list_mutex);
b45ce3213   zhanglin   sock: fix potenti...
3322
3323
3324
  	ret = assign_proto_idx(prot);
  	if (ret) {
  		mutex_unlock(&proto_list_mutex);
0f5907af3   Miaohe Lin   net: Fix potentia...
3325
  		goto out_free_timewait_sock_slab;
b45ce3213   zhanglin   sock: fix potenti...
3326
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3327
  	list_add(&prot->node, &proto_list);
36b77a520   Glauber Costa   net: fix sleeping...
3328
  	mutex_unlock(&proto_list_mutex);
b45ce3213   zhanglin   sock: fix potenti...
3329
  	return ret;
b733c007e   Pavel Emelyanov   [NET]: Clean prot...
3330

0f5907af3   Miaohe Lin   net: Fix potentia...
3331
  out_free_timewait_sock_slab:
b45ce3213   zhanglin   sock: fix potenti...
3332
  	if (alloc_slab && prot->twsk_prot)
0f5907af3   Miaohe Lin   net: Fix potentia...
3333
  		tw_prot_cleanup(prot->twsk_prot);
8feaf0c0a   Arnaldo Carvalho de Melo   [INET]: Generalis...
3334
  out_free_request_sock_slab:
b45ce3213   zhanglin   sock: fix potenti...
3335
3336
  	if (alloc_slab) {
  		req_prot_cleanup(prot->rsk_prot);
0159dfd3d   Eric Dumazet   net: add req_prot...
3337

b45ce3213   zhanglin   sock: fix potenti...
3338
3339
3340
  		kmem_cache_destroy(prot->slab);
  		prot->slab = NULL;
  	}
b733c007e   Pavel Emelyanov   [NET]: Clean prot...
3341
  out:
b45ce3213   zhanglin   sock: fix potenti...
3342
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3343
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3344
3345
3346
3347
  EXPORT_SYMBOL(proto_register);
  
  void proto_unregister(struct proto *prot)
  {
36b77a520   Glauber Costa   net: fix sleeping...
3348
  	mutex_lock(&proto_list_mutex);
13ff3d6fa   Pavel Emelyanov   [SOCK]: Enumerate...
3349
  	release_proto_idx(prot);
0a3f4358a   Patrick McHardy   [NET]: proto_unre...
3350
  	list_del(&prot->node);
36b77a520   Glauber Costa   net: fix sleeping...
3351
  	mutex_unlock(&proto_list_mutex);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3352

adf78edac   Julia Lawall   net: core: drop n...
3353
3354
  	kmem_cache_destroy(prot->slab);
  	prot->slab = NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3355

0159dfd3d   Eric Dumazet   net: add req_prot...
3356
  	req_prot_cleanup(prot->rsk_prot);
0f5907af3   Miaohe Lin   net: Fix potentia...
3357
  	tw_prot_cleanup(prot->twsk_prot);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3358
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3359
  EXPORT_SYMBOL(proto_unregister);
bf2ae2e4b   Xin Long   sock_diag: reques...
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
  int sock_load_diag_module(int family, int protocol)
  {
  	if (!protocol) {
  		if (!sock_is_registered(family))
  			return -ENOENT;
  
  		return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
  				      NETLINK_SOCK_DIAG, family);
  	}
  
  #ifdef CONFIG_INET
  	if (family == AF_INET &&
c34c12877   Andrei Vagin   sock_diag: fix au...
3372
  	    protocol != IPPROTO_RAW &&
3f935c75e   Paolo Abeni   inet_diag: suppor...
3373
  	    protocol < MAX_INET_PROTOS &&
bf2ae2e4b   Xin Long   sock_diag: reques...
3374
3375
3376
3377
3378
3379
3380
3381
  	    !rcu_access_pointer(inet_protos[protocol]))
  		return -ENOENT;
  #endif
  
  	return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
  			      NETLINK_SOCK_DIAG, family, protocol);
  }
  EXPORT_SYMBOL(sock_load_diag_module);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3382
  #ifdef CONFIG_PROC_FS
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3383
  static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a520   Glauber Costa   net: fix sleeping...
3384
  	__acquires(proto_list_mutex)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3385
  {
36b77a520   Glauber Costa   net: fix sleeping...
3386
  	mutex_lock(&proto_list_mutex);
60f0438a8   Pavel Emelianov   [NET]: Make some ...
3387
  	return seq_list_start_head(&proto_list, *pos);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3388
3389
3390
3391
  }
  
  static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
  {
60f0438a8   Pavel Emelianov   [NET]: Make some ...
3392
  	return seq_list_next(v, &proto_list, pos);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3393
3394
3395
  }
  
  static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a520   Glauber Costa   net: fix sleeping...
3396
  	__releases(proto_list_mutex)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3397
  {
36b77a520   Glauber Costa   net: fix sleeping...
3398
  	mutex_unlock(&proto_list_mutex);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3399
3400
3401
3402
3403
3404
  }
  
  static char proto_method_implemented(const void *method)
  {
  	return method == NULL ? 'n' : 'y';
  }
180d8cd94   Glauber Costa   foundations of pe...
3405
3406
  static long sock_prot_memory_allocated(struct proto *proto)
  {
cb75a36c8   Jeffrin Jose   net: Fixed a codi...
3407
  	return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd94   Glauber Costa   foundations of pe...
3408
  }
7a512eb86   Alexey Dobriyan   net: make sock_pr...
3409
  static const char *sock_prot_memory_pressure(struct proto *proto)
180d8cd94   Glauber Costa   foundations of pe...
3410
3411
3412
3413
  {
  	return proto->memory_pressure != NULL ?
  	proto_memory_pressure(proto) ? "yes" : "no" : "NI";
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3414
3415
3416
  
  static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
  {
180d8cd94   Glauber Costa   foundations of pe...
3417

8d987e5c7   Eric Dumazet   net: avoid limits...
3418
  	seq_printf(seq, "%-9s %4u %6d  %6ld   %-3s %6u   %-3s  %-10s "
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3419
3420
3421
3422
  			"%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c
  ",
  		   proto->name,
  		   proto->obj_size,
14e943db1   Eric Dumazet   net: make /proc/n...
3423
  		   sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd94   Glauber Costa   foundations of pe...
3424
3425
  		   sock_prot_memory_allocated(proto),
  		   sock_prot_memory_pressure(proto),
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
  		   proto->max_header,
  		   proto->slab == NULL ? "no" : "yes",
  		   module_name(proto->owner),
  		   proto_method_implemented(proto->close),
  		   proto_method_implemented(proto->connect),
  		   proto_method_implemented(proto->disconnect),
  		   proto_method_implemented(proto->accept),
  		   proto_method_implemented(proto->ioctl),
  		   proto_method_implemented(proto->init),
  		   proto_method_implemented(proto->destroy),
  		   proto_method_implemented(proto->shutdown),
  		   proto_method_implemented(proto->setsockopt),
  		   proto_method_implemented(proto->getsockopt),
  		   proto_method_implemented(proto->sendmsg),
  		   proto_method_implemented(proto->recvmsg),
  		   proto_method_implemented(proto->sendpage),
  		   proto_method_implemented(proto->bind),
  		   proto_method_implemented(proto->backlog_rcv),
  		   proto_method_implemented(proto->hash),
  		   proto_method_implemented(proto->unhash),
  		   proto_method_implemented(proto->get_port),
  		   proto_method_implemented(proto->enter_memory_pressure));
  }
  
  static int proto_seq_show(struct seq_file *seq, void *v)
  {
60f0438a8   Pavel Emelianov   [NET]: Make some ...
3452
  	if (v == &proto_list)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
  		seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
  			   "protocol",
  			   "size",
  			   "sockets",
  			   "memory",
  			   "press",
  			   "maxhdr",
  			   "slab",
  			   "module",
  			   "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em
  ");
  	else
60f0438a8   Pavel Emelianov   [NET]: Make some ...
3465
  		proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3466
3467
  	return 0;
  }
f690808e1   Stephen Hemminger   [NET]: make seq_o...
3468
  static const struct seq_operations proto_seq_ops = {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3469
3470
3471
3472
3473
  	.start  = proto_seq_start,
  	.next   = proto_seq_next,
  	.stop   = proto_seq_stop,
  	.show   = proto_seq_show,
  };
14e943db1   Eric Dumazet   net: make /proc/n...
3474
3475
  static __net_init int proto_init_net(struct net *net)
  {
c35063722   Christoph Hellwig   proc: introduce p...
3476
3477
  	if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
  			sizeof(struct seq_net_private)))
14e943db1   Eric Dumazet   net: make /proc/n...
3478
3479
3480
3481
3482
3483
3484
  		return -ENOMEM;
  
  	return 0;
  }
  
  static __net_exit void proto_exit_net(struct net *net)
  {
ece31ffd5   Gao feng   net: proc: change...
3485
  	remove_proc_entry("protocols", net->proc_net);
14e943db1   Eric Dumazet   net: make /proc/n...
3486
3487
3488
3489
3490
3491
  }
  
  
  static __net_initdata struct pernet_operations proto_net_ops = {
  	.init = proto_init_net,
  	.exit = proto_exit_net,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3492
3493
3494
3495
  };
  
  static int __init proto_init(void)
  {
14e943db1   Eric Dumazet   net: make /proc/n...
3496
  	return register_pernet_subsys(&proto_net_ops);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3497
3498
3499
3500
3501
  }
  
  subsys_initcall(proto_init);
  
  #endif /* PROC_FS */
7db6b048d   Sridhar Samudrala   net: Commonize bu...
3502
3503
3504
3505
3506
  
  #ifdef CONFIG_NET_RX_BUSY_POLL
  bool sk_busy_loop_end(void *p, unsigned long start_time)
  {
  	struct sock *sk = p;
3f926af3f   Eric Dumazet   net: use skb_queu...
3507
  	return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
7db6b048d   Sridhar Samudrala   net: Commonize bu...
3508
3509
3510
3511
  	       sk_busy_loop_timeout(sk, start_time);
  }
  EXPORT_SYMBOL(sk_busy_loop_end);
  #endif /* CONFIG_NET_RX_BUSY_POLL */
c0425a424   Christoph Hellwig   net: add a new bi...
3512
3513
3514
3515
3516
3517
3518
3519
  
  int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len)
  {
  	if (!sk->sk_prot->bind_add)
  		return -EOPNOTSUPP;
  	return sk->sk_prot->bind_add(sk, addr, addr_len);
  }
  EXPORT_SYMBOL(sock_bind_add);