Blame view

fs/file.c 21.9 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
6
7
  /*
   *  linux/fs/file.c
   *
   *  Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes
   *
   *  Manage the dynamic fd arrays in the process files_struct.
   */
fe17f22d7   Al Viro   take purely descr...
8
  #include <linux/syscalls.h>
630d9c472   Paul Gortmaker   fs: reduce the us...
9
  #include <linux/export.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
10
11
  #include <linux/fs.h>
  #include <linux/mm.h>
6d4831c28   Andrew Morton   vfs: avoid large ...
12
  #include <linux/mmzone.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
13
  #include <linux/time.h>
d43c36dc6   Alexey Dobriyan   headers: remove s...
14
  #include <linux/sched.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
15
16
17
  #include <linux/slab.h>
  #include <linux/vmalloc.h>
  #include <linux/file.h>
9f3acc314   Al Viro   [PATCH] split lin...
18
  #include <linux/fdtable.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
19
  #include <linux/bitops.h>
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
20
21
22
23
  #include <linux/interrupt.h>
  #include <linux/spinlock.h>
  #include <linux/rcupdate.h>
  #include <linux/workqueue.h>
9cfe015aa   Eric Dumazet   get rid of NR_OPE...
24
  int sysctl_nr_open __read_mostly = 1024*1024;
eceea0b3d   Al Viro   [PATCH] avoid mul...
25
26
  int sysctl_nr_open_min = BITS_PER_LONG;
  int sysctl_nr_open_max = 1024 * 1024; /* raised later */
9cfe015aa   Eric Dumazet   get rid of NR_OPE...
27

1fd36adcd   David Howells   Replace the fd_se...
28
  static void *alloc_fdmem(size_t size)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
29
  {
6d4831c28   Andrew Morton   vfs: avoid large ...
30
31
32
33
34
35
36
37
38
  	/*
  	 * Very large allocations can stress page reclaim, so fall back to
  	 * vmalloc() if the allocation size will be considered "large" by the VM.
  	 */
  	if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
  		void *data = kmalloc(size, GFP_KERNEL|__GFP_NOWARN);
  		if (data != NULL)
  			return data;
  	}
a892e2d7d   Changli Gao   vfs: use kmalloc(...
39
  	return vmalloc(size);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
40
  }
a892e2d7d   Changli Gao   vfs: use kmalloc(...
41
  static void free_fdmem(void *ptr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
42
  {
a892e2d7d   Changli Gao   vfs: use kmalloc(...
43
  	is_vmalloc_addr(ptr) ? vfree(ptr) : kfree(ptr);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
44
  }
a892e2d7d   Changli Gao   vfs: use kmalloc(...
45
  static void __free_fdtable(struct fdtable *fdt)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
46
  {
a892e2d7d   Changli Gao   vfs: use kmalloc(...
47
48
49
  	free_fdmem(fdt->fd);
  	free_fdmem(fdt->open_fds);
  	kfree(fdt);
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
50
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
51

7cf4dc3c8   Al Viro   move files_struct...
52
  static void free_fdtable_rcu(struct rcu_head *rcu)
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
53
  {
ac3e3c5b1   Al Viro   don't bother with...
54
  	__free_fdtable(container_of(rcu, struct fdtable, rcu));
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
55
  }
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
56
57
58
59
  /*
   * Expand the fdset in the files_struct.  Called with the files spinlock
   * held for write.
   */
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
60
  static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt)
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
61
  {
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
62
  	unsigned int cpy, set;
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
63

5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
64
  	BUG_ON(nfdt->max_fds < ofdt->max_fds);
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
65
66
67
68
69
70
71
72
73
74
75
76
  
  	cpy = ofdt->max_fds * sizeof(struct file *);
  	set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *);
  	memcpy(nfdt->fd, ofdt->fd, cpy);
  	memset((char *)(nfdt->fd) + cpy, 0, set);
  
  	cpy = ofdt->max_fds / BITS_PER_BYTE;
  	set = (nfdt->max_fds - ofdt->max_fds) / BITS_PER_BYTE;
  	memcpy(nfdt->open_fds, ofdt->open_fds, cpy);
  	memset((char *)(nfdt->open_fds) + cpy, 0, set);
  	memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy);
  	memset((char *)(nfdt->close_on_exec) + cpy, 0, set);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
77
  }
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
78
  static struct fdtable * alloc_fdtable(unsigned int nr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
79
  {
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
80
  	struct fdtable *fdt;
1fd36adcd   David Howells   Replace the fd_se...
81
  	void *data;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
82

ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
83
  	/*
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
84
85
86
87
88
  	 * Figure out how many fds we actually want to support in this fdtable.
  	 * Allocation steps are keyed to the size of the fdarray, since it
  	 * grows far faster than any of the other dynamic data. We try to fit
  	 * the fdarray into comfortable page-tuned chunks: starting at 1024B
  	 * and growing in powers of two from there on.
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
89
  	 */
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
90
91
92
  	nr /= (1024 / sizeof(struct file *));
  	nr = roundup_pow_of_two(nr + 1);
  	nr *= (1024 / sizeof(struct file *));
5c598b342   Al Viro   [PATCH] fix sysct...
93
94
95
96
97
98
99
100
101
102
  	/*
  	 * Note that this can drive nr *below* what we had passed if sysctl_nr_open
  	 * had been set lower between the check in expand_files() and here.  Deal
  	 * with that in caller, it's cheaper that way.
  	 *
  	 * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
  	 * bitmaps handling below becomes unpleasant, to put it mildly...
  	 */
  	if (unlikely(nr > sysctl_nr_open))
  		nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
bbea9f696   Vadim Lobanov   [PATCH] fdtable: ...
103

5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
104
105
  	fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL);
  	if (!fdt)
bbea9f696   Vadim Lobanov   [PATCH] fdtable: ...
106
  		goto out;
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
107
108
109
110
  	fdt->max_fds = nr;
  	data = alloc_fdmem(nr * sizeof(struct file *));
  	if (!data)
  		goto out_fdt;
1fd36adcd   David Howells   Replace the fd_se...
111
112
113
  	fdt->fd = data;
  
  	data = alloc_fdmem(max_t(size_t,
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
114
115
116
  				 2 * nr / BITS_PER_BYTE, L1_CACHE_BYTES));
  	if (!data)
  		goto out_arr;
1fd36adcd   David Howells   Replace the fd_se...
117
  	fdt->open_fds = data;
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
118
  	data += nr / BITS_PER_BYTE;
1fd36adcd   David Howells   Replace the fd_se...
119
  	fdt->close_on_exec = data;
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
120

ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
121
  	return fdt;
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
122
123
  
  out_arr:
a892e2d7d   Changli Gao   vfs: use kmalloc(...
124
  	free_fdmem(fdt->fd);
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
125
  out_fdt:
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
126
  	kfree(fdt);
5466b456e   Vadim Lobanov   [PATCH] fdtable: ...
127
  out:
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
128
129
  	return NULL;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
130

ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
131
  /*
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
132
133
134
135
136
   * Expand the file descriptor table.
   * This function will allocate a new fdtable and both fd array and fdset, of
   * the given size.
   * Return <0 error code on error; 1 on successful completion.
   * The files->file_lock should be held on entry, and will be held on exit.
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
137
138
139
140
141
   */
  static int expand_fdtable(struct files_struct *files, int nr)
  	__releases(files->file_lock)
  	__acquires(files->file_lock)
  {
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
142
  	struct fdtable *new_fdt, *cur_fdt;
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
143
144
  
  	spin_unlock(&files->file_lock);
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
145
  	new_fdt = alloc_fdtable(nr);
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
146
  	spin_lock(&files->file_lock);
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
147
148
  	if (!new_fdt)
  		return -ENOMEM;
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
149
  	/*
5c598b342   Al Viro   [PATCH] fix sysct...
150
151
152
153
  	 * extremely unlikely race - sysctl_nr_open decreased between the check in
  	 * caller and alloc_fdtable().  Cheaper to catch it here...
  	 */
  	if (unlikely(new_fdt->max_fds <= nr)) {
a892e2d7d   Changli Gao   vfs: use kmalloc(...
154
  		__free_fdtable(new_fdt);
5c598b342   Al Viro   [PATCH] fix sysct...
155
156
157
  		return -EMFILE;
  	}
  	/*
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
158
159
  	 * Check again since another task may have expanded the fd table while
  	 * we dropped the lock
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
160
  	 */
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
161
  	cur_fdt = files_fdtable(files);
bbea9f696   Vadim Lobanov   [PATCH] fdtable: ...
162
  	if (nr >= cur_fdt->max_fds) {
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
163
164
165
  		/* Continue as planned */
  		copy_fdtable(new_fdt, cur_fdt);
  		rcu_assign_pointer(files->fdt, new_fdt);
ac3e3c5b1   Al Viro   don't bother with...
166
  		if (cur_fdt != &files->fdtab)
1983e781d   Al Viro   trim free_fdtable...
167
  			call_rcu(&cur_fdt->rcu, free_fdtable_rcu);
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
168
  	} else {
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
169
  		/* Somebody else expanded, so undo our attempt */
a892e2d7d   Changli Gao   vfs: use kmalloc(...
170
  		__free_fdtable(new_fdt);
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
171
  	}
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
172
  	return 1;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
173
174
175
176
  }
  
  /*
   * Expand files.
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
177
178
179
180
181
   * This function will expand the file structures, if the requested size exceeds
   * the current capacity and there is room for expansion.
   * Return <0 error code on error; 0 when nothing done; 1 when files were
   * expanded and execution may have blocked.
   * The files->file_lock should be held on entry, and will be held on exit.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
182
   */
ad47bd725   Al Viro   make expand_files...
183
  static int expand_files(struct files_struct *files, int nr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
184
  {
badf16621   Dipankar Sarma   [PATCH] files: br...
185
  	struct fdtable *fdt;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
186

badf16621   Dipankar Sarma   [PATCH] files: br...
187
  	fdt = files_fdtable(files);
4e1e018ec   Al Viro   [PATCH] fix RLIM_...
188

74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
189
  	/* Do we need to expand? */
bbea9f696   Vadim Lobanov   [PATCH] fdtable: ...
190
  	if (nr < fdt->max_fds)
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
191
  		return 0;
4e1e018ec   Al Viro   [PATCH] fix RLIM_...
192

74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
193
  	/* Can we expand? */
9cfe015aa   Eric Dumazet   get rid of NR_OPE...
194
  	if (nr >= sysctl_nr_open)
74d392aaa   Vadim Lobanov   [PATCH] Clean up ...
195
196
197
198
  		return -EMFILE;
  
  	/* All good, so we try */
  	return expand_fdtable(files, nr);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
199
  }
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
200

b8318b01a   Al Viro   take __{set,clear...
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  static inline void __set_close_on_exec(int fd, struct fdtable *fdt)
  {
  	__set_bit(fd, fdt->close_on_exec);
  }
  
  static inline void __clear_close_on_exec(int fd, struct fdtable *fdt)
  {
  	__clear_bit(fd, fdt->close_on_exec);
  }
  
  static inline void __set_open_fd(int fd, struct fdtable *fdt)
  {
  	__set_bit(fd, fdt->open_fds);
  }
  
  static inline void __clear_open_fd(int fd, struct fdtable *fdt)
  {
  	__clear_bit(fd, fdt->open_fds);
  }
02afc6267   Al Viro   [PATCH] dup_fd() ...
220
221
222
223
224
225
  static int count_open_files(struct fdtable *fdt)
  {
  	int size = fdt->max_fds;
  	int i;
  
  	/* Find the last open fd */
1fd36adcd   David Howells   Replace the fd_se...
226
227
  	for (i = size / BITS_PER_LONG; i > 0; ) {
  		if (fdt->open_fds[--i])
02afc6267   Al Viro   [PATCH] dup_fd() ...
228
229
  			break;
  	}
1fd36adcd   David Howells   Replace the fd_se...
230
  	i = (i + 1) * BITS_PER_LONG;
02afc6267   Al Viro   [PATCH] dup_fd() ...
231
232
  	return i;
  }
02afc6267   Al Viro   [PATCH] dup_fd() ...
233
234
235
236
237
238
239
240
241
242
243
244
245
  /*
   * Allocate a new files structure and copy contents from the
   * passed in files structure.
   * errorp will be valid only when the returned files_struct is NULL.
   */
  struct files_struct *dup_fd(struct files_struct *oldf, int *errorp)
  {
  	struct files_struct *newf;
  	struct file **old_fds, **new_fds;
  	int open_files, size, i;
  	struct fdtable *old_fdt, *new_fdt;
  
  	*errorp = -ENOMEM;
afbec7fff   Al Viro   [PATCH] dup_fd() ...
246
  	newf = kmem_cache_alloc(files_cachep, GFP_KERNEL);
02afc6267   Al Viro   [PATCH] dup_fd() ...
247
248
  	if (!newf)
  		goto out;
afbec7fff   Al Viro   [PATCH] dup_fd() ...
249
250
251
252
253
254
  	atomic_set(&newf->count, 1);
  
  	spin_lock_init(&newf->file_lock);
  	newf->next_fd = 0;
  	new_fdt = &newf->fdtab;
  	new_fdt->max_fds = NR_OPEN_DEFAULT;
1fd36adcd   David Howells   Replace the fd_se...
255
256
  	new_fdt->close_on_exec = newf->close_on_exec_init;
  	new_fdt->open_fds = newf->open_fds_init;
afbec7fff   Al Viro   [PATCH] dup_fd() ...
257
  	new_fdt->fd = &newf->fd_array[0];
afbec7fff   Al Viro   [PATCH] dup_fd() ...
258

02afc6267   Al Viro   [PATCH] dup_fd() ...
259
260
  	spin_lock(&oldf->file_lock);
  	old_fdt = files_fdtable(oldf);
02afc6267   Al Viro   [PATCH] dup_fd() ...
261
262
263
264
  	open_files = count_open_files(old_fdt);
  
  	/*
  	 * Check whether we need to allocate a larger fd array and fd set.
02afc6267   Al Viro   [PATCH] dup_fd() ...
265
  	 */
adbecb128   Al Viro   [PATCH] dup_fd() ...
266
  	while (unlikely(open_files > new_fdt->max_fds)) {
02afc6267   Al Viro   [PATCH] dup_fd() ...
267
  		spin_unlock(&oldf->file_lock);
9dec3c4d3   Al Viro   [PATCH] dup_fd() ...
268

a892e2d7d   Changli Gao   vfs: use kmalloc(...
269
270
  		if (new_fdt != &newf->fdtab)
  			__free_fdtable(new_fdt);
adbecb128   Al Viro   [PATCH] dup_fd() ...
271

9dec3c4d3   Al Viro   [PATCH] dup_fd() ...
272
273
274
275
276
277
278
279
  		new_fdt = alloc_fdtable(open_files - 1);
  		if (!new_fdt) {
  			*errorp = -ENOMEM;
  			goto out_release;
  		}
  
  		/* beyond sysctl_nr_open; nothing to do */
  		if (unlikely(new_fdt->max_fds < open_files)) {
a892e2d7d   Changli Gao   vfs: use kmalloc(...
280
  			__free_fdtable(new_fdt);
9dec3c4d3   Al Viro   [PATCH] dup_fd() ...
281
  			*errorp = -EMFILE;
02afc6267   Al Viro   [PATCH] dup_fd() ...
282
  			goto out_release;
9dec3c4d3   Al Viro   [PATCH] dup_fd() ...
283
  		}
9dec3c4d3   Al Viro   [PATCH] dup_fd() ...
284

02afc6267   Al Viro   [PATCH] dup_fd() ...
285
286
287
288
289
290
291
  		/*
  		 * Reacquire the oldf lock and a pointer to its fd table
  		 * who knows it may have a new bigger fd table. We need
  		 * the latest pointer.
  		 */
  		spin_lock(&oldf->file_lock);
  		old_fdt = files_fdtable(oldf);
adbecb128   Al Viro   [PATCH] dup_fd() ...
292
  		open_files = count_open_files(old_fdt);
02afc6267   Al Viro   [PATCH] dup_fd() ...
293
294
295
296
  	}
  
  	old_fds = old_fdt->fd;
  	new_fds = new_fdt->fd;
1fd36adcd   David Howells   Replace the fd_se...
297
298
  	memcpy(new_fdt->open_fds, old_fdt->open_fds, open_files / 8);
  	memcpy(new_fdt->close_on_exec, old_fdt->close_on_exec, open_files / 8);
02afc6267   Al Viro   [PATCH] dup_fd() ...
299
300
301
302
303
304
305
306
307
308
309
310
  
  	for (i = open_files; i != 0; i--) {
  		struct file *f = *old_fds++;
  		if (f) {
  			get_file(f);
  		} else {
  			/*
  			 * The fd may be claimed in the fd bitmap but not yet
  			 * instantiated in the files array if a sibling thread
  			 * is partway through open().  So make sure that this
  			 * fd is available to the new process.
  			 */
1dce27c5a   David Howells   Wrap accesses to ...
311
  			__clear_open_fd(open_files - i, new_fdt);
02afc6267   Al Viro   [PATCH] dup_fd() ...
312
313
314
315
316
317
318
319
320
321
322
323
  		}
  		rcu_assign_pointer(*new_fds++, f);
  	}
  	spin_unlock(&oldf->file_lock);
  
  	/* compute the remainder to be cleared */
  	size = (new_fdt->max_fds - open_files) * sizeof(struct file *);
  
  	/* This is long word aligned thus could use a optimized version */
  	memset(new_fds, 0, size);
  
  	if (new_fdt->max_fds > open_files) {
1fd36adcd   David Howells   Replace the fd_se...
324
325
  		int left = (new_fdt->max_fds - open_files) / 8;
  		int start = open_files / BITS_PER_LONG;
02afc6267   Al Viro   [PATCH] dup_fd() ...
326

1fd36adcd   David Howells   Replace the fd_se...
327
328
  		memset(&new_fdt->open_fds[start], 0, left);
  		memset(&new_fdt->close_on_exec[start], 0, left);
02afc6267   Al Viro   [PATCH] dup_fd() ...
329
  	}
afbec7fff   Al Viro   [PATCH] dup_fd() ...
330
  	rcu_assign_pointer(newf->fdt, new_fdt);
02afc6267   Al Viro   [PATCH] dup_fd() ...
331
332
333
334
335
336
337
  	return newf;
  
  out_release:
  	kmem_cache_free(files_cachep, newf);
  out:
  	return NULL;
  }
7cf4dc3c8   Al Viro   move files_struct...
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
  static void close_files(struct files_struct * files)
  {
  	int i, j;
  	struct fdtable *fdt;
  
  	j = 0;
  
  	/*
  	 * It is safe to dereference the fd table without RCU or
  	 * ->file_lock because this is the last reference to the
  	 * files structure.  But use RCU to shut RCU-lockdep up.
  	 */
  	rcu_read_lock();
  	fdt = files_fdtable(files);
  	rcu_read_unlock();
  	for (;;) {
  		unsigned long set;
  		i = j * BITS_PER_LONG;
  		if (i >= fdt->max_fds)
  			break;
  		set = fdt->open_fds[j++];
  		while (set) {
  			if (set & 1) {
  				struct file * file = xchg(&fdt->fd[i], NULL);
  				if (file) {
  					filp_close(file, files);
  					cond_resched();
  				}
  			}
  			i++;
  			set >>= 1;
  		}
  	}
  }
  
  struct files_struct *get_files_struct(struct task_struct *task)
  {
  	struct files_struct *files;
  
  	task_lock(task);
  	files = task->files;
  	if (files)
  		atomic_inc(&files->count);
  	task_unlock(task);
  
  	return files;
  }
  
  void put_files_struct(struct files_struct *files)
  {
  	struct fdtable *fdt;
  
  	if (atomic_dec_and_test(&files->count)) {
  		close_files(files);
b9e02af0a   Al Viro   don't bother with...
392
  		/* not really needed, since nobody can see us */
7cf4dc3c8   Al Viro   move files_struct...
393
394
  		rcu_read_lock();
  		fdt = files_fdtable(files);
7cf4dc3c8   Al Viro   move files_struct...
395
  		rcu_read_unlock();
b9e02af0a   Al Viro   don't bother with...
396
397
398
399
  		/* free the arrays if they are not embedded */
  		if (fdt != &files->fdtab)
  			__free_fdtable(fdt);
  		kmem_cache_free(files_cachep, files);
7cf4dc3c8   Al Viro   move files_struct...
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
  	}
  }
  
  void reset_files_struct(struct files_struct *files)
  {
  	struct task_struct *tsk = current;
  	struct files_struct *old;
  
  	old = tsk->files;
  	task_lock(tsk);
  	tsk->files = files;
  	task_unlock(tsk);
  	put_files_struct(old);
  }
  
  void exit_files(struct task_struct *tsk)
  {
  	struct files_struct * files = tsk->files;
  
  	if (files) {
  		task_lock(tsk);
  		tsk->files = NULL;
  		task_unlock(tsk);
  		put_files_struct(files);
  	}
  }
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
426
427
  void __init files_defer_init(void)
  {
eceea0b3d   Al Viro   [PATCH] avoid mul...
428
429
  	sysctl_nr_open_max = min((size_t)INT_MAX, ~(size_t)0/sizeof(void *)) &
  			     -BITS_PER_LONG;
ab2af1f50   Dipankar Sarma   [PATCH] files: fi...
430
  }
f52111b15   Al Viro   [PATCH] take init...
431
432
433
434
435
436
437
  
  struct files_struct init_files = {
  	.count		= ATOMIC_INIT(1),
  	.fdt		= &init_files.fdtab,
  	.fdtab		= {
  		.max_fds	= NR_OPEN_DEFAULT,
  		.fd		= &init_files.fd_array[0],
1fd36adcd   David Howells   Replace the fd_se...
438
439
  		.close_on_exec	= init_files.close_on_exec_init,
  		.open_fds	= init_files.open_fds_init,
f52111b15   Al Viro   [PATCH] take init...
440
  	},
eece09ec2   Thomas Gleixner   locking: Various ...
441
  	.file_lock	= __SPIN_LOCK_UNLOCKED(init_files.file_lock),
f52111b15   Al Viro   [PATCH] take init...
442
  };
1027abe88   Al Viro   [PATCH] merge loc...
443
444
445
446
  
  /*
   * allocate a file descriptor, mark it busy.
   */
dcfadfa4e   Al Viro   new helper: __all...
447
448
  int __alloc_fd(struct files_struct *files,
  	       unsigned start, unsigned end, unsigned flags)
1027abe88   Al Viro   [PATCH] merge loc...
449
  {
1027abe88   Al Viro   [PATCH] merge loc...
450
451
452
453
454
455
456
457
458
459
460
461
  	unsigned int fd;
  	int error;
  	struct fdtable *fdt;
  
  	spin_lock(&files->file_lock);
  repeat:
  	fdt = files_fdtable(files);
  	fd = start;
  	if (fd < files->next_fd)
  		fd = files->next_fd;
  
  	if (fd < fdt->max_fds)
1fd36adcd   David Howells   Replace the fd_se...
462
  		fd = find_next_zero_bit(fdt->open_fds, fdt->max_fds, fd);
1027abe88   Al Viro   [PATCH] merge loc...
463

f33ff9927   Al Viro   take rlimit check...
464
465
466
467
468
469
470
  	/*
  	 * N.B. For clone tasks sharing a files structure, this test
  	 * will limit the total number of files that can be opened.
  	 */
  	error = -EMFILE;
  	if (fd >= end)
  		goto out;
1027abe88   Al Viro   [PATCH] merge loc...
471
472
473
474
475
476
477
478
479
480
481
482
483
  	error = expand_files(files, fd);
  	if (error < 0)
  		goto out;
  
  	/*
  	 * If we needed to expand the fs array we
  	 * might have blocked - try again.
  	 */
  	if (error)
  		goto repeat;
  
  	if (start <= files->next_fd)
  		files->next_fd = fd + 1;
1dce27c5a   David Howells   Wrap accesses to ...
484
  	__set_open_fd(fd, fdt);
1027abe88   Al Viro   [PATCH] merge loc...
485
  	if (flags & O_CLOEXEC)
1dce27c5a   David Howells   Wrap accesses to ...
486
  		__set_close_on_exec(fd, fdt);
1027abe88   Al Viro   [PATCH] merge loc...
487
  	else
1dce27c5a   David Howells   Wrap accesses to ...
488
  		__clear_close_on_exec(fd, fdt);
1027abe88   Al Viro   [PATCH] merge loc...
489
490
491
  	error = fd;
  #if 1
  	/* Sanity check */
7dc521579   Paul E. McKenney   vfs: Apply lockde...
492
  	if (rcu_dereference_raw(fdt->fd[fd]) != NULL) {
1027abe88   Al Viro   [PATCH] merge loc...
493
494
495
496
497
498
499
500
501
502
  		printk(KERN_WARNING "alloc_fd: slot %d not NULL!
  ", fd);
  		rcu_assign_pointer(fdt->fd[fd], NULL);
  	}
  #endif
  
  out:
  	spin_unlock(&files->file_lock);
  	return error;
  }
ad47bd725   Al Viro   make expand_files...
503
  static int alloc_fd(unsigned start, unsigned flags)
dcfadfa4e   Al Viro   new helper: __all...
504
505
506
  {
  	return __alloc_fd(current->files, start, rlimit(RLIMIT_NOFILE), flags);
  }
1a7bd2265   Al Viro   make get_unused_f...
507
  int get_unused_fd_flags(unsigned flags)
1027abe88   Al Viro   [PATCH] merge loc...
508
  {
dcfadfa4e   Al Viro   new helper: __all...
509
  	return __alloc_fd(current->files, 0, rlimit(RLIMIT_NOFILE), flags);
1027abe88   Al Viro   [PATCH] merge loc...
510
  }
1a7bd2265   Al Viro   make get_unused_f...
511
  EXPORT_SYMBOL(get_unused_fd_flags);
56007cae9   Al Viro   move put_unused_f...
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
  
  static void __put_unused_fd(struct files_struct *files, unsigned int fd)
  {
  	struct fdtable *fdt = files_fdtable(files);
  	__clear_open_fd(fd, fdt);
  	if (fd < files->next_fd)
  		files->next_fd = fd;
  }
  
  void put_unused_fd(unsigned int fd)
  {
  	struct files_struct *files = current->files;
  	spin_lock(&files->file_lock);
  	__put_unused_fd(files, fd);
  	spin_unlock(&files->file_lock);
  }
  
  EXPORT_SYMBOL(put_unused_fd);
  
  /*
   * Install a file pointer in the fd array.
   *
   * The VFS is full of places where we drop the files lock between
   * setting the open_fds bitmap and installing the file in the file
   * array.  At any such point, we are vulnerable to a dup2() race
   * installing a file in the array before us.  We need to detect this and
   * fput() the struct file we are about to overwrite in this case.
   *
   * It should never happen - if we allow dup2() do it, _really_ bad things
   * will follow.
f869e8a7f   Al Viro   expose a low-leve...
542
543
544
545
546
547
548
   *
   * NOTE: __fd_install() variant is really, really low-level; don't
   * use it unless you are forced to by truly lousy API shoved down
   * your throat.  'files' *MUST* be either current->files or obtained
   * by get_files_struct(current) done by whoever had given it to you,
   * or really bad things will happen.  Normally you want to use
   * fd_install() instead.
56007cae9   Al Viro   move put_unused_f...
549
   */
f869e8a7f   Al Viro   expose a low-leve...
550
551
  void __fd_install(struct files_struct *files, unsigned int fd,
  		struct file *file)
56007cae9   Al Viro   move put_unused_f...
552
  {
56007cae9   Al Viro   move put_unused_f...
553
554
555
556
557
558
559
  	struct fdtable *fdt;
  	spin_lock(&files->file_lock);
  	fdt = files_fdtable(files);
  	BUG_ON(fdt->fd[fd] != NULL);
  	rcu_assign_pointer(fdt->fd[fd], file);
  	spin_unlock(&files->file_lock);
  }
f869e8a7f   Al Viro   expose a low-leve...
560
561
562
563
  void fd_install(unsigned int fd, struct file *file)
  {
  	__fd_install(current->files, fd, file);
  }
56007cae9   Al Viro   move put_unused_f...
564
  EXPORT_SYMBOL(fd_install);
0ee8cdfe6   Al Viro   take fget() and f...
565

483ce1d4b   Al Viro   take descriptor-r...
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
  /*
   * The same warnings as for __alloc_fd()/__fd_install() apply here...
   */
  int __close_fd(struct files_struct *files, unsigned fd)
  {
  	struct file *file;
  	struct fdtable *fdt;
  
  	spin_lock(&files->file_lock);
  	fdt = files_fdtable(files);
  	if (fd >= fdt->max_fds)
  		goto out_unlock;
  	file = fdt->fd[fd];
  	if (!file)
  		goto out_unlock;
  	rcu_assign_pointer(fdt->fd[fd], NULL);
  	__clear_close_on_exec(fd, fdt);
  	__put_unused_fd(files, fd);
  	spin_unlock(&files->file_lock);
  	return filp_close(file, files);
  
  out_unlock:
  	spin_unlock(&files->file_lock);
  	return -EBADF;
  }
6a6d27de3   Al Viro   take close-on-exe...
591
592
593
594
595
596
  void do_close_on_exec(struct files_struct *files)
  {
  	unsigned i;
  	struct fdtable *fdt;
  
  	/* exec unshares first */
6a6d27de3   Al Viro   take close-on-exe...
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
  	spin_lock(&files->file_lock);
  	for (i = 0; ; i++) {
  		unsigned long set;
  		unsigned fd = i * BITS_PER_LONG;
  		fdt = files_fdtable(files);
  		if (fd >= fdt->max_fds)
  			break;
  		set = fdt->close_on_exec[i];
  		if (!set)
  			continue;
  		fdt->close_on_exec[i] = 0;
  		for ( ; set ; fd++, set >>= 1) {
  			struct file *file;
  			if (!(set & 1))
  				continue;
  			file = fdt->fd[fd];
  			if (!file)
  				continue;
  			rcu_assign_pointer(fdt->fd[fd], NULL);
  			__put_unused_fd(files, fd);
  			spin_unlock(&files->file_lock);
  			filp_close(file, files);
  			cond_resched();
  			spin_lock(&files->file_lock);
  		}
  
  	}
  	spin_unlock(&files->file_lock);
  }
0ee8cdfe6   Al Viro   take fget() and f...
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
  struct file *fget(unsigned int fd)
  {
  	struct file *file;
  	struct files_struct *files = current->files;
  
  	rcu_read_lock();
  	file = fcheck_files(files, fd);
  	if (file) {
  		/* File object ref couldn't be taken */
  		if (file->f_mode & FMODE_PATH ||
  		    !atomic_long_inc_not_zero(&file->f_count))
  			file = NULL;
  	}
  	rcu_read_unlock();
  
  	return file;
  }
  
  EXPORT_SYMBOL(fget);
  
  struct file *fget_raw(unsigned int fd)
  {
  	struct file *file;
  	struct files_struct *files = current->files;
  
  	rcu_read_lock();
  	file = fcheck_files(files, fd);
  	if (file) {
  		/* File object ref couldn't be taken */
  		if (!atomic_long_inc_not_zero(&file->f_count))
  			file = NULL;
  	}
  	rcu_read_unlock();
  
  	return file;
  }
  
  EXPORT_SYMBOL(fget_raw);
  
  /*
   * Lightweight file lookup - no refcnt increment if fd table isn't shared.
   *
   * You can use this instead of fget if you satisfy all of the following
   * conditions:
   * 1) You must call fput_light before exiting the syscall and returning control
   *    to userspace (i.e. you cannot remember the returned struct file * after
   *    returning to userspace).
   * 2) You must not call filp_close on the returned struct file * in between
   *    calls to fget_light and fput_light.
   * 3) You must not clone the current task in between the calls to fget_light
   *    and fput_light.
   *
   * The fput_needed flag returned by fget_light should be passed to the
   * corresponding fput_light.
   */
  struct file *fget_light(unsigned int fd, int *fput_needed)
  {
  	struct file *file;
  	struct files_struct *files = current->files;
  
  	*fput_needed = 0;
  	if (atomic_read(&files->count) == 1) {
  		file = fcheck_files(files, fd);
  		if (file && (file->f_mode & FMODE_PATH))
  			file = NULL;
  	} else {
  		rcu_read_lock();
  		file = fcheck_files(files, fd);
  		if (file) {
  			if (!(file->f_mode & FMODE_PATH) &&
  			    atomic_long_inc_not_zero(&file->f_count))
  				*fput_needed = 1;
  			else
  				/* Didn't get the reference, someone's freed */
  				file = NULL;
  		}
  		rcu_read_unlock();
  	}
  
  	return file;
  }
4557c669e   Al Viro   export fget_light
707
  EXPORT_SYMBOL(fget_light);
0ee8cdfe6   Al Viro   take fget() and f...
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
  
  struct file *fget_raw_light(unsigned int fd, int *fput_needed)
  {
  	struct file *file;
  	struct files_struct *files = current->files;
  
  	*fput_needed = 0;
  	if (atomic_read(&files->count) == 1) {
  		file = fcheck_files(files, fd);
  	} else {
  		rcu_read_lock();
  		file = fcheck_files(files, fd);
  		if (file) {
  			if (atomic_long_inc_not_zero(&file->f_count))
  				*fput_needed = 1;
  			else
  				/* Didn't get the reference, someone's freed */
  				file = NULL;
  		}
  		rcu_read_unlock();
  	}
  
  	return file;
  }
fe17f22d7   Al Viro   take purely descr...
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
  
  void set_close_on_exec(unsigned int fd, int flag)
  {
  	struct files_struct *files = current->files;
  	struct fdtable *fdt;
  	spin_lock(&files->file_lock);
  	fdt = files_fdtable(files);
  	if (flag)
  		__set_close_on_exec(fd, fdt);
  	else
  		__clear_close_on_exec(fd, fdt);
  	spin_unlock(&files->file_lock);
  }
  
  bool get_close_on_exec(unsigned int fd)
  {
  	struct files_struct *files = current->files;
  	struct fdtable *fdt;
  	bool res;
  	rcu_read_lock();
  	fdt = files_fdtable(files);
  	res = close_on_exec(fd, fdt);
  	rcu_read_unlock();
  	return res;
  }
8280d1617   Al Viro   new helper: repla...
757
758
  static int do_dup2(struct files_struct *files,
  	struct file *file, unsigned fd, unsigned flags)
fe17f22d7   Al Viro   take purely descr...
759
  {
8280d1617   Al Viro   new helper: repla...
760
  	struct file *tofree;
fe17f22d7   Al Viro   take purely descr...
761
  	struct fdtable *fdt;
fe17f22d7   Al Viro   take purely descr...
762
763
764
765
766
767
768
769
770
771
772
773
774
775
  	/*
  	 * We need to detect attempts to do dup2() over allocated but still
  	 * not finished descriptor.  NB: OpenBSD avoids that at the price of
  	 * extra work in their equivalent of fget() - they insert struct
  	 * file immediately after grabbing descriptor, mark it larval if
  	 * more work (e.g. actual opening) is needed and make sure that
  	 * fget() treats larval files as absent.  Potentially interesting,
  	 * but while extra work in fget() is trivial, locking implications
  	 * and amount of surgery on open()-related paths in VFS are not.
  	 * FreeBSD fails with -EBADF in the same situation, NetBSD "solution"
  	 * deadlocks in rather amusing ways, AFAICS.  All of that is out of
  	 * scope of POSIX or SUS, since neither considers shared descriptor
  	 * tables and this condition does not arise without those.
  	 */
fe17f22d7   Al Viro   take purely descr...
776
  	fdt = files_fdtable(files);
8280d1617   Al Viro   new helper: repla...
777
778
779
  	tofree = fdt->fd[fd];
  	if (!tofree && fd_is_open(fd, fdt))
  		goto Ebusy;
fe17f22d7   Al Viro   take purely descr...
780
  	get_file(file);
8280d1617   Al Viro   new helper: repla...
781
782
  	rcu_assign_pointer(fdt->fd[fd], file);
  	__set_open_fd(fd, fdt);
fe17f22d7   Al Viro   take purely descr...
783
  	if (flags & O_CLOEXEC)
8280d1617   Al Viro   new helper: repla...
784
  		__set_close_on_exec(fd, fdt);
fe17f22d7   Al Viro   take purely descr...
785
  	else
8280d1617   Al Viro   new helper: repla...
786
  		__clear_close_on_exec(fd, fdt);
fe17f22d7   Al Viro   take purely descr...
787
788
789
790
  	spin_unlock(&files->file_lock);
  
  	if (tofree)
  		filp_close(tofree, files);
8280d1617   Al Viro   new helper: repla...
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
  	return fd;
  
  Ebusy:
  	spin_unlock(&files->file_lock);
  	return -EBUSY;
  }
  
  int replace_fd(unsigned fd, struct file *file, unsigned flags)
  {
  	int err;
  	struct files_struct *files = current->files;
  
  	if (!file)
  		return __close_fd(files, fd);
  
  	if (fd >= rlimit(RLIMIT_NOFILE))
08f05c497   Al Viro   Return the right ...
807
  		return -EBADF;
8280d1617   Al Viro   new helper: repla...
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
  
  	spin_lock(&files->file_lock);
  	err = expand_files(files, fd);
  	if (unlikely(err < 0))
  		goto out_unlock;
  	return do_dup2(files, file, fd, flags);
  
  out_unlock:
  	spin_unlock(&files->file_lock);
  	return err;
  }
  
  SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags)
  {
  	int err = -EBADF;
  	struct file *file;
  	struct files_struct *files = current->files;
  
  	if ((flags & ~O_CLOEXEC) != 0)
  		return -EINVAL;
aed976475   Richard W.M. Jones   dup3: Return an e...
828
829
  	if (unlikely(oldfd == newfd))
  		return -EINVAL;
8280d1617   Al Viro   new helper: repla...
830
  	if (newfd >= rlimit(RLIMIT_NOFILE))
08f05c497   Al Viro   Return the right ...
831
  		return -EBADF;
8280d1617   Al Viro   new helper: repla...
832
833
834
835
836
837
838
839
840
841
842
843
  
  	spin_lock(&files->file_lock);
  	err = expand_files(files, newfd);
  	file = fcheck(oldfd);
  	if (unlikely(!file))
  		goto Ebadf;
  	if (unlikely(err < 0)) {
  		if (err == -EMFILE)
  			goto Ebadf;
  		goto out_unlock;
  	}
  	return do_dup2(files, file, newfd, flags);
fe17f22d7   Al Viro   take purely descr...
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
  
  Ebadf:
  	err = -EBADF;
  out_unlock:
  	spin_unlock(&files->file_lock);
  	return err;
  }
  
  SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd)
  {
  	if (unlikely(newfd == oldfd)) { /* corner case */
  		struct files_struct *files = current->files;
  		int retval = oldfd;
  
  		rcu_read_lock();
  		if (!fcheck_files(files, oldfd))
  			retval = -EBADF;
  		rcu_read_unlock();
  		return retval;
  	}
  	return sys_dup3(oldfd, newfd, 0);
  }
  
  SYSCALL_DEFINE1(dup, unsigned int, fildes)
  {
  	int ret = -EBADF;
  	struct file *file = fget_raw(fildes);
  
  	if (file) {
  		ret = get_unused_fd();
  		if (ret >= 0)
  			fd_install(ret, file);
  		else
  			fput(file);
  	}
  	return ret;
  }
  
  int f_dupfd(unsigned int from, struct file *file, unsigned flags)
  {
  	int err;
  	if (from >= rlimit(RLIMIT_NOFILE))
  		return -EINVAL;
  	err = alloc_fd(from, flags);
  	if (err >= 0) {
  		get_file(file);
  		fd_install(err, file);
  	}
  	return err;
  }
c3c073f80   Al Viro   new helper: itera...
894
895
896
897
898
899
  
  int iterate_fd(struct files_struct *files, unsigned n,
  		int (*f)(const void *, struct file *, unsigned),
  		const void *p)
  {
  	struct fdtable *fdt;
c3c073f80   Al Viro   new helper: itera...
900
901
902
903
  	int res = 0;
  	if (!files)
  		return 0;
  	spin_lock(&files->file_lock);
a77cfcb42   Al Viro   fix off-by-one in...
904
905
906
907
908
909
910
911
  	for (fdt = files_fdtable(files); n < fdt->max_fds; n++) {
  		struct file *file;
  		file = rcu_dereference_check_fdtable(files, fdt->fd[n]);
  		if (!file)
  			continue;
  		res = f(p, file, n);
  		if (res)
  			break;
c3c073f80   Al Viro   new helper: itera...
912
913
914
915
916
  	}
  	spin_unlock(&files->file_lock);
  	return res;
  }
  EXPORT_SYMBOL(iterate_fd);