Blame view

mm/nommu.c 48.5 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
6
7
8
  /*
   *  linux/mm/nommu.c
   *
   *  Replacement code for mm functions to support CPU's that don't
   *  have any form of memory management unit (thus no virtual memory).
   *
   *  See Documentation/nommu-mmap.txt
   *
8feae1311   David Howells   NOMMU: Make VMAs ...
9
   *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
10
11
12
   *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
   *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
   *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
29c185e5c   Paul Mundt   nommu: Provide st...
13
   *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
14
   */
b1de0d139   Mitchel Humpherys   mm: convert some ...
15
  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
b95f1b31b   Paul Gortmaker   mm: Map most file...
16
  #include <linux/export.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
17
  #include <linux/mm.h>
615d6e875   Davidlohr Bueso   mm: per-thread vm...
18
  #include <linux/vmacache.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
19
20
21
22
23
24
25
  #include <linux/mman.h>
  #include <linux/swap.h>
  #include <linux/file.h>
  #include <linux/highmem.h>
  #include <linux/pagemap.h>
  #include <linux/slab.h>
  #include <linux/vmalloc.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
26
27
  #include <linux/blkdev.h>
  #include <linux/backing-dev.h>
3b32123d7   Gideon Israel Dsouza   mm: use macros fr...
28
  #include <linux/compiler.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
29
30
31
32
  #include <linux/mount.h>
  #include <linux/personality.h>
  #include <linux/security.h>
  #include <linux/syscalls.h>
120a795da   Al Viro   audit mmap
33
  #include <linux/audit.h>
b1de0d139   Mitchel Humpherys   mm: convert some ...
34
  #include <linux/printk.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
35
36
37
38
  
  #include <asm/uaccess.h>
  #include <asm/tlb.h>
  #include <asm/tlbflush.h>
eb8cdec4a   Bernd Schmidt   nommu: add suppor...
39
  #include <asm/mmu_context.h>
8feae1311   David Howells   NOMMU: Make VMAs ...
40
  #include "internal.h"
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
41
  void *high_memory;
944b68749   Arnd Bergmann   mm: export "high_...
42
  EXPORT_SYMBOL(high_memory);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
43
44
  struct page *mem_map;
  unsigned long max_mapnr;
5b8bf3072   gchen gchen   mm/nommu.c: expor...
45
  EXPORT_SYMBOL(max_mapnr);
4266c97a3   Hugh Dickins   nommu: fix two bu...
46
  unsigned long highest_memmap_pfn;
fc4d5c292   David Howells   nommu: make the i...
47
  int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
48
  int heap_stack_gap = 0;
33e5d7697   David Howells   nommu: fix a numb...
49
  atomic_long_t mmap_pages_allocated;
8feae1311   David Howells   NOMMU: Make VMAs ...
50

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
51
  EXPORT_SYMBOL(mem_map);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
52

8feae1311   David Howells   NOMMU: Make VMAs ...
53
54
55
56
  /* list of mapped, potentially shareable regions */
  static struct kmem_cache *vm_region_jar;
  struct rb_root nommu_region_tree = RB_ROOT;
  DECLARE_RWSEM(nommu_region_sem);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
57

f0f37e2f7   Alexey Dobriyan   const: mark struc...
58
  const struct vm_operations_struct generic_file_vm_ops = {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
59
60
61
  };
  
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
62
63
64
65
66
67
68
69
   * Return the total memory allocated for this pointer, not
   * just what the caller asked for.
   *
   * Doesn't have to be accurate, i.e. may have races.
   */
  unsigned int kobjsize(const void *objp)
  {
  	struct page *page;
4016a1390   Michael Hennerich   mm/nommu.c: retur...
70
71
72
73
  	/*
  	 * If the object we have should not have ksize performed on it,
  	 * return size of 0
  	 */
5a1603be5   Paul Mundt   nommu: Correct ko...
74
  	if (!objp || !virt_addr_valid(objp))
6cfd53fc0   Paul Mundt   nommu: fix kobjsi...
75
76
77
  		return 0;
  
  	page = virt_to_head_page(objp);
6cfd53fc0   Paul Mundt   nommu: fix kobjsi...
78
79
80
81
82
  
  	/*
  	 * If the allocator sets PageSlab, we know the pointer came from
  	 * kmalloc().
  	 */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
83
84
  	if (PageSlab(page))
  		return ksize(objp);
6cfd53fc0   Paul Mundt   nommu: fix kobjsi...
85
  	/*
ab2e83ead   Paul Mundt   NOMMU: Teach kobj...
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  	 * If it's not a compound page, see if we have a matching VMA
  	 * region. This test is intentionally done in reverse order,
  	 * so if there's no VMA, we still fall through and hand back
  	 * PAGE_SIZE for 0-order pages.
  	 */
  	if (!PageCompound(page)) {
  		struct vm_area_struct *vma;
  
  		vma = find_vma(current->mm, (unsigned long)objp);
  		if (vma)
  			return vma->vm_end - vma->vm_start;
  	}
  
  	/*
6cfd53fc0   Paul Mundt   nommu: fix kobjsi...
100
  	 * The ksize() function is only guaranteed to work for pointers
5a1603be5   Paul Mundt   nommu: Correct ko...
101
  	 * returned by kmalloc(). So handle arbitrary pointers here.
6cfd53fc0   Paul Mundt   nommu: fix kobjsi...
102
  	 */
5a1603be5   Paul Mundt   nommu: Correct ko...
103
  	return PAGE_SIZE << compound_order(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
104
  }
28a35716d   Michel Lespinasse   mm: use long type...
105
106
107
108
  long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  		      unsigned long start, unsigned long nr_pages,
  		      unsigned int foll_flags, struct page **pages,
  		      struct vm_area_struct **vmas, int *nonblocking)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
109
  {
910e46da4   Sonic Zhang   [PATCH] Check if ...
110
  	struct vm_area_struct *vma;
7b4d5b8b3   David Howells   [PATCH] NOMMU: Ch...
111
112
113
114
  	unsigned long vm_flags;
  	int i;
  
  	/* calculate required read or write permissions.
58fa879e1   Hugh Dickins   mm: FOLL flags fo...
115
  	 * If FOLL_FORCE is set, we only require the "MAY" flags.
7b4d5b8b3   David Howells   [PATCH] NOMMU: Ch...
116
  	 */
58fa879e1   Hugh Dickins   mm: FOLL flags fo...
117
118
119
120
  	vm_flags  = (foll_flags & FOLL_WRITE) ?
  			(VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
  	vm_flags &= (foll_flags & FOLL_FORCE) ?
  			(VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
121

9d73777e5   Peter Zijlstra   clarify get_user_...
122
  	for (i = 0; i < nr_pages; i++) {
7561e8ca0   David Howells   NOMMU: Revert 'no...
123
  		vma = find_vma(mm, start);
7b4d5b8b3   David Howells   [PATCH] NOMMU: Ch...
124
125
126
127
  		if (!vma)
  			goto finish_or_fault;
  
  		/* protect what we can, including chardevs */
1c3aff1ce   Hugh Dickins   mm: remove unused...
128
129
  		if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
  		    !(vm_flags & vma->vm_flags))
7b4d5b8b3   David Howells   [PATCH] NOMMU: Ch...
130
  			goto finish_or_fault;
910e46da4   Sonic Zhang   [PATCH] Check if ...
131

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
132
133
134
  		if (pages) {
  			pages[i] = virt_to_page(start);
  			if (pages[i])
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
135
  				get_page(pages[i]);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
136
137
  		}
  		if (vmas)
910e46da4   Sonic Zhang   [PATCH] Check if ...
138
  			vmas[i] = vma;
e1ee65d85   David Howells   NOMMU: Fix __get_...
139
  		start = (start + PAGE_SIZE) & PAGE_MASK;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
140
  	}
7b4d5b8b3   David Howells   [PATCH] NOMMU: Ch...
141
142
143
144
145
  
  	return i;
  
  finish_or_fault:
  	return i ? : -EFAULT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
146
  }
b291f0003   Nick Piggin   mlock: mlocked pa...
147

b291f0003   Nick Piggin   mlock: mlocked pa...
148
149
150
151
152
153
154
  /*
   * get a list of pages in an address range belonging to the specified process
   * and indicate the VMA that covers each page
   * - this is potentially dodgy as we may end incrementing the page count of a
   *   slab page or a secondary page from a compound page
   * - don't permit access to VMAs that don't support it, such as I/O mappings
   */
c12d2da56   Ingo Molnar   mm/gup: Remove th...
155
  long get_user_pages(unsigned long start, unsigned long nr_pages,
28a35716d   Michel Lespinasse   mm: use long type...
156
157
  		    int write, int force, struct page **pages,
  		    struct vm_area_struct **vmas)
b291f0003   Nick Piggin   mlock: mlocked pa...
158
159
160
161
  {
  	int flags = 0;
  
  	if (write)
58fa879e1   Hugh Dickins   mm: FOLL flags fo...
162
  		flags |= FOLL_WRITE;
b291f0003   Nick Piggin   mlock: mlocked pa...
163
  	if (force)
58fa879e1   Hugh Dickins   mm: FOLL flags fo...
164
  		flags |= FOLL_FORCE;
b291f0003   Nick Piggin   mlock: mlocked pa...
165

cde70140f   Dave Hansen   mm/gup: Overload ...
166
167
  	return __get_user_pages(current, current->mm, start, nr_pages, flags,
  				pages, vmas, NULL);
b291f0003   Nick Piggin   mlock: mlocked pa...
168
  }
c12d2da56   Ingo Molnar   mm/gup: Remove th...
169
  EXPORT_SYMBOL(get_user_pages);
66aa2b4b1   Greg Ungerer   [PATCH] uclinux: ...
170

c12d2da56   Ingo Molnar   mm/gup: Remove th...
171
  long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
cde70140f   Dave Hansen   mm/gup: Overload ...
172
173
  			    int write, int force, struct page **pages,
  			    int *locked)
f0818f472   Andrea Arcangeli   mm: gup: add get_...
174
  {
c12d2da56   Ingo Molnar   mm/gup: Remove th...
175
  	return get_user_pages(start, nr_pages, write, force, pages, NULL);
f0818f472   Andrea Arcangeli   mm: gup: add get_...
176
  }
c12d2da56   Ingo Molnar   mm/gup: Remove th...
177
  EXPORT_SYMBOL(get_user_pages_locked);
f0818f472   Andrea Arcangeli   mm: gup: add get_...
178

0fd71a56f   Andrea Arcangeli   mm: gup: add __ge...
179
180
181
182
  long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
  			       unsigned long start, unsigned long nr_pages,
  			       int write, int force, struct page **pages,
  			       unsigned int gup_flags)
f0818f472   Andrea Arcangeli   mm: gup: add get_...
183
184
185
  {
  	long ret;
  	down_read(&mm->mmap_sem);
cde70140f   Dave Hansen   mm/gup: Overload ...
186
187
  	ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages,
  				NULL, NULL);
f0818f472   Andrea Arcangeli   mm: gup: add get_...
188
189
190
  	up_read(&mm->mmap_sem);
  	return ret;
  }
0fd71a56f   Andrea Arcangeli   mm: gup: add __ge...
191
  EXPORT_SYMBOL(__get_user_pages_unlocked);
c12d2da56   Ingo Molnar   mm/gup: Remove th...
192
  long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
0fd71a56f   Andrea Arcangeli   mm: gup: add __ge...
193
194
  			     int write, int force, struct page **pages)
  {
cde70140f   Dave Hansen   mm/gup: Overload ...
195
196
  	return __get_user_pages_unlocked(current, current->mm, start, nr_pages,
  					 write, force, pages, 0);
0fd71a56f   Andrea Arcangeli   mm: gup: add __ge...
197
  }
c12d2da56   Ingo Molnar   mm/gup: Remove th...
198
  EXPORT_SYMBOL(get_user_pages_unlocked);
f0818f472   Andrea Arcangeli   mm: gup: add get_...
199

dfc2f91ac   Paul Mundt   nommu: provide fo...
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  /**
   * follow_pfn - look up PFN at a user virtual address
   * @vma: memory mapping
   * @address: user virtual address
   * @pfn: location to store found PFN
   *
   * Only IO mappings and raw PFN mappings are allowed.
   *
   * Returns zero and the pfn at @pfn on success, -ve otherwise.
   */
  int follow_pfn(struct vm_area_struct *vma, unsigned long address,
  	unsigned long *pfn)
  {
  	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  		return -EINVAL;
  
  	*pfn = address >> PAGE_SHIFT;
  	return 0;
  }
  EXPORT_SYMBOL(follow_pfn);
f1c4069e1   Joonsoo Kim   mm, vmalloc: expo...
220
  LIST_HEAD(vmap_area_list);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
221

b3bdda02a   Christoph Lameter   vmalloc: add cons...
222
  void vfree(const void *addr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
223
224
225
  {
  	kfree(addr);
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
226
  EXPORT_SYMBOL(vfree);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
227

dd0fc66fb   Al Viro   [PATCH] gfp flags...
228
  void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
229
230
  {
  	/*
8518609de   Robert P. J. Day   Explain clearly w...
231
232
  	 *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
  	 * returns only a logical address.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
233
  	 */
84097518d   Nick Piggin   [PATCH] mm: nommu...
234
  	return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
235
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
236
  EXPORT_SYMBOL(__vmalloc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
237

f905bc447   Paul Mundt   nommu: add new vm...
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
  void *vmalloc_user(unsigned long size)
  {
  	void *ret;
  
  	ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  			PAGE_KERNEL);
  	if (ret) {
  		struct vm_area_struct *vma;
  
  		down_write(&current->mm->mmap_sem);
  		vma = find_vma(current->mm, (unsigned long)ret);
  		if (vma)
  			vma->vm_flags |= VM_USERMAP;
  		up_write(&current->mm->mmap_sem);
  	}
  
  	return ret;
  }
  EXPORT_SYMBOL(vmalloc_user);
b3bdda02a   Christoph Lameter   vmalloc: add cons...
257
  struct page *vmalloc_to_page(const void *addr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
258
259
260
  {
  	return virt_to_page(addr);
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
261
  EXPORT_SYMBOL(vmalloc_to_page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
262

b3bdda02a   Christoph Lameter   vmalloc: add cons...
263
  unsigned long vmalloc_to_pfn(const void *addr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
264
265
266
  {
  	return page_to_pfn(virt_to_page(addr));
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
267
  EXPORT_SYMBOL(vmalloc_to_pfn);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
268
269
270
  
  long vread(char *buf, char *addr, unsigned long count)
  {
9bde916bc   Chen Gang   mm/nommu.c: add a...
271
272
273
  	/* Don't allow overflow */
  	if ((unsigned long) buf + count < count)
  		count = -(unsigned long) buf;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
274
275
276
277
278
279
280
281
282
283
284
  	memcpy(buf, addr, count);
  	return count;
  }
  
  long vwrite(char *buf, char *addr, unsigned long count)
  {
  	/* Don't allow overflow */
  	if ((unsigned long) addr + count < count)
  		count = -(unsigned long) addr;
  
  	memcpy(addr, buf, count);
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
285
  	return count;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
286
287
288
  }
  
  /*
e1c05067c   Masahiro Yamada   treewide: fix typ...
289
   *	vmalloc  -  allocate virtually contiguous memory
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
290
291
292
293
   *
   *	@size:		allocation size
   *
   *	Allocate enough pages to cover @size from the page level
e1c05067c   Masahiro Yamada   treewide: fix typ...
294
   *	allocator and map them into contiguous kernel virtual space.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
295
   *
c1c8897f8   Michael Opdenacker   Spelling fix: "co...
296
   *	For tight control over page level allocator and protection flags
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
297
298
299
300
301
302
   *	use __vmalloc() instead.
   */
  void *vmalloc(unsigned long size)
  {
         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
  }
f61388822   Andrew Morton   [PATCH] nommu: im...
303
  EXPORT_SYMBOL(vmalloc);
e1ca7788d   Dave Young   mm: add vzalloc()...
304
  /*
e1c05067c   Masahiro Yamada   treewide: fix typ...
305
   *	vzalloc - allocate virtually contiguous memory with zero fill
e1ca7788d   Dave Young   mm: add vzalloc()...
306
307
308
309
   *
   *	@size:		allocation size
   *
   *	Allocate enough pages to cover @size from the page level
e1c05067c   Masahiro Yamada   treewide: fix typ...
310
   *	allocator and map them into contiguous kernel virtual space.
e1ca7788d   Dave Young   mm: add vzalloc()...
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
   *	The memory allocated is set to zero.
   *
   *	For tight control over page level allocator and protection flags
   *	use __vmalloc() instead.
   */
  void *vzalloc(unsigned long size)
  {
  	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  			PAGE_KERNEL);
  }
  EXPORT_SYMBOL(vzalloc);
  
  /**
   * vmalloc_node - allocate memory on a specific node
   * @size:	allocation size
   * @node:	numa node
   *
   * Allocate enough pages to cover @size from the page level
   * allocator and map them into contiguous kernel virtual space.
   *
   * For tight control over page level allocator and protection flags
   * use __vmalloc() instead.
   */
f61388822   Andrew Morton   [PATCH] nommu: im...
334
335
336
337
  void *vmalloc_node(unsigned long size, int node)
  {
  	return vmalloc(size);
  }
9a14f653d   Paul Mundt   nommu: Fix up vma...
338
  EXPORT_SYMBOL(vmalloc_node);
e1ca7788d   Dave Young   mm: add vzalloc()...
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
  
  /**
   * vzalloc_node - allocate memory on a specific node with zero fill
   * @size:	allocation size
   * @node:	numa node
   *
   * Allocate enough pages to cover @size from the page level
   * allocator and map them into contiguous kernel virtual space.
   * The memory allocated is set to zero.
   *
   * For tight control over page level allocator and protection flags
   * use __vmalloc() instead.
   */
  void *vzalloc_node(unsigned long size, int node)
  {
  	return vzalloc(size);
  }
  EXPORT_SYMBOL(vzalloc_node);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
357

1af446edf   Paul Mundt   nommu: Provide vm...
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
  #ifndef PAGE_KERNEL_EXEC
  # define PAGE_KERNEL_EXEC PAGE_KERNEL
  #endif
  
  /**
   *	vmalloc_exec  -  allocate virtually contiguous, executable memory
   *	@size:		allocation size
   *
   *	Kernel-internal function to allocate enough pages to cover @size
   *	the page level allocator and map them into contiguous and
   *	executable kernel virtual space.
   *
   *	For tight control over page level allocator and protection flags
   *	use __vmalloc() instead.
   */
  
  void *vmalloc_exec(unsigned long size)
  {
  	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
378
379
  /**
   * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
380
381
382
   *	@size:		allocation size
   *
   *	Allocate enough 32bit PA addressable pages to cover @size from the
e1c05067c   Masahiro Yamada   treewide: fix typ...
383
   *	page level allocator and map them into contiguous kernel virtual space.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
384
385
386
387
388
   */
  void *vmalloc_32(unsigned long size)
  {
  	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
389
390
391
392
393
394
395
396
  EXPORT_SYMBOL(vmalloc_32);
  
  /**
   * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
   *	@size:		allocation size
   *
   * The resulting memory area is 32bit addressable and zeroed so it can be
   * mapped to userspace without leaking data.
f905bc447   Paul Mundt   nommu: add new vm...
397
398
399
   *
   * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
   * remap_vmalloc_range() are permissible.
b50731732   Paul Mundt   nommu: vmalloc_32...
400
401
402
   */
  void *vmalloc_32_user(unsigned long size)
  {
f905bc447   Paul Mundt   nommu: add new vm...
403
404
405
406
407
  	/*
  	 * We'll have to sort out the ZONE_DMA bits for 64-bit,
  	 * but for now this can simply use vmalloc_user() directly.
  	 */
  	return vmalloc_user(size);
b50731732   Paul Mundt   nommu: vmalloc_32...
408
409
  }
  EXPORT_SYMBOL(vmalloc_32_user);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
410
411
412
413
414
415
  
  void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
  {
  	BUG();
  	return NULL;
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
416
  EXPORT_SYMBOL(vmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
417

b3bdda02a   Christoph Lameter   vmalloc: add cons...
418
  void vunmap(const void *addr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
419
420
421
  {
  	BUG();
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
422
  EXPORT_SYMBOL(vunmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
423

eb6434d9e   Paul Mundt   nommu: Stub in vm...
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
  void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
  {
  	BUG();
  	return NULL;
  }
  EXPORT_SYMBOL(vm_map_ram);
  
  void vm_unmap_ram(const void *mem, unsigned int count)
  {
  	BUG();
  }
  EXPORT_SYMBOL(vm_unmap_ram);
  
  void vm_unmap_aliases(void)
  {
  }
  EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
441
  /*
1eeb66a1b   Christoph Hellwig   move die notifier...
442
443
444
   * Implement a stub for vmalloc_sync_all() if the architecture chose not to
   * have one.
   */
3b32123d7   Gideon Israel Dsouza   mm: use macros fr...
445
  void __weak vmalloc_sync_all(void)
1eeb66a1b   Christoph Hellwig   move die notifier...
446
447
  {
  }
29c185e5c   Paul Mundt   nommu: Provide st...
448
449
450
451
452
453
454
455
456
457
458
459
  /**
   *	alloc_vm_area - allocate a range of kernel address space
   *	@size:		size of the area
   *
   *	Returns:	NULL on failure, vm_struct on success
   *
   *	This function reserves a range of kernel address space, and
   *	allocates pagetables to map that range.  No actual mappings
   *	are created.  If the kernel address space is not shared
   *	between processes, it syncs the pagetable across all
   *	processes.
   */
cd12909cb   David Vrabel   xen: map foreign ...
460
  struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
29c185e5c   Paul Mundt   nommu: Provide st...
461
462
463
464
465
466
467
468
469
470
471
  {
  	BUG();
  	return NULL;
  }
  EXPORT_SYMBOL_GPL(alloc_vm_area);
  
  void free_vm_area(struct vm_struct *area)
  {
  	BUG();
  }
  EXPORT_SYMBOL_GPL(free_vm_area);
b50731732   Paul Mundt   nommu: vmalloc_32...
472
473
474
475
476
477
  int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  		   struct page *page)
  {
  	return -EINVAL;
  }
  EXPORT_SYMBOL(vm_insert_page);
1eeb66a1b   Christoph Hellwig   move die notifier...
478
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
479
480
481
482
483
484
   *  sys_brk() for the most part doesn't need the global kernel
   *  lock, except when an application is doing something nasty
   *  like trying to un-brk an area that has already been mapped
   *  to a regular file.  in this case, the unmapping will need
   *  to invoke file system routines that need the global lock.
   */
6a6160a7b   Heiko Carstens   [CVE-2009-0029] S...
485
  SYSCALL_DEFINE1(brk, unsigned long, brk)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
  {
  	struct mm_struct *mm = current->mm;
  
  	if (brk < mm->start_brk || brk > mm->context.end_brk)
  		return mm->brk;
  
  	if (mm->brk == brk)
  		return mm->brk;
  
  	/*
  	 * Always allow shrinking brk
  	 */
  	if (brk <= mm->brk) {
  		mm->brk = brk;
  		return brk;
  	}
  
  	/*
  	 * Ok, looks good - let it rip.
  	 */
cfe79c00a   Mike Frysinger   NOMMU: Avoiding d...
506
  	flush_icache_range(mm->brk, brk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
507
508
  	return mm->brk = brk;
  }
8feae1311   David Howells   NOMMU: Make VMAs ...
509
510
511
512
  /*
   * initialise the VMA and region record slabs
   */
  void __init mmap_init(void)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
513
  {
00a62ce91   KOSAKI Motohiro   mm: fix Committed...
514
  	int ret;
908c7f194   Tejun Heo   percpu_counter: a...
515
  	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
00a62ce91   KOSAKI Motohiro   mm: fix Committed...
516
  	VM_BUG_ON(ret);
5d097056c   Vladimir Davydov   kmemcg: account c...
517
  	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
518
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
519

3034097a5   David Howells   [PATCH] NOMMU: Or...
520
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
521
522
   * validate the region tree
   * - the caller must hold the region lock
3034097a5   David Howells   [PATCH] NOMMU: Or...
523
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
524
525
  #ifdef CONFIG_DEBUG_NOMMU_REGIONS
  static noinline void validate_nommu_regions(void)
3034097a5   David Howells   [PATCH] NOMMU: Or...
526
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
527
528
  	struct vm_region *region, *last;
  	struct rb_node *p, *lastp;
3034097a5   David Howells   [PATCH] NOMMU: Or...
529

8feae1311   David Howells   NOMMU: Make VMAs ...
530
531
532
533
534
  	lastp = rb_first(&nommu_region_tree);
  	if (!lastp)
  		return;
  
  	last = rb_entry(lastp, struct vm_region, vm_rb);
c9427bc04   Geliang Tang   mm/nommu.c: drop ...
535
536
  	BUG_ON(last->vm_end <= last->vm_start);
  	BUG_ON(last->vm_top < last->vm_end);
8feae1311   David Howells   NOMMU: Make VMAs ...
537
538
539
540
  
  	while ((p = rb_next(lastp))) {
  		region = rb_entry(p, struct vm_region, vm_rb);
  		last = rb_entry(lastp, struct vm_region, vm_rb);
c9427bc04   Geliang Tang   mm/nommu.c: drop ...
541
542
543
  		BUG_ON(region->vm_end <= region->vm_start);
  		BUG_ON(region->vm_top < region->vm_end);
  		BUG_ON(region->vm_start < last->vm_top);
3034097a5   David Howells   [PATCH] NOMMU: Or...
544

8feae1311   David Howells   NOMMU: Make VMAs ...
545
546
  		lastp = p;
  	}
3034097a5   David Howells   [PATCH] NOMMU: Or...
547
  }
8feae1311   David Howells   NOMMU: Make VMAs ...
548
  #else
33e5d7697   David Howells   nommu: fix a numb...
549
550
551
  static void validate_nommu_regions(void)
  {
  }
8feae1311   David Howells   NOMMU: Make VMAs ...
552
  #endif
3034097a5   David Howells   [PATCH] NOMMU: Or...
553
554
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
555
   * add a region into the global tree
3034097a5   David Howells   [PATCH] NOMMU: Or...
556
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
557
  static void add_nommu_region(struct vm_region *region)
3034097a5   David Howells   [PATCH] NOMMU: Or...
558
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
559
560
  	struct vm_region *pregion;
  	struct rb_node **p, *parent;
3034097a5   David Howells   [PATCH] NOMMU: Or...
561

8feae1311   David Howells   NOMMU: Make VMAs ...
562
  	validate_nommu_regions();
8feae1311   David Howells   NOMMU: Make VMAs ...
563
564
565
566
567
568
569
570
571
572
573
574
575
  	parent = NULL;
  	p = &nommu_region_tree.rb_node;
  	while (*p) {
  		parent = *p;
  		pregion = rb_entry(parent, struct vm_region, vm_rb);
  		if (region->vm_start < pregion->vm_start)
  			p = &(*p)->rb_left;
  		else if (region->vm_start > pregion->vm_start)
  			p = &(*p)->rb_right;
  		else if (pregion == region)
  			return;
  		else
  			BUG();
3034097a5   David Howells   [PATCH] NOMMU: Or...
576
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
577
578
  	rb_link_node(&region->vm_rb, parent, p);
  	rb_insert_color(&region->vm_rb, &nommu_region_tree);
3034097a5   David Howells   [PATCH] NOMMU: Or...
579

8feae1311   David Howells   NOMMU: Make VMAs ...
580
  	validate_nommu_regions();
3034097a5   David Howells   [PATCH] NOMMU: Or...
581
  }
3034097a5   David Howells   [PATCH] NOMMU: Or...
582
583
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
584
   * delete a region from the global tree
930e652a2   David Howells   [PATCH] NOMMU: Ma...
585
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
586
  static void delete_nommu_region(struct vm_region *region)
930e652a2   David Howells   [PATCH] NOMMU: Ma...
587
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
588
  	BUG_ON(!nommu_region_tree.rb_node);
930e652a2   David Howells   [PATCH] NOMMU: Ma...
589

8feae1311   David Howells   NOMMU: Make VMAs ...
590
591
592
  	validate_nommu_regions();
  	rb_erase(&region->vm_rb, &nommu_region_tree);
  	validate_nommu_regions();
57c8f63e8   Greg Ungerer   nommu: stub expan...
593
  }
930e652a2   David Howells   [PATCH] NOMMU: Ma...
594
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
595
   * free a contiguous series of pages
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
596
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
597
  static void free_page_series(unsigned long from, unsigned long to)
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
598
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
599
600
  	for (; from < to; from += PAGE_SIZE) {
  		struct page *page = virt_to_page(from);
33e5d7697   David Howells   nommu: fix a numb...
601
  		atomic_long_dec(&mmap_pages_allocated);
8feae1311   David Howells   NOMMU: Make VMAs ...
602
  		put_page(page);
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
603
  	}
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
604
605
606
  }
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
607
   * release a reference to a region
33e5d7697   David Howells   nommu: fix a numb...
608
   * - the caller must hold the region semaphore for writing, which this releases
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
609
   * - the region may not have been added to the tree yet, in which case vm_top
8feae1311   David Howells   NOMMU: Make VMAs ...
610
   *   will equal vm_start
3034097a5   David Howells   [PATCH] NOMMU: Or...
611
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
612
613
  static void __put_nommu_region(struct vm_region *region)
  	__releases(nommu_region_sem)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
614
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
615
  	BUG_ON(!nommu_region_tree.rb_node);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
616

1e2ae599d   David Howells   nommu: struct vm_...
617
  	if (--region->vm_usage == 0) {
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
618
  		if (region->vm_top > region->vm_start)
8feae1311   David Howells   NOMMU: Make VMAs ...
619
620
621
622
623
624
625
626
  			delete_nommu_region(region);
  		up_write(&nommu_region_sem);
  
  		if (region->vm_file)
  			fput(region->vm_file);
  
  		/* IO memory and memory shared directly out of the pagecache
  		 * from ramfs/tmpfs mustn't be released here */
22cc877b3   Leon Romanovsky   mm: nommu: refact...
627
  		if (region->vm_flags & VM_MAPPED_COPY)
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
628
  			free_page_series(region->vm_start, region->vm_top);
8feae1311   David Howells   NOMMU: Make VMAs ...
629
630
631
  		kmem_cache_free(vm_region_jar, region);
  	} else {
  		up_write(&nommu_region_sem);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
632
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
633
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
634

8feae1311   David Howells   NOMMU: Make VMAs ...
635
636
637
638
639
640
641
  /*
   * release a reference to a region
   */
  static void put_nommu_region(struct vm_region *region)
  {
  	down_write(&nommu_region_sem);
  	__put_nommu_region(region);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
642
  }
3034097a5   David Howells   [PATCH] NOMMU: Or...
643
  /*
eb8cdec4a   Bernd Schmidt   nommu: add suppor...
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
   * update protection on a vma
   */
  static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
  {
  #ifdef CONFIG_MPU
  	struct mm_struct *mm = vma->vm_mm;
  	long start = vma->vm_start & PAGE_MASK;
  	while (start < vma->vm_end) {
  		protect_page(mm, start, flags);
  		start += PAGE_SIZE;
  	}
  	update_protections(mm);
  #endif
  }
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
660
661
662
663
   * add a VMA into a process's mm_struct in the appropriate place in the list
   * and tree and add to the address space's page tree also if not an anonymous
   * page
   * - should be called with mm->mmap_sem held writelocked
3034097a5   David Howells   [PATCH] NOMMU: Or...
664
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
665
  static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
666
  {
6038def0d   Namhyung Kim   mm: nommu: sort m...
667
  	struct vm_area_struct *pvma, *prev;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
668
  	struct address_space *mapping;
6038def0d   Namhyung Kim   mm: nommu: sort m...
669
  	struct rb_node **p, *parent, *rb_prev;
8feae1311   David Howells   NOMMU: Make VMAs ...
670

8feae1311   David Howells   NOMMU: Make VMAs ...
671
672
673
674
  	BUG_ON(!vma->vm_region);
  
  	mm->map_count++;
  	vma->vm_mm = mm;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
675

eb8cdec4a   Bernd Schmidt   nommu: add suppor...
676
  	protect_vma(vma, vma->vm_flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
677
678
679
  	/* add the VMA to the mapping */
  	if (vma->vm_file) {
  		mapping = vma->vm_file->f_mapping;
83cde9e8b   Davidlohr Bueso   mm: use new helpe...
680
  		i_mmap_lock_write(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
681
  		flush_dcache_mmap_lock(mapping);
6b2dbba8b   Michel Lespinasse   mm: replace vma p...
682
  		vma_interval_tree_insert(vma, &mapping->i_mmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
683
  		flush_dcache_mmap_unlock(mapping);
83cde9e8b   Davidlohr Bueso   mm: use new helpe...
684
  		i_mmap_unlock_write(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
685
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
686
  	/* add the VMA to the tree */
6038def0d   Namhyung Kim   mm: nommu: sort m...
687
  	parent = rb_prev = NULL;
8feae1311   David Howells   NOMMU: Make VMAs ...
688
  	p = &mm->mm_rb.rb_node;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
689
690
691
  	while (*p) {
  		parent = *p;
  		pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
8feae1311   David Howells   NOMMU: Make VMAs ...
692
693
694
  		/* sort by: start addr, end addr, VMA struct addr in that order
  		 * (the latter is necessary as we may get identical VMAs) */
  		if (vma->vm_start < pvma->vm_start)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
695
  			p = &(*p)->rb_left;
6038def0d   Namhyung Kim   mm: nommu: sort m...
696
697
  		else if (vma->vm_start > pvma->vm_start) {
  			rb_prev = parent;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
698
  			p = &(*p)->rb_right;
6038def0d   Namhyung Kim   mm: nommu: sort m...
699
  		} else if (vma->vm_end < pvma->vm_end)
8feae1311   David Howells   NOMMU: Make VMAs ...
700
  			p = &(*p)->rb_left;
6038def0d   Namhyung Kim   mm: nommu: sort m...
701
702
  		else if (vma->vm_end > pvma->vm_end) {
  			rb_prev = parent;
8feae1311   David Howells   NOMMU: Make VMAs ...
703
  			p = &(*p)->rb_right;
6038def0d   Namhyung Kim   mm: nommu: sort m...
704
  		} else if (vma < pvma)
8feae1311   David Howells   NOMMU: Make VMAs ...
705
  			p = &(*p)->rb_left;
6038def0d   Namhyung Kim   mm: nommu: sort m...
706
707
  		else if (vma > pvma) {
  			rb_prev = parent;
8feae1311   David Howells   NOMMU: Make VMAs ...
708
  			p = &(*p)->rb_right;
6038def0d   Namhyung Kim   mm: nommu: sort m...
709
  		} else
8feae1311   David Howells   NOMMU: Make VMAs ...
710
  			BUG();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
711
712
713
  	}
  
  	rb_link_node(&vma->vm_rb, parent, p);
8feae1311   David Howells   NOMMU: Make VMAs ...
714
715
716
  	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
  
  	/* add VMA to the VMA list also */
6038def0d   Namhyung Kim   mm: nommu: sort m...
717
718
719
  	prev = NULL;
  	if (rb_prev)
  		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
8feae1311   David Howells   NOMMU: Make VMAs ...
720

6038def0d   Namhyung Kim   mm: nommu: sort m...
721
  	__vma_link_list(mm, vma, prev, parent);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
722
  }
3034097a5   David Howells   [PATCH] NOMMU: Or...
723
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
724
   * delete a VMA from its owning mm_struct and address space
3034097a5   David Howells   [PATCH] NOMMU: Or...
725
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
726
  static void delete_vma_from_mm(struct vm_area_struct *vma)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
727
  {
615d6e875   Davidlohr Bueso   mm: per-thread vm...
728
  	int i;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
729
  	struct address_space *mapping;
8feae1311   David Howells   NOMMU: Make VMAs ...
730
  	struct mm_struct *mm = vma->vm_mm;
615d6e875   Davidlohr Bueso   mm: per-thread vm...
731
  	struct task_struct *curr = current;
8feae1311   David Howells   NOMMU: Make VMAs ...
732

eb8cdec4a   Bernd Schmidt   nommu: add suppor...
733
  	protect_vma(vma, 0);
8feae1311   David Howells   NOMMU: Make VMAs ...
734
  	mm->map_count--;
615d6e875   Davidlohr Bueso   mm: per-thread vm...
735
736
737
  	for (i = 0; i < VMACACHE_SIZE; i++) {
  		/* if the vma is cached, invalidate the entire cache */
  		if (curr->vmacache[i] == vma) {
e020d5bd8   Steven Miao   mm: nommu: per-th...
738
  			vmacache_invalidate(mm);
615d6e875   Davidlohr Bueso   mm: per-thread vm...
739
740
741
  			break;
  		}
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
742
743
744
745
  
  	/* remove the VMA from the mapping */
  	if (vma->vm_file) {
  		mapping = vma->vm_file->f_mapping;
83cde9e8b   Davidlohr Bueso   mm: use new helpe...
746
  		i_mmap_lock_write(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
747
  		flush_dcache_mmap_lock(mapping);
6b2dbba8b   Michel Lespinasse   mm: replace vma p...
748
  		vma_interval_tree_remove(vma, &mapping->i_mmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
749
  		flush_dcache_mmap_unlock(mapping);
83cde9e8b   Davidlohr Bueso   mm: use new helpe...
750
  		i_mmap_unlock_write(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
751
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
752
753
  	/* remove from the MM's tree and list */
  	rb_erase(&vma->vm_rb, &mm->mm_rb);
b951bf2c4   Namhyung Kim   mm: nommu: don't ...
754
755
756
757
758
759
760
761
  
  	if (vma->vm_prev)
  		vma->vm_prev->vm_next = vma->vm_next;
  	else
  		mm->mmap = vma->vm_next;
  
  	if (vma->vm_next)
  		vma->vm_next->vm_prev = vma->vm_prev;
8feae1311   David Howells   NOMMU: Make VMAs ...
762
763
764
765
766
767
768
  }
  
  /*
   * destroy a VMA record
   */
  static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
769
770
  	if (vma->vm_ops && vma->vm_ops->close)
  		vma->vm_ops->close(vma);
e9714acf8   Konstantin Khlebnikov   mm: kill vma flag...
771
  	if (vma->vm_file)
8feae1311   David Howells   NOMMU: Make VMAs ...
772
  		fput(vma->vm_file);
8feae1311   David Howells   NOMMU: Make VMAs ...
773
774
775
776
777
778
779
780
781
782
783
  	put_nommu_region(vma->vm_region);
  	kmem_cache_free(vm_area_cachep, vma);
  }
  
  /*
   * look up the first VMA in which addr resides, NULL if none
   * - should be called with mm->mmap_sem at least held readlocked
   */
  struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
  {
  	struct vm_area_struct *vma;
8feae1311   David Howells   NOMMU: Make VMAs ...
784
785
  
  	/* check the cache first */
615d6e875   Davidlohr Bueso   mm: per-thread vm...
786
787
  	vma = vmacache_find(mm, addr);
  	if (likely(vma))
8feae1311   David Howells   NOMMU: Make VMAs ...
788
  		return vma;
e922c4c53   Namhyung Kim   mm: nommu: find v...
789
  	/* trawl the list (there may be multiple mappings in which addr
8feae1311   David Howells   NOMMU: Make VMAs ...
790
  	 * resides) */
e922c4c53   Namhyung Kim   mm: nommu: find v...
791
  	for (vma = mm->mmap; vma; vma = vma->vm_next) {
8feae1311   David Howells   NOMMU: Make VMAs ...
792
793
794
  		if (vma->vm_start > addr)
  			return NULL;
  		if (vma->vm_end > addr) {
615d6e875   Davidlohr Bueso   mm: per-thread vm...
795
  			vmacache_update(addr, vma);
8feae1311   David Howells   NOMMU: Make VMAs ...
796
797
798
799
800
801
802
803
804
805
806
807
808
809
  			return vma;
  		}
  	}
  
  	return NULL;
  }
  EXPORT_SYMBOL(find_vma);
  
  /*
   * find a VMA
   * - we don't extend stack VMAs under NOMMU conditions
   */
  struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
  {
7561e8ca0   David Howells   NOMMU: Revert 'no...
810
  	return find_vma(mm, addr);
8feae1311   David Howells   NOMMU: Make VMAs ...
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
  }
  
  /*
   * expand a stack to a given address
   * - not supported under NOMMU conditions
   */
  int expand_stack(struct vm_area_struct *vma, unsigned long address)
  {
  	return -ENOMEM;
  }
  
  /*
   * look up the first VMA exactly that exactly matches addr
   * - should be called with mm->mmap_sem at least held readlocked
   */
  static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
  					     unsigned long addr,
  					     unsigned long len)
  {
  	struct vm_area_struct *vma;
8feae1311   David Howells   NOMMU: Make VMAs ...
831
832
833
  	unsigned long end = addr + len;
  
  	/* check the cache first */
615d6e875   Davidlohr Bueso   mm: per-thread vm...
834
835
  	vma = vmacache_find_exact(mm, addr, end);
  	if (vma)
8feae1311   David Howells   NOMMU: Make VMAs ...
836
  		return vma;
e922c4c53   Namhyung Kim   mm: nommu: find v...
837
  	/* trawl the list (there may be multiple mappings in which addr
8feae1311   David Howells   NOMMU: Make VMAs ...
838
  	 * resides) */
e922c4c53   Namhyung Kim   mm: nommu: find v...
839
  	for (vma = mm->mmap; vma; vma = vma->vm_next) {
8feae1311   David Howells   NOMMU: Make VMAs ...
840
841
842
843
844
  		if (vma->vm_start < addr)
  			continue;
  		if (vma->vm_start > addr)
  			return NULL;
  		if (vma->vm_end == end) {
615d6e875   Davidlohr Bueso   mm: per-thread vm...
845
  			vmacache_update(addr, vma);
8feae1311   David Howells   NOMMU: Make VMAs ...
846
847
848
849
850
  			return vma;
  		}
  	}
  
  	return NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
851
852
853
854
855
856
857
858
859
860
861
862
863
864
  }
  
  /*
   * determine whether a mapping should be permitted and, if so, what sort of
   * mapping we're capable of supporting
   */
  static int validate_mmap_request(struct file *file,
  				 unsigned long addr,
  				 unsigned long len,
  				 unsigned long prot,
  				 unsigned long flags,
  				 unsigned long pgoff,
  				 unsigned long *_capabilities)
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
865
  	unsigned long capabilities, rlen;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
866
867
868
  	int ret;
  
  	/* do the simple checks first */
22cc877b3   Leon Romanovsky   mm: nommu: refact...
869
  	if (flags & MAP_FIXED)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
870
  		return -EINVAL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
871
872
873
874
  
  	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
  	    (flags & MAP_TYPE) != MAP_SHARED)
  		return -EINVAL;
f81cff0d4   Mike Frysinger   [PATCH] uclinux: ...
875
  	if (!len)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
876
  		return -EINVAL;
f81cff0d4   Mike Frysinger   [PATCH] uclinux: ...
877
  	/* Careful about overflows.. */
8feae1311   David Howells   NOMMU: Make VMAs ...
878
879
  	rlen = PAGE_ALIGN(len);
  	if (!rlen || rlen > TASK_SIZE)
f81cff0d4   Mike Frysinger   [PATCH] uclinux: ...
880
  		return -ENOMEM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
881
  	/* offset overflow? */
8feae1311   David Howells   NOMMU: Make VMAs ...
882
  	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
f81cff0d4   Mike Frysinger   [PATCH] uclinux: ...
883
  		return -EOVERFLOW;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
884
885
  
  	if (file) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
886
  		/* files must support mmap */
72c2d5319   Al Viro   file->f_op is nev...
887
  		if (!file->f_op->mmap)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
888
889
890
891
892
893
  			return -ENODEV;
  
  		/* work out if what we've got could possibly be shared
  		 * - we support chardevs that provide their own "memory"
  		 * - we support files/blockdevs that are memory backed
  		 */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
894
895
896
  		if (file->f_op->mmap_capabilities) {
  			capabilities = file->f_op->mmap_capabilities(file);
  		} else {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
897
898
  			/* no explicit capabilities set, so assume some
  			 * defaults */
496ad9aa8   Al Viro   new helper: file_...
899
  			switch (file_inode(file)->i_mode & S_IFMT) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
900
901
  			case S_IFREG:
  			case S_IFBLK:
b4caecd48   Christoph Hellwig   fs: introduce f_o...
902
  				capabilities = NOMMU_MAP_COPY;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
903
904
905
906
  				break;
  
  			case S_IFCHR:
  				capabilities =
b4caecd48   Christoph Hellwig   fs: introduce f_o...
907
908
909
  					NOMMU_MAP_DIRECT |
  					NOMMU_MAP_READ |
  					NOMMU_MAP_WRITE;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
910
911
912
913
914
915
916
917
918
919
  				break;
  
  			default:
  				return -EINVAL;
  			}
  		}
  
  		/* eliminate any capabilities that we can't support on this
  		 * device */
  		if (!file->f_op->get_unmapped_area)
b4caecd48   Christoph Hellwig   fs: introduce f_o...
920
  			capabilities &= ~NOMMU_MAP_DIRECT;
6e242a1ce   Al Viro   nommu: use __vfs_...
921
  		if (!(file->f_mode & FMODE_CAN_READ))
b4caecd48   Christoph Hellwig   fs: introduce f_o...
922
  			capabilities &= ~NOMMU_MAP_COPY;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
923

28d7a6ae9   Graff Yang   nommu: check fd r...
924
925
926
  		/* The file shall have been opened with read permission. */
  		if (!(file->f_mode & FMODE_READ))
  			return -EACCES;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
927
928
929
930
931
  		if (flags & MAP_SHARED) {
  			/* do checks for writing, appending and locking */
  			if ((prot & PROT_WRITE) &&
  			    !(file->f_mode & FMODE_WRITE))
  				return -EACCES;
496ad9aa8   Al Viro   new helper: file_...
932
  			if (IS_APPEND(file_inode(file)) &&
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
933
934
  			    (file->f_mode & FMODE_WRITE))
  				return -EACCES;
d7a06983a   Jeff Layton   locks: fix locks_...
935
  			if (locks_verify_locked(file))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
936
  				return -EAGAIN;
b4caecd48   Christoph Hellwig   fs: introduce f_o...
937
  			if (!(capabilities & NOMMU_MAP_DIRECT))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
938
  				return -ENODEV;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
939
  			/* we mustn't privatise shared mappings */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
940
  			capabilities &= ~NOMMU_MAP_COPY;
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
941
  		} else {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
942
943
  			/* we're going to read the file into private memory we
  			 * allocate */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
944
  			if (!(capabilities & NOMMU_MAP_COPY))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
945
946
947
948
949
  				return -ENODEV;
  
  			/* we don't permit a private writable mapping to be
  			 * shared with the backing device */
  			if (prot & PROT_WRITE)
b4caecd48   Christoph Hellwig   fs: introduce f_o...
950
  				capabilities &= ~NOMMU_MAP_DIRECT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
951
  		}
b4caecd48   Christoph Hellwig   fs: introduce f_o...
952
953
954
955
  		if (capabilities & NOMMU_MAP_DIRECT) {
  			if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
  			    ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
  			    ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
3c7b20454   Bernd Schmidt   nommu: allow priv...
956
  			    ) {
b4caecd48   Christoph Hellwig   fs: introduce f_o...
957
  				capabilities &= ~NOMMU_MAP_DIRECT;
3c7b20454   Bernd Schmidt   nommu: allow priv...
958
  				if (flags & MAP_SHARED) {
22cc877b3   Leon Romanovsky   mm: nommu: refact...
959
960
  					pr_warn("MAP_SHARED not completely supported on !MMU
  ");
3c7b20454   Bernd Schmidt   nommu: allow priv...
961
962
963
964
  					return -EINVAL;
  				}
  			}
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
965
966
  		/* handle executable mappings and implied executable
  		 * mappings */
90f8572b0   Eric W. Biederman   vfs: Commit to ne...
967
  		if (path_noexec(&file->f_path)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
968
969
  			if (prot & PROT_EXEC)
  				return -EPERM;
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
970
  		} else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
971
972
  			/* handle implication of PROT_EXEC by PROT_READ */
  			if (current->personality & READ_IMPLIES_EXEC) {
b4caecd48   Christoph Hellwig   fs: introduce f_o...
973
  				if (capabilities & NOMMU_MAP_EXEC)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
974
975
  					prot |= PROT_EXEC;
  			}
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
976
  		} else if ((prot & PROT_READ) &&
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
977
  			 (prot & PROT_EXEC) &&
b4caecd48   Christoph Hellwig   fs: introduce f_o...
978
  			 !(capabilities & NOMMU_MAP_EXEC)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
979
980
  			 ) {
  			/* backing file is not executable, try to copy */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
981
  			capabilities &= ~NOMMU_MAP_DIRECT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
982
  		}
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
983
  	} else {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
984
985
986
  		/* anonymous mappings are always memory backed and can be
  		 * privately mapped
  		 */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
987
  		capabilities = NOMMU_MAP_COPY;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
988
989
990
991
992
993
994
995
  
  		/* handle PROT_EXEC implication by PROT_READ */
  		if ((prot & PROT_READ) &&
  		    (current->personality & READ_IMPLIES_EXEC))
  			prot |= PROT_EXEC;
  	}
  
  	/* allow the security API to have its say */
e5467859f   Al Viro   split ->file_mmap...
996
997
998
  	ret = security_mmap_addr(addr);
  	if (ret < 0)
  		return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
  
  	/* looks okay */
  	*_capabilities = capabilities;
  	return 0;
  }
  
  /*
   * we've determined that we can make the mapping, now translate what we
   * now know into VMA flags
   */
  static unsigned long determine_vm_flags(struct file *file,
  					unsigned long prot,
  					unsigned long flags,
  					unsigned long capabilities)
  {
  	unsigned long vm_flags;
e6bfb7095   Dave Hansen   mm/core, arch, po...
1015
  	vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1016
  	/* vm_flags |= mm->def_flags; */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1017
  	if (!(capabilities & NOMMU_MAP_DIRECT)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1018
  		/* attempt to share read-only copies of mapped file chunks */
3c7b20454   Bernd Schmidt   nommu: allow priv...
1019
  		vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1020
1021
  		if (file && !(prot & PROT_WRITE))
  			vm_flags |= VM_MAYSHARE;
3c7b20454   Bernd Schmidt   nommu: allow priv...
1022
  	} else {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1023
1024
1025
  		/* overlay a shareable mapping on the backing device or inode
  		 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
  		 * romfs/cramfs */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1026
  		vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1027
  		if (flags & MAP_SHARED)
3c7b20454   Bernd Schmidt   nommu: allow priv...
1028
  			vm_flags |= VM_SHARED;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1029
1030
1031
1032
1033
1034
  	}
  
  	/* refuse to let anyone share private mappings with this process if
  	 * it's being traced - otherwise breakpoints set in it may interfere
  	 * with another untraced process
  	 */
a288eecce   Tejun Heo   ptrace: kill triv...
1035
  	if ((flags & MAP_PRIVATE) && current->ptrace)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1036
1037
1038
1039
1040
1041
  		vm_flags &= ~VM_MAYSHARE;
  
  	return vm_flags;
  }
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
1042
1043
   * set up a shared mapping on a file (the driver or filesystem provides and
   * pins the storage)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1044
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
1045
  static int do_mmap_shared_file(struct vm_area_struct *vma)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1046
1047
1048
1049
  {
  	int ret;
  
  	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1050
1051
  	if (ret == 0) {
  		vma->vm_region->vm_top = vma->vm_region->vm_end;
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1052
  		return 0;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1053
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1054
1055
  	if (ret != -ENOSYS)
  		return ret;
3fa30460e   David Howells   nommu: fix an inc...
1056
1057
1058
  	/* getting -ENOSYS indicates that direct mmap isn't possible (as
  	 * opposed to tried but failed) so we can only give a suitable error as
  	 * it's not possible to make a private copy if MAP_SHARED was given */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1059
1060
1061
1062
1063
1064
  	return -ENODEV;
  }
  
  /*
   * set up a private mapping or an anonymous shared mapping
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
1065
1066
  static int do_mmap_private(struct vm_area_struct *vma,
  			   struct vm_region *region,
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1067
1068
  			   unsigned long len,
  			   unsigned long capabilities)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1069
  {
dbc8358c7   Joonsoo Kim   mm/nommu: use all...
1070
  	unsigned long total, point;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1071
  	void *base;
8feae1311   David Howells   NOMMU: Make VMAs ...
1072
  	int ret, order;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1073
1074
1075
1076
1077
  
  	/* invoke the file's mapping function so that it can keep track of
  	 * shared mappings on devices or memory
  	 * - VM_MAYSHARE will be set if it may attempt to share
  	 */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1078
  	if (capabilities & NOMMU_MAP_DIRECT) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1079
  		ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1080
  		if (ret == 0) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1081
  			/* shouldn't return success if we're not sharing */
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1082
1083
  			BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
  			vma->vm_region->vm_top = vma->vm_region->vm_end;
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1084
  			return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1085
  		}
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1086
1087
  		if (ret != -ENOSYS)
  			return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1088
1089
1090
1091
1092
  
  		/* getting an ENOSYS error indicates that direct mmap isn't
  		 * possible (as opposed to tried but failed) so we'll try to
  		 * make a private copy of the data and map that instead */
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
1093

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1094
1095
1096
1097
  	/* allocate some memory to hold the mapping
  	 * - note that this may not return a page-aligned address if the object
  	 *   we're allocating is smaller than a page
  	 */
f67d9b157   Bob Liu   nommu: add page a...
1098
  	order = get_order(len);
8feae1311   David Howells   NOMMU: Make VMAs ...
1099
  	total = 1 << order;
f67d9b157   Bob Liu   nommu: add page a...
1100
  	point = len >> PAGE_SHIFT;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1101

dbc8358c7   Joonsoo Kim   mm/nommu: use all...
1102
  	/* we don't want to allocate a power-of-2 sized page set */
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1103
  	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
dbc8358c7   Joonsoo Kim   mm/nommu: use all...
1104
  		total = point;
8feae1311   David Howells   NOMMU: Make VMAs ...
1105

da616534e   Joonsoo Kim   mm/nommu: fix mem...
1106
  	base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
dbc8358c7   Joonsoo Kim   mm/nommu: use all...
1107
1108
1109
1110
  	if (!base)
  		goto enomem;
  
  	atomic_long_add(total, &mmap_pages_allocated);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1111

8feae1311   David Howells   NOMMU: Make VMAs ...
1112
1113
  	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
  	region->vm_start = (unsigned long) base;
f67d9b157   Bob Liu   nommu: add page a...
1114
  	region->vm_end   = region->vm_start + len;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1115
  	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
8feae1311   David Howells   NOMMU: Make VMAs ...
1116
1117
1118
  
  	vma->vm_start = region->vm_start;
  	vma->vm_end   = region->vm_start + len;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
  
  	if (vma->vm_file) {
  		/* read the contents of a file into the copy */
  		mm_segment_t old_fs;
  		loff_t fpos;
  
  		fpos = vma->vm_pgoff;
  		fpos <<= PAGE_SHIFT;
  
  		old_fs = get_fs();
  		set_fs(KERNEL_DS);
6e242a1ce   Al Viro   nommu: use __vfs_...
1130
  		ret = __vfs_read(vma->vm_file, base, len, &fpos);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1131
1132
1133
1134
1135
1136
  		set_fs(old_fs);
  
  		if (ret < 0)
  			goto error_free;
  
  		/* clear the last little bit */
f67d9b157   Bob Liu   nommu: add page a...
1137
1138
  		if (ret < len)
  			memset(base + ret, 0, len - ret);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1139

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1140
1141
1142
1143
1144
  	}
  
  	return 0;
  
  error_free:
7223bb4a8   Namhyung Kim   mm: nommu: fix a ...
1145
  	free_page_series(region->vm_start, region->vm_top);
8feae1311   David Howells   NOMMU: Make VMAs ...
1146
1147
  	region->vm_start = vma->vm_start = 0;
  	region->vm_end   = vma->vm_end = 0;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1148
  	region->vm_top   = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1149
1150
1151
  	return ret;
  
  enomem:
b1de0d139   Mitchel Humpherys   mm: convert some ...
1152
1153
  	pr_err("Allocation of length %lu from process %d (%s) failed
  ",
05ae6fa31   Greg Ungerer   uclinux: add proc...
1154
  	       len, current->pid, current->comm);
7bf02ea22   David Rientjes   arch, mm: filter ...
1155
  	show_free_areas(0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1156
1157
1158
1159
1160
1161
  	return -ENOMEM;
  }
  
  /*
   * handle mapping creation for uClinux
   */
1fcfd8db7   Oleg Nesterov   mm, mpx: add "vm_...
1162
1163
1164
1165
1166
1167
1168
1169
  unsigned long do_mmap(struct file *file,
  			unsigned long addr,
  			unsigned long len,
  			unsigned long prot,
  			unsigned long flags,
  			vm_flags_t vm_flags,
  			unsigned long pgoff,
  			unsigned long *populate)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1170
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
1171
1172
  	struct vm_area_struct *vma;
  	struct vm_region *region;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1173
  	struct rb_node *rb;
1fcfd8db7   Oleg Nesterov   mm, mpx: add "vm_...
1174
  	unsigned long capabilities, result;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1175
  	int ret;
41badc15c   Michel Lespinasse   mm: make do_mmap_...
1176
  	*populate = 0;
bebeb3d68   Michel Lespinasse   mm: introduce mm_...
1177

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1178
1179
1180
1181
  	/* decide whether we should attempt the mapping, and if so what sort of
  	 * mapping */
  	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
  				    &capabilities);
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1182
  	if (ret < 0)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1183
  		return ret;
06aab5a30   David Howells   NOMMU: Ignore mma...
1184
1185
  	/* we ignore the address hint */
  	addr = 0;
f67d9b157   Bob Liu   nommu: add page a...
1186
  	len = PAGE_ALIGN(len);
06aab5a30   David Howells   NOMMU: Ignore mma...
1187

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1188
1189
  	/* we've determined that we can make the mapping, now translate what we
  	 * now know into VMA flags */
1fcfd8db7   Oleg Nesterov   mm, mpx: add "vm_...
1190
  	vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1191

8feae1311   David Howells   NOMMU: Make VMAs ...
1192
1193
1194
1195
1196
1197
1198
1199
  	/* we're going to need to record the mapping */
  	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
  	if (!region)
  		goto error_getting_region;
  
  	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  	if (!vma)
  		goto error_getting_vma;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1200

1e2ae599d   David Howells   nommu: struct vm_...
1201
  	region->vm_usage = 1;
8feae1311   David Howells   NOMMU: Make VMAs ...
1202
1203
  	region->vm_flags = vm_flags;
  	region->vm_pgoff = pgoff;
5beb49305   Rik van Riel   mm: change anon_v...
1204
  	INIT_LIST_HEAD(&vma->anon_vma_chain);
8feae1311   David Howells   NOMMU: Make VMAs ...
1205
1206
  	vma->vm_flags = vm_flags;
  	vma->vm_pgoff = pgoff;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1207

8feae1311   David Howells   NOMMU: Make VMAs ...
1208
  	if (file) {
cb0942b81   Al Viro   make get_file() r...
1209
1210
  		region->vm_file = get_file(file);
  		vma->vm_file = get_file(file);
8feae1311   David Howells   NOMMU: Make VMAs ...
1211
1212
1213
1214
1215
  	}
  
  	down_write(&nommu_region_sem);
  
  	/* if we want to share, we need to check for regions created by other
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1216
  	 * mmap() calls that overlap with our proposed mapping
8feae1311   David Howells   NOMMU: Make VMAs ...
1217
  	 * - we can only share with a superset match on most regular files
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1218
1219
1220
1221
1222
1223
  	 * - shared mappings on character devices and memory backed files are
  	 *   permitted to overlap inexactly as far as we are concerned for in
  	 *   these cases, sharing is handled in the driver or filesystem rather
  	 *   than here
  	 */
  	if (vm_flags & VM_MAYSHARE) {
8feae1311   David Howells   NOMMU: Make VMAs ...
1224
1225
  		struct vm_region *pregion;
  		unsigned long pglen, rpglen, pgend, rpgend, start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1226

8feae1311   David Howells   NOMMU: Make VMAs ...
1227
1228
  		pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  		pgend = pgoff + pglen;
165b23927   David Howells   [PATCH] NOMMU: ma...
1229

8feae1311   David Howells   NOMMU: Make VMAs ...
1230
1231
  		for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
  			pregion = rb_entry(rb, struct vm_region, vm_rb);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1232

8feae1311   David Howells   NOMMU: Make VMAs ...
1233
  			if (!(pregion->vm_flags & VM_MAYSHARE))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1234
1235
1236
  				continue;
  
  			/* search for overlapping mappings on the same file */
496ad9aa8   Al Viro   new helper: file_...
1237
1238
  			if (file_inode(pregion->vm_file) !=
  			    file_inode(file))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1239
  				continue;
8feae1311   David Howells   NOMMU: Make VMAs ...
1240
  			if (pregion->vm_pgoff >= pgend)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1241
  				continue;
8feae1311   David Howells   NOMMU: Make VMAs ...
1242
1243
1244
1245
  			rpglen = pregion->vm_end - pregion->vm_start;
  			rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  			rpgend = pregion->vm_pgoff + rpglen;
  			if (pgoff >= rpgend)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1246
  				continue;
8feae1311   David Howells   NOMMU: Make VMAs ...
1247
1248
1249
1250
1251
  			/* handle inexactly overlapping matches between
  			 * mappings */
  			if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
  			    !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
  				/* new mapping is not a subset of the region */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1252
  				if (!(capabilities & NOMMU_MAP_DIRECT))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1253
1254
1255
  					goto sharing_violation;
  				continue;
  			}
8feae1311   David Howells   NOMMU: Make VMAs ...
1256
  			/* we've found a region we can share */
1e2ae599d   David Howells   nommu: struct vm_...
1257
  			pregion->vm_usage++;
8feae1311   David Howells   NOMMU: Make VMAs ...
1258
1259
1260
1261
1262
  			vma->vm_region = pregion;
  			start = pregion->vm_start;
  			start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
  			vma->vm_start = start;
  			vma->vm_end = start + len;
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1263
  			if (pregion->vm_flags & VM_MAPPED_COPY)
8feae1311   David Howells   NOMMU: Make VMAs ...
1264
  				vma->vm_flags |= VM_MAPPED_COPY;
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1265
  			else {
8feae1311   David Howells   NOMMU: Make VMAs ...
1266
1267
1268
1269
1270
  				ret = do_mmap_shared_file(vma);
  				if (ret < 0) {
  					vma->vm_region = NULL;
  					vma->vm_start = 0;
  					vma->vm_end = 0;
1e2ae599d   David Howells   nommu: struct vm_...
1271
  					pregion->vm_usage--;
8feae1311   David Howells   NOMMU: Make VMAs ...
1272
1273
1274
1275
1276
1277
1278
1279
1280
  					pregion = NULL;
  					goto error_just_free;
  				}
  			}
  			fput(region->vm_file);
  			kmem_cache_free(vm_region_jar, region);
  			region = pregion;
  			result = start;
  			goto share;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1281
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1282
1283
1284
1285
  		/* obtain the address at which to make a shared mapping
  		 * - this is the hook for quasi-memory character devices to
  		 *   tell us the location of a shared mapping
  		 */
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1286
  		if (capabilities & NOMMU_MAP_DIRECT) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1287
1288
  			addr = file->f_op->get_unmapped_area(file, addr, len,
  							     pgoff, flags);
bb005a59e   Namhyung Kim   mm: nommu: fix a ...
1289
  			if (IS_ERR_VALUE(addr)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1290
  				ret = addr;
bb005a59e   Namhyung Kim   mm: nommu: fix a ...
1291
  				if (ret != -ENOSYS)
8feae1311   David Howells   NOMMU: Make VMAs ...
1292
  					goto error_just_free;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1293
1294
1295
1296
  
  				/* the driver refused to tell us where to site
  				 * the mapping so we'll have to attempt to copy
  				 * it */
bb005a59e   Namhyung Kim   mm: nommu: fix a ...
1297
  				ret = -ENODEV;
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1298
  				if (!(capabilities & NOMMU_MAP_COPY))
8feae1311   David Howells   NOMMU: Make VMAs ...
1299
  					goto error_just_free;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1300

b4caecd48   Christoph Hellwig   fs: introduce f_o...
1301
  				capabilities &= ~NOMMU_MAP_DIRECT;
8feae1311   David Howells   NOMMU: Make VMAs ...
1302
1303
1304
  			} else {
  				vma->vm_start = region->vm_start = addr;
  				vma->vm_end = region->vm_end = addr + len;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1305
1306
1307
  			}
  		}
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
1308
  	vma->vm_region = region;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1309

645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1310
  	/* set up the mapping
b4caecd48   Christoph Hellwig   fs: introduce f_o...
1311
  	 * - the region is filled in if NOMMU_MAP_DIRECT is still set
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1312
  	 */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1313
  	if (file && vma->vm_flags & VM_SHARED)
8feae1311   David Howells   NOMMU: Make VMAs ...
1314
  		ret = do_mmap_shared_file(vma);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1315
  	else
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1316
  		ret = do_mmap_private(vma, region, len, capabilities);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1317
  	if (ret < 0)
645d83c5d   David Howells   NOMMU: Fix MAP_PR...
1318
1319
  		goto error_just_free;
  	add_nommu_region(region);
8feae1311   David Howells   NOMMU: Make VMAs ...
1320

ea6376395   Jie Zhang   nommu: fix malloc...
1321
1322
1323
1324
  	/* clear anonymous mappings that don't ask for uninitialized data */
  	if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
  		memset((void *)region->vm_start, 0,
  		       region->vm_end - region->vm_start);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1325
  	/* okay... we have a mapping; now we have to register it */
8feae1311   David Howells   NOMMU: Make VMAs ...
1326
  	result = vma->vm_start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1327

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1328
  	current->mm->total_vm += len >> PAGE_SHIFT;
8feae1311   David Howells   NOMMU: Make VMAs ...
1329
1330
  share:
  	add_vma_to_mm(current->mm, vma);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1331

cfe79c00a   Mike Frysinger   NOMMU: Avoiding d...
1332
1333
1334
1335
1336
1337
  	/* we flush the region from the icache only when the first executable
  	 * mapping of it is made  */
  	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
  		flush_icache_range(region->vm_start, region->vm_end);
  		region->vm_icache_flushed = true;
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1338

cfe79c00a   Mike Frysinger   NOMMU: Avoiding d...
1339
  	up_write(&nommu_region_sem);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1340

8feae1311   David Howells   NOMMU: Make VMAs ...
1341
  	return result;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1342

8feae1311   David Howells   NOMMU: Make VMAs ...
1343
1344
1345
  error_just_free:
  	up_write(&nommu_region_sem);
  error:
89a864027   David Howells   NOMMU: Don't pass...
1346
1347
  	if (region->vm_file)
  		fput(region->vm_file);
8feae1311   David Howells   NOMMU: Make VMAs ...
1348
  	kmem_cache_free(vm_region_jar, region);
89a864027   David Howells   NOMMU: Don't pass...
1349
1350
  	if (vma->vm_file)
  		fput(vma->vm_file);
8feae1311   David Howells   NOMMU: Make VMAs ...
1351
  	kmem_cache_free(vm_area_cachep, vma);
8feae1311   David Howells   NOMMU: Make VMAs ...
1352
1353
1354
1355
  	return ret;
  
  sharing_violation:
  	up_write(&nommu_region_sem);
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1356
1357
  	pr_warn("Attempt to share mismatched mappings
  ");
8feae1311   David Howells   NOMMU: Make VMAs ...
1358
1359
  	ret = -EINVAL;
  	goto error;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1360

8feae1311   David Howells   NOMMU: Make VMAs ...
1361
1362
  error_getting_vma:
  	kmem_cache_free(vm_region_jar, region);
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1363
1364
1365
  	pr_warn("Allocation of vma for %lu byte allocation from process %d failed
  ",
  			len, current->pid);
7bf02ea22   David Rientjes   arch, mm: filter ...
1366
  	show_free_areas(0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1367
  	return -ENOMEM;
8feae1311   David Howells   NOMMU: Make VMAs ...
1368
  error_getting_region:
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1369
1370
1371
  	pr_warn("Allocation of vm region for %lu byte allocation from process %d failed
  ",
  			len, current->pid);
7bf02ea22   David Rientjes   arch, mm: filter ...
1372
  	show_free_areas(0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1373
1374
  	return -ENOMEM;
  }
6be5ceb02   Linus Torvalds   VM: add "vm_mmap(...
1375

66f0dc481   Hugh Dickins   mm: move sys_mmap...
1376
1377
1378
1379
1380
1381
  SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
  		unsigned long, prot, unsigned long, flags,
  		unsigned long, fd, unsigned long, pgoff)
  {
  	struct file *file = NULL;
  	unsigned long retval = -EBADF;
120a795da   Al Viro   audit mmap
1382
  	audit_mmap_fd(fd, flags);
66f0dc481   Hugh Dickins   mm: move sys_mmap...
1383
1384
1385
1386
1387
1388
1389
  	if (!(flags & MAP_ANONYMOUS)) {
  		file = fget(fd);
  		if (!file)
  			goto out;
  	}
  
  	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
ad1ed2937   Greg Ungerer   nommu: fix compil...
1390
  	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
66f0dc481   Hugh Dickins   mm: move sys_mmap...
1391
1392
1393
1394
1395
1396
  
  	if (file)
  		fput(file);
  out:
  	return retval;
  }
a4679373c   Christoph Hellwig   Add generic sys_o...
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
  #ifdef __ARCH_WANT_SYS_OLD_MMAP
  struct mmap_arg_struct {
  	unsigned long addr;
  	unsigned long len;
  	unsigned long prot;
  	unsigned long flags;
  	unsigned long fd;
  	unsigned long offset;
  };
  
  SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
  {
  	struct mmap_arg_struct a;
  
  	if (copy_from_user(&a, arg, sizeof(a)))
  		return -EFAULT;
1824cb753   Alexander Kuleshov   mm/nommu: use off...
1413
  	if (offset_in_page(a.offset))
a4679373c   Christoph Hellwig   Add generic sys_o...
1414
1415
1416
1417
1418
1419
  		return -EINVAL;
  
  	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
  			      a.offset >> PAGE_SHIFT);
  }
  #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1420
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
1421
1422
   * split a vma into two pieces at address 'addr', a new vma is allocated either
   * for the first part or the tail.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1423
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
1424
1425
  int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
  	      unsigned long addr, int new_below)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1426
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
1427
1428
1429
  	struct vm_area_struct *new;
  	struct vm_region *region;
  	unsigned long npages;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1430

779c10232   David Howells   nommu: remove a s...
1431
1432
1433
  	/* we're only permitted to split anonymous regions (these should have
  	 * only a single usage on the region) */
  	if (vma->vm_file)
8feae1311   David Howells   NOMMU: Make VMAs ...
1434
  		return -ENOMEM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1435

8feae1311   David Howells   NOMMU: Make VMAs ...
1436
1437
  	if (mm->map_count >= sysctl_max_map_count)
  		return -ENOMEM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1438

8feae1311   David Howells   NOMMU: Make VMAs ...
1439
1440
1441
  	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
  	if (!region)
  		return -ENOMEM;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1442

8feae1311   David Howells   NOMMU: Make VMAs ...
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
  	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  	if (!new) {
  		kmem_cache_free(vm_region_jar, region);
  		return -ENOMEM;
  	}
  
  	/* most fields are the same, copy all, and then fixup */
  	*new = *vma;
  	*region = *vma->vm_region;
  	new->vm_region = region;
  
  	npages = (addr - vma->vm_start) >> PAGE_SHIFT;
  
  	if (new_below) {
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1457
  		region->vm_top = region->vm_end = new->vm_end = addr;
8feae1311   David Howells   NOMMU: Make VMAs ...
1458
1459
1460
  	} else {
  		region->vm_start = new->vm_start = addr;
  		region->vm_pgoff = new->vm_pgoff += npages;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1461
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
  
  	if (new->vm_ops && new->vm_ops->open)
  		new->vm_ops->open(new);
  
  	delete_vma_from_mm(vma);
  	down_write(&nommu_region_sem);
  	delete_nommu_region(vma->vm_region);
  	if (new_below) {
  		vma->vm_region->vm_start = vma->vm_start = addr;
  		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
  	} else {
  		vma->vm_region->vm_end = vma->vm_end = addr;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1474
  		vma->vm_region->vm_top = addr;
8feae1311   David Howells   NOMMU: Make VMAs ...
1475
1476
1477
1478
1479
1480
1481
  	}
  	add_nommu_region(vma->vm_region);
  	add_nommu_region(new->vm_region);
  	up_write(&nommu_region_sem);
  	add_vma_to_mm(mm, vma);
  	add_vma_to_mm(mm, new);
  	return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1482
  }
3034097a5   David Howells   [PATCH] NOMMU: Or...
1483
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
1484
1485
   * shrink a VMA by removing the specified chunk from either the beginning or
   * the end
3034097a5   David Howells   [PATCH] NOMMU: Or...
1486
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
1487
1488
1489
  static int shrink_vma(struct mm_struct *mm,
  		      struct vm_area_struct *vma,
  		      unsigned long from, unsigned long to)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1490
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
1491
  	struct vm_region *region;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1492

8feae1311   David Howells   NOMMU: Make VMAs ...
1493
1494
1495
1496
1497
1498
1499
1500
  	/* adjust the VMA's pointers, which may reposition it in the MM's tree
  	 * and list */
  	delete_vma_from_mm(vma);
  	if (from > vma->vm_start)
  		vma->vm_end = from;
  	else
  		vma->vm_start = to;
  	add_vma_to_mm(mm, vma);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1501

8feae1311   David Howells   NOMMU: Make VMAs ...
1502
1503
  	/* cut the backing region down to size */
  	region = vma->vm_region;
1e2ae599d   David Howells   nommu: struct vm_...
1504
  	BUG_ON(region->vm_usage != 1);
8feae1311   David Howells   NOMMU: Make VMAs ...
1505
1506
1507
  
  	down_write(&nommu_region_sem);
  	delete_nommu_region(region);
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1508
1509
1510
1511
  	if (from > region->vm_start) {
  		to = region->vm_top;
  		region->vm_top = region->vm_end = from;
  	} else {
8feae1311   David Howells   NOMMU: Make VMAs ...
1512
  		region->vm_start = to;
dd8632a12   Paul Mundt   NOMMU: Make mmap ...
1513
  	}
8feae1311   David Howells   NOMMU: Make VMAs ...
1514
1515
1516
1517
1518
1519
  	add_nommu_region(region);
  	up_write(&nommu_region_sem);
  
  	free_page_series(from, to);
  	return 0;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1520

8feae1311   David Howells   NOMMU: Make VMAs ...
1521
1522
1523
1524
1525
1526
1527
1528
  /*
   * release a mapping
   * - under NOMMU conditions the chunk to be unmapped must be backed by a single
   *   VMA, though it need not cover the whole VMA
   */
  int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
  {
  	struct vm_area_struct *vma;
f67d9b157   Bob Liu   nommu: add page a...
1529
  	unsigned long end;
8feae1311   David Howells   NOMMU: Make VMAs ...
1530
  	int ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1531

f67d9b157   Bob Liu   nommu: add page a...
1532
  	len = PAGE_ALIGN(len);
8feae1311   David Howells   NOMMU: Make VMAs ...
1533
1534
  	if (len == 0)
  		return -EINVAL;
365e9c87a   Hugh Dickins   [PATCH] mm: updat...
1535

f67d9b157   Bob Liu   nommu: add page a...
1536
  	end = start + len;
8feae1311   David Howells   NOMMU: Make VMAs ...
1537
1538
1539
  	/* find the first potentially overlapping VMA */
  	vma = find_vma(mm, start);
  	if (!vma) {
ac7149045   Choi Gi-yong   mm: fix 'ERROR: d...
1540
  		static int limit;
33e5d7697   David Howells   nommu: fix a numb...
1541
  		if (limit < 5) {
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1542
1543
1544
1545
  			pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx
  ",
  					current->pid, current->comm,
  					start, start + len - 1);
33e5d7697   David Howells   nommu: fix a numb...
1546
1547
  			limit++;
  		}
8feae1311   David Howells   NOMMU: Make VMAs ...
1548
1549
  		return -EINVAL;
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1550

8feae1311   David Howells   NOMMU: Make VMAs ...
1551
1552
1553
  	/* we're allowed to split an anonymous VMA but not a file-backed one */
  	if (vma->vm_file) {
  		do {
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1554
  			if (start > vma->vm_start)
8feae1311   David Howells   NOMMU: Make VMAs ...
1555
  				return -EINVAL;
8feae1311   David Howells   NOMMU: Make VMAs ...
1556
1557
  			if (end == vma->vm_end)
  				goto erase_whole_vma;
d75a310c4   Namhyung Kim   mm: nommu: check ...
1558
1559
  			vma = vma->vm_next;
  		} while (vma);
8feae1311   David Howells   NOMMU: Make VMAs ...
1560
1561
1562
1563
1564
  		return -EINVAL;
  	} else {
  		/* the chunk must be a subset of the VMA found */
  		if (start == vma->vm_start && end == vma->vm_end)
  			goto erase_whole_vma;
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1565
  		if (start < vma->vm_start || end > vma->vm_end)
8feae1311   David Howells   NOMMU: Make VMAs ...
1566
  			return -EINVAL;
1824cb753   Alexander Kuleshov   mm/nommu: use off...
1567
  		if (offset_in_page(start))
8feae1311   David Howells   NOMMU: Make VMAs ...
1568
  			return -EINVAL;
1824cb753   Alexander Kuleshov   mm/nommu: use off...
1569
  		if (end != vma->vm_end && offset_in_page(end))
8feae1311   David Howells   NOMMU: Make VMAs ...
1570
  			return -EINVAL;
8feae1311   David Howells   NOMMU: Make VMAs ...
1571
1572
  		if (start != vma->vm_start && end != vma->vm_end) {
  			ret = split_vma(mm, vma, start, 1);
22cc877b3   Leon Romanovsky   mm: nommu: refact...
1573
  			if (ret < 0)
8feae1311   David Howells   NOMMU: Make VMAs ...
1574
  				return ret;
8feae1311   David Howells   NOMMU: Make VMAs ...
1575
1576
1577
  		}
  		return shrink_vma(mm, vma, start, end);
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1578

8feae1311   David Howells   NOMMU: Make VMAs ...
1579
1580
1581
  erase_whole_vma:
  	delete_vma_from_mm(vma);
  	delete_vma(mm, vma);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1582
1583
  	return 0;
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
1584
  EXPORT_SYMBOL(do_munmap);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1585

bfce281c2   Al Viro   kill mm argument ...
1586
  int vm_munmap(unsigned long addr, size_t len)
3034097a5   David Howells   [PATCH] NOMMU: Or...
1587
  {
bfce281c2   Al Viro   kill mm argument ...
1588
  	struct mm_struct *mm = current->mm;
3034097a5   David Howells   [PATCH] NOMMU: Or...
1589
  	int ret;
3034097a5   David Howells   [PATCH] NOMMU: Or...
1590
1591
1592
1593
1594
1595
  
  	down_write(&mm->mmap_sem);
  	ret = do_munmap(mm, addr, len);
  	up_write(&mm->mmap_sem);
  	return ret;
  }
a46ef99d8   Linus Torvalds   VM: add "vm_munma...
1596
1597
1598
1599
  EXPORT_SYMBOL(vm_munmap);
  
  SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
  {
bfce281c2   Al Viro   kill mm argument ...
1600
  	return vm_munmap(addr, len);
a46ef99d8   Linus Torvalds   VM: add "vm_munma...
1601
  }
3034097a5   David Howells   [PATCH] NOMMU: Or...
1602
1603
  
  /*
8feae1311   David Howells   NOMMU: Make VMAs ...
1604
   * release all the mappings made in a process's VM space
3034097a5   David Howells   [PATCH] NOMMU: Or...
1605
   */
8feae1311   David Howells   NOMMU: Make VMAs ...
1606
  void exit_mmap(struct mm_struct *mm)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1607
  {
8feae1311   David Howells   NOMMU: Make VMAs ...
1608
  	struct vm_area_struct *vma;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1609

8feae1311   David Howells   NOMMU: Make VMAs ...
1610
1611
  	if (!mm)
  		return;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1612

8feae1311   David Howells   NOMMU: Make VMAs ...
1613
  	mm->total_vm = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1614

8feae1311   David Howells   NOMMU: Make VMAs ...
1615
1616
1617
1618
  	while ((vma = mm->mmap)) {
  		mm->mmap = vma->vm_next;
  		delete_vma_from_mm(vma);
  		delete_vma(mm, vma);
04c349615   Steven J. Magnani   nommu: yield CPU ...
1619
  		cond_resched();
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1620
1621
  	}
  }
e4eb1ff61   Linus Torvalds   VM: add "vm_brk()...
1622
  unsigned long vm_brk(unsigned long addr, unsigned long len)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1623
1624
1625
1626
1627
  {
  	return -ENOMEM;
  }
  
  /*
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1628
1629
   * expand (or shrink) an existing mapping, potentially moving it at the same
   * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1630
   *
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1631
   * under NOMMU conditions, we only permit changing a mapping's size, and only
8feae1311   David Howells   NOMMU: Make VMAs ...
1632
1633
   * as long as it stays within the region allocated by do_mmap_private() and the
   * block is not shareable
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1634
   *
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1635
   * MREMAP_FIXED is not supported under NOMMU conditions
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1636
   */
4b377bab2   Al Viro   make do_mremap() ...
1637
  static unsigned long do_mremap(unsigned long addr,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1638
1639
1640
  			unsigned long old_len, unsigned long new_len,
  			unsigned long flags, unsigned long new_addr)
  {
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1641
  	struct vm_area_struct *vma;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1642
1643
  
  	/* insanity checks first */
f67d9b157   Bob Liu   nommu: add page a...
1644
1645
  	old_len = PAGE_ALIGN(old_len);
  	new_len = PAGE_ALIGN(new_len);
8feae1311   David Howells   NOMMU: Make VMAs ...
1646
  	if (old_len == 0 || new_len == 0)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1647
  		return (unsigned long) -EINVAL;
1824cb753   Alexander Kuleshov   mm/nommu: use off...
1648
  	if (offset_in_page(addr))
8feae1311   David Howells   NOMMU: Make VMAs ...
1649
  		return -EINVAL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1650
1651
  	if (flags & MREMAP_FIXED && new_addr != addr)
  		return (unsigned long) -EINVAL;
8feae1311   David Howells   NOMMU: Make VMAs ...
1652
  	vma = find_vma_exact(current->mm, addr, old_len);
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1653
1654
  	if (!vma)
  		return (unsigned long) -EINVAL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1655

6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1656
  	if (vma->vm_end != vma->vm_start + old_len)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1657
  		return (unsigned long) -EFAULT;
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1658
  	if (vma->vm_flags & VM_MAYSHARE)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1659
  		return (unsigned long) -EPERM;
8feae1311   David Howells   NOMMU: Make VMAs ...
1660
  	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1661
1662
1663
  		return (unsigned long) -ENOMEM;
  
  	/* all checks complete - do it */
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1664
  	vma->vm_end = vma->vm_start + new_len;
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1665
1666
  	return vma->vm_start;
  }
6a6160a7b   Heiko Carstens   [CVE-2009-0029] S...
1667
1668
1669
  SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
  		unsigned long, new_len, unsigned long, flags,
  		unsigned long, new_addr)
6fa5f80bc   David Howells   [PATCH] NOMMU: Ma...
1670
1671
1672
1673
1674
1675
1676
  {
  	unsigned long ret;
  
  	down_write(&current->mm->mmap_sem);
  	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
  	up_write(&current->mm->mmap_sem);
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1677
  }
240aadeed   Michel Lespinasse   mm: accelerate mm...
1678
1679
1680
  struct page *follow_page_mask(struct vm_area_struct *vma,
  			      unsigned long address, unsigned int flags,
  			      unsigned int *page_mask)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1681
  {
240aadeed   Michel Lespinasse   mm: accelerate mm...
1682
  	*page_mask = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1683
1684
  	return NULL;
  }
8f3b1327a   Bob Liu   mm/nommu.c: fix r...
1685
1686
  int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  		unsigned long pfn, unsigned long size, pgprot_t prot)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1687
  {
8f3b1327a   Bob Liu   mm/nommu.c: fix r...
1688
1689
  	if (addr != (pfn << PAGE_SHIFT))
  		return -EINVAL;
314e51b98   Konstantin Khlebnikov   mm: kill vma flag...
1690
  	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
66aa2b4b1   Greg Ungerer   [PATCH] uclinux: ...
1691
  	return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1692
  }
22c4af409   Luke Yang   [PATCH] nommu: ex...
1693
  EXPORT_SYMBOL(remap_pfn_range);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1694

3c0b9de6d   Linus Torvalds   vm: add no-mmu vm...
1695
1696
1697
1698
1699
1700
1701
1702
1703
  int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
  {
  	unsigned long pfn = start >> PAGE_SHIFT;
  	unsigned long vm_len = vma->vm_end - vma->vm_start;
  
  	pfn += vma->vm_pgoff;
  	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
  }
  EXPORT_SYMBOL(vm_iomap_memory);
f905bc447   Paul Mundt   nommu: add new vm...
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
  int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  			unsigned long pgoff)
  {
  	unsigned int size = vma->vm_end - vma->vm_start;
  
  	if (!(vma->vm_flags & VM_USERMAP))
  		return -EINVAL;
  
  	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
  	vma->vm_end = vma->vm_start + size;
  
  	return 0;
  }
  EXPORT_SYMBOL(remap_vmalloc_range);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1718
1719
1720
1721
1722
  unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
  	unsigned long len, unsigned long pgoff, unsigned long flags)
  {
  	return -ENOMEM;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1723
1724
1725
1726
1727
  void unmap_mapping_range(struct address_space *mapping,
  			 loff_t const holebegin, loff_t const holelen,
  			 int even_cows)
  {
  }
22c4af409   Luke Yang   [PATCH] nommu: ex...
1728
  EXPORT_SYMBOL(unmap_mapping_range);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1729

d0217ac04   Nick Piggin   mm: fault feedbac...
1730
  int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
b0e15190e   David Howells   [PATCH] NOMMU: Ma...
1731
1732
  {
  	BUG();
d0217ac04   Nick Piggin   mm: fault feedbac...
1733
  	return 0;
b0e15190e   David Howells   [PATCH] NOMMU: Ma...
1734
  }
b50731732   Paul Mundt   nommu: vmalloc_32...
1735
  EXPORT_SYMBOL(filemap_fault);
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1736

f1820361f   Kirill A. Shutemov   mm: implement ->m...
1737
1738
1739
1740
1741
  void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
  {
  	BUG();
  }
  EXPORT_SYMBOL(filemap_map_pages);
f55f199b7   Mike Frysinger   NOMMU: implement ...
1742
1743
  static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
  		unsigned long addr, void *buf, int len, int write)
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1744
  {
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1745
  	struct vm_area_struct *vma;
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1746
1747
1748
1749
  
  	down_read(&mm->mmap_sem);
  
  	/* the access must start within one of the target process's mappings */
0159b141d   David Howells   [PATCH] NOMMU: Us...
1750
1751
  	vma = find_vma(mm, addr);
  	if (vma) {
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1752
1753
1754
1755
1756
  		/* don't overrun this mapping */
  		if (addr + len >= vma->vm_end)
  			len = vma->vm_end - addr;
  
  		/* only read or write mappings where it is permitted */
d00c7b993   David Howells   [PATCH] NOMMU: Pe...
1757
  		if (write && vma->vm_flags & VM_MAYWRITE)
7959722b9   Jie Zhang   NOMMU: Use copy_*...
1758
1759
  			copy_to_user_page(vma, NULL, addr,
  					 (void *) addr, buf, len);
d00c7b993   David Howells   [PATCH] NOMMU: Pe...
1760
  		else if (!write && vma->vm_flags & VM_MAYREAD)
7959722b9   Jie Zhang   NOMMU: Use copy_*...
1761
1762
  			copy_from_user_page(vma, NULL, addr,
  					    buf, (void *) addr, len);
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1763
1764
1765
1766
1767
1768
1769
  		else
  			len = 0;
  	} else {
  		len = 0;
  	}
  
  	up_read(&mm->mmap_sem);
f55f199b7   Mike Frysinger   NOMMU: implement ...
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
  
  	return len;
  }
  
  /**
   * @access_remote_vm - access another process' address space
   * @mm:		the mm_struct of the target address space
   * @addr:	start address to access
   * @buf:	source or destination buffer
   * @len:	number of bytes to transfer
   * @write:	whether the access is a write
   *
   * The caller must hold a reference on @mm.
   */
  int access_remote_vm(struct mm_struct *mm, unsigned long addr,
  		void *buf, int len, int write)
  {
  	return __access_remote_vm(NULL, mm, addr, buf, len, write);
  }
  
  /*
   * Access another process' address space.
   * - source/target buffer must be kernel space
   */
  int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
  {
  	struct mm_struct *mm;
  
  	if (addr + len < addr)
  		return 0;
  
  	mm = get_task_mm(tsk);
  	if (!mm)
  		return 0;
  
  	len = __access_remote_vm(tsk, mm, addr, buf, len, write);
0ec76a110   David Howells   [PATCH] NOMMU: Ch...
1806
1807
1808
  	mmput(mm);
  	return len;
  }
7e6608724   David Howells   nommu: fix shared...
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
  
  /**
   * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
   * @inode: The inode to check
   * @size: The current filesize of the inode
   * @newsize: The proposed filesize of the inode
   *
   * Check the shared mappings on an inode on behalf of a shrinking truncate to
   * make sure that that any outstanding VMAs aren't broken and then shrink the
   * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
   * automatically grant mappings that are too large.
   */
  int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
  				size_t newsize)
  {
  	struct vm_area_struct *vma;
7e6608724   David Howells   nommu: fix shared...
1825
1826
1827
1828
1829
1830
1831
1832
  	struct vm_region *region;
  	pgoff_t low, high;
  	size_t r_size, r_top;
  
  	low = newsize >> PAGE_SHIFT;
  	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
  
  	down_write(&nommu_region_sem);
1acf2e040   Davidlohr Bueso   mm/nommu: share t...
1833
  	i_mmap_lock_read(inode->i_mapping);
7e6608724   David Howells   nommu: fix shared...
1834
1835
  
  	/* search for VMAs that fall within the dead zone */
6b2dbba8b   Michel Lespinasse   mm: replace vma p...
1836
  	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
7e6608724   David Howells   nommu: fix shared...
1837
1838
1839
  		/* found one - only interested if it's shared out of the page
  		 * cache */
  		if (vma->vm_flags & VM_SHARED) {
1acf2e040   Davidlohr Bueso   mm/nommu: share t...
1840
  			i_mmap_unlock_read(inode->i_mapping);
7e6608724   David Howells   nommu: fix shared...
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
  			up_write(&nommu_region_sem);
  			return -ETXTBSY; /* not quite true, but near enough */
  		}
  	}
  
  	/* reduce any regions that overlap the dead zone - if in existence,
  	 * these will be pointed to by VMAs that don't overlap the dead zone
  	 *
  	 * we don't check for any regions that start beyond the EOF as there
  	 * shouldn't be any
  	 */
1acf2e040   Davidlohr Bueso   mm/nommu: share t...
1852
  	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
7e6608724   David Howells   nommu: fix shared...
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
  		if (!(vma->vm_flags & VM_SHARED))
  			continue;
  
  		region = vma->vm_region;
  		r_size = region->vm_top - region->vm_start;
  		r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
  
  		if (r_top > newsize) {
  			region->vm_top -= r_top - newsize;
  			if (region->vm_end > region->vm_top)
  				region->vm_end = region->vm_top;
  		}
  	}
1acf2e040   Davidlohr Bueso   mm/nommu: share t...
1866
  	i_mmap_unlock_read(inode->i_mapping);
7e6608724   David Howells   nommu: fix shared...
1867
1868
1869
  	up_write(&nommu_region_sem);
  	return 0;
  }
c9b1d0981   Andrew Shewmaker   mm: limit growth ...
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
  
  /*
   * Initialise sysctl_user_reserve_kbytes.
   *
   * This is intended to prevent a user from starting a single memory hogging
   * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
   * mode.
   *
   * The default value is min(3% of free memory, 128MB)
   * 128MB is enough to recover with sshd/login, bash, and top/kill.
   */
  static int __meminit init_user_reserve(void)
  {
  	unsigned long free_kbytes;
  
  	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
  
  	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
  	return 0;
  }
a4bc6fc79   Paul Gortmaker   mm: replace modul...
1890
  subsys_initcall(init_user_reserve);
4eeab4f55   Andrew Shewmaker   mm: replace hardc...
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
  
  /*
   * Initialise sysctl_admin_reserve_kbytes.
   *
   * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
   * to log in and kill a memory hogging process.
   *
   * Systems with more than 256MB will reserve 8MB, enough to recover
   * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
   * only reserve 3% of free pages by default.
   */
  static int __meminit init_admin_reserve(void)
  {
  	unsigned long free_kbytes;
  
  	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
  
  	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
  	return 0;
  }
a4bc6fc79   Paul Gortmaker   mm: replace modul...
1911
  subsys_initcall(init_admin_reserve);