Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit e1ca7788dec6773b1a2bce51b7141948f2b8bccf

Authored by Dave Young 2010-10-27 05:22:06 +0800

Committed by Linus Torvalds 2010-10-27 07:52:10 +0800

mm: add vzalloc() and vzalloc_node() helpers

Add vzalloc() and vzalloc_node() to encapsulate the
vmalloc-then-memset-zero operation.

Use __GFP_ZERO to zero fill the allocated memory.

Signed-off-by: Dave Young <hidave.darkstar@gmail.com>
Cc: Christoph Lameter <cl@linux-foundation.org>
Acked-by: Greg Ungerer <gerg@snapgear.com>
Cc: David Howells <dhowells@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 3 changed files with 94 additions and 3 deletions Inline Diff

include/linux/vmalloc.h
mm/nommu.c
mm/vmalloc.c

include/linux/vmalloc.h

Diff comments View file @ e1ca778

 #ifndef _LINUX_VMALLOC_H
 #define _LINUX_VMALLOC_H
 #include <linux/spinlock.h>
 #include <linux/init.h>
 #include <asm/page.h>		/* pgprot_t */
 struct vm_area_struct;		/* vma defining user mapping in mm_types.h */
 extern bool vmap_lazy_unmap;
 /* bits in flags of vmalloc's vm_struct below */
 #define VM_IOREMAP	0x00000001	/* ioremap() and friends */
 #define VM_ALLOC	0x00000002	/* vmalloc() */
 #define VM_MAP		0x00000004	/* vmap()ed pages */
 #define VM_USERMAP	0x00000008	/* suitable for remap_vmalloc_range */
 #define VM_VPAGES	0x00000010	/* buffer for pages was vmalloc'ed */
 /* bits [20..32] reserved for arch specific ioremap internals */
 /*
  * Maximum alignment for ioremap() regions.
  * Can be overriden by arch-specific value.
  */
 #ifndef IOREMAP_MAX_ORDER
 #define IOREMAP_MAX_ORDER	(7 + PAGE_SHIFT)	/* 128 pages */
 #endif
 struct vm_struct {
 	struct vm_struct	*next;
 	void			*addr;
 	unsigned long		size;
 	unsigned long		flags;
 	struct page		**pages;
 	unsigned int		nr_pages;
 	phys_addr_t		phys_addr;
 	void			*caller;
 };
 /*
  *	Highlevel APIs for driver use
  */
 extern void vm_unmap_ram(const void *mem, unsigned int count);
 extern void *vm_map_ram(struct page **pages, unsigned int count,
 				int node, pgprot_t prot);
 extern void vm_unmap_aliases(void);
 #ifdef CONFIG_MMU
 extern void __init vmalloc_init(void);
 #else
 static inline void vmalloc_init(void)
 {
 }
 #endif
 extern void *vmalloc(unsigned long size);
+extern void *vzalloc(unsigned long size);
 extern void *vmalloc_user(unsigned long size);
 extern void *vmalloc_node(unsigned long size, int node);
+extern void *vzalloc_node(unsigned long size, int node);
 extern void *vmalloc_exec(unsigned long size);
 extern void *vmalloc_32(unsigned long size);
 extern void *vmalloc_32_user(unsigned long size);
 extern void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot);
 extern void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask,
 				pgprot_t prot);
 extern void vfree(const void *addr);
 extern void *vmap(struct page **pages, unsigned int count,
 			unsigned long flags, pgprot_t prot);
 extern void vunmap(const void *addr);
 extern int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
 							unsigned long pgoff);
 void vmalloc_sync_all(void);
 /*
  *	Lowlevel-APIs (not for driver use!)
  */
 static inline size_t get_vm_area_size(const struct vm_struct *area)
 {
 	/* return actual size without guard page */
 	return area->size - PAGE_SIZE;
 }
 extern struct vm_struct *get_vm_area(unsigned long size, unsigned long flags);
 extern struct vm_struct *get_vm_area_caller(unsigned long size,
 					unsigned long flags, void *caller);
 extern struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 					unsigned long start, unsigned long end);
 extern struct vm_struct *__get_vm_area_caller(unsigned long size,
 					unsigned long flags,
 					unsigned long start, unsigned long end,
 					void *caller);
 extern struct vm_struct *get_vm_area_node(unsigned long size,
 					  unsigned long flags, int node,
 					  gfp_t gfp_mask);
 extern struct vm_struct *remove_vm_area(const void *addr);
 extern int map_vm_area(struct vm_struct *area, pgprot_t prot,
 			struct page ***pages);
 extern int map_kernel_range_noflush(unsigned long start, unsigned long size,
 				    pgprot_t prot, struct page **pages);
 extern void unmap_kernel_range_noflush(unsigned long addr, unsigned long size);
 extern void unmap_kernel_range(unsigned long addr, unsigned long size);
 /* Allocate/destroy a 'vmalloc' VM area. */
 extern struct vm_struct *alloc_vm_area(size_t size);
 extern void free_vm_area(struct vm_struct *area);
 /* for /dev/kmem */
 extern long vread(char *buf, char *addr, unsigned long count);
 extern long vwrite(char *buf, char *addr, unsigned long count);
 /*
  *	Internals.  Dont't use..
  */
 extern rwlock_t vmlist_lock;
 extern struct vm_struct *vmlist;
 extern __init void vm_area_register_early(struct vm_struct *vm, size_t align);
 #ifdef CONFIG_SMP
 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 				     const size_t *sizes, int nr_vms,
 				     size_t align, gfp_t gfp_mask);
 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms);
 #endif
 #endif /* _LINUX_VMALLOC_H */

mm/nommu.c

Diff comments View file @ e1ca778

1	/*	1	/*
2	* linux/mm/nommu.c	2	* linux/mm/nommu.c
3	*	3	*
4	* Replacement code for mm functions to support CPU's that don't	4	* Replacement code for mm functions to support CPU's that don't
5	* have any form of memory management unit (thus no virtual memory).	5	* have any form of memory management unit (thus no virtual memory).
6	*	6	*
7	* See Documentation/nommu-mmap.txt	7	* See Documentation/nommu-mmap.txt
8	*	8	*
9	* Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>	9	* Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
10	* Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>	10	* Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
11	* Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>	11	* Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
12	* Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>	12	* Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com>
13	* Copyright (c) 2007-2009 Paul Mundt <lethal@linux-sh.org>	13	* Copyright (c) 2007-2009 Paul Mundt <lethal@linux-sh.org>
14	*/	14	*/
15		15
16	#include <linux/module.h>	16	#include <linux/module.h>
17	#include <linux/mm.h>	17	#include <linux/mm.h>
18	#include <linux/mman.h>	18	#include <linux/mman.h>
19	#include <linux/swap.h>	19	#include <linux/swap.h>
20	#include <linux/file.h>	20	#include <linux/file.h>
21	#include <linux/highmem.h>	21	#include <linux/highmem.h>
22	#include <linux/pagemap.h>	22	#include <linux/pagemap.h>
23	#include <linux/slab.h>	23	#include <linux/slab.h>
24	#include <linux/vmalloc.h>	24	#include <linux/vmalloc.h>
25	#include <linux/tracehook.h>	25	#include <linux/tracehook.h>
26	#include <linux/blkdev.h>	26	#include <linux/blkdev.h>
27	#include <linux/backing-dev.h>	27	#include <linux/backing-dev.h>
28	#include <linux/mount.h>	28	#include <linux/mount.h>
29	#include <linux/personality.h>	29	#include <linux/personality.h>
30	#include <linux/security.h>	30	#include <linux/security.h>
31	#include <linux/syscalls.h>	31	#include <linux/syscalls.h>
32		32
33	#include <asm/uaccess.h>	33	#include <asm/uaccess.h>
34	#include <asm/tlb.h>	34	#include <asm/tlb.h>
35	#include <asm/tlbflush.h>	35	#include <asm/tlbflush.h>
36	#include <asm/mmu_context.h>	36	#include <asm/mmu_context.h>
37	#include "internal.h"	37	#include "internal.h"
38		38
39	#if 0	39	#if 0
40	#define kenter(FMT, ...) \	40	#define kenter(FMT, ...) \
41	printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)	41	printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
42	#define kleave(FMT, ...) \	42	#define kleave(FMT, ...) \
43	printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)	43	printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
44	#define kdebug(FMT, ...) \	44	#define kdebug(FMT, ...) \
45	printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)	45	printk(KERN_DEBUG "xxx" FMT"yyy\n", ##__VA_ARGS__)
46	#else	46	#else
47	#define kenter(FMT, ...) \	47	#define kenter(FMT, ...) \
48	no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)	48	no_printk(KERN_DEBUG "==> %s("FMT")\n", __func__, ##__VA_ARGS__)
49	#define kleave(FMT, ...) \	49	#define kleave(FMT, ...) \
50	no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)	50	no_printk(KERN_DEBUG "<== %s()"FMT"\n", __func__, ##__VA_ARGS__)
51	#define kdebug(FMT, ...) \	51	#define kdebug(FMT, ...) \
52	no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)	52	no_printk(KERN_DEBUG FMT"\n", ##__VA_ARGS__)
53	#endif	53	#endif
54		54
55	void *high_memory;	55	void *high_memory;
56	struct page *mem_map;	56	struct page *mem_map;
57	unsigned long max_mapnr;	57	unsigned long max_mapnr;
58	unsigned long num_physpages;	58	unsigned long num_physpages;
59	unsigned long highest_memmap_pfn;	59	unsigned long highest_memmap_pfn;
60	struct percpu_counter vm_committed_as;	60	struct percpu_counter vm_committed_as;
61	int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */	61	int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
62	int sysctl_overcommit_ratio = 50; /* default is 50% */	62	int sysctl_overcommit_ratio = 50; /* default is 50% */
63	int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;	63	int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
64	int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;	64	int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
65	int heap_stack_gap = 0;	65	int heap_stack_gap = 0;
66		66
67	atomic_long_t mmap_pages_allocated;	67	atomic_long_t mmap_pages_allocated;
68		68
69	EXPORT_SYMBOL(mem_map);	69	EXPORT_SYMBOL(mem_map);
70	EXPORT_SYMBOL(num_physpages);	70	EXPORT_SYMBOL(num_physpages);
71		71
72	/* list of mapped, potentially shareable regions */	72	/* list of mapped, potentially shareable regions */
73	static struct kmem_cache *vm_region_jar;	73	static struct kmem_cache *vm_region_jar;
74	struct rb_root nommu_region_tree = RB_ROOT;	74	struct rb_root nommu_region_tree = RB_ROOT;
75	DECLARE_RWSEM(nommu_region_sem);	75	DECLARE_RWSEM(nommu_region_sem);
76		76
77	const struct vm_operations_struct generic_file_vm_ops = {	77	const struct vm_operations_struct generic_file_vm_ops = {
78	};	78	};
79		79
80	/*	80	/*
81	* Return the total memory allocated for this pointer, not	81	* Return the total memory allocated for this pointer, not
82	* just what the caller asked for.	82	* just what the caller asked for.
83	*	83	*
84	* Doesn't have to be accurate, i.e. may have races.	84	* Doesn't have to be accurate, i.e. may have races.
85	*/	85	*/
86	unsigned int kobjsize(const void *objp)	86	unsigned int kobjsize(const void *objp)
87	{	87	{
88	struct page *page;	88	struct page *page;
89		89
90	/*	90	/*
91	* If the object we have should not have ksize performed on it,	91	* If the object we have should not have ksize performed on it,
92	* return size of 0	92	* return size of 0
93	*/	93	*/
94	if (!objp \|\| !virt_addr_valid(objp))	94	if (!objp \|\| !virt_addr_valid(objp))
95	return 0;	95	return 0;
96		96
97	page = virt_to_head_page(objp);	97	page = virt_to_head_page(objp);
98		98
99	/*	99	/*
100	* If the allocator sets PageSlab, we know the pointer came from	100	* If the allocator sets PageSlab, we know the pointer came from
101	* kmalloc().	101	* kmalloc().
102	*/	102	*/
103	if (PageSlab(page))	103	if (PageSlab(page))
104	return ksize(objp);	104	return ksize(objp);
105		105
106	/*	106	/*
107	* If it's not a compound page, see if we have a matching VMA	107	* If it's not a compound page, see if we have a matching VMA
108	* region. This test is intentionally done in reverse order,	108	* region. This test is intentionally done in reverse order,
109	* so if there's no VMA, we still fall through and hand back	109	* so if there's no VMA, we still fall through and hand back
110	* PAGE_SIZE for 0-order pages.	110	* PAGE_SIZE for 0-order pages.
111	*/	111	*/
112	if (!PageCompound(page)) {	112	if (!PageCompound(page)) {
113	struct vm_area_struct *vma;	113	struct vm_area_struct *vma;
114		114
115	vma = find_vma(current->mm, (unsigned long)objp);	115	vma = find_vma(current->mm, (unsigned long)objp);
116	if (vma)	116	if (vma)
117	return vma->vm_end - vma->vm_start;	117	return vma->vm_end - vma->vm_start;
118	}	118	}
119		119
120	/*	120	/*
121	* The ksize() function is only guaranteed to work for pointers	121	* The ksize() function is only guaranteed to work for pointers
122	* returned by kmalloc(). So handle arbitrary pointers here.	122	* returned by kmalloc(). So handle arbitrary pointers here.
123	*/	123	*/
124	return PAGE_SIZE << compound_order(page);	124	return PAGE_SIZE << compound_order(page);
125	}	125	}
126		126
127	int __get_user_pages(struct task_struct tsk, struct mm_struct mm,	127	int __get_user_pages(struct task_struct tsk, struct mm_struct mm,
128	unsigned long start, int nr_pages, unsigned int foll_flags,	128	unsigned long start, int nr_pages, unsigned int foll_flags,
129	struct page pages, struct vm_area_struct vmas)	129	struct page pages, struct vm_area_struct vmas)
130	{	130	{
131	struct vm_area_struct *vma;	131	struct vm_area_struct *vma;
132	unsigned long vm_flags;	132	unsigned long vm_flags;
133	int i;	133	int i;
134		134
135	/* calculate required read or write permissions.	135	/* calculate required read or write permissions.
136	* If FOLL_FORCE is set, we only require the "MAY" flags.	136	* If FOLL_FORCE is set, we only require the "MAY" flags.
137	*/	137	*/
138	vm_flags = (foll_flags & FOLL_WRITE) ?	138	vm_flags = (foll_flags & FOLL_WRITE) ?
139	(VM_WRITE \| VM_MAYWRITE) : (VM_READ \| VM_MAYREAD);	139	(VM_WRITE \| VM_MAYWRITE) : (VM_READ \| VM_MAYREAD);
140	vm_flags &= (foll_flags & FOLL_FORCE) ?	140	vm_flags &= (foll_flags & FOLL_FORCE) ?
141	(VM_MAYREAD \| VM_MAYWRITE) : (VM_READ \| VM_WRITE);	141	(VM_MAYREAD \| VM_MAYWRITE) : (VM_READ \| VM_WRITE);
142		142
143	for (i = 0; i < nr_pages; i++) {	143	for (i = 0; i < nr_pages; i++) {
144	vma = find_vma(mm, start);	144	vma = find_vma(mm, start);
145	if (!vma)	145	if (!vma)
146	goto finish_or_fault;	146	goto finish_or_fault;
147		147
148	/* protect what we can, including chardevs */	148	/* protect what we can, including chardevs */
149	if ((vma->vm_flags & (VM_IO \| VM_PFNMAP)) \|\|	149	if ((vma->vm_flags & (VM_IO \| VM_PFNMAP)) \|\|
150	!(vm_flags & vma->vm_flags))	150	!(vm_flags & vma->vm_flags))
151	goto finish_or_fault;	151	goto finish_or_fault;
152		152
153	if (pages) {	153	if (pages) {
154	pages[i] = virt_to_page(start);	154	pages[i] = virt_to_page(start);
155	if (pages[i])	155	if (pages[i])
156	page_cache_get(pages[i]);	156	page_cache_get(pages[i]);
157	}	157	}
158	if (vmas)	158	if (vmas)
159	vmas[i] = vma;	159	vmas[i] = vma;
160	start = (start + PAGE_SIZE) & PAGE_MASK;	160	start = (start + PAGE_SIZE) & PAGE_MASK;
161	}	161	}
162		162
163	return i;	163	return i;
164		164
165	finish_or_fault:	165	finish_or_fault:
166	return i ? : -EFAULT;	166	return i ? : -EFAULT;
167	}	167	}
168		168
169	/*	169	/*
170	* get a list of pages in an address range belonging to the specified process	170	* get a list of pages in an address range belonging to the specified process
171	* and indicate the VMA that covers each page	171	* and indicate the VMA that covers each page
172	* - this is potentially dodgy as we may end incrementing the page count of a	172	* - this is potentially dodgy as we may end incrementing the page count of a
173	* slab page or a secondary page from a compound page	173	* slab page or a secondary page from a compound page
174	* - don't permit access to VMAs that don't support it, such as I/O mappings	174	* - don't permit access to VMAs that don't support it, such as I/O mappings
175	*/	175	*/
176	int get_user_pages(struct task_struct tsk, struct mm_struct mm,	176	int get_user_pages(struct task_struct tsk, struct mm_struct mm,
177	unsigned long start, int nr_pages, int write, int force,	177	unsigned long start, int nr_pages, int write, int force,
178	struct page pages, struct vm_area_struct vmas)	178	struct page pages, struct vm_area_struct vmas)
179	{	179	{
180	int flags = 0;	180	int flags = 0;
181		181
182	if (write)	182	if (write)
183	flags \|= FOLL_WRITE;	183	flags \|= FOLL_WRITE;
184	if (force)	184	if (force)
185	flags \|= FOLL_FORCE;	185	flags \|= FOLL_FORCE;
186		186
187	return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas);	187	return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
188	}	188	}
189	EXPORT_SYMBOL(get_user_pages);	189	EXPORT_SYMBOL(get_user_pages);
190		190
191	/**	191	/**
192	* follow_pfn - look up PFN at a user virtual address	192	* follow_pfn - look up PFN at a user virtual address
193	* @vma: memory mapping	193	* @vma: memory mapping
194	* @address: user virtual address	194	* @address: user virtual address
195	* @pfn: location to store found PFN	195	* @pfn: location to store found PFN
196	*	196	*
197	* Only IO mappings and raw PFN mappings are allowed.	197	* Only IO mappings and raw PFN mappings are allowed.
198	*	198	*
199	* Returns zero and the pfn at @pfn on success, -ve otherwise.	199	* Returns zero and the pfn at @pfn on success, -ve otherwise.
200	*/	200	*/
201	int follow_pfn(struct vm_area_struct *vma, unsigned long address,	201	int follow_pfn(struct vm_area_struct *vma, unsigned long address,
202	unsigned long *pfn)	202	unsigned long *pfn)
203	{	203	{
204	if (!(vma->vm_flags & (VM_IO \| VM_PFNMAP)))	204	if (!(vma->vm_flags & (VM_IO \| VM_PFNMAP)))
205	return -EINVAL;	205	return -EINVAL;
206		206
207	*pfn = address >> PAGE_SHIFT;	207	*pfn = address >> PAGE_SHIFT;
208	return 0;	208	return 0;
209	}	209	}
210	EXPORT_SYMBOL(follow_pfn);	210	EXPORT_SYMBOL(follow_pfn);
211		211
212	DEFINE_RWLOCK(vmlist_lock);	212	DEFINE_RWLOCK(vmlist_lock);
213	struct vm_struct *vmlist;	213	struct vm_struct *vmlist;
214		214
215	void vfree(const void *addr)	215	void vfree(const void *addr)
216	{	216	{
217	kfree(addr);	217	kfree(addr);
218	}	218	}
219	EXPORT_SYMBOL(vfree);	219	EXPORT_SYMBOL(vfree);
220		220
221	void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)	221	void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
222	{	222	{
223	/*	223	/*
224	* You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()	224	* You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
225	* returns only a logical address.	225	* returns only a logical address.
226	*/	226	*/
227	return kmalloc(size, (gfp_mask \| __GFP_COMP) & ~__GFP_HIGHMEM);	227	return kmalloc(size, (gfp_mask \| __GFP_COMP) & ~__GFP_HIGHMEM);
228	}	228	}
229	EXPORT_SYMBOL(__vmalloc);	229	EXPORT_SYMBOL(__vmalloc);
230		230
231	void *vmalloc_user(unsigned long size)	231	void *vmalloc_user(unsigned long size)
232	{	232	{
233	void *ret;	233	void *ret;
234		234
235	ret = __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO,	235	ret = __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO,
236	PAGE_KERNEL);	236	PAGE_KERNEL);
237	if (ret) {	237	if (ret) {
238	struct vm_area_struct *vma;	238	struct vm_area_struct *vma;
239		239
240	down_write(&current->mm->mmap_sem);	240	down_write(&current->mm->mmap_sem);
241	vma = find_vma(current->mm, (unsigned long)ret);	241	vma = find_vma(current->mm, (unsigned long)ret);
242	if (vma)	242	if (vma)
243	vma->vm_flags \|= VM_USERMAP;	243	vma->vm_flags \|= VM_USERMAP;
244	up_write(&current->mm->mmap_sem);	244	up_write(&current->mm->mmap_sem);
245	}	245	}
246		246
247	return ret;	247	return ret;
248	}	248	}
249	EXPORT_SYMBOL(vmalloc_user);	249	EXPORT_SYMBOL(vmalloc_user);
250		250
251	struct page vmalloc_to_page(const void addr)	251	struct page vmalloc_to_page(const void addr)
252	{	252	{
253	return virt_to_page(addr);	253	return virt_to_page(addr);
254	}	254	}
255	EXPORT_SYMBOL(vmalloc_to_page);	255	EXPORT_SYMBOL(vmalloc_to_page);
256		256
257	unsigned long vmalloc_to_pfn(const void *addr)	257	unsigned long vmalloc_to_pfn(const void *addr)
258	{	258	{
259	return page_to_pfn(virt_to_page(addr));	259	return page_to_pfn(virt_to_page(addr));
260	}	260	}
261	EXPORT_SYMBOL(vmalloc_to_pfn);	261	EXPORT_SYMBOL(vmalloc_to_pfn);
262		262
263	long vread(char buf, char addr, unsigned long count)	263	long vread(char buf, char addr, unsigned long count)
264	{	264	{
265	memcpy(buf, addr, count);	265	memcpy(buf, addr, count);
266	return count;	266	return count;
267	}	267	}
268		268
269	long vwrite(char buf, char addr, unsigned long count)	269	long vwrite(char buf, char addr, unsigned long count)
270	{	270	{
271	/* Don't allow overflow */	271	/* Don't allow overflow */
272	if ((unsigned long) addr + count < count)	272	if ((unsigned long) addr + count < count)
273	count = -(unsigned long) addr;	273	count = -(unsigned long) addr;
274		274
275	memcpy(addr, buf, count);	275	memcpy(addr, buf, count);
276	return(count);	276	return(count);
277	}	277	}
278		278
279	/*	279	/*
280	* vmalloc - allocate virtually continguos memory	280	* vmalloc - allocate virtually continguos memory
281	*	281	*
282	* @size: allocation size	282	* @size: allocation size
283	*	283	*
284	* Allocate enough pages to cover @size from the page level	284	* Allocate enough pages to cover @size from the page level
285	* allocator and map them into continguos kernel virtual space.	285	* allocator and map them into continguos kernel virtual space.
286	*	286	*
287	* For tight control over page level allocator and protection flags	287	* For tight control over page level allocator and protection flags
288	* use __vmalloc() instead.	288	* use __vmalloc() instead.
289	*/	289	*/
290	void *vmalloc(unsigned long size)	290	void *vmalloc(unsigned long size)
291	{	291	{
292	return __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM, PAGE_KERNEL);	292	return __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM, PAGE_KERNEL);
293	}	293	}
294	EXPORT_SYMBOL(vmalloc);	294	EXPORT_SYMBOL(vmalloc);
295		295
		296	/*
		297	* vzalloc - allocate virtually continguos memory with zero fill
		298	*
		299	* @size: allocation size
		300	*
		301	* Allocate enough pages to cover @size from the page level
		302	* allocator and map them into continguos kernel virtual space.
		303	* The memory allocated is set to zero.
		304	*
		305	* For tight control over page level allocator and protection flags
		306	* use __vmalloc() instead.
		307	*/
		308	void *vzalloc(unsigned long size)
		309	{
		310	return __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO,
		311	PAGE_KERNEL);
		312	}
		313	EXPORT_SYMBOL(vzalloc);
		314
		315	/**
		316	* vmalloc_node - allocate memory on a specific node
		317	* @size: allocation size
		318	* @node: numa node
		319	*
		320	* Allocate enough pages to cover @size from the page level
		321	* allocator and map them into contiguous kernel virtual space.
		322	*
		323	* For tight control over page level allocator and protection flags
		324	* use __vmalloc() instead.
		325	*/
296	void *vmalloc_node(unsigned long size, int node)	326	void *vmalloc_node(unsigned long size, int node)
297	{	327	{
298	return vmalloc(size);	328	return vmalloc(size);
299	}	329	}
300	EXPORT_SYMBOL(vmalloc_node);	330
		331	/**
		332	* vzalloc_node - allocate memory on a specific node with zero fill
		333	* @size: allocation size
		334	* @node: numa node
		335	*
		336	* Allocate enough pages to cover @size from the page level
		337	* allocator and map them into contiguous kernel virtual space.
		338	* The memory allocated is set to zero.
		339	*
		340	* For tight control over page level allocator and protection flags
		341	* use __vmalloc() instead.
		342	*/
		343	void *vzalloc_node(unsigned long size, int node)
		344	{
		345	return vzalloc(size);
		346	}
		347	EXPORT_SYMBOL(vzalloc_node);
301		348
302	#ifndef PAGE_KERNEL_EXEC	349	#ifndef PAGE_KERNEL_EXEC
303	# define PAGE_KERNEL_EXEC PAGE_KERNEL	350	# define PAGE_KERNEL_EXEC PAGE_KERNEL
304	#endif	351	#endif
305		352
306	/**	353	/**
307	* vmalloc_exec - allocate virtually contiguous, executable memory	354	* vmalloc_exec - allocate virtually contiguous, executable memory
308	* @size: allocation size	355	* @size: allocation size
309	*	356	*
310	* Kernel-internal function to allocate enough pages to cover @size	357	* Kernel-internal function to allocate enough pages to cover @size
311	* the page level allocator and map them into contiguous and	358	* the page level allocator and map them into contiguous and
312	* executable kernel virtual space.	359	* executable kernel virtual space.
313	*	360	*
314	* For tight control over page level allocator and protection flags	361	* For tight control over page level allocator and protection flags
315	* use __vmalloc() instead.	362	* use __vmalloc() instead.
316	*/	363	*/
317		364
318	void *vmalloc_exec(unsigned long size)	365	void *vmalloc_exec(unsigned long size)
319	{	366	{
320	return __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM, PAGE_KERNEL_EXEC);	367	return __vmalloc(size, GFP_KERNEL \| __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
321	}	368	}
322		369
323	/**	370	/**
324	* vmalloc_32 - allocate virtually contiguous memory (32bit addressable)	371	* vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
325	* @size: allocation size	372	* @size: allocation size
326	*	373	*
327	* Allocate enough 32bit PA addressable pages to cover @size from the	374	* Allocate enough 32bit PA addressable pages to cover @size from the
328	* page level allocator and map them into continguos kernel virtual space.	375	* page level allocator and map them into continguos kernel virtual space.
329	*/	376	*/
330	void *vmalloc_32(unsigned long size)	377	void *vmalloc_32(unsigned long size)
331	{	378	{
332	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);	379	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
333	}	380	}
334	EXPORT_SYMBOL(vmalloc_32);	381	EXPORT_SYMBOL(vmalloc_32);
335		382
336	/**	383	/**
337	* vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory	384	* vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
338	* @size: allocation size	385	* @size: allocation size
339	*	386	*
340	* The resulting memory area is 32bit addressable and zeroed so it can be	387	* The resulting memory area is 32bit addressable and zeroed so it can be
341	* mapped to userspace without leaking data.	388	* mapped to userspace without leaking data.
342	*	389	*
343	* VM_USERMAP is set on the corresponding VMA so that subsequent calls to	390	* VM_USERMAP is set on the corresponding VMA so that subsequent calls to
344	* remap_vmalloc_range() are permissible.	391	* remap_vmalloc_range() are permissible.
345	*/	392	*/
346	void *vmalloc_32_user(unsigned long size)	393	void *vmalloc_32_user(unsigned long size)
347	{	394	{
348	/*	395	/*
349	* We'll have to sort out the ZONE_DMA bits for 64-bit,	396	* We'll have to sort out the ZONE_DMA bits for 64-bit,
350	* but for now this can simply use vmalloc_user() directly.	397	* but for now this can simply use vmalloc_user() directly.
351	*/	398	*/
352	return vmalloc_user(size);	399	return vmalloc_user(size);
353	}	400	}
354	EXPORT_SYMBOL(vmalloc_32_user);	401	EXPORT_SYMBOL(vmalloc_32_user);
355		402
356	void vmap(struct page *pages, unsigned int count, unsigned long flags, pgprot_t prot)	403	void vmap(struct page *pages, unsigned int count, unsigned long flags, pgprot_t prot)
357	{	404	{
358	BUG();	405	BUG();
359	return NULL;	406	return NULL;
360	}	407	}
361	EXPORT_SYMBOL(vmap);	408	EXPORT_SYMBOL(vmap);
362		409
363	void vunmap(const void *addr)	410	void vunmap(const void *addr)
364	{	411	{
365	BUG();	412	BUG();
366	}	413	}
367	EXPORT_SYMBOL(vunmap);	414	EXPORT_SYMBOL(vunmap);
368		415
369	void vm_map_ram(struct page *pages, unsigned int count, int node, pgprot_t prot)	416	void vm_map_ram(struct page *pages, unsigned int count, int node, pgprot_t prot)
370	{	417	{
371	BUG();	418	BUG();
372	return NULL;	419	return NULL;
373	}	420	}
374	EXPORT_SYMBOL(vm_map_ram);	421	EXPORT_SYMBOL(vm_map_ram);
375		422
376	void vm_unmap_ram(const void *mem, unsigned int count)	423	void vm_unmap_ram(const void *mem, unsigned int count)
377	{	424	{
378	BUG();	425	BUG();
379	}	426	}
380	EXPORT_SYMBOL(vm_unmap_ram);	427	EXPORT_SYMBOL(vm_unmap_ram);
381		428
382	void vm_unmap_aliases(void)	429	void vm_unmap_aliases(void)
383	{	430	{
384	}	431	}
385	EXPORT_SYMBOL_GPL(vm_unmap_aliases);	432	EXPORT_SYMBOL_GPL(vm_unmap_aliases);
386		433
387	/*	434	/*
388	* Implement a stub for vmalloc_sync_all() if the architecture chose not to	435	* Implement a stub for vmalloc_sync_all() if the architecture chose not to
389	* have one.	436	* have one.
390	*/	437	*/
391	void __attribute__((weak)) vmalloc_sync_all(void)	438	void __attribute__((weak)) vmalloc_sync_all(void)
392	{	439	{
393	}	440	}
394		441
395	int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,	442	int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
396	struct page *page)	443	struct page *page)
397	{	444	{
398	return -EINVAL;	445	return -EINVAL;
399	}	446	}
400	EXPORT_SYMBOL(vm_insert_page);	447	EXPORT_SYMBOL(vm_insert_page);
401		448
402	/*	449	/*
403	* sys_brk() for the most part doesn't need the global kernel	450	* sys_brk() for the most part doesn't need the global kernel
404	* lock, except when an application is doing something nasty	451	* lock, except when an application is doing something nasty
405	* like trying to un-brk an area that has already been mapped	452	* like trying to un-brk an area that has already been mapped
406	* to a regular file. in this case, the unmapping will need	453	* to a regular file. in this case, the unmapping will need
407	* to invoke file system routines that need the global lock.	454	* to invoke file system routines that need the global lock.
408	*/	455	*/
409	SYSCALL_DEFINE1(brk, unsigned long, brk)	456	SYSCALL_DEFINE1(brk, unsigned long, brk)
410	{	457	{
411	struct mm_struct *mm = current->mm;	458	struct mm_struct *mm = current->mm;
412		459
413	if (brk < mm->start_brk \|\| brk > mm->context.end_brk)	460	if (brk < mm->start_brk \|\| brk > mm->context.end_brk)
414	return mm->brk;	461	return mm->brk;
415		462
416	if (mm->brk == brk)	463	if (mm->brk == brk)
417	return mm->brk;	464	return mm->brk;
418		465
419	/*	466	/*
420	* Always allow shrinking brk	467	* Always allow shrinking brk
421	*/	468	*/
422	if (brk <= mm->brk) {	469	if (brk <= mm->brk) {
423	mm->brk = brk;	470	mm->brk = brk;
424	return brk;	471	return brk;
425	}	472	}
426		473
427	/*	474	/*
428	* Ok, looks good - let it rip.	475	* Ok, looks good - let it rip.
429	*/	476	*/
430	flush_icache_range(mm->brk, brk);	477	flush_icache_range(mm->brk, brk);
431	return mm->brk = brk;	478	return mm->brk = brk;
432	}	479	}
433		480
434	/*	481	/*
435	* initialise the VMA and region record slabs	482	* initialise the VMA and region record slabs
436	*/	483	*/
437	void __init mmap_init(void)	484	void __init mmap_init(void)
438	{	485	{
439	int ret;	486	int ret;
440		487
441	ret = percpu_counter_init(&vm_committed_as, 0);	488	ret = percpu_counter_init(&vm_committed_as, 0);
442	VM_BUG_ON(ret);	489	VM_BUG_ON(ret);
443	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);	490	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC);
444	}	491	}
445		492
446	/*	493	/*
447	* validate the region tree	494	* validate the region tree
448	* - the caller must hold the region lock	495	* - the caller must hold the region lock
449	*/	496	*/
450	#ifdef CONFIG_DEBUG_NOMMU_REGIONS	497	#ifdef CONFIG_DEBUG_NOMMU_REGIONS
451	static noinline void validate_nommu_regions(void)	498	static noinline void validate_nommu_regions(void)
452	{	499	{
453	struct vm_region region, last;	500	struct vm_region region, last;
454	struct rb_node p, lastp;	501	struct rb_node p, lastp;
455		502
456	lastp = rb_first(&nommu_region_tree);	503	lastp = rb_first(&nommu_region_tree);
457	if (!lastp)	504	if (!lastp)
458	return;	505	return;
459		506
460	last = rb_entry(lastp, struct vm_region, vm_rb);	507	last = rb_entry(lastp, struct vm_region, vm_rb);
461	BUG_ON(unlikely(last->vm_end <= last->vm_start));	508	BUG_ON(unlikely(last->vm_end <= last->vm_start));
462	BUG_ON(unlikely(last->vm_top < last->vm_end));	509	BUG_ON(unlikely(last->vm_top < last->vm_end));
463		510
464	while ((p = rb_next(lastp))) {	511	while ((p = rb_next(lastp))) {
465	region = rb_entry(p, struct vm_region, vm_rb);	512	region = rb_entry(p, struct vm_region, vm_rb);
466	last = rb_entry(lastp, struct vm_region, vm_rb);	513	last = rb_entry(lastp, struct vm_region, vm_rb);
467		514
468	BUG_ON(unlikely(region->vm_end <= region->vm_start));	515	BUG_ON(unlikely(region->vm_end <= region->vm_start));
469	BUG_ON(unlikely(region->vm_top < region->vm_end));	516	BUG_ON(unlikely(region->vm_top < region->vm_end));
470	BUG_ON(unlikely(region->vm_start < last->vm_top));	517	BUG_ON(unlikely(region->vm_start < last->vm_top));
471		518
472	lastp = p;	519	lastp = p;
473	}	520	}
474	}	521	}
475	#else	522	#else
476	static void validate_nommu_regions(void)	523	static void validate_nommu_regions(void)
477	{	524	{
478	}	525	}
479	#endif	526	#endif
480		527
481	/*	528	/*
482	* add a region into the global tree	529	* add a region into the global tree
483	*/	530	*/
484	static void add_nommu_region(struct vm_region *region)	531	static void add_nommu_region(struct vm_region *region)
485	{	532	{
486	struct vm_region *pregion;	533	struct vm_region *pregion;
487	struct rb_node *p, parent;	534	struct rb_node *p, parent;
488		535
489	validate_nommu_regions();	536	validate_nommu_regions();
490		537
491	parent = NULL;	538	parent = NULL;
492	p = &nommu_region_tree.rb_node;	539	p = &nommu_region_tree.rb_node;
493	while (*p) {	540	while (*p) {
494	parent = *p;	541	parent = *p;
495	pregion = rb_entry(parent, struct vm_region, vm_rb);	542	pregion = rb_entry(parent, struct vm_region, vm_rb);
496	if (region->vm_start < pregion->vm_start)	543	if (region->vm_start < pregion->vm_start)
497	p = &(*p)->rb_left;	544	p = &(*p)->rb_left;
498	else if (region->vm_start > pregion->vm_start)	545	else if (region->vm_start > pregion->vm_start)
499	p = &(*p)->rb_right;	546	p = &(*p)->rb_right;
500	else if (pregion == region)	547	else if (pregion == region)
501	return;	548	return;
502	else	549	else
503	BUG();	550	BUG();
504	}	551	}
505		552
506	rb_link_node(&region->vm_rb, parent, p);	553	rb_link_node(&region->vm_rb, parent, p);
507	rb_insert_color(&region->vm_rb, &nommu_region_tree);	554	rb_insert_color(&region->vm_rb, &nommu_region_tree);
508		555
509	validate_nommu_regions();	556	validate_nommu_regions();
510	}	557	}
511		558
512	/*	559	/*
513	* delete a region from the global tree	560	* delete a region from the global tree
514	*/	561	*/
515	static void delete_nommu_region(struct vm_region *region)	562	static void delete_nommu_region(struct vm_region *region)
516	{	563	{
517	BUG_ON(!nommu_region_tree.rb_node);	564	BUG_ON(!nommu_region_tree.rb_node);
518		565
519	validate_nommu_regions();	566	validate_nommu_regions();
520	rb_erase(&region->vm_rb, &nommu_region_tree);	567	rb_erase(&region->vm_rb, &nommu_region_tree);
521	validate_nommu_regions();	568	validate_nommu_regions();
522	}	569	}
523		570
524	/*	571	/*
525	* free a contiguous series of pages	572	* free a contiguous series of pages
526	*/	573	*/
527	static void free_page_series(unsigned long from, unsigned long to)	574	static void free_page_series(unsigned long from, unsigned long to)
528	{	575	{
529	for (; from < to; from += PAGE_SIZE) {	576	for (; from < to; from += PAGE_SIZE) {
530	struct page *page = virt_to_page(from);	577	struct page *page = virt_to_page(from);
531		578
532	kdebug("- free %lx", from);	579	kdebug("- free %lx", from);
533	atomic_long_dec(&mmap_pages_allocated);	580	atomic_long_dec(&mmap_pages_allocated);
534	if (page_count(page) != 1)	581	if (page_count(page) != 1)
535	kdebug("free page %p: refcount not one: %d",	582	kdebug("free page %p: refcount not one: %d",
536	page, page_count(page));	583	page, page_count(page));
537	put_page(page);	584	put_page(page);
538	}	585	}
539	}	586	}
540		587
541	/*	588	/*
542	* release a reference to a region	589	* release a reference to a region
543	* - the caller must hold the region semaphore for writing, which this releases	590	* - the caller must hold the region semaphore for writing, which this releases
544	* - the region may not have been added to the tree yet, in which case vm_top	591	* - the region may not have been added to the tree yet, in which case vm_top
545	* will equal vm_start	592	* will equal vm_start
546	*/	593	*/
547	static void __put_nommu_region(struct vm_region *region)	594	static void __put_nommu_region(struct vm_region *region)
548	__releases(nommu_region_sem)	595	__releases(nommu_region_sem)
549	{	596	{
550	kenter("%p{%d}", region, region->vm_usage);	597	kenter("%p{%d}", region, region->vm_usage);
551		598
552	BUG_ON(!nommu_region_tree.rb_node);	599	BUG_ON(!nommu_region_tree.rb_node);
553		600
554	if (--region->vm_usage == 0) {	601	if (--region->vm_usage == 0) {
555	if (region->vm_top > region->vm_start)	602	if (region->vm_top > region->vm_start)
556	delete_nommu_region(region);	603	delete_nommu_region(region);
557	up_write(&nommu_region_sem);	604	up_write(&nommu_region_sem);
558		605
559	if (region->vm_file)	606	if (region->vm_file)
560	fput(region->vm_file);	607	fput(region->vm_file);
561		608
562	/* IO memory and memory shared directly out of the pagecache	609	/* IO memory and memory shared directly out of the pagecache
563	* from ramfs/tmpfs mustn't be released here */	610	* from ramfs/tmpfs mustn't be released here */
564	if (region->vm_flags & VM_MAPPED_COPY) {	611	if (region->vm_flags & VM_MAPPED_COPY) {
565	kdebug("free series");	612	kdebug("free series");
566	free_page_series(region->vm_start, region->vm_top);	613	free_page_series(region->vm_start, region->vm_top);
567	}	614	}
568	kmem_cache_free(vm_region_jar, region);	615	kmem_cache_free(vm_region_jar, region);
569	} else {	616	} else {
570	up_write(&nommu_region_sem);	617	up_write(&nommu_region_sem);
571	}	618	}
572	}	619	}
573		620
574	/*	621	/*
575	* release a reference to a region	622	* release a reference to a region
576	*/	623	*/
577	static void put_nommu_region(struct vm_region *region)	624	static void put_nommu_region(struct vm_region *region)
578	{	625	{
579	down_write(&nommu_region_sem);	626	down_write(&nommu_region_sem);
580	__put_nommu_region(region);	627	__put_nommu_region(region);
581	}	628	}
582		629
583	/*	630	/*
584	* update protection on a vma	631	* update protection on a vma
585	*/	632	*/
586	static void protect_vma(struct vm_area_struct *vma, unsigned long flags)	633	static void protect_vma(struct vm_area_struct *vma, unsigned long flags)
587	{	634	{
588	#ifdef CONFIG_MPU	635	#ifdef CONFIG_MPU
589	struct mm_struct *mm = vma->vm_mm;	636	struct mm_struct *mm = vma->vm_mm;
590	long start = vma->vm_start & PAGE_MASK;	637	long start = vma->vm_start & PAGE_MASK;
591	while (start < vma->vm_end) {	638	while (start < vma->vm_end) {
592	protect_page(mm, start, flags);	639	protect_page(mm, start, flags);
593	start += PAGE_SIZE;	640	start += PAGE_SIZE;
594	}	641	}
595	update_protections(mm);	642	update_protections(mm);
596	#endif	643	#endif
597	}	644	}
598		645
599	/*	646	/*
600	* add a VMA into a process's mm_struct in the appropriate place in the list	647	* add a VMA into a process's mm_struct in the appropriate place in the list
601	* and tree and add to the address space's page tree also if not an anonymous	648	* and tree and add to the address space's page tree also if not an anonymous
602	* page	649	* page
603	* - should be called with mm->mmap_sem held writelocked	650	* - should be called with mm->mmap_sem held writelocked
604	*/	651	*/
605	static void add_vma_to_mm(struct mm_struct mm, struct vm_area_struct vma)	652	static void add_vma_to_mm(struct mm_struct mm, struct vm_area_struct vma)
606	{	653	{
607	struct vm_area_struct pvma, pp, next;	654	struct vm_area_struct pvma, pp, next;
608	struct address_space *mapping;	655	struct address_space *mapping;
609	struct rb_node *p, parent;	656	struct rb_node *p, parent;
610		657
611	kenter(",%p", vma);	658	kenter(",%p", vma);
612		659
613	BUG_ON(!vma->vm_region);	660	BUG_ON(!vma->vm_region);
614		661
615	mm->map_count++;	662	mm->map_count++;
616	vma->vm_mm = mm;	663	vma->vm_mm = mm;
617		664
618	protect_vma(vma, vma->vm_flags);	665	protect_vma(vma, vma->vm_flags);
619		666
620	/* add the VMA to the mapping */	667	/* add the VMA to the mapping */
621	if (vma->vm_file) {	668	if (vma->vm_file) {
622	mapping = vma->vm_file->f_mapping;	669	mapping = vma->vm_file->f_mapping;
623		670
624	flush_dcache_mmap_lock(mapping);	671	flush_dcache_mmap_lock(mapping);
625	vma_prio_tree_insert(vma, &mapping->i_mmap);	672	vma_prio_tree_insert(vma, &mapping->i_mmap);
626	flush_dcache_mmap_unlock(mapping);	673	flush_dcache_mmap_unlock(mapping);
627	}	674	}
628		675
629	/* add the VMA to the tree */	676	/* add the VMA to the tree */
630	parent = NULL;	677	parent = NULL;
631	p = &mm->mm_rb.rb_node;	678	p = &mm->mm_rb.rb_node;
632	while (*p) {	679	while (*p) {
633	parent = *p;	680	parent = *p;
634	pvma = rb_entry(parent, struct vm_area_struct, vm_rb);	681	pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
635		682
636	/* sort by: start addr, end addr, VMA struct addr in that order	683	/* sort by: start addr, end addr, VMA struct addr in that order
637	* (the latter is necessary as we may get identical VMAs) */	684	* (the latter is necessary as we may get identical VMAs) */
638	if (vma->vm_start < pvma->vm_start)	685	if (vma->vm_start < pvma->vm_start)
639	p = &(*p)->rb_left;	686	p = &(*p)->rb_left;
640	else if (vma->vm_start > pvma->vm_start)	687	else if (vma->vm_start > pvma->vm_start)
641	p = &(*p)->rb_right;	688	p = &(*p)->rb_right;
642	else if (vma->vm_end < pvma->vm_end)	689	else if (vma->vm_end < pvma->vm_end)
643	p = &(*p)->rb_left;	690	p = &(*p)->rb_left;
644	else if (vma->vm_end > pvma->vm_end)	691	else if (vma->vm_end > pvma->vm_end)
645	p = &(*p)->rb_right;	692	p = &(*p)->rb_right;
646	else if (vma < pvma)	693	else if (vma < pvma)
647	p = &(*p)->rb_left;	694	p = &(*p)->rb_left;
648	else if (vma > pvma)	695	else if (vma > pvma)
649	p = &(*p)->rb_right;	696	p = &(*p)->rb_right;
650	else	697	else
651	BUG();	698	BUG();
652	}	699	}
653		700
654	rb_link_node(&vma->vm_rb, parent, p);	701	rb_link_node(&vma->vm_rb, parent, p);
655	rb_insert_color(&vma->vm_rb, &mm->mm_rb);	702	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
656		703
657	/* add VMA to the VMA list also */	704	/* add VMA to the VMA list also */
658	for (pp = &mm->mmap; (pvma = pp); pp = &(pp)->vm_next) {	705	for (pp = &mm->mmap; (pvma = pp); pp = &(pp)->vm_next) {
659	if (pvma->vm_start > vma->vm_start)	706	if (pvma->vm_start > vma->vm_start)
660	break;	707	break;
661	if (pvma->vm_start < vma->vm_start)	708	if (pvma->vm_start < vma->vm_start)
662	continue;	709	continue;
663	if (pvma->vm_end < vma->vm_end)	710	if (pvma->vm_end < vma->vm_end)
664	break;	711	break;
665	}	712	}
666		713
667	next = *pp;	714	next = *pp;
668	*pp = vma;	715	*pp = vma;
669	vma->vm_next = next;	716	vma->vm_next = next;
670	if (next)	717	if (next)
671	next->vm_prev = vma;	718	next->vm_prev = vma;
672	}	719	}
673		720
674	/*	721	/*
675	* delete a VMA from its owning mm_struct and address space	722	* delete a VMA from its owning mm_struct and address space
676	*/	723	*/
677	static void delete_vma_from_mm(struct vm_area_struct *vma)	724	static void delete_vma_from_mm(struct vm_area_struct *vma)
678	{	725	{
679	struct vm_area_struct **pp;	726	struct vm_area_struct **pp;
680	struct address_space *mapping;	727	struct address_space *mapping;
681	struct mm_struct *mm = vma->vm_mm;	728	struct mm_struct *mm = vma->vm_mm;
682		729
683	kenter("%p", vma);	730	kenter("%p", vma);
684		731
685	protect_vma(vma, 0);	732	protect_vma(vma, 0);
686		733
687	mm->map_count--;	734	mm->map_count--;
688	if (mm->mmap_cache == vma)	735	if (mm->mmap_cache == vma)
689	mm->mmap_cache = NULL;	736	mm->mmap_cache = NULL;
690		737
691	/* remove the VMA from the mapping */	738	/* remove the VMA from the mapping */
692	if (vma->vm_file) {	739	if (vma->vm_file) {
693	mapping = vma->vm_file->f_mapping;	740	mapping = vma->vm_file->f_mapping;
694		741
695	flush_dcache_mmap_lock(mapping);	742	flush_dcache_mmap_lock(mapping);
696	vma_prio_tree_remove(vma, &mapping->i_mmap);	743	vma_prio_tree_remove(vma, &mapping->i_mmap);
697	flush_dcache_mmap_unlock(mapping);	744	flush_dcache_mmap_unlock(mapping);
698	}	745	}
699		746
700	/* remove from the MM's tree and list */	747	/* remove from the MM's tree and list */
701	rb_erase(&vma->vm_rb, &mm->mm_rb);	748	rb_erase(&vma->vm_rb, &mm->mm_rb);
702	for (pp = &mm->mmap; pp; pp = &(pp)->vm_next) {	749	for (pp = &mm->mmap; pp; pp = &(pp)->vm_next) {
703	if (*pp == vma) {	750	if (*pp == vma) {
704	*pp = vma->vm_next;	751	*pp = vma->vm_next;
705	break;	752	break;
706	}	753	}
707	}	754	}
708		755
709	vma->vm_mm = NULL;	756	vma->vm_mm = NULL;
710	}	757	}
711		758
712	/*	759	/*
713	* destroy a VMA record	760	* destroy a VMA record
714	*/	761	*/
715	static void delete_vma(struct mm_struct mm, struct vm_area_struct vma)	762	static void delete_vma(struct mm_struct mm, struct vm_area_struct vma)
716	{	763	{
717	kenter("%p", vma);	764	kenter("%p", vma);
718	if (vma->vm_ops && vma->vm_ops->close)	765	if (vma->vm_ops && vma->vm_ops->close)
719	vma->vm_ops->close(vma);	766	vma->vm_ops->close(vma);
720	if (vma->vm_file) {	767	if (vma->vm_file) {
721	fput(vma->vm_file);	768	fput(vma->vm_file);
722	if (vma->vm_flags & VM_EXECUTABLE)	769	if (vma->vm_flags & VM_EXECUTABLE)
723	removed_exe_file_vma(mm);	770	removed_exe_file_vma(mm);
724	}	771	}
725	put_nommu_region(vma->vm_region);	772	put_nommu_region(vma->vm_region);
726	kmem_cache_free(vm_area_cachep, vma);	773	kmem_cache_free(vm_area_cachep, vma);
727	}	774	}
728		775
729	/*	776	/*
730	* look up the first VMA in which addr resides, NULL if none	777	* look up the first VMA in which addr resides, NULL if none
731	* - should be called with mm->mmap_sem at least held readlocked	778	* - should be called with mm->mmap_sem at least held readlocked
732	*/	779	*/
733	struct vm_area_struct find_vma(struct mm_struct mm, unsigned long addr)	780	struct vm_area_struct find_vma(struct mm_struct mm, unsigned long addr)
734	{	781	{
735	struct vm_area_struct *vma;	782	struct vm_area_struct *vma;
736	struct rb_node *n = mm->mm_rb.rb_node;	783	struct rb_node *n = mm->mm_rb.rb_node;
737		784
738	/* check the cache first */	785	/* check the cache first */
739	vma = mm->mmap_cache;	786	vma = mm->mmap_cache;
740	if (vma && vma->vm_start <= addr && vma->vm_end > addr)	787	if (vma && vma->vm_start <= addr && vma->vm_end > addr)
741	return vma;	788	return vma;
742		789
743	/* trawl the tree (there may be multiple mappings in which addr	790	/* trawl the tree (there may be multiple mappings in which addr
744	* resides) */	791	* resides) */
745	for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {	792	for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
746	vma = rb_entry(n, struct vm_area_struct, vm_rb);	793	vma = rb_entry(n, struct vm_area_struct, vm_rb);
747	if (vma->vm_start > addr)	794	if (vma->vm_start > addr)
748	return NULL;	795	return NULL;
749	if (vma->vm_end > addr) {	796	if (vma->vm_end > addr) {
750	mm->mmap_cache = vma;	797	mm->mmap_cache = vma;
751	return vma;	798	return vma;
752	}	799	}
753	}	800	}
754		801
755	return NULL;	802	return NULL;
756	}	803	}
757	EXPORT_SYMBOL(find_vma);	804	EXPORT_SYMBOL(find_vma);
758		805
759	/*	806	/*
760	* find a VMA	807	* find a VMA
761	* - we don't extend stack VMAs under NOMMU conditions	808	* - we don't extend stack VMAs under NOMMU conditions
762	*/	809	*/
763	struct vm_area_struct find_extend_vma(struct mm_struct mm, unsigned long addr)	810	struct vm_area_struct find_extend_vma(struct mm_struct mm, unsigned long addr)
764	{	811	{
765	return find_vma(mm, addr);	812	return find_vma(mm, addr);
766	}	813	}
767		814
768	/*	815	/*
769	* expand a stack to a given address	816	* expand a stack to a given address
770	* - not supported under NOMMU conditions	817	* - not supported under NOMMU conditions
771	*/	818	*/
772	int expand_stack(struct vm_area_struct *vma, unsigned long address)	819	int expand_stack(struct vm_area_struct *vma, unsigned long address)
773	{	820	{
774	return -ENOMEM;	821	return -ENOMEM;
775	}	822	}
776		823
777	/*	824	/*
778	* look up the first VMA exactly that exactly matches addr	825	* look up the first VMA exactly that exactly matches addr
779	* - should be called with mm->mmap_sem at least held readlocked	826	* - should be called with mm->mmap_sem at least held readlocked
780	*/	827	*/
781	static struct vm_area_struct find_vma_exact(struct mm_struct mm,	828	static struct vm_area_struct find_vma_exact(struct mm_struct mm,
782	unsigned long addr,	829	unsigned long addr,
783	unsigned long len)	830	unsigned long len)
784	{	831	{
785	struct vm_area_struct *vma;	832	struct vm_area_struct *vma;
786	struct rb_node *n = mm->mm_rb.rb_node;	833	struct rb_node *n = mm->mm_rb.rb_node;
787	unsigned long end = addr + len;	834	unsigned long end = addr + len;
788		835
789	/* check the cache first */	836	/* check the cache first */
790	vma = mm->mmap_cache;	837	vma = mm->mmap_cache;
791	if (vma && vma->vm_start == addr && vma->vm_end == end)	838	if (vma && vma->vm_start == addr && vma->vm_end == end)
792	return vma;	839	return vma;
793		840
794	/* trawl the tree (there may be multiple mappings in which addr	841	/* trawl the tree (there may be multiple mappings in which addr
795	* resides) */	842	* resides) */
796	for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {	843	for (n = rb_first(&mm->mm_rb); n; n = rb_next(n)) {
797	vma = rb_entry(n, struct vm_area_struct, vm_rb);	844	vma = rb_entry(n, struct vm_area_struct, vm_rb);
798	if (vma->vm_start < addr)	845	if (vma->vm_start < addr)
799	continue;	846	continue;
800	if (vma->vm_start > addr)	847	if (vma->vm_start > addr)
801	return NULL;	848	return NULL;
802	if (vma->vm_end == end) {	849	if (vma->vm_end == end) {
803	mm->mmap_cache = vma;	850	mm->mmap_cache = vma;
804	return vma;	851	return vma;
805	}	852	}
806	}	853	}
807		854
808	return NULL;	855	return NULL;
809	}	856	}
810		857
811	/*	858	/*
812	* determine whether a mapping should be permitted and, if so, what sort of	859	* determine whether a mapping should be permitted and, if so, what sort of
813	* mapping we're capable of supporting	860	* mapping we're capable of supporting
814	*/	861	*/
815	static int validate_mmap_request(struct file *file,	862	static int validate_mmap_request(struct file *file,
816	unsigned long addr,	863	unsigned long addr,
817	unsigned long len,	864	unsigned long len,
818	unsigned long prot,	865	unsigned long prot,
819	unsigned long flags,	866	unsigned long flags,
820	unsigned long pgoff,	867	unsigned long pgoff,
821	unsigned long *_capabilities)	868	unsigned long *_capabilities)
822	{	869	{
823	unsigned long capabilities, rlen;	870	unsigned long capabilities, rlen;
824	unsigned long reqprot = prot;	871	unsigned long reqprot = prot;
825	int ret;	872	int ret;
826		873
827	/* do the simple checks first */	874	/* do the simple checks first */
828	if (flags & MAP_FIXED) {	875	if (flags & MAP_FIXED) {
829	printk(KERN_DEBUG	876	printk(KERN_DEBUG
830	"%d: Can't do fixed-address/overlay mmap of RAM\n",	877	"%d: Can't do fixed-address/overlay mmap of RAM\n",
831	current->pid);	878	current->pid);
832	return -EINVAL;	879	return -EINVAL;
833	}	880	}
834		881
835	if ((flags & MAP_TYPE) != MAP_PRIVATE &&	882	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
836	(flags & MAP_TYPE) != MAP_SHARED)	883	(flags & MAP_TYPE) != MAP_SHARED)
837	return -EINVAL;	884	return -EINVAL;
838		885
839	if (!len)	886	if (!len)
840	return -EINVAL;	887	return -EINVAL;
841		888
842	/* Careful about overflows.. */	889	/* Careful about overflows.. */
843	rlen = PAGE_ALIGN(len);	890	rlen = PAGE_ALIGN(len);
844	if (!rlen \|\| rlen > TASK_SIZE)	891	if (!rlen \|\| rlen > TASK_SIZE)
845	return -ENOMEM;	892	return -ENOMEM;
846		893
847	/* offset overflow? */	894	/* offset overflow? */
848	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)	895	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
849	return -EOVERFLOW;	896	return -EOVERFLOW;
850		897
851	if (file) {	898	if (file) {
852	/* validate file mapping requests */	899	/* validate file mapping requests */
853	struct address_space *mapping;	900	struct address_space *mapping;
854		901
855	/* files must support mmap */	902	/* files must support mmap */
856	if (!file->f_op \|\| !file->f_op->mmap)	903	if (!file->f_op \|\| !file->f_op->mmap)
857	return -ENODEV;	904	return -ENODEV;
858		905
859	/* work out if what we've got could possibly be shared	906	/* work out if what we've got could possibly be shared
860	* - we support chardevs that provide their own "memory"	907	* - we support chardevs that provide their own "memory"
861	* - we support files/blockdevs that are memory backed	908	* - we support files/blockdevs that are memory backed
862	*/	909	*/
863	mapping = file->f_mapping;	910	mapping = file->f_mapping;
864	if (!mapping)	911	if (!mapping)
865	mapping = file->f_path.dentry->d_inode->i_mapping;	912	mapping = file->f_path.dentry->d_inode->i_mapping;
866		913
867	capabilities = 0;	914	capabilities = 0;
868	if (mapping && mapping->backing_dev_info)	915	if (mapping && mapping->backing_dev_info)
869	capabilities = mapping->backing_dev_info->capabilities;	916	capabilities = mapping->backing_dev_info->capabilities;
870		917
871	if (!capabilities) {	918	if (!capabilities) {
872	/* no explicit capabilities set, so assume some	919	/* no explicit capabilities set, so assume some
873	* defaults */	920	* defaults */
874	switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) {	921	switch (file->f_path.dentry->d_inode->i_mode & S_IFMT) {
875	case S_IFREG:	922	case S_IFREG:
876	case S_IFBLK:	923	case S_IFBLK:
877	capabilities = BDI_CAP_MAP_COPY;	924	capabilities = BDI_CAP_MAP_COPY;
878	break;	925	break;
879		926
880	case S_IFCHR:	927	case S_IFCHR:
881	capabilities =	928	capabilities =
882	BDI_CAP_MAP_DIRECT \|	929	BDI_CAP_MAP_DIRECT \|
883	BDI_CAP_READ_MAP \|	930	BDI_CAP_READ_MAP \|
884	BDI_CAP_WRITE_MAP;	931	BDI_CAP_WRITE_MAP;
885	break;	932	break;
886		933
887	default:	934	default:
888	return -EINVAL;	935	return -EINVAL;
889	}	936	}
890	}	937	}
891		938
892	/* eliminate any capabilities that we can't support on this	939	/* eliminate any capabilities that we can't support on this
893	* device */	940	* device */
894	if (!file->f_op->get_unmapped_area)	941	if (!file->f_op->get_unmapped_area)
895	capabilities &= ~BDI_CAP_MAP_DIRECT;	942	capabilities &= ~BDI_CAP_MAP_DIRECT;
896	if (!file->f_op->read)	943	if (!file->f_op->read)
897	capabilities &= ~BDI_CAP_MAP_COPY;	944	capabilities &= ~BDI_CAP_MAP_COPY;
898		945
899	/* The file shall have been opened with read permission. */	946	/* The file shall have been opened with read permission. */
900	if (!(file->f_mode & FMODE_READ))	947	if (!(file->f_mode & FMODE_READ))
901	return -EACCES;	948	return -EACCES;
902		949
903	if (flags & MAP_SHARED) {	950	if (flags & MAP_SHARED) {
904	/* do checks for writing, appending and locking */	951	/* do checks for writing, appending and locking */
905	if ((prot & PROT_WRITE) &&	952	if ((prot & PROT_WRITE) &&
906	!(file->f_mode & FMODE_WRITE))	953	!(file->f_mode & FMODE_WRITE))
907	return -EACCES;	954	return -EACCES;
908		955
909	if (IS_APPEND(file->f_path.dentry->d_inode) &&	956	if (IS_APPEND(file->f_path.dentry->d_inode) &&
910	(file->f_mode & FMODE_WRITE))	957	(file->f_mode & FMODE_WRITE))
911	return -EACCES;	958	return -EACCES;
912		959
913	if (locks_verify_locked(file->f_path.dentry->d_inode))	960	if (locks_verify_locked(file->f_path.dentry->d_inode))
914	return -EAGAIN;	961	return -EAGAIN;
915		962
916	if (!(capabilities & BDI_CAP_MAP_DIRECT))	963	if (!(capabilities & BDI_CAP_MAP_DIRECT))
917	return -ENODEV;	964	return -ENODEV;
918		965
919	/* we mustn't privatise shared mappings */	966	/* we mustn't privatise shared mappings */
920	capabilities &= ~BDI_CAP_MAP_COPY;	967	capabilities &= ~BDI_CAP_MAP_COPY;
921	}	968	}
922	else {	969	else {
923	/* we're going to read the file into private memory we	970	/* we're going to read the file into private memory we
924	* allocate */	971	* allocate */
925	if (!(capabilities & BDI_CAP_MAP_COPY))	972	if (!(capabilities & BDI_CAP_MAP_COPY))
926	return -ENODEV;	973	return -ENODEV;
927		974
928	/* we don't permit a private writable mapping to be	975	/* we don't permit a private writable mapping to be
929	* shared with the backing device */	976	* shared with the backing device */
930	if (prot & PROT_WRITE)	977	if (prot & PROT_WRITE)
931	capabilities &= ~BDI_CAP_MAP_DIRECT;	978	capabilities &= ~BDI_CAP_MAP_DIRECT;
932	}	979	}
933		980
934	if (capabilities & BDI_CAP_MAP_DIRECT) {	981	if (capabilities & BDI_CAP_MAP_DIRECT) {
935	if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) \|\|	982	if (((prot & PROT_READ) && !(capabilities & BDI_CAP_READ_MAP)) \|\|
936	((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) \|\|	983	((prot & PROT_WRITE) && !(capabilities & BDI_CAP_WRITE_MAP)) \|\|
937	((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP))	984	((prot & PROT_EXEC) && !(capabilities & BDI_CAP_EXEC_MAP))
938	) {	985	) {
939	capabilities &= ~BDI_CAP_MAP_DIRECT;	986	capabilities &= ~BDI_CAP_MAP_DIRECT;
940	if (flags & MAP_SHARED) {	987	if (flags & MAP_SHARED) {
941	printk(KERN_WARNING	988	printk(KERN_WARNING
942	"MAP_SHARED not completely supported on !MMU\n");	989	"MAP_SHARED not completely supported on !MMU\n");
943	return -EINVAL;	990	return -EINVAL;
944	}	991	}
945	}	992	}
946	}	993	}
947		994
948	/* handle executable mappings and implied executable	995	/* handle executable mappings and implied executable
949	* mappings */	996	* mappings */
950	if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {	997	if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
951	if (prot & PROT_EXEC)	998	if (prot & PROT_EXEC)
952	return -EPERM;	999	return -EPERM;
953	}	1000	}
954	else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {	1001	else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
955	/* handle implication of PROT_EXEC by PROT_READ */	1002	/* handle implication of PROT_EXEC by PROT_READ */
956	if (current->personality & READ_IMPLIES_EXEC) {	1003	if (current->personality & READ_IMPLIES_EXEC) {
957	if (capabilities & BDI_CAP_EXEC_MAP)	1004	if (capabilities & BDI_CAP_EXEC_MAP)
958	prot \|= PROT_EXEC;	1005	prot \|= PROT_EXEC;
959	}	1006	}
960	}	1007	}
961	else if ((prot & PROT_READ) &&	1008	else if ((prot & PROT_READ) &&
962	(prot & PROT_EXEC) &&	1009	(prot & PROT_EXEC) &&
963	!(capabilities & BDI_CAP_EXEC_MAP)	1010	!(capabilities & BDI_CAP_EXEC_MAP)
964	) {	1011	) {
965	/* backing file is not executable, try to copy */	1012	/* backing file is not executable, try to copy */
966	capabilities &= ~BDI_CAP_MAP_DIRECT;	1013	capabilities &= ~BDI_CAP_MAP_DIRECT;
967	}	1014	}
968	}	1015	}
969	else {	1016	else {
970	/* anonymous mappings are always memory backed and can be	1017	/* anonymous mappings are always memory backed and can be
971	* privately mapped	1018	* privately mapped
972	*/	1019	*/
973	capabilities = BDI_CAP_MAP_COPY;	1020	capabilities = BDI_CAP_MAP_COPY;
974		1021
975	/* handle PROT_EXEC implication by PROT_READ */	1022	/* handle PROT_EXEC implication by PROT_READ */
976	if ((prot & PROT_READ) &&	1023	if ((prot & PROT_READ) &&
977	(current->personality & READ_IMPLIES_EXEC))	1024	(current->personality & READ_IMPLIES_EXEC))
978	prot \|= PROT_EXEC;	1025	prot \|= PROT_EXEC;
979	}	1026	}
980		1027
981	/* allow the security API to have its say */	1028	/* allow the security API to have its say */
982	ret = security_file_mmap(file, reqprot, prot, flags, addr, 0);	1029	ret = security_file_mmap(file, reqprot, prot, flags, addr, 0);
983	if (ret < 0)	1030	if (ret < 0)
984	return ret;	1031	return ret;
985		1032
986	/* looks okay */	1033	/* looks okay */
987	*_capabilities = capabilities;	1034	*_capabilities = capabilities;
988	return 0;	1035	return 0;
989	}	1036	}
990		1037
991	/*	1038	/*
992	* we've determined that we can make the mapping, now translate what we	1039	* we've determined that we can make the mapping, now translate what we
993	* now know into VMA flags	1040	* now know into VMA flags
994	*/	1041	*/
995	static unsigned long determine_vm_flags(struct file *file,	1042	static unsigned long determine_vm_flags(struct file *file,
996	unsigned long prot,	1043	unsigned long prot,
997	unsigned long flags,	1044	unsigned long flags,
998	unsigned long capabilities)	1045	unsigned long capabilities)
999	{	1046	{
1000	unsigned long vm_flags;	1047	unsigned long vm_flags;
1001		1048
1002	vm_flags = calc_vm_prot_bits(prot) \| calc_vm_flag_bits(flags);	1049	vm_flags = calc_vm_prot_bits(prot) \| calc_vm_flag_bits(flags);
1003	/* vm_flags \|= mm->def_flags; */	1050	/* vm_flags \|= mm->def_flags; */
1004		1051
1005	if (!(capabilities & BDI_CAP_MAP_DIRECT)) {	1052	if (!(capabilities & BDI_CAP_MAP_DIRECT)) {
1006	/* attempt to share read-only copies of mapped file chunks */	1053	/* attempt to share read-only copies of mapped file chunks */
1007	vm_flags \|= VM_MAYREAD \| VM_MAYWRITE \| VM_MAYEXEC;	1054	vm_flags \|= VM_MAYREAD \| VM_MAYWRITE \| VM_MAYEXEC;
1008	if (file && !(prot & PROT_WRITE))	1055	if (file && !(prot & PROT_WRITE))
1009	vm_flags \|= VM_MAYSHARE;	1056	vm_flags \|= VM_MAYSHARE;
1010	} else {	1057	} else {
1011	/* overlay a shareable mapping on the backing device or inode	1058	/* overlay a shareable mapping on the backing device or inode
1012	* if possible - used for chardevs, ramfs/tmpfs/shmfs and	1059	* if possible - used for chardevs, ramfs/tmpfs/shmfs and
1013	* romfs/cramfs */	1060	* romfs/cramfs */
1014	vm_flags \|= VM_MAYSHARE \| (capabilities & BDI_CAP_VMFLAGS);	1061	vm_flags \|= VM_MAYSHARE \| (capabilities & BDI_CAP_VMFLAGS);
1015	if (flags & MAP_SHARED)	1062	if (flags & MAP_SHARED)
1016	vm_flags \|= VM_SHARED;	1063	vm_flags \|= VM_SHARED;
1017	}	1064	}
1018		1065
1019	/* refuse to let anyone share private mappings with this process if	1066	/* refuse to let anyone share private mappings with this process if
1020	* it's being traced - otherwise breakpoints set in it may interfere	1067	* it's being traced - otherwise breakpoints set in it may interfere
1021	* with another untraced process	1068	* with another untraced process
1022	*/	1069	*/
1023	if ((flags & MAP_PRIVATE) && tracehook_expect_breakpoints(current))	1070	if ((flags & MAP_PRIVATE) && tracehook_expect_breakpoints(current))
1024	vm_flags &= ~VM_MAYSHARE;	1071	vm_flags &= ~VM_MAYSHARE;
1025		1072
1026	return vm_flags;	1073	return vm_flags;
1027	}	1074	}
1028		1075
1029	/*	1076	/*
1030	* set up a shared mapping on a file (the driver or filesystem provides and	1077	* set up a shared mapping on a file (the driver or filesystem provides and
1031	* pins the storage)	1078	* pins the storage)
1032	*/	1079	*/
1033	static int do_mmap_shared_file(struct vm_area_struct *vma)	1080	static int do_mmap_shared_file(struct vm_area_struct *vma)
1034	{	1081	{
1035	int ret;	1082	int ret;
1036		1083
1037	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);	1084	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1038	if (ret == 0) {	1085	if (ret == 0) {
1039	vma->vm_region->vm_top = vma->vm_region->vm_end;	1086	vma->vm_region->vm_top = vma->vm_region->vm_end;
1040	return 0;	1087	return 0;
1041	}	1088	}
1042	if (ret != -ENOSYS)	1089	if (ret != -ENOSYS)
1043	return ret;	1090	return ret;
1044		1091
1045	/* getting -ENOSYS indicates that direct mmap isn't possible (as	1092	/* getting -ENOSYS indicates that direct mmap isn't possible (as
1046	* opposed to tried but failed) so we can only give a suitable error as	1093	* opposed to tried but failed) so we can only give a suitable error as
1047	* it's not possible to make a private copy if MAP_SHARED was given */	1094	* it's not possible to make a private copy if MAP_SHARED was given */
1048	return -ENODEV;	1095	return -ENODEV;
1049	}	1096	}
1050		1097
1051	/*	1098	/*
1052	* set up a private mapping or an anonymous shared mapping	1099	* set up a private mapping or an anonymous shared mapping
1053	*/	1100	*/
1054	static int do_mmap_private(struct vm_area_struct *vma,	1101	static int do_mmap_private(struct vm_area_struct *vma,
1055	struct vm_region *region,	1102	struct vm_region *region,
1056	unsigned long len,	1103	unsigned long len,
1057	unsigned long capabilities)	1104	unsigned long capabilities)
1058	{	1105	{
1059	struct page *pages;	1106	struct page *pages;
1060	unsigned long total, point, n, rlen;	1107	unsigned long total, point, n, rlen;
1061	void *base;	1108	void *base;
1062	int ret, order;	1109	int ret, order;
1063		1110
1064	/* invoke the file's mapping function so that it can keep track of	1111	/* invoke the file's mapping function so that it can keep track of
1065	* shared mappings on devices or memory	1112	* shared mappings on devices or memory
1066	* - VM_MAYSHARE will be set if it may attempt to share	1113	* - VM_MAYSHARE will be set if it may attempt to share
1067	*/	1114	*/
1068	if (capabilities & BDI_CAP_MAP_DIRECT) {	1115	if (capabilities & BDI_CAP_MAP_DIRECT) {
1069	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);	1116	ret = vma->vm_file->f_op->mmap(vma->vm_file, vma);
1070	if (ret == 0) {	1117	if (ret == 0) {
1071	/* shouldn't return success if we're not sharing */	1118	/* shouldn't return success if we're not sharing */
1072	BUG_ON(!(vma->vm_flags & VM_MAYSHARE));	1119	BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
1073	vma->vm_region->vm_top = vma->vm_region->vm_end;	1120	vma->vm_region->vm_top = vma->vm_region->vm_end;
1074	return 0;	1121	return 0;
1075	}	1122	}
1076	if (ret != -ENOSYS)	1123	if (ret != -ENOSYS)
1077	return ret;	1124	return ret;
1078		1125
1079	/* getting an ENOSYS error indicates that direct mmap isn't	1126	/* getting an ENOSYS error indicates that direct mmap isn't
1080	* possible (as opposed to tried but failed) so we'll try to	1127	* possible (as opposed to tried but failed) so we'll try to
1081	* make a private copy of the data and map that instead */	1128	* make a private copy of the data and map that instead */
1082	}	1129	}
1083		1130
1084	rlen = PAGE_ALIGN(len);	1131	rlen = PAGE_ALIGN(len);
1085		1132
1086	/* allocate some memory to hold the mapping	1133	/* allocate some memory to hold the mapping
1087	* - note that this may not return a page-aligned address if the object	1134	* - note that this may not return a page-aligned address if the object
1088	* we're allocating is smaller than a page	1135	* we're allocating is smaller than a page
1089	*/	1136	*/
1090	order = get_order(rlen);	1137	order = get_order(rlen);
1091	kdebug("alloc order %d for %lx", order, len);	1138	kdebug("alloc order %d for %lx", order, len);
1092		1139
1093	pages = alloc_pages(GFP_KERNEL, order);	1140	pages = alloc_pages(GFP_KERNEL, order);
1094	if (!pages)	1141	if (!pages)
1095	goto enomem;	1142	goto enomem;
1096		1143
1097	total = 1 << order;	1144	total = 1 << order;
1098	atomic_long_add(total, &mmap_pages_allocated);	1145	atomic_long_add(total, &mmap_pages_allocated);
1099		1146
1100	point = rlen >> PAGE_SHIFT;	1147	point = rlen >> PAGE_SHIFT;
1101		1148
1102	/* we allocated a power-of-2 sized page set, so we may want to trim off	1149	/* we allocated a power-of-2 sized page set, so we may want to trim off
1103	* the excess */	1150	* the excess */
1104	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) {	1151	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) {
1105	while (total > point) {	1152	while (total > point) {
1106	order = ilog2(total - point);	1153	order = ilog2(total - point);
1107	n = 1 << order;	1154	n = 1 << order;
1108	kdebug("shave %lu/%lu @%lu", n, total - point, total);	1155	kdebug("shave %lu/%lu @%lu", n, total - point, total);
1109	atomic_long_sub(n, &mmap_pages_allocated);	1156	atomic_long_sub(n, &mmap_pages_allocated);
1110	total -= n;	1157	total -= n;
1111	set_page_refcounted(pages + total);	1158	set_page_refcounted(pages + total);
1112	__free_pages(pages + total, order);	1159	__free_pages(pages + total, order);
1113	}	1160	}
1114	}	1161	}
1115		1162
1116	for (point = 1; point < total; point++)	1163	for (point = 1; point < total; point++)
1117	set_page_refcounted(&pages[point]);	1164	set_page_refcounted(&pages[point]);
1118		1165
1119	base = page_address(pages);	1166	base = page_address(pages);
1120	region->vm_flags = vma->vm_flags \|= VM_MAPPED_COPY;	1167	region->vm_flags = vma->vm_flags \|= VM_MAPPED_COPY;
1121	region->vm_start = (unsigned long) base;	1168	region->vm_start = (unsigned long) base;
1122	region->vm_end = region->vm_start + rlen;	1169	region->vm_end = region->vm_start + rlen;
1123	region->vm_top = region->vm_start + (total << PAGE_SHIFT);	1170	region->vm_top = region->vm_start + (total << PAGE_SHIFT);
1124		1171
1125	vma->vm_start = region->vm_start;	1172	vma->vm_start = region->vm_start;
1126	vma->vm_end = region->vm_start + len;	1173	vma->vm_end = region->vm_start + len;
1127		1174
1128	if (vma->vm_file) {	1175	if (vma->vm_file) {
1129	/* read the contents of a file into the copy */	1176	/* read the contents of a file into the copy */
1130	mm_segment_t old_fs;	1177	mm_segment_t old_fs;
1131	loff_t fpos;	1178	loff_t fpos;
1132		1179
1133	fpos = vma->vm_pgoff;	1180	fpos = vma->vm_pgoff;
1134	fpos <<= PAGE_SHIFT;	1181	fpos <<= PAGE_SHIFT;
1135		1182
1136	old_fs = get_fs();	1183	old_fs = get_fs();
1137	set_fs(KERNEL_DS);	1184	set_fs(KERNEL_DS);
1138	ret = vma->vm_file->f_op->read(vma->vm_file, base, rlen, &fpos);	1185	ret = vma->vm_file->f_op->read(vma->vm_file, base, rlen, &fpos);
1139	set_fs(old_fs);	1186	set_fs(old_fs);
1140		1187
1141	if (ret < 0)	1188	if (ret < 0)
1142	goto error_free;	1189	goto error_free;
1143		1190
1144	/* clear the last little bit */	1191	/* clear the last little bit */
1145	if (ret < rlen)	1192	if (ret < rlen)
1146	memset(base + ret, 0, rlen - ret);	1193	memset(base + ret, 0, rlen - ret);
1147		1194
1148	}	1195	}
1149		1196
1150	return 0;	1197	return 0;
1151		1198
1152	error_free:	1199	error_free:
1153	free_page_series(region->vm_start, region->vm_end);	1200	free_page_series(region->vm_start, region->vm_end);
1154	region->vm_start = vma->vm_start = 0;	1201	region->vm_start = vma->vm_start = 0;
1155	region->vm_end = vma->vm_end = 0;	1202	region->vm_end = vma->vm_end = 0;
1156	region->vm_top = 0;	1203	region->vm_top = 0;
1157	return ret;	1204	return ret;
1158		1205
1159	enomem:	1206	enomem:
1160	printk("Allocation of length %lu from process %d (%s) failed\n",	1207	printk("Allocation of length %lu from process %d (%s) failed\n",
1161	len, current->pid, current->comm);	1208	len, current->pid, current->comm);
1162	show_free_areas();	1209	show_free_areas();
1163	return -ENOMEM;	1210	return -ENOMEM;
1164	}	1211	}
1165		1212
1166	/*	1213	/*
1167	* handle mapping creation for uClinux	1214	* handle mapping creation for uClinux
1168	*/	1215	*/
1169	unsigned long do_mmap_pgoff(struct file *file,	1216	unsigned long do_mmap_pgoff(struct file *file,
1170	unsigned long addr,	1217	unsigned long addr,
1171	unsigned long len,	1218	unsigned long len,
1172	unsigned long prot,	1219	unsigned long prot,
1173	unsigned long flags,	1220	unsigned long flags,
1174	unsigned long pgoff)	1221	unsigned long pgoff)
1175	{	1222	{
1176	struct vm_area_struct *vma;	1223	struct vm_area_struct *vma;
1177	struct vm_region *region;	1224	struct vm_region *region;
1178	struct rb_node *rb;	1225	struct rb_node *rb;
1179	unsigned long capabilities, vm_flags, result;	1226	unsigned long capabilities, vm_flags, result;
1180	int ret;	1227	int ret;
1181		1228
1182	kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);	1229	kenter(",%lx,%lx,%lx,%lx,%lx", addr, len, prot, flags, pgoff);
1183		1230
1184	/* decide whether we should attempt the mapping, and if so what sort of	1231	/* decide whether we should attempt the mapping, and if so what sort of
1185	* mapping */	1232	* mapping */
1186	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,	1233	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1187	&capabilities);	1234	&capabilities);
1188	if (ret < 0) {	1235	if (ret < 0) {
1189	kleave(" = %d [val]", ret);	1236	kleave(" = %d [val]", ret);
1190	return ret;	1237	return ret;
1191	}	1238	}
1192		1239
1193	/* we ignore the address hint */	1240	/* we ignore the address hint */
1194	addr = 0;	1241	addr = 0;
1195		1242
1196	/* we've determined that we can make the mapping, now translate what we	1243	/* we've determined that we can make the mapping, now translate what we
1197	* now know into VMA flags */	1244	* now know into VMA flags */
1198	vm_flags = determine_vm_flags(file, prot, flags, capabilities);	1245	vm_flags = determine_vm_flags(file, prot, flags, capabilities);
1199		1246
1200	/* we're going to need to record the mapping */	1247	/* we're going to need to record the mapping */
1201	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);	1248	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1202	if (!region)	1249	if (!region)
1203	goto error_getting_region;	1250	goto error_getting_region;
1204		1251
1205	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);	1252	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1206	if (!vma)	1253	if (!vma)
1207	goto error_getting_vma;	1254	goto error_getting_vma;
1208		1255
1209	region->vm_usage = 1;	1256	region->vm_usage = 1;
1210	region->vm_flags = vm_flags;	1257	region->vm_flags = vm_flags;
1211	region->vm_pgoff = pgoff;	1258	region->vm_pgoff = pgoff;
1212		1259
1213	INIT_LIST_HEAD(&vma->anon_vma_chain);	1260	INIT_LIST_HEAD(&vma->anon_vma_chain);
1214	vma->vm_flags = vm_flags;	1261	vma->vm_flags = vm_flags;
1215	vma->vm_pgoff = pgoff;	1262	vma->vm_pgoff = pgoff;
1216		1263
1217	if (file) {	1264	if (file) {
1218	region->vm_file = file;	1265	region->vm_file = file;
1219	get_file(file);	1266	get_file(file);
1220	vma->vm_file = file;	1267	vma->vm_file = file;
1221	get_file(file);	1268	get_file(file);
1222	if (vm_flags & VM_EXECUTABLE) {	1269	if (vm_flags & VM_EXECUTABLE) {
1223	added_exe_file_vma(current->mm);	1270	added_exe_file_vma(current->mm);
1224	vma->vm_mm = current->mm;	1271	vma->vm_mm = current->mm;
1225	}	1272	}
1226	}	1273	}
1227		1274
1228	down_write(&nommu_region_sem);	1275	down_write(&nommu_region_sem);
1229		1276
1230	/* if we want to share, we need to check for regions created by other	1277	/* if we want to share, we need to check for regions created by other
1231	* mmap() calls that overlap with our proposed mapping	1278	* mmap() calls that overlap with our proposed mapping
1232	* - we can only share with a superset match on most regular files	1279	* - we can only share with a superset match on most regular files
1233	* - shared mappings on character devices and memory backed files are	1280	* - shared mappings on character devices and memory backed files are
1234	* permitted to overlap inexactly as far as we are concerned for in	1281	* permitted to overlap inexactly as far as we are concerned for in
1235	* these cases, sharing is handled in the driver or filesystem rather	1282	* these cases, sharing is handled in the driver or filesystem rather
1236	* than here	1283	* than here
1237	*/	1284	*/
1238	if (vm_flags & VM_MAYSHARE) {	1285	if (vm_flags & VM_MAYSHARE) {
1239	struct vm_region *pregion;	1286	struct vm_region *pregion;
1240	unsigned long pglen, rpglen, pgend, rpgend, start;	1287	unsigned long pglen, rpglen, pgend, rpgend, start;
1241		1288
1242	pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;	1289	pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1243	pgend = pgoff + pglen;	1290	pgend = pgoff + pglen;
1244		1291
1245	for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {	1292	for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1246	pregion = rb_entry(rb, struct vm_region, vm_rb);	1293	pregion = rb_entry(rb, struct vm_region, vm_rb);
1247		1294
1248	if (!(pregion->vm_flags & VM_MAYSHARE))	1295	if (!(pregion->vm_flags & VM_MAYSHARE))
1249	continue;	1296	continue;
1250		1297
1251	/* search for overlapping mappings on the same file */	1298	/* search for overlapping mappings on the same file */
1252	if (pregion->vm_file->f_path.dentry->d_inode !=	1299	if (pregion->vm_file->f_path.dentry->d_inode !=
1253	file->f_path.dentry->d_inode)	1300	file->f_path.dentry->d_inode)
1254	continue;	1301	continue;
1255		1302
1256	if (pregion->vm_pgoff >= pgend)	1303	if (pregion->vm_pgoff >= pgend)
1257	continue;	1304	continue;
1258		1305
1259	rpglen = pregion->vm_end - pregion->vm_start;	1306	rpglen = pregion->vm_end - pregion->vm_start;
1260	rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;	1307	rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1261	rpgend = pregion->vm_pgoff + rpglen;	1308	rpgend = pregion->vm_pgoff + rpglen;
1262	if (pgoff >= rpgend)	1309	if (pgoff >= rpgend)
1263	continue;	1310	continue;
1264		1311
1265	/* handle inexactly overlapping matches between	1312	/* handle inexactly overlapping matches between
1266	* mappings */	1313	* mappings */
1267	if ((pregion->vm_pgoff != pgoff \|\| rpglen != pglen) &&	1314	if ((pregion->vm_pgoff != pgoff \|\| rpglen != pglen) &&
1268	!(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {	1315	!(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1269	/* new mapping is not a subset of the region */	1316	/* new mapping is not a subset of the region */
1270	if (!(capabilities & BDI_CAP_MAP_DIRECT))	1317	if (!(capabilities & BDI_CAP_MAP_DIRECT))
1271	goto sharing_violation;	1318	goto sharing_violation;
1272	continue;	1319	continue;
1273	}	1320	}
1274		1321
1275	/* we've found a region we can share */	1322	/* we've found a region we can share */
1276	pregion->vm_usage++;	1323	pregion->vm_usage++;
1277	vma->vm_region = pregion;	1324	vma->vm_region = pregion;
1278	start = pregion->vm_start;	1325	start = pregion->vm_start;
1279	start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;	1326	start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1280	vma->vm_start = start;	1327	vma->vm_start = start;
1281	vma->vm_end = start + len;	1328	vma->vm_end = start + len;
1282		1329
1283	if (pregion->vm_flags & VM_MAPPED_COPY) {	1330	if (pregion->vm_flags & VM_MAPPED_COPY) {
1284	kdebug("share copy");	1331	kdebug("share copy");
1285	vma->vm_flags \|= VM_MAPPED_COPY;	1332	vma->vm_flags \|= VM_MAPPED_COPY;
1286	} else {	1333	} else {
1287	kdebug("share mmap");	1334	kdebug("share mmap");
1288	ret = do_mmap_shared_file(vma);	1335	ret = do_mmap_shared_file(vma);
1289	if (ret < 0) {	1336	if (ret < 0) {
1290	vma->vm_region = NULL;	1337	vma->vm_region = NULL;
1291	vma->vm_start = 0;	1338	vma->vm_start = 0;
1292	vma->vm_end = 0;	1339	vma->vm_end = 0;
1293	pregion->vm_usage--;	1340	pregion->vm_usage--;
1294	pregion = NULL;	1341	pregion = NULL;
1295	goto error_just_free;	1342	goto error_just_free;
1296	}	1343	}
1297	}	1344	}
1298	fput(region->vm_file);	1345	fput(region->vm_file);
1299	kmem_cache_free(vm_region_jar, region);	1346	kmem_cache_free(vm_region_jar, region);
1300	region = pregion;	1347	region = pregion;
1301	result = start;	1348	result = start;
1302	goto share;	1349	goto share;
1303	}	1350	}
1304		1351
1305	/* obtain the address at which to make a shared mapping	1352	/* obtain the address at which to make a shared mapping
1306	* - this is the hook for quasi-memory character devices to	1353	* - this is the hook for quasi-memory character devices to
1307	* tell us the location of a shared mapping	1354	* tell us the location of a shared mapping
1308	*/	1355	*/
1309	if (capabilities & BDI_CAP_MAP_DIRECT) {	1356	if (capabilities & BDI_CAP_MAP_DIRECT) {
1310	addr = file->f_op->get_unmapped_area(file, addr, len,	1357	addr = file->f_op->get_unmapped_area(file, addr, len,
1311	pgoff, flags);	1358	pgoff, flags);
1312	if (IS_ERR((void *) addr)) {	1359	if (IS_ERR((void *) addr)) {
1313	ret = addr;	1360	ret = addr;
1314	if (ret != (unsigned long) -ENOSYS)	1361	if (ret != (unsigned long) -ENOSYS)
1315	goto error_just_free;	1362	goto error_just_free;
1316		1363
1317	/* the driver refused to tell us where to site	1364	/* the driver refused to tell us where to site
1318	* the mapping so we'll have to attempt to copy	1365	* the mapping so we'll have to attempt to copy
1319	* it */	1366	* it */
1320	ret = (unsigned long) -ENODEV;	1367	ret = (unsigned long) -ENODEV;
1321	if (!(capabilities & BDI_CAP_MAP_COPY))	1368	if (!(capabilities & BDI_CAP_MAP_COPY))
1322	goto error_just_free;	1369	goto error_just_free;
1323		1370
1324	capabilities &= ~BDI_CAP_MAP_DIRECT;	1371	capabilities &= ~BDI_CAP_MAP_DIRECT;
1325	} else {	1372	} else {
1326	vma->vm_start = region->vm_start = addr;	1373	vma->vm_start = region->vm_start = addr;
1327	vma->vm_end = region->vm_end = addr + len;	1374	vma->vm_end = region->vm_end = addr + len;
1328	}	1375	}
1329	}	1376	}
1330	}	1377	}
1331		1378
1332	vma->vm_region = region;	1379	vma->vm_region = region;
1333		1380
1334	/* set up the mapping	1381	/* set up the mapping
1335	* - the region is filled in if BDI_CAP_MAP_DIRECT is still set	1382	* - the region is filled in if BDI_CAP_MAP_DIRECT is still set
1336	*/	1383	*/
1337	if (file && vma->vm_flags & VM_SHARED)	1384	if (file && vma->vm_flags & VM_SHARED)
1338	ret = do_mmap_shared_file(vma);	1385	ret = do_mmap_shared_file(vma);
1339	else	1386	else
1340	ret = do_mmap_private(vma, region, len, capabilities);	1387	ret = do_mmap_private(vma, region, len, capabilities);
1341	if (ret < 0)	1388	if (ret < 0)
1342	goto error_just_free;	1389	goto error_just_free;
1343	add_nommu_region(region);	1390	add_nommu_region(region);
1344		1391
1345	/* clear anonymous mappings that don't ask for uninitialized data */	1392	/* clear anonymous mappings that don't ask for uninitialized data */
1346	if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))	1393	if (!vma->vm_file && !(flags & MAP_UNINITIALIZED))
1347	memset((void *)region->vm_start, 0,	1394	memset((void *)region->vm_start, 0,
1348	region->vm_end - region->vm_start);	1395	region->vm_end - region->vm_start);
1349		1396
1350	/* okay... we have a mapping; now we have to register it */	1397	/* okay... we have a mapping; now we have to register it */
1351	result = vma->vm_start;	1398	result = vma->vm_start;
1352		1399
1353	current->mm->total_vm += len >> PAGE_SHIFT;	1400	current->mm->total_vm += len >> PAGE_SHIFT;
1354		1401
1355	share:	1402	share:
1356	add_vma_to_mm(current->mm, vma);	1403	add_vma_to_mm(current->mm, vma);
1357		1404
1358	/* we flush the region from the icache only when the first executable	1405	/* we flush the region from the icache only when the first executable
1359	* mapping of it is made */	1406	* mapping of it is made */
1360	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {	1407	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1361	flush_icache_range(region->vm_start, region->vm_end);	1408	flush_icache_range(region->vm_start, region->vm_end);
1362	region->vm_icache_flushed = true;	1409	region->vm_icache_flushed = true;
1363	}	1410	}
1364		1411
1365	up_write(&nommu_region_sem);	1412	up_write(&nommu_region_sem);
1366		1413
1367	kleave(" = %lx", result);	1414	kleave(" = %lx", result);
1368	return result;	1415	return result;
1369		1416
1370	error_just_free:	1417	error_just_free:
1371	up_write(&nommu_region_sem);	1418	up_write(&nommu_region_sem);
1372	error:	1419	error:
1373	if (region->vm_file)	1420	if (region->vm_file)
1374	fput(region->vm_file);	1421	fput(region->vm_file);
1375	kmem_cache_free(vm_region_jar, region);	1422	kmem_cache_free(vm_region_jar, region);
1376	if (vma->vm_file)	1423	if (vma->vm_file)
1377	fput(vma->vm_file);	1424	fput(vma->vm_file);
1378	if (vma->vm_flags & VM_EXECUTABLE)	1425	if (vma->vm_flags & VM_EXECUTABLE)
1379	removed_exe_file_vma(vma->vm_mm);	1426	removed_exe_file_vma(vma->vm_mm);
1380	kmem_cache_free(vm_area_cachep, vma);	1427	kmem_cache_free(vm_area_cachep, vma);
1381	kleave(" = %d", ret);	1428	kleave(" = %d", ret);
1382	return ret;	1429	return ret;
1383		1430
1384	sharing_violation:	1431	sharing_violation:
1385	up_write(&nommu_region_sem);	1432	up_write(&nommu_region_sem);
1386	printk(KERN_WARNING "Attempt to share mismatched mappings\n");	1433	printk(KERN_WARNING "Attempt to share mismatched mappings\n");
1387	ret = -EINVAL;	1434	ret = -EINVAL;
1388	goto error;	1435	goto error;
1389		1436
1390	error_getting_vma:	1437	error_getting_vma:
1391	kmem_cache_free(vm_region_jar, region);	1438	kmem_cache_free(vm_region_jar, region);
1392	printk(KERN_WARNING "Allocation of vma for %lu byte allocation"	1439	printk(KERN_WARNING "Allocation of vma for %lu byte allocation"
1393	" from process %d failed\n",	1440	" from process %d failed\n",
1394	len, current->pid);	1441	len, current->pid);
1395	show_free_areas();	1442	show_free_areas();
1396	return -ENOMEM;	1443	return -ENOMEM;
1397		1444
1398	error_getting_region:	1445	error_getting_region:
1399	printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"	1446	printk(KERN_WARNING "Allocation of vm region for %lu byte allocation"
1400	" from process %d failed\n",	1447	" from process %d failed\n",
1401	len, current->pid);	1448	len, current->pid);
1402	show_free_areas();	1449	show_free_areas();
1403	return -ENOMEM;	1450	return -ENOMEM;
1404	}	1451	}
1405	EXPORT_SYMBOL(do_mmap_pgoff);	1452	EXPORT_SYMBOL(do_mmap_pgoff);
1406		1453
1407	SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,	1454	SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1408	unsigned long, prot, unsigned long, flags,	1455	unsigned long, prot, unsigned long, flags,
1409	unsigned long, fd, unsigned long, pgoff)	1456	unsigned long, fd, unsigned long, pgoff)
1410	{	1457	{
1411	struct file *file = NULL;	1458	struct file *file = NULL;
1412	unsigned long retval = -EBADF;	1459	unsigned long retval = -EBADF;
1413		1460
1414	if (!(flags & MAP_ANONYMOUS)) {	1461	if (!(flags & MAP_ANONYMOUS)) {
1415	file = fget(fd);	1462	file = fget(fd);
1416	if (!file)	1463	if (!file)
1417	goto out;	1464	goto out;
1418	}	1465	}
1419		1466
1420	flags &= ~(MAP_EXECUTABLE \| MAP_DENYWRITE);	1467	flags &= ~(MAP_EXECUTABLE \| MAP_DENYWRITE);
1421		1468
1422	down_write(&current->mm->mmap_sem);	1469	down_write(&current->mm->mmap_sem);
1423	retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);	1470	retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1424	up_write(&current->mm->mmap_sem);	1471	up_write(&current->mm->mmap_sem);
1425		1472
1426	if (file)	1473	if (file)
1427	fput(file);	1474	fput(file);
1428	out:	1475	out:
1429	return retval;	1476	return retval;
1430	}	1477	}
1431		1478
1432	#ifdef __ARCH_WANT_SYS_OLD_MMAP	1479	#ifdef __ARCH_WANT_SYS_OLD_MMAP
1433	struct mmap_arg_struct {	1480	struct mmap_arg_struct {
1434	unsigned long addr;	1481	unsigned long addr;
1435	unsigned long len;	1482	unsigned long len;
1436	unsigned long prot;	1483	unsigned long prot;
1437	unsigned long flags;	1484	unsigned long flags;
1438	unsigned long fd;	1485	unsigned long fd;
1439	unsigned long offset;	1486	unsigned long offset;
1440	};	1487	};
1441		1488
1442	SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)	1489	SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1443	{	1490	{
1444	struct mmap_arg_struct a;	1491	struct mmap_arg_struct a;
1445		1492
1446	if (copy_from_user(&a, arg, sizeof(a)))	1493	if (copy_from_user(&a, arg, sizeof(a)))
1447	return -EFAULT;	1494	return -EFAULT;
1448	if (a.offset & ~PAGE_MASK)	1495	if (a.offset & ~PAGE_MASK)
1449	return -EINVAL;	1496	return -EINVAL;
1450		1497
1451	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,	1498	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1452	a.offset >> PAGE_SHIFT);	1499	a.offset >> PAGE_SHIFT);
1453	}	1500	}
1454	#endif /* __ARCH_WANT_SYS_OLD_MMAP */	1501	#endif /* __ARCH_WANT_SYS_OLD_MMAP */
1455		1502
1456	/*	1503	/*
1457	* split a vma into two pieces at address 'addr', a new vma is allocated either	1504	* split a vma into two pieces at address 'addr', a new vma is allocated either
1458	* for the first part or the tail.	1505	* for the first part or the tail.
1459	*/	1506	*/
1460	int split_vma(struct mm_struct mm, struct vm_area_struct vma,	1507	int split_vma(struct mm_struct mm, struct vm_area_struct vma,
1461	unsigned long addr, int new_below)	1508	unsigned long addr, int new_below)
1462	{	1509	{
1463	struct vm_area_struct *new;	1510	struct vm_area_struct *new;
1464	struct vm_region *region;	1511	struct vm_region *region;
1465	unsigned long npages;	1512	unsigned long npages;
1466		1513
1467	kenter("");	1514	kenter("");
1468		1515
1469	/* we're only permitted to split anonymous regions (these should have	1516	/* we're only permitted to split anonymous regions (these should have
1470	* only a single usage on the region) */	1517	* only a single usage on the region) */
1471	if (vma->vm_file)	1518	if (vma->vm_file)
1472	return -ENOMEM;	1519	return -ENOMEM;
1473		1520
1474	if (mm->map_count >= sysctl_max_map_count)	1521	if (mm->map_count >= sysctl_max_map_count)
1475	return -ENOMEM;	1522	return -ENOMEM;
1476		1523
1477	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);	1524	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1478	if (!region)	1525	if (!region)
1479	return -ENOMEM;	1526	return -ENOMEM;
1480		1527
1481	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);	1528	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1482	if (!new) {	1529	if (!new) {
1483	kmem_cache_free(vm_region_jar, region);	1530	kmem_cache_free(vm_region_jar, region);
1484	return -ENOMEM;	1531	return -ENOMEM;
1485	}	1532	}
1486		1533
1487	/* most fields are the same, copy all, and then fixup */	1534	/* most fields are the same, copy all, and then fixup */
1488	new = vma;	1535	new = vma;
1489	region = vma->vm_region;	1536	region = vma->vm_region;
1490	new->vm_region = region;	1537	new->vm_region = region;
1491		1538
1492	npages = (addr - vma->vm_start) >> PAGE_SHIFT;	1539	npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1493		1540
1494	if (new_below) {	1541	if (new_below) {
1495	region->vm_top = region->vm_end = new->vm_end = addr;	1542	region->vm_top = region->vm_end = new->vm_end = addr;
1496	} else {	1543	} else {
1497	region->vm_start = new->vm_start = addr;	1544	region->vm_start = new->vm_start = addr;
1498	region->vm_pgoff = new->vm_pgoff += npages;	1545	region->vm_pgoff = new->vm_pgoff += npages;
1499	}	1546	}
1500		1547
1501	if (new->vm_ops && new->vm_ops->open)	1548	if (new->vm_ops && new->vm_ops->open)
1502	new->vm_ops->open(new);	1549	new->vm_ops->open(new);
1503		1550
1504	delete_vma_from_mm(vma);	1551	delete_vma_from_mm(vma);
1505	down_write(&nommu_region_sem);	1552	down_write(&nommu_region_sem);
1506	delete_nommu_region(vma->vm_region);	1553	delete_nommu_region(vma->vm_region);
1507	if (new_below) {	1554	if (new_below) {
1508	vma->vm_region->vm_start = vma->vm_start = addr;	1555	vma->vm_region->vm_start = vma->vm_start = addr;
1509	vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;	1556	vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1510	} else {	1557	} else {
1511	vma->vm_region->vm_end = vma->vm_end = addr;	1558	vma->vm_region->vm_end = vma->vm_end = addr;
1512	vma->vm_region->vm_top = addr;	1559	vma->vm_region->vm_top = addr;
1513	}	1560	}
1514	add_nommu_region(vma->vm_region);	1561	add_nommu_region(vma->vm_region);
1515	add_nommu_region(new->vm_region);	1562	add_nommu_region(new->vm_region);
1516	up_write(&nommu_region_sem);	1563	up_write(&nommu_region_sem);
1517	add_vma_to_mm(mm, vma);	1564	add_vma_to_mm(mm, vma);
1518	add_vma_to_mm(mm, new);	1565	add_vma_to_mm(mm, new);
1519	return 0;	1566	return 0;
1520	}	1567	}
1521		1568
1522	/*	1569	/*
1523	* shrink a VMA by removing the specified chunk from either the beginning or	1570	* shrink a VMA by removing the specified chunk from either the beginning or
1524	* the end	1571	* the end
1525	*/	1572	*/
1526	static int shrink_vma(struct mm_struct *mm,	1573	static int shrink_vma(struct mm_struct *mm,
1527	struct vm_area_struct *vma,	1574	struct vm_area_struct *vma,
1528	unsigned long from, unsigned long to)	1575	unsigned long from, unsigned long to)
1529	{	1576	{
1530	struct vm_region *region;	1577	struct vm_region *region;
1531		1578
1532	kenter("");	1579	kenter("");
1533		1580
1534	/* adjust the VMA's pointers, which may reposition it in the MM's tree	1581	/* adjust the VMA's pointers, which may reposition it in the MM's tree
1535	* and list */	1582	* and list */
1536	delete_vma_from_mm(vma);	1583	delete_vma_from_mm(vma);
1537	if (from > vma->vm_start)	1584	if (from > vma->vm_start)
1538	vma->vm_end = from;	1585	vma->vm_end = from;
1539	else	1586	else
1540	vma->vm_start = to;	1587	vma->vm_start = to;
1541	add_vma_to_mm(mm, vma);	1588	add_vma_to_mm(mm, vma);
1542		1589
1543	/* cut the backing region down to size */	1590	/* cut the backing region down to size */
1544	region = vma->vm_region;	1591	region = vma->vm_region;
1545	BUG_ON(region->vm_usage != 1);	1592	BUG_ON(region->vm_usage != 1);
1546		1593
1547	down_write(&nommu_region_sem);	1594	down_write(&nommu_region_sem);
1548	delete_nommu_region(region);	1595	delete_nommu_region(region);
1549	if (from > region->vm_start) {	1596	if (from > region->vm_start) {
1550	to = region->vm_top;	1597	to = region->vm_top;
1551	region->vm_top = region->vm_end = from;	1598	region->vm_top = region->vm_end = from;
1552	} else {	1599	} else {
1553	region->vm_start = to;	1600	region->vm_start = to;
1554	}	1601	}
1555	add_nommu_region(region);	1602	add_nommu_region(region);
1556	up_write(&nommu_region_sem);	1603	up_write(&nommu_region_sem);
1557		1604
1558	free_page_series(from, to);	1605	free_page_series(from, to);
1559	return 0;	1606	return 0;
1560	}	1607	}
1561		1608
1562	/*	1609	/*
1563	* release a mapping	1610	* release a mapping
1564	* - under NOMMU conditions the chunk to be unmapped must be backed by a single	1611	* - under NOMMU conditions the chunk to be unmapped must be backed by a single
1565	* VMA, though it need not cover the whole VMA	1612	* VMA, though it need not cover the whole VMA
1566	*/	1613	*/
1567	int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)	1614	int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
1568	{	1615	{
1569	struct vm_area_struct *vma;	1616	struct vm_area_struct *vma;
1570	struct rb_node *rb;	1617	struct rb_node *rb;
1571	unsigned long end = start + len;	1618	unsigned long end = start + len;
1572	int ret;	1619	int ret;
1573		1620
1574	kenter(",%lx,%zx", start, len);	1621	kenter(",%lx,%zx", start, len);
1575		1622
1576	if (len == 0)	1623	if (len == 0)
1577	return -EINVAL;	1624	return -EINVAL;
1578		1625
1579	/* find the first potentially overlapping VMA */	1626	/* find the first potentially overlapping VMA */
1580	vma = find_vma(mm, start);	1627	vma = find_vma(mm, start);
1581	if (!vma) {	1628	if (!vma) {
1582	static int limit = 0;	1629	static int limit = 0;
1583	if (limit < 5) {	1630	if (limit < 5) {
1584	printk(KERN_WARNING	1631	printk(KERN_WARNING
1585	"munmap of memory not mmapped by process %d"	1632	"munmap of memory not mmapped by process %d"
1586	" (%s): 0x%lx-0x%lx\n",	1633	" (%s): 0x%lx-0x%lx\n",
1587	current->pid, current->comm,	1634	current->pid, current->comm,
1588	start, start + len - 1);	1635	start, start + len - 1);
1589	limit++;	1636	limit++;
1590	}	1637	}
1591	return -EINVAL;	1638	return -EINVAL;
1592	}	1639	}
1593		1640
1594	/* we're allowed to split an anonymous VMA but not a file-backed one */	1641	/* we're allowed to split an anonymous VMA but not a file-backed one */
1595	if (vma->vm_file) {	1642	if (vma->vm_file) {
1596	do {	1643	do {
1597	if (start > vma->vm_start) {	1644	if (start > vma->vm_start) {
1598	kleave(" = -EINVAL [miss]");	1645	kleave(" = -EINVAL [miss]");
1599	return -EINVAL;	1646	return -EINVAL;
1600	}	1647	}
1601	if (end == vma->vm_end)	1648	if (end == vma->vm_end)
1602	goto erase_whole_vma;	1649	goto erase_whole_vma;
1603	rb = rb_next(&vma->vm_rb);	1650	rb = rb_next(&vma->vm_rb);
1604	vma = rb_entry(rb, struct vm_area_struct, vm_rb);	1651	vma = rb_entry(rb, struct vm_area_struct, vm_rb);
1605	} while (rb);	1652	} while (rb);
1606	kleave(" = -EINVAL [split file]");	1653	kleave(" = -EINVAL [split file]");
1607	return -EINVAL;	1654	return -EINVAL;
1608	} else {	1655	} else {
1609	/* the chunk must be a subset of the VMA found */	1656	/* the chunk must be a subset of the VMA found */
1610	if (start == vma->vm_start && end == vma->vm_end)	1657	if (start == vma->vm_start && end == vma->vm_end)
1611	goto erase_whole_vma;	1658	goto erase_whole_vma;
1612	if (start < vma->vm_start \|\| end > vma->vm_end) {	1659	if (start < vma->vm_start \|\| end > vma->vm_end) {
1613	kleave(" = -EINVAL [superset]");	1660	kleave(" = -EINVAL [superset]");
1614	return -EINVAL;	1661	return -EINVAL;
1615	}	1662	}
1616	if (start & ~PAGE_MASK) {	1663	if (start & ~PAGE_MASK) {
1617	kleave(" = -EINVAL [unaligned start]");	1664	kleave(" = -EINVAL [unaligned start]");
1618	return -EINVAL;	1665	return -EINVAL;
1619	}	1666	}
1620	if (end != vma->vm_end && end & ~PAGE_MASK) {	1667	if (end != vma->vm_end && end & ~PAGE_MASK) {
1621	kleave(" = -EINVAL [unaligned split]");	1668	kleave(" = -EINVAL [unaligned split]");
1622	return -EINVAL;	1669	return -EINVAL;
1623	}	1670	}
1624	if (start != vma->vm_start && end != vma->vm_end) {	1671	if (start != vma->vm_start && end != vma->vm_end) {
1625	ret = split_vma(mm, vma, start, 1);	1672	ret = split_vma(mm, vma, start, 1);
1626	if (ret < 0) {	1673	if (ret < 0) {
1627	kleave(" = %d [split]", ret);	1674	kleave(" = %d [split]", ret);
1628	return ret;	1675	return ret;
1629	}	1676	}
1630	}	1677	}
1631	return shrink_vma(mm, vma, start, end);	1678	return shrink_vma(mm, vma, start, end);
1632	}	1679	}
1633		1680
1634	erase_whole_vma:	1681	erase_whole_vma:
1635	delete_vma_from_mm(vma);	1682	delete_vma_from_mm(vma);
1636	delete_vma(mm, vma);	1683	delete_vma(mm, vma);
1637	kleave(" = 0");	1684	kleave(" = 0");
1638	return 0;	1685	return 0;
1639	}	1686	}
1640	EXPORT_SYMBOL(do_munmap);	1687	EXPORT_SYMBOL(do_munmap);
1641		1688
1642	SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)	1689	SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1643	{	1690	{
1644	int ret;	1691	int ret;
1645	struct mm_struct *mm = current->mm;	1692	struct mm_struct *mm = current->mm;
1646		1693
1647	down_write(&mm->mmap_sem);	1694	down_write(&mm->mmap_sem);
1648	ret = do_munmap(mm, addr, len);	1695	ret = do_munmap(mm, addr, len);
1649	up_write(&mm->mmap_sem);	1696	up_write(&mm->mmap_sem);
1650	return ret;	1697	return ret;
1651	}	1698	}
1652		1699
1653	/*	1700	/*
1654	* release all the mappings made in a process's VM space	1701	* release all the mappings made in a process's VM space
1655	*/	1702	*/
1656	void exit_mmap(struct mm_struct *mm)	1703	void exit_mmap(struct mm_struct *mm)
1657	{	1704	{
1658	struct vm_area_struct *vma;	1705	struct vm_area_struct *vma;
1659		1706
1660	if (!mm)	1707	if (!mm)
1661	return;	1708	return;
1662		1709
1663	kenter("");	1710	kenter("");
1664		1711
1665	mm->total_vm = 0;	1712	mm->total_vm = 0;
1666		1713
1667	while ((vma = mm->mmap)) {	1714	while ((vma = mm->mmap)) {
1668	mm->mmap = vma->vm_next;	1715	mm->mmap = vma->vm_next;
1669	delete_vma_from_mm(vma);	1716	delete_vma_from_mm(vma);
1670	delete_vma(mm, vma);	1717	delete_vma(mm, vma);
1671	}	1718	}
1672		1719
1673	kleave("");	1720	kleave("");
1674	}	1721	}
1675		1722
1676	unsigned long do_brk(unsigned long addr, unsigned long len)	1723	unsigned long do_brk(unsigned long addr, unsigned long len)
1677	{	1724	{
1678	return -ENOMEM;	1725	return -ENOMEM;
1679	}	1726	}
1680		1727
1681	/*	1728	/*
1682	* expand (or shrink) an existing mapping, potentially moving it at the same	1729	* expand (or shrink) an existing mapping, potentially moving it at the same
1683	* time (controlled by the MREMAP_MAYMOVE flag and available VM space)	1730	* time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1684	*	1731	*
1685	* under NOMMU conditions, we only permit changing a mapping's size, and only	1732	* under NOMMU conditions, we only permit changing a mapping's size, and only
1686	* as long as it stays within the region allocated by do_mmap_private() and the	1733	* as long as it stays within the region allocated by do_mmap_private() and the
1687	* block is not shareable	1734	* block is not shareable
1688	*	1735	*
1689	* MREMAP_FIXED is not supported under NOMMU conditions	1736	* MREMAP_FIXED is not supported under NOMMU conditions
1690	*/	1737	*/
1691	unsigned long do_mremap(unsigned long addr,	1738	unsigned long do_mremap(unsigned long addr,
1692	unsigned long old_len, unsigned long new_len,	1739	unsigned long old_len, unsigned long new_len,
1693	unsigned long flags, unsigned long new_addr)	1740	unsigned long flags, unsigned long new_addr)
1694	{	1741	{
1695	struct vm_area_struct *vma;	1742	struct vm_area_struct *vma;
1696		1743
1697	/* insanity checks first */	1744	/* insanity checks first */
1698	if (old_len == 0 \|\| new_len == 0)	1745	if (old_len == 0 \|\| new_len == 0)
1699	return (unsigned long) -EINVAL;	1746	return (unsigned long) -EINVAL;
1700		1747
1701	if (addr & ~PAGE_MASK)	1748	if (addr & ~PAGE_MASK)
1702	return -EINVAL;	1749	return -EINVAL;
1703		1750
1704	if (flags & MREMAP_FIXED && new_addr != addr)	1751	if (flags & MREMAP_FIXED && new_addr != addr)
1705	return (unsigned long) -EINVAL;	1752	return (unsigned long) -EINVAL;
1706		1753
1707	vma = find_vma_exact(current->mm, addr, old_len);	1754	vma = find_vma_exact(current->mm, addr, old_len);
1708	if (!vma)	1755	if (!vma)
1709	return (unsigned long) -EINVAL;	1756	return (unsigned long) -EINVAL;
1710		1757
1711	if (vma->vm_end != vma->vm_start + old_len)	1758	if (vma->vm_end != vma->vm_start + old_len)
1712	return (unsigned long) -EFAULT;	1759	return (unsigned long) -EFAULT;
1713		1760
1714	if (vma->vm_flags & VM_MAYSHARE)	1761	if (vma->vm_flags & VM_MAYSHARE)
1715	return (unsigned long) -EPERM;	1762	return (unsigned long) -EPERM;
1716		1763
1717	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)	1764	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1718	return (unsigned long) -ENOMEM;	1765	return (unsigned long) -ENOMEM;
1719		1766
1720	/* all checks complete - do it */	1767	/* all checks complete - do it */
1721	vma->vm_end = vma->vm_start + new_len;	1768	vma->vm_end = vma->vm_start + new_len;
1722	return vma->vm_start;	1769	return vma->vm_start;
1723	}	1770	}
1724	EXPORT_SYMBOL(do_mremap);	1771	EXPORT_SYMBOL(do_mremap);
1725		1772
1726	SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,	1773	SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1727	unsigned long, new_len, unsigned long, flags,	1774	unsigned long, new_len, unsigned long, flags,
1728	unsigned long, new_addr)	1775	unsigned long, new_addr)
1729	{	1776	{
1730	unsigned long ret;	1777	unsigned long ret;
1731		1778
1732	down_write(&current->mm->mmap_sem);	1779	down_write(&current->mm->mmap_sem);
1733	ret = do_mremap(addr, old_len, new_len, flags, new_addr);	1780	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1734	up_write(&current->mm->mmap_sem);	1781	up_write(&current->mm->mmap_sem);
1735	return ret;	1782	return ret;
1736	}	1783	}
1737		1784
1738	struct page follow_page(struct vm_area_struct vma, unsigned long address,	1785	struct page follow_page(struct vm_area_struct vma, unsigned long address,
1739	unsigned int foll_flags)	1786	unsigned int foll_flags)
1740	{	1787	{
1741	return NULL;	1788	return NULL;
1742	}	1789	}
1743		1790
1744	int remap_pfn_range(struct vm_area_struct *vma, unsigned long from,	1791	int remap_pfn_range(struct vm_area_struct *vma, unsigned long from,
1745	unsigned long to, unsigned long size, pgprot_t prot)	1792	unsigned long to, unsigned long size, pgprot_t prot)
1746	{	1793	{
1747	vma->vm_start = vma->vm_pgoff << PAGE_SHIFT;	1794	vma->vm_start = vma->vm_pgoff << PAGE_SHIFT;
1748	return 0;	1795	return 0;
1749	}	1796	}
1750	EXPORT_SYMBOL(remap_pfn_range);	1797	EXPORT_SYMBOL(remap_pfn_range);
1751		1798
1752	int remap_vmalloc_range(struct vm_area_struct vma, void addr,	1799	int remap_vmalloc_range(struct vm_area_struct vma, void addr,
1753	unsigned long pgoff)	1800	unsigned long pgoff)
1754	{	1801	{
1755	unsigned int size = vma->vm_end - vma->vm_start;	1802	unsigned int size = vma->vm_end - vma->vm_start;
1756		1803
1757	if (!(vma->vm_flags & VM_USERMAP))	1804	if (!(vma->vm_flags & VM_USERMAP))
1758	return -EINVAL;	1805	return -EINVAL;
1759		1806
1760	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));	1807	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1761	vma->vm_end = vma->vm_start + size;	1808	vma->vm_end = vma->vm_start + size;
1762		1809
1763	return 0;	1810	return 0;
1764	}	1811	}
1765	EXPORT_SYMBOL(remap_vmalloc_range);	1812	EXPORT_SYMBOL(remap_vmalloc_range);
1766		1813
1767	void swap_unplug_io_fn(struct backing_dev_info bdi, struct page page)	1814	void swap_unplug_io_fn(struct backing_dev_info bdi, struct page page)
1768	{	1815	{
1769	}	1816	}
1770		1817
1771	unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,	1818	unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
1772	unsigned long len, unsigned long pgoff, unsigned long flags)	1819	unsigned long len, unsigned long pgoff, unsigned long flags)
1773	{	1820	{
1774	return -ENOMEM;	1821	return -ENOMEM;
1775	}	1822	}
1776		1823
1777	void arch_unmap_area(struct mm_struct *mm, unsigned long addr)	1824	void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
1778	{	1825	{
1779	}	1826	}
1780		1827
1781	void unmap_mapping_range(struct address_space *mapping,	1828	void unmap_mapping_range(struct address_space *mapping,
1782	loff_t const holebegin, loff_t const holelen,	1829	loff_t const holebegin, loff_t const holelen,
1783	int even_cows)	1830	int even_cows)
1784	{	1831	{
1785	}	1832	}
1786	EXPORT_SYMBOL(unmap_mapping_range);	1833	EXPORT_SYMBOL(unmap_mapping_range);
1787		1834
1788	/*	1835	/*
1789	* Check that a process has enough memory to allocate a new virtual	1836	* Check that a process has enough memory to allocate a new virtual
1790	* mapping. 0 means there is enough memory for the allocation to	1837	* mapping. 0 means there is enough memory for the allocation to
1791	* succeed and -ENOMEM implies there is not.	1838	* succeed and -ENOMEM implies there is not.
1792	*	1839	*
1793	* We currently support three overcommit policies, which are set via the	1840	* We currently support three overcommit policies, which are set via the
1794	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting	1841	* vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
1795	*	1842	*
1796	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.	1843	* Strict overcommit modes added 2002 Feb 26 by Alan Cox.
1797	* Additional code 2002 Jul 20 by Robert Love.	1844	* Additional code 2002 Jul 20 by Robert Love.
1798	*	1845	*
1799	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.	1846	* cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
1800	*	1847	*
1801	* Note this is a helper function intended to be used by LSMs which	1848	* Note this is a helper function intended to be used by LSMs which
1802	* wish to use this logic.	1849	* wish to use this logic.
1803	*/	1850	*/
1804	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)	1851	int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1805	{	1852	{
1806	unsigned long free, allowed;	1853	unsigned long free, allowed;
1807		1854
1808	vm_acct_memory(pages);	1855	vm_acct_memory(pages);
1809		1856
1810	/*	1857	/*
1811	* Sometimes we want to use more memory than we have	1858	* Sometimes we want to use more memory than we have
1812	*/	1859	*/
1813	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)	1860	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
1814	return 0;	1861	return 0;
1815		1862
1816	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {	1863	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
1817	unsigned long n;	1864	unsigned long n;
1818		1865
1819	free = global_page_state(NR_FILE_PAGES);	1866	free = global_page_state(NR_FILE_PAGES);
1820	free += nr_swap_pages;	1867	free += nr_swap_pages;
1821		1868
1822	/*	1869	/*
1823	* Any slabs which are created with the	1870	* Any slabs which are created with the
1824	* SLAB_RECLAIM_ACCOUNT flag claim to have contents	1871	* SLAB_RECLAIM_ACCOUNT flag claim to have contents
1825	* which are reclaimable, under pressure. The dentry	1872	* which are reclaimable, under pressure. The dentry
1826	* cache and most inode caches should fall into this	1873	* cache and most inode caches should fall into this
1827	*/	1874	*/
1828	free += global_page_state(NR_SLAB_RECLAIMABLE);	1875	free += global_page_state(NR_SLAB_RECLAIMABLE);
1829		1876
1830	/*	1877	/*
1831	* Leave the last 3% for root	1878	* Leave the last 3% for root
1832	*/	1879	*/
1833	if (!cap_sys_admin)	1880	if (!cap_sys_admin)
1834	free -= free / 32;	1881	free -= free / 32;
1835		1882
1836	if (free > pages)	1883	if (free > pages)
1837	return 0;	1884	return 0;
1838		1885
1839	/*	1886	/*
1840	* nr_free_pages() is very expensive on large systems,	1887	* nr_free_pages() is very expensive on large systems,
1841	* only call if we're about to fail.	1888	* only call if we're about to fail.
1842	*/	1889	*/
1843	n = nr_free_pages();	1890	n = nr_free_pages();
1844		1891
1845	/*	1892	/*
1846	* Leave reserved pages. The pages are not for anonymous pages.	1893	* Leave reserved pages. The pages are not for anonymous pages.
1847	*/	1894	*/
1848	if (n <= totalreserve_pages)	1895	if (n <= totalreserve_pages)
1849	goto error;	1896	goto error;
1850	else	1897	else
1851	n -= totalreserve_pages;	1898	n -= totalreserve_pages;
1852		1899
1853	/*	1900	/*
1854	* Leave the last 3% for root	1901	* Leave the last 3% for root
1855	*/	1902	*/
1856	if (!cap_sys_admin)	1903	if (!cap_sys_admin)
1857	n -= n / 32;	1904	n -= n / 32;
1858	free += n;	1905	free += n;
1859		1906
1860	if (free > pages)	1907	if (free > pages)
1861	return 0;	1908	return 0;
1862		1909
1863	goto error;	1910	goto error;
1864	}	1911	}
1865		1912
1866	allowed = totalram_pages * sysctl_overcommit_ratio / 100;	1913	allowed = totalram_pages * sysctl_overcommit_ratio / 100;
1867	/*	1914	/*
1868	* Leave the last 3% for root	1915	* Leave the last 3% for root
1869	*/	1916	*/
1870	if (!cap_sys_admin)	1917	if (!cap_sys_admin)
1871	allowed -= allowed / 32;	1918	allowed -= allowed / 32;
1872	allowed += total_swap_pages;	1919	allowed += total_swap_pages;
1873		1920
1874	/* Don't let a single process grow too big:	1921	/* Don't let a single process grow too big:
1875	leave 3% of the size of this process for other processes */	1922	leave 3% of the size of this process for other processes */
1876	if (mm)	1923	if (mm)
1877	allowed -= mm->total_vm / 32;	1924	allowed -= mm->total_vm / 32;
1878		1925
1879	if (percpu_counter_read_positive(&vm_committed_as) < allowed)	1926	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
1880	return 0;	1927	return 0;
1881		1928
1882	error:	1929	error:
1883	vm_unacct_memory(pages);	1930	vm_unacct_memory(pages);
1884		1931
1885	return -ENOMEM;	1932	return -ENOMEM;
1886	}	1933	}
1887		1934
1888	int in_gate_area_no_task(unsigned long addr)	1935	int in_gate_area_no_task(unsigned long addr)
1889	{	1936	{
1890	return 0;	1937	return 0;
1891	}	1938	}
1892		1939
1893	int filemap_fault(struct vm_area_struct vma, struct vm_fault vmf)	1940	int filemap_fault(struct vm_area_struct vma, struct vm_fault vmf)
1894	{	1941	{
1895	BUG();	1942	BUG();
1896	return 0;	1943	return 0;
1897	}	1944	}
1898	EXPORT_SYMBOL(filemap_fault);	1945	EXPORT_SYMBOL(filemap_fault);
1899		1946
1900	/*	1947	/*
1901	* Access another process' address space.	1948	* Access another process' address space.
1902	* - source/target buffer must be kernel space	1949	* - source/target buffer must be kernel space
1903	*/	1950	*/
1904	int access_process_vm(struct task_struct tsk, unsigned long addr, void buf, int len, int write)	1951	int access_process_vm(struct task_struct tsk, unsigned long addr, void buf, int len, int write)
1905	{	1952	{
1906	struct vm_area_struct *vma;	1953	struct vm_area_struct *vma;
1907	struct mm_struct *mm;	1954	struct mm_struct *mm;
1908		1955
1909	if (addr + len < addr)	1956	if (addr + len < addr)
1910	return 0;	1957	return 0;
1911		1958
1912	mm = get_task_mm(tsk);	1959	mm = get_task_mm(tsk);
1913	if (!mm)	1960	if (!mm)
1914	return 0;	1961	return 0;
1915		1962
1916	down_read(&mm->mmap_sem);	1963	down_read(&mm->mmap_sem);
1917		1964
1918	/* the access must start within one of the target process's mappings */	1965	/* the access must start within one of the target process's mappings */
1919	vma = find_vma(mm, addr);	1966	vma = find_vma(mm, addr);
1920	if (vma) {	1967	if (vma) {
1921	/* don't overrun this mapping */	1968	/* don't overrun this mapping */
1922	if (addr + len >= vma->vm_end)	1969	if (addr + len >= vma->vm_end)
1923	len = vma->vm_end - addr;	1970	len = vma->vm_end - addr;
1924		1971
1925	/* only read or write mappings where it is permitted */	1972	/* only read or write mappings where it is permitted */
1926	if (write && vma->vm_flags & VM_MAYWRITE)	1973	if (write && vma->vm_flags & VM_MAYWRITE)
1927	copy_to_user_page(vma, NULL, addr,	1974	copy_to_user_page(vma, NULL, addr,
1928	(void *) addr, buf, len);	1975	(void *) addr, buf, len);
1929	else if (!write && vma->vm_flags & VM_MAYREAD)	1976	else if (!write && vma->vm_flags & VM_MAYREAD)
1930	copy_from_user_page(vma, NULL, addr,	1977	copy_from_user_page(vma, NULL, addr,
1931	buf, (void *) addr, len);	1978	buf, (void *) addr, len);
1932	else	1979	else
1933	len = 0;	1980	len = 0;
1934	} else {	1981	} else {
1935	len = 0;	1982	len = 0;
1936	}	1983	}
1937		1984
1938	up_read(&mm->mmap_sem);	1985	up_read(&mm->mmap_sem);
1939	mmput(mm);	1986	mmput(mm);
1940	return len;	1987	return len;
1941	}	1988	}
1942		1989
1943	/**	1990	/**
1944	* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode	1991	* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1945	* @inode: The inode to check	1992	* @inode: The inode to check
1946	* @size: The current filesize of the inode	1993	* @size: The current filesize of the inode
1947	* @newsize: The proposed filesize of the inode	1994	* @newsize: The proposed filesize of the inode
1948	*	1995	*
1949	* Check the shared mappings on an inode on behalf of a shrinking truncate to	1996	* Check the shared mappings on an inode on behalf of a shrinking truncate to
1950	* make sure that that any outstanding VMAs aren't broken and then shrink the	1997	* make sure that that any outstanding VMAs aren't broken and then shrink the
1951	* vm_regions that extend that beyond so that do_mmap_pgoff() doesn't	1998	* vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
1952	* automatically grant mappings that are too large.	1999	* automatically grant mappings that are too large.
1953	*/	2000	*/
1954	int nommu_shrink_inode_mappings(struct inode *inode, size_t size,	2001	int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1955	size_t newsize)	2002	size_t newsize)
1956	{	2003	{
1957	struct vm_area_struct *vma;	2004	struct vm_area_struct *vma;
1958	struct prio_tree_iter iter;	2005	struct prio_tree_iter iter;
1959	struct vm_region *region;	2006	struct vm_region *region;
1960	pgoff_t low, high;	2007	pgoff_t low, high;
1961	size_t r_size, r_top;	2008	size_t r_size, r_top;
1962		2009
1963	low = newsize >> PAGE_SHIFT;	2010	low = newsize >> PAGE_SHIFT;
1964	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;	2011	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1965		2012
1966	down_write(&nommu_region_sem);	2013	down_write(&nommu_region_sem);
1967		2014
1968	/* search for VMAs that fall within the dead zone */	2015	/* search for VMAs that fall within the dead zone */
1969	vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap,	2016	vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap,
1970	low, high) {	2017	low, high) {
1971	/* found one - only interested if it's shared out of the page	2018	/* found one - only interested if it's shared out of the page
1972	* cache */	2019	* cache */
1973	if (vma->vm_flags & VM_SHARED) {	2020	if (vma->vm_flags & VM_SHARED) {
1974	up_write(&nommu_region_sem);	2021	up_write(&nommu_region_sem);
1975	return -ETXTBSY; /* not quite true, but near enough */	2022	return -ETXTBSY; /* not quite true, but near enough */
1976	}	2023	}
1977	}	2024	}
1978		2025
1979	/* reduce any regions that overlap the dead zone - if in existence,	2026	/* reduce any regions that overlap the dead zone - if in existence,
1980	* these will be pointed to by VMAs that don't overlap the dead zone	2027	* these will be pointed to by VMAs that don't overlap the dead zone
1981	*	2028	*
1982	* we don't check for any regions that start beyond the EOF as there	2029	* we don't check for any regions that start beyond the EOF as there
1983	* shouldn't be any	2030	* shouldn't be any
1984	*/	2031	*/
1985	vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap,	2032	vma_prio_tree_foreach(vma, &iter, &inode->i_mapping->i_mmap,
1986	0, ULONG_MAX) {	2033	0, ULONG_MAX) {
1987	if (!(vma->vm_flags & VM_SHARED))	2034	if (!(vma->vm_flags & VM_SHARED))
1988	continue;	2035	continue;
1989		2036
1990	region = vma->vm_region;	2037	region = vma->vm_region;
1991	r_size = region->vm_top - region->vm_start;	2038	r_size = region->vm_top - region->vm_start;
1992	r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;	2039	r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1993		2040
1994	if (r_top > newsize) {	2041	if (r_top > newsize) {
1995	region->vm_top -= r_top - newsize;	2042	region->vm_top -= r_top - newsize;
1996	if (region->vm_end > region->vm_top)	2043	if (region->vm_end > region->vm_top)
1997	region->vm_end = region->vm_top;	2044	region->vm_end = region->vm_top;
1998	}	2045	}
1999	}	2046	}
2000		2047
2001	up_write(&nommu_region_sem);	2048	up_write(&nommu_region_sem);
2002	return 0;	2049	return 0;
2003	}	2050	}
2004		2051

mm/vmalloc.c

Diff comments View file @ e1ca778

 /*
  *  linux/mm/vmalloc.c
  *
  *  Copyright (C) 1993  Linus Torvalds
  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  *  Numa awareness, Christoph Lameter, SGI, June 2005
  */
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/highmem.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/debugobjects.h>
 #include <linux/kallsyms.h>
 #include <linux/list.h>
 #include <linux/rbtree.h>
 #include <linux/radix-tree.h>
 #include <linux/rcupdate.h>
 #include <linux/pfn.h>
 #include <linux/kmemleak.h>
 #include <asm/atomic.h>
 #include <asm/uaccess.h>
 #include <asm/tlbflush.h>
 #include <asm/shmparam.h>
 bool vmap_lazy_unmap __read_mostly = true;
 /*** Page table manipulation functions ***/
 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
 {
 	pte_t *pte;
 	pte = pte_offset_kernel(pmd, addr);
 	do {
 		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
 		WARN_ON(!pte_none(ptent) && !pte_present(ptent));
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 }
 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
 {
 	pmd_t *pmd;
 	unsigned long next;
 	pmd = pmd_offset(pud, addr);
 	do {
 		next = pmd_addr_end(addr, end);
 		if (pmd_none_or_clear_bad(pmd))
 			continue;
 		vunmap_pte_range(pmd, addr, next);
 	} while (pmd++, addr = next, addr != end);
 }
 static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
 {
 	pud_t *pud;
 	unsigned long next;
 	pud = pud_offset(pgd, addr);
 	do {
 		next = pud_addr_end(addr, end);
 		if (pud_none_or_clear_bad(pud))
 			continue;
 		vunmap_pmd_range(pud, addr, next);
 	} while (pud++, addr = next, addr != end);
 }
 static void vunmap_page_range(unsigned long addr, unsigned long end)
 {
 	pgd_t *pgd;
 	unsigned long next;
 	BUG_ON(addr >= end);
 	pgd = pgd_offset_k(addr);
 	do {
 		next = pgd_addr_end(addr, end);
 		if (pgd_none_or_clear_bad(pgd))
 			continue;
 		vunmap_pud_range(pgd, addr, next);
 	} while (pgd++, addr = next, addr != end);
 }
 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
 {
 	pte_t *pte;
 	/*
 	 * nr is a running index into the array which helps higher level
 	 * callers keep track of where we're up to.
 	 */
 	pte = pte_alloc_kernel(pmd, addr);
 	if (!pte)
 		return -ENOMEM;
 	do {
 		struct page *page = pages[*nr];
 		if (WARN_ON(!pte_none(*pte)))
 			return -EBUSY;
 		if (WARN_ON(!page))
 			return -ENOMEM;
 		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
 		(*nr)++;
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 	return 0;
 }
 static int vmap_pmd_range(pud_t *pud, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
 {
 	pmd_t *pmd;
 	unsigned long next;
 	pmd = pmd_alloc(&init_mm, pud, addr);
 	if (!pmd)
 		return -ENOMEM;
 	do {
 		next = pmd_addr_end(addr, end);
 		if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
 			return -ENOMEM;
 	} while (pmd++, addr = next, addr != end);
 	return 0;
 }
 static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
 {
 	pud_t *pud;
 	unsigned long next;
 	pud = pud_alloc(&init_mm, pgd, addr);
 	if (!pud)
 		return -ENOMEM;
 	do {
 		next = pud_addr_end(addr, end);
 		if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
 			return -ENOMEM;
 	} while (pud++, addr = next, addr != end);
 	return 0;
 }
 /*
  * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
  * will have pfns corresponding to the "pages" array.
  *
  * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
  */
 static int vmap_page_range_noflush(unsigned long start, unsigned long end,
 				   pgprot_t prot, struct page **pages)
 {
 	pgd_t *pgd;
 	unsigned long next;
 	unsigned long addr = start;
 	int err = 0;
 	int nr = 0;
 	BUG_ON(addr >= end);
 	pgd = pgd_offset_k(addr);
 	do {
 		next = pgd_addr_end(addr, end);
 		err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
 		if (err)
 			return err;
 	} while (pgd++, addr = next, addr != end);
 	return nr;
 }
 static int vmap_page_range(unsigned long start, unsigned long end,
 			   pgprot_t prot, struct page **pages)
 {
 	int ret;
 	ret = vmap_page_range_noflush(start, end, prot, pages);
 	flush_cache_vmap(start, end);
 	return ret;
 }
 int is_vmalloc_or_module_addr(const void *x)
 {
 	/*
 	 * ARM, x86-64 and sparc64 put modules in a special place,
 	 * and fall back on vmalloc() if that fails. Others
 	 * just put it in the vmalloc space.
 	 */
 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
 	unsigned long addr = (unsigned long)x;
 	if (addr >= MODULES_VADDR && addr < MODULES_END)
 		return 1;
 #endif
 	return is_vmalloc_addr(x);
 }
 /*
  * Walk a vmap address to the struct page it maps.
  */
 struct page *vmalloc_to_page(const void *vmalloc_addr)
 {
 	unsigned long addr = (unsigned long) vmalloc_addr;
 	struct page *page = NULL;
 	pgd_t *pgd = pgd_offset_k(addr);
 	/*
 	 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
 	 * architectures that do not vmalloc module space
 	 */
 	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
 	if (!pgd_none(*pgd)) {
 		pud_t *pud = pud_offset(pgd, addr);
 		if (!pud_none(*pud)) {
 			pmd_t *pmd = pmd_offset(pud, addr);
 			if (!pmd_none(*pmd)) {
 				pte_t *ptep, pte;
 				ptep = pte_offset_map(pmd, addr);
 				pte = *ptep;
 				if (pte_present(pte))
 					page = pte_page(pte);
 				pte_unmap(ptep);
 			}
 		}
 	}
 	return page;
 }
 EXPORT_SYMBOL(vmalloc_to_page);
 /*
  * Map a vmalloc()-space virtual address to the physical page frame number.
  */
 unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
 {
 	return page_to_pfn(vmalloc_to_page(vmalloc_addr));
 }
 EXPORT_SYMBOL(vmalloc_to_pfn);
 /*** Global kva allocator ***/
 #define VM_LAZY_FREE	0x01
 #define VM_LAZY_FREEING	0x02
 #define VM_VM_AREA	0x04
 struct vmap_area {
 	unsigned long va_start;
 	unsigned long va_end;
 	unsigned long flags;
 	struct rb_node rb_node;		/* address sorted rbtree */
 	struct list_head list;		/* address sorted list */
 	struct list_head purge_list;	/* "lazy purge" list */
 	void *private;
 	struct rcu_head rcu_head;
 };
 static DEFINE_SPINLOCK(vmap_area_lock);
 static struct rb_root vmap_area_root = RB_ROOT;
 static LIST_HEAD(vmap_area_list);
 static unsigned long vmap_area_pcpu_hole;
 static struct vmap_area *__find_vmap_area(unsigned long addr)
 {
 	struct rb_node *n = vmap_area_root.rb_node;
 	while (n) {
 		struct vmap_area *va;
 		va = rb_entry(n, struct vmap_area, rb_node);
 		if (addr < va->va_start)
 			n = n->rb_left;
 		else if (addr > va->va_start)
 			n = n->rb_right;
 		else
 			return va;
 	}
 	return NULL;
 }
 static void __insert_vmap_area(struct vmap_area *va)
 {
 	struct rb_node **p = &vmap_area_root.rb_node;
 	struct rb_node *parent = NULL;
 	struct rb_node *tmp;
 	while (*p) {
 		struct vmap_area *tmp_va;
 		parent = *p;
 		tmp_va = rb_entry(parent, struct vmap_area, rb_node);
 		if (va->va_start < tmp_va->va_end)
 			p = &(*p)->rb_left;
 		else if (va->va_end > tmp_va->va_start)
 			p = &(*p)->rb_right;
 		else
 			BUG();
 	}
 	rb_link_node(&va->rb_node, parent, p);
 	rb_insert_color(&va->rb_node, &vmap_area_root);
 	/* address-sort this list so it is usable like the vmlist */
 	tmp = rb_prev(&va->rb_node);
 	if (tmp) {
 		struct vmap_area *prev;
 		prev = rb_entry(tmp, struct vmap_area, rb_node);
 		list_add_rcu(&va->list, &prev->list);
 	} else
 		list_add_rcu(&va->list, &vmap_area_list);
 }
 static void purge_vmap_area_lazy(void);
 /*
  * Allocate a region of KVA of the specified size and alignment, within the
  * vstart and vend.
  */
 static struct vmap_area *alloc_vmap_area(unsigned long size,
 				unsigned long align,
 				unsigned long vstart, unsigned long vend,
 				int node, gfp_t gfp_mask)
 {
 	struct vmap_area *va;
 	struct rb_node *n;
 	unsigned long addr;
 	int purged = 0;
 	BUG_ON(!size);
 	BUG_ON(size & ~PAGE_MASK);
 	va = kmalloc_node(sizeof(struct vmap_area),
 			gfp_mask & GFP_RECLAIM_MASK, node);
 	if (unlikely(!va))
 		return ERR_PTR(-ENOMEM);
 retry:
 	addr = ALIGN(vstart, align);
 	spin_lock(&vmap_area_lock);
 	if (addr + size - 1 < addr)
 		goto overflow;
 	/* XXX: could have a last_hole cache */
 	n = vmap_area_root.rb_node;
 	if (n) {
 		struct vmap_area *first = NULL;
 		do {
 			struct vmap_area *tmp;
 			tmp = rb_entry(n, struct vmap_area, rb_node);
 			if (tmp->va_end >= addr) {
 				if (!first && tmp->va_start < addr + size)
 					first = tmp;
 				n = n->rb_left;
 			} else {
 				first = tmp;
 				n = n->rb_right;
 			}
 		} while (n);
 		if (!first)
 			goto found;
 		if (first->va_end < addr) {
 			n = rb_next(&first->rb_node);
 			if (n)
 				first = rb_entry(n, struct vmap_area, rb_node);
 			else
 				goto found;
 		}
 		while (addr + size > first->va_start && addr + size <= vend) {
 			addr = ALIGN(first->va_end + PAGE_SIZE, align);
 			if (addr + size - 1 < addr)
 				goto overflow;
 			n = rb_next(&first->rb_node);
 			if (n)
 				first = rb_entry(n, struct vmap_area, rb_node);
 			else
 				goto found;
 		}
 	}
 found:
 	if (addr + size > vend) {
 overflow:
 		spin_unlock(&vmap_area_lock);
 		if (!purged) {
 			purge_vmap_area_lazy();
 			purged = 1;
 			goto retry;
 		}
 		if (printk_ratelimit())
 			printk(KERN_WARNING
 				"vmap allocation for size %lu failed: "
 				"use vmalloc=<size> to increase size.\n", size);
 		kfree(va);
 		return ERR_PTR(-EBUSY);
 	}
 	BUG_ON(addr & (align-1));
 	va->va_start = addr;
 	va->va_end = addr + size;
 	va->flags = 0;
 	__insert_vmap_area(va);
 	spin_unlock(&vmap_area_lock);
 	return va;
 }
 static void rcu_free_va(struct rcu_head *head)
 {
 	struct vmap_area *va = container_of(head, struct vmap_area, rcu_head);
 	kfree(va);
 }
 static void __free_vmap_area(struct vmap_area *va)
 {
 	BUG_ON(RB_EMPTY_NODE(&va->rb_node));
 	rb_erase(&va->rb_node, &vmap_area_root);
 	RB_CLEAR_NODE(&va->rb_node);
 	list_del_rcu(&va->list);
 	/*
 	 * Track the highest possible candidate for pcpu area
 	 * allocation.  Areas outside of vmalloc area can be returned
 	 * here too, consider only end addresses which fall inside
 	 * vmalloc area proper.
 	 */
 	if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
 		vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
 	call_rcu(&va->rcu_head, rcu_free_va);
 }
 /*
  * Free a region of KVA allocated by alloc_vmap_area
  */
 static void free_vmap_area(struct vmap_area *va)
 {
 	spin_lock(&vmap_area_lock);
 	__free_vmap_area(va);
 	spin_unlock(&vmap_area_lock);
 }
 /*
  * Clear the pagetable entries of a given vmap_area
  */
 static void unmap_vmap_area(struct vmap_area *va)
 {
 	vunmap_page_range(va->va_start, va->va_end);
 }
 static void vmap_debug_free_range(unsigned long start, unsigned long end)
 {
 	/*
 	 * Unmap page tables and force a TLB flush immediately if
 	 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
 	 * bugs similarly to those in linear kernel virtual address
 	 * space after a page has been freed.
 	 *
 	 * All the lazy freeing logic is still retained, in order to
 	 * minimise intrusiveness of this debugging feature.
 	 *
 	 * This is going to be *slow* (linear kernel virtual address
 	 * debugging doesn't do a broadcast TLB flush so it is a lot
 	 * faster).
 	 */
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	vunmap_page_range(start, end);
 	flush_tlb_kernel_range(start, end);
 #endif
 }
 /*
  * lazy_max_pages is the maximum amount of virtual address space we gather up
  * before attempting to purge with a TLB flush.
  *
  * There is a tradeoff here: a larger number will cover more kernel page tables
  * and take slightly longer to purge, but it will linearly reduce the number of
  * global TLB flushes that must be performed. It would seem natural to scale
  * this number up linearly with the number of CPUs (because vmapping activity
  * could also scale linearly with the number of CPUs), however it is likely
  * that in practice, workloads might be constrained in other ways that mean
  * vmap activity will not scale linearly with CPUs. Also, I want to be
  * conservative and not introduce a big latency on huge systems, so go with
  * a less aggressive log scale. It will still be an improvement over the old
  * code, and it will be simple to change the scale factor if we find that it
  * becomes a problem on bigger systems.
  */
 static unsigned long lazy_max_pages(void)
 {
 	unsigned int log;
 	if (!vmap_lazy_unmap)
 		return 0;
 	log = fls(num_online_cpus());
 	return log * (32UL * 1024 * 1024 / PAGE_SIZE);
 }
 static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
 /* for per-CPU blocks */
 static void purge_fragmented_blocks_allcpus(void);
 /*
  * called before a call to iounmap() if the caller wants vm_area_struct's
  * immediately freed.
  */
 void set_iounmap_nonlazy(void)
 {
 	atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
 }
 /*
  * Purges all lazily-freed vmap areas.
  *
  * If sync is 0 then don't purge if there is already a purge in progress.
  * If force_flush is 1, then flush kernel TLBs between *start and *end even
  * if we found no lazy vmap areas to unmap (callers can use this to optimise
  * their own TLB flushing).
  * Returns with *start = min(*start, lowest purged address)
  *              *end = max(*end, highest purged address)
  */
 static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
 					int sync, int force_flush)
 {
 	static DEFINE_SPINLOCK(purge_lock);
 	LIST_HEAD(valist);
 	struct vmap_area *va;
 	struct vmap_area *n_va;
 	int nr = 0;
 	/*
 	 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
 	 * should not expect such behaviour. This just simplifies locking for
 	 * the case that isn't actually used at the moment anyway.
 	 */
 	if (!sync && !force_flush) {
 		if (!spin_trylock(&purge_lock))
 			return;
 	} else
 		spin_lock(&purge_lock);
 	if (sync)
 		purge_fragmented_blocks_allcpus();
 	rcu_read_lock();
 	list_for_each_entry_rcu(va, &vmap_area_list, list) {
 		if (va->flags & VM_LAZY_FREE) {
 			if (va->va_start < *start)
 				*start = va->va_start;
 			if (va->va_end > *end)
 				*end = va->va_end;
 			nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
 			unmap_vmap_area(va);
 			list_add_tail(&va->purge_list, &valist);
 			va->flags |= VM_LAZY_FREEING;
 			va->flags &= ~VM_LAZY_FREE;
 		}
 	}
 	rcu_read_unlock();
 	if (nr)
 		atomic_sub(nr, &vmap_lazy_nr);
 	if (nr || force_flush)
 		flush_tlb_kernel_range(*start, *end);
 	if (nr) {
 		spin_lock(&vmap_area_lock);
 		list_for_each_entry_safe(va, n_va, &valist, purge_list)
 			__free_vmap_area(va);
 		spin_unlock(&vmap_area_lock);
 	}
 	spin_unlock(&purge_lock);
 }
 /*
  * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
  * is already purging.
  */
 static void try_purge_vmap_area_lazy(void)
 {
 	unsigned long start = ULONG_MAX, end = 0;
 	__purge_vmap_area_lazy(&start, &end, 0, 0);
 }
 /*
  * Kick off a purge of the outstanding lazy areas.
  */
 static void purge_vmap_area_lazy(void)
 {
 	unsigned long start = ULONG_MAX, end = 0;
 	__purge_vmap_area_lazy(&start, &end, 1, 0);
 }
 /*
  * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
  * called for the correct range previously.
  */
 static void free_unmap_vmap_area_noflush(struct vmap_area *va)
 {
 	va->flags |= VM_LAZY_FREE;
 	atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
 	if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
 		try_purge_vmap_area_lazy();
 }
 /*
  * Free and unmap a vmap area
  */
 static void free_unmap_vmap_area(struct vmap_area *va)
 {
 	flush_cache_vunmap(va->va_start, va->va_end);
 	free_unmap_vmap_area_noflush(va);
 }
 static struct vmap_area *find_vmap_area(unsigned long addr)
 {
 	struct vmap_area *va;
 	spin_lock(&vmap_area_lock);
 	va = __find_vmap_area(addr);
 	spin_unlock(&vmap_area_lock);
 	return va;
 }
 static void free_unmap_vmap_area_addr(unsigned long addr)
 {
 	struct vmap_area *va;
 	va = find_vmap_area(addr);
 	BUG_ON(!va);
 	free_unmap_vmap_area(va);
 }
 /*** Per cpu kva allocator ***/
 /*
  * vmap space is limited especially on 32 bit architectures. Ensure there is
  * room for at least 16 percpu vmap blocks per CPU.
  */
 /*
  * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
  * to #define VMALLOC_SPACE		(VMALLOC_END-VMALLOC_START). Guess
  * instead (we just need a rough idea)
  */
 #if BITS_PER_LONG == 32
 #define VMALLOC_SPACE		(128UL*1024*1024)
 #else
 #define VMALLOC_SPACE		(128UL*1024*1024*1024)
 #endif
 #define VMALLOC_PAGES		(VMALLOC_SPACE / PAGE_SIZE)
 #define VMAP_MAX_ALLOC		BITS_PER_LONG	/* 256K with 4K pages */
 #define VMAP_BBMAP_BITS_MAX	1024	/* 4MB with 4K pages */
 #define VMAP_BBMAP_BITS_MIN	(VMAP_MAX_ALLOC*2)
 #define VMAP_MIN(x, y)		((x) < (y) ? (x) : (y)) /* can't use min() */
 #define VMAP_MAX(x, y)		((x) > (y) ? (x) : (y)) /* can't use max() */
 #define VMAP_BBMAP_BITS		VMAP_MIN(VMAP_BBMAP_BITS_MAX,		\
 					VMAP_MAX(VMAP_BBMAP_BITS_MIN,	\
 						VMALLOC_PAGES / NR_CPUS / 16))
 #define VMAP_BLOCK_SIZE		(VMAP_BBMAP_BITS * PAGE_SIZE)
 static bool vmap_initialized __read_mostly = false;
 struct vmap_block_queue {
 	spinlock_t lock;
 	struct list_head free;
 };
 struct vmap_block {
 	spinlock_t lock;
 	struct vmap_area *va;
 	struct vmap_block_queue *vbq;
 	unsigned long free, dirty;
 	DECLARE_BITMAP(alloc_map, VMAP_BBMAP_BITS);
 	DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
 	struct list_head free_list;
 	struct rcu_head rcu_head;
 	struct list_head purge;
 };
 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
 /*
  * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
  * in the free path. Could get rid of this if we change the API to return a
  * "cookie" from alloc, to be passed to free. But no big deal yet.
  */
 static DEFINE_SPINLOCK(vmap_block_tree_lock);
 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
 /*
  * We should probably have a fallback mechanism to allocate virtual memory
  * out of partially filled vmap blocks. However vmap block sizing should be
  * fairly reasonable according to the vmalloc size, so it shouldn't be a
  * big problem.
  */
 static unsigned long addr_to_vb_idx(unsigned long addr)
 {
 	addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
 	addr /= VMAP_BLOCK_SIZE;
 	return addr;
 }
 static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
 {
 	struct vmap_block_queue *vbq;
 	struct vmap_block *vb;
 	struct vmap_area *va;
 	unsigned long vb_idx;
 	int node, err;
 	node = numa_node_id();
 	vb = kmalloc_node(sizeof(struct vmap_block),
 			gfp_mask & GFP_RECLAIM_MASK, node);
 	if (unlikely(!vb))
 		return ERR_PTR(-ENOMEM);
 	va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
 					VMALLOC_START, VMALLOC_END,
 					node, gfp_mask);
 	if (unlikely(IS_ERR(va))) {
 		kfree(vb);
 		return ERR_CAST(va);
 	}
 	err = radix_tree_preload(gfp_mask);
 	if (unlikely(err)) {
 		kfree(vb);
 		free_vmap_area(va);
 		return ERR_PTR(err);
 	}
 	spin_lock_init(&vb->lock);
 	vb->va = va;
 	vb->free = VMAP_BBMAP_BITS;
 	vb->dirty = 0;
 	bitmap_zero(vb->alloc_map, VMAP_BBMAP_BITS);
 	bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
 	INIT_LIST_HEAD(&vb->free_list);
 	vb_idx = addr_to_vb_idx(va->va_start);
 	spin_lock(&vmap_block_tree_lock);
 	err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
 	spin_unlock(&vmap_block_tree_lock);
 	BUG_ON(err);
 	radix_tree_preload_end();
 	vbq = &get_cpu_var(vmap_block_queue);
 	vb->vbq = vbq;
 	spin_lock(&vbq->lock);
 	list_add_rcu(&vb->free_list, &vbq->free);
 	spin_unlock(&vbq->lock);
 	put_cpu_var(vmap_block_queue);
 	return vb;
 }
 static void rcu_free_vb(struct rcu_head *head)
 {
 	struct vmap_block *vb = container_of(head, struct vmap_block, rcu_head);
 	kfree(vb);
 }
 static void free_vmap_block(struct vmap_block *vb)
 {
 	struct vmap_block *tmp;
 	unsigned long vb_idx;
 	vb_idx = addr_to_vb_idx(vb->va->va_start);
 	spin_lock(&vmap_block_tree_lock);
 	tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
 	spin_unlock(&vmap_block_tree_lock);
 	BUG_ON(tmp != vb);
 	free_unmap_vmap_area_noflush(vb->va);
 	call_rcu(&vb->rcu_head, rcu_free_vb);
 }
 static void purge_fragmented_blocks(int cpu)
 {
 	LIST_HEAD(purge);
 	struct vmap_block *vb;
 	struct vmap_block *n_vb;
 	struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
 	rcu_read_lock();
 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
 		if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
 			continue;
 		spin_lock(&vb->lock);
 		if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
 			vb->free = 0; /* prevent further allocs after releasing lock */
 			vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
 			bitmap_fill(vb->alloc_map, VMAP_BBMAP_BITS);
 			bitmap_fill(vb->dirty_map, VMAP_BBMAP_BITS);
 			spin_lock(&vbq->lock);
 			list_del_rcu(&vb->free_list);
 			spin_unlock(&vbq->lock);
 			spin_unlock(&vb->lock);
 			list_add_tail(&vb->purge, &purge);
 		} else
 			spin_unlock(&vb->lock);
 	}
 	rcu_read_unlock();
 	list_for_each_entry_safe(vb, n_vb, &purge, purge) {
 		list_del(&vb->purge);
 		free_vmap_block(vb);
 	}
 }
 static void purge_fragmented_blocks_thiscpu(void)
 {
 	purge_fragmented_blocks(smp_processor_id());
 }
 static void purge_fragmented_blocks_allcpus(void)
 {
 	int cpu;
 	for_each_possible_cpu(cpu)
 		purge_fragmented_blocks(cpu);
 }
 static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
 {
 	struct vmap_block_queue *vbq;
 	struct vmap_block *vb;
 	unsigned long addr = 0;
 	unsigned int order;
 	int purge = 0;
 	BUG_ON(size & ~PAGE_MASK);
 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
 	order = get_order(size);
 again:
 	rcu_read_lock();
 	vbq = &get_cpu_var(vmap_block_queue);
 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
 		int i;
 		spin_lock(&vb->lock);
 		if (vb->free < 1UL << order)
 			goto next;
 		i = bitmap_find_free_region(vb->alloc_map,
 						VMAP_BBMAP_BITS, order);
 		if (i < 0) {
 			if (vb->free + vb->dirty == VMAP_BBMAP_BITS) {
 				/* fragmented and no outstanding allocations */
 				BUG_ON(vb->dirty != VMAP_BBMAP_BITS);
 				purge = 1;
 			}
 			goto next;
 		}
 		addr = vb->va->va_start + (i << PAGE_SHIFT);
 		BUG_ON(addr_to_vb_idx(addr) !=
 				addr_to_vb_idx(vb->va->va_start));
 		vb->free -= 1UL << order;
 		if (vb->free == 0) {
 			spin_lock(&vbq->lock);
 			list_del_rcu(&vb->free_list);
 			spin_unlock(&vbq->lock);
 		}
 		spin_unlock(&vb->lock);
 		break;
 next:
 		spin_unlock(&vb->lock);
 	}
 	if (purge)
 		purge_fragmented_blocks_thiscpu();
 	put_cpu_var(vmap_block_queue);
 	rcu_read_unlock();
 	if (!addr) {
 		vb = new_vmap_block(gfp_mask);
 		if (IS_ERR(vb))
 			return vb;
 		goto again;
 	}
 	return (void *)addr;
 }
 static void vb_free(const void *addr, unsigned long size)
 {
 	unsigned long offset;
 	unsigned long vb_idx;
 	unsigned int order;
 	struct vmap_block *vb;
 	BUG_ON(size & ~PAGE_MASK);
 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
 	flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
 	order = get_order(size);
 	offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
 	vb_idx = addr_to_vb_idx((unsigned long)addr);
 	rcu_read_lock();
 	vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
 	rcu_read_unlock();
 	BUG_ON(!vb);
 	spin_lock(&vb->lock);
 	BUG_ON(bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order));
 	vb->dirty += 1UL << order;
 	if (vb->dirty == VMAP_BBMAP_BITS) {
 		BUG_ON(vb->free);
 		spin_unlock(&vb->lock);
 		free_vmap_block(vb);
 	} else
 		spin_unlock(&vb->lock);
 }
 /**
  * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
  *
  * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
  * to amortize TLB flushing overheads. What this means is that any page you
  * have now, may, in a former life, have been mapped into kernel virtual
  * address by the vmap layer and so there might be some CPUs with TLB entries
  * still referencing that page (additional to the regular 1:1 kernel mapping).
  *
  * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
  * be sure that none of the pages we have control over will have any aliases
  * from the vmap layer.
  */
 void vm_unmap_aliases(void)
 {
 	unsigned long start = ULONG_MAX, end = 0;
 	int cpu;
 	int flush = 0;
 	if (unlikely(!vmap_initialized))
 		return;
 	for_each_possible_cpu(cpu) {
 		struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
 		struct vmap_block *vb;
 		rcu_read_lock();
 		list_for_each_entry_rcu(vb, &vbq->free, free_list) {
 			int i;
 			spin_lock(&vb->lock);
 			i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS);
 			while (i < VMAP_BBMAP_BITS) {
 				unsigned long s, e;
 				int j;
 				j = find_next_zero_bit(vb->dirty_map,
 					VMAP_BBMAP_BITS, i);
 				s = vb->va->va_start + (i << PAGE_SHIFT);
 				e = vb->va->va_start + (j << PAGE_SHIFT);
 				vunmap_page_range(s, e);
 				flush = 1;
 				if (s < start)
 					start = s;
 				if (e > end)
 					end = e;
 				i = j;
 				i = find_next_bit(vb->dirty_map,
 							VMAP_BBMAP_BITS, i);
 			}
 			spin_unlock(&vb->lock);
 		}
 		rcu_read_unlock();
 	}
 	__purge_vmap_area_lazy(&start, &end, 1, flush);
 }
 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
 /**
  * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
  * @mem: the pointer returned by vm_map_ram
  * @count: the count passed to that vm_map_ram call (cannot unmap partial)
  */
 void vm_unmap_ram(const void *mem, unsigned int count)
 {
 	unsigned long size = count << PAGE_SHIFT;
 	unsigned long addr = (unsigned long)mem;
 	BUG_ON(!addr);
 	BUG_ON(addr < VMALLOC_START);
 	BUG_ON(addr > VMALLOC_END);
 	BUG_ON(addr & (PAGE_SIZE-1));
 	debug_check_no_locks_freed(mem, size);
 	vmap_debug_free_range(addr, addr+size);
 	if (likely(count <= VMAP_MAX_ALLOC))
 		vb_free(mem, size);
 	else
 		free_unmap_vmap_area_addr(addr);
 }
 EXPORT_SYMBOL(vm_unmap_ram);
 /**
  * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
  * @pages: an array of pointers to the pages to be mapped
  * @count: number of pages
  * @node: prefer to allocate data structures on this node
  * @prot: memory protection to use. PAGE_KERNEL for regular RAM
  *
  * Returns: a pointer to the address that has been mapped, or %NULL on failure
  */
 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
 {
 	unsigned long size = count << PAGE_SHIFT;
 	unsigned long addr;
 	void *mem;
 	if (likely(count <= VMAP_MAX_ALLOC)) {
 		mem = vb_alloc(size, GFP_KERNEL);
 		if (IS_ERR(mem))
 			return NULL;
 		addr = (unsigned long)mem;
 	} else {
 		struct vmap_area *va;
 		va = alloc_vmap_area(size, PAGE_SIZE,
 				VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
 		if (IS_ERR(va))
 			return NULL;
 		addr = va->va_start;
 		mem = (void *)addr;
 	}
 	if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
 		vm_unmap_ram(mem, count);
 		return NULL;
 	}
 	return mem;
 }
 EXPORT_SYMBOL(vm_map_ram);
 /**
  * vm_area_register_early - register vmap area early during boot
  * @vm: vm_struct to register
  * @align: requested alignment
  *
  * This function is used to register kernel vm area before
  * vmalloc_init() is called.  @vm->size and @vm->flags should contain
  * proper values on entry and other fields should be zero.  On return,
  * vm->addr contains the allocated address.
  *
  * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  */
 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
 {
 	static size_t vm_init_off __initdata;
 	unsigned long addr;
 	addr = ALIGN(VMALLOC_START + vm_init_off, align);
 	vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
 	vm->addr = (void *)addr;
 	vm->next = vmlist;
 	vmlist = vm;
 }
 void __init vmalloc_init(void)
 {
 	struct vmap_area *va;
 	struct vm_struct *tmp;
 	int i;
 	for_each_possible_cpu(i) {
 		struct vmap_block_queue *vbq;
 		vbq = &per_cpu(vmap_block_queue, i);
 		spin_lock_init(&vbq->lock);
 		INIT_LIST_HEAD(&vbq->free);
 	}
 	/* Import existing vmlist entries. */
 	for (tmp = vmlist; tmp; tmp = tmp->next) {
 		va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
 		va->flags = tmp->flags | VM_VM_AREA;
 		va->va_start = (unsigned long)tmp->addr;
 		va->va_end = va->va_start + tmp->size;
 		__insert_vmap_area(va);
 	}
 	vmap_area_pcpu_hole = VMALLOC_END;
 	vmap_initialized = true;
 }
 /**
  * map_kernel_range_noflush - map kernel VM area with the specified pages
  * @addr: start of the VM area to map
  * @size: size of the VM area to map
  * @prot: page protection flags to use
  * @pages: pages to map
  *
  * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size
  * specify should have been allocated using get_vm_area() and its
  * friends.
  *
  * NOTE:
  * This function does NOT do any cache flushing.  The caller is
  * responsible for calling flush_cache_vmap() on to-be-mapped areas
  * before calling this function.
  *
  * RETURNS:
  * The number of pages mapped on success, -errno on failure.
  */
 int map_kernel_range_noflush(unsigned long addr, unsigned long size,
 			     pgprot_t prot, struct page **pages)
 {
 	return vmap_page_range_noflush(addr, addr + size, prot, pages);
 }
 /**
  * unmap_kernel_range_noflush - unmap kernel VM area
  * @addr: start of the VM area to unmap
  * @size: size of the VM area to unmap
  *
  * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size
  * specify should have been allocated using get_vm_area() and its
  * friends.
  *
  * NOTE:
  * This function does NOT do any cache flushing.  The caller is
  * responsible for calling flush_cache_vunmap() on to-be-mapped areas
  * before calling this function and flush_tlb_kernel_range() after.
  */
 void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
 {
 	vunmap_page_range(addr, addr + size);
 }
 /**
  * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
  * @addr: start of the VM area to unmap
  * @size: size of the VM area to unmap
  *
  * Similar to unmap_kernel_range_noflush() but flushes vcache before
  * the unmapping and tlb after.
  */
 void unmap_kernel_range(unsigned long addr, unsigned long size)
 {
 	unsigned long end = addr + size;
 	flush_cache_vunmap(addr, end);
 	vunmap_page_range(addr, end);
 	flush_tlb_kernel_range(addr, end);
 }
 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
 {
 	unsigned long addr = (unsigned long)area->addr;
 	unsigned long end = addr + area->size - PAGE_SIZE;
 	int err;
 	err = vmap_page_range(addr, end, prot, *pages);
 	if (err > 0) {
 		*pages += err;
 		err = 0;
 	}
 	return err;
 }
 EXPORT_SYMBOL_GPL(map_vm_area);
 /*** Old vmalloc interfaces ***/
 DEFINE_RWLOCK(vmlist_lock);
 struct vm_struct *vmlist;
 static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
 			      unsigned long flags, void *caller)
 {
 	struct vm_struct *tmp, **p;
 	vm->flags = flags;
 	vm->addr = (void *)va->va_start;
 	vm->size = va->va_end - va->va_start;
 	vm->caller = caller;
 	va->private = vm;
 	va->flags |= VM_VM_AREA;
 	write_lock(&vmlist_lock);
 	for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
 		if (tmp->addr >= vm->addr)
 			break;
 	}
 	vm->next = *p;
 	*p = vm;
 	write_unlock(&vmlist_lock);
 }
 static struct vm_struct *__get_vm_area_node(unsigned long size,
 		unsigned long align, unsigned long flags, unsigned long start,
 		unsigned long end, int node, gfp_t gfp_mask, void *caller)
 {
 	static struct vmap_area *va;
 	struct vm_struct *area;
 	BUG_ON(in_interrupt());
 	if (flags & VM_IOREMAP) {
 		int bit = fls(size);
 		if (bit > IOREMAP_MAX_ORDER)
 			bit = IOREMAP_MAX_ORDER;
 		else if (bit < PAGE_SHIFT)
 			bit = PAGE_SHIFT;
 		align = 1ul << bit;
 	}
 	size = PAGE_ALIGN(size);
 	if (unlikely(!size))
 		return NULL;
 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
 	if (unlikely(!area))
 		return NULL;
 	/*
 	 * We always allocate a guard page.
 	 */
 	size += PAGE_SIZE;
 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
 	if (IS_ERR(va)) {
 		kfree(area);
 		return NULL;
 	}
 	insert_vmalloc_vm(area, va, flags, caller);
 	return area;
 }
 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 				unsigned long start, unsigned long end)
 {
 	return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
 						__builtin_return_address(0));
 }
 EXPORT_SYMBOL_GPL(__get_vm_area);
 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
 				       unsigned long start, unsigned long end,
 				       void *caller)
 {
 	return __get_vm_area_node(size, 1, flags, start, end, -1, GFP_KERNEL,
 				  caller);
 }
 /**
  *	get_vm_area  -  reserve a contiguous kernel virtual area
  *	@size:		size of the area
  *	@flags:		%VM_IOREMAP for I/O mappings or VM_ALLOC
  *
  *	Search an area of @size in the kernel virtual mapping area,
  *	and reserved it for out purposes.  Returns the area descriptor
  *	on success or %NULL on failure.
  */
 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
 {
 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
 				-1, GFP_KERNEL, __builtin_return_address(0));
 }
 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
 				void *caller)
 {
 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
 						-1, GFP_KERNEL, caller);
 }
 struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
 				   int node, gfp_t gfp_mask)
 {
 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
 				  node, gfp_mask, __builtin_return_address(0));
 }
 static struct vm_struct *find_vm_area(const void *addr)
 {
 	struct vmap_area *va;
 	va = find_vmap_area((unsigned long)addr);
 	if (va && va->flags & VM_VM_AREA)
 		return va->private;
 	return NULL;
 }
 /**
  *	remove_vm_area  -  find and remove a continuous kernel virtual area
  *	@addr:		base address
  *
  *	Search for the kernel VM area starting at @addr, and remove it.
  *	This function returns the found VM area, but using it is NOT safe
  *	on SMP machines, except for its size or flags.
  */
 struct vm_struct *remove_vm_area(const void *addr)
 {
 	struct vmap_area *va;
 	va = find_vmap_area((unsigned long)addr);
 	if (va && va->flags & VM_VM_AREA) {
 		struct vm_struct *vm = va->private;
 		struct vm_struct *tmp, **p;
 		/*
 		 * remove from list and disallow access to this vm_struct
 		 * before unmap. (address range confliction is maintained by
 		 * vmap.)
 		 */
 		write_lock(&vmlist_lock);
 		for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
 			;
 		*p = tmp->next;
 		write_unlock(&vmlist_lock);
 		vmap_debug_free_range(va->va_start, va->va_end);
 		free_unmap_vmap_area(va);
 		vm->size -= PAGE_SIZE;
 		return vm;
 	}
 	return NULL;
 }
 static void __vunmap(const void *addr, int deallocate_pages)
 {
 	struct vm_struct *area;
 	if (!addr)
 		return;
 	if ((PAGE_SIZE-1) & (unsigned long)addr) {
 		WARN(1, KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 		return;
 	}
 	area = remove_vm_area(addr);
 	if (unlikely(!area)) {
 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
 				addr);
 		return;
 	}
 	debug_check_no_locks_freed(addr, area->size);
 	debug_check_no_obj_freed(addr, area->size);
 	if (deallocate_pages) {
 		int i;
 		for (i = 0; i < area->nr_pages; i++) {
 			struct page *page = area->pages[i];
 			BUG_ON(!page);
 			__free_page(page);
 		}
 		if (area->flags & VM_VPAGES)
 			vfree(area->pages);
 		else
 			kfree(area->pages);
 	}
 	kfree(area);
 	return;
 }
 /**
  *	vfree  -  release memory allocated by vmalloc()
  *	@addr:		memory base address
  *
  *	Free the virtually continuous memory area starting at @addr, as
  *	obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  *	NULL, no operation is performed.
  *
  *	Must not be called in interrupt context.
  */
 void vfree(const void *addr)
 {
 	BUG_ON(in_interrupt());
 	kmemleak_free(addr);
 	__vunmap(addr, 1);
 }
 EXPORT_SYMBOL(vfree);
 /**
  *	vunmap  -  release virtual mapping obtained by vmap()
  *	@addr:		memory base address
  *
  *	Free the virtually contiguous memory area starting at @addr,
  *	which was created from the page array passed to vmap().
  *
  *	Must not be called in interrupt context.
  */
 void vunmap(const void *addr)
 {
 	BUG_ON(in_interrupt());
 	might_sleep();
 	__vunmap(addr, 0);
 }
 EXPORT_SYMBOL(vunmap);
 /**
  *	vmap  -  map an array of pages into virtually contiguous space
  *	@pages:		array of page pointers
  *	@count:		number of pages to map
  *	@flags:		vm_area->flags
  *	@prot:		page protection for the mapping
  *
  *	Maps @count pages from @pages into contiguous kernel virtual
  *	space.
  */
 void *vmap(struct page **pages, unsigned int count,
 		unsigned long flags, pgprot_t prot)
 {
 	struct vm_struct *area;
 	might_sleep();
 	if (count > totalram_pages)
 		return NULL;
 	area = get_vm_area_caller((count << PAGE_SHIFT), flags,
 					__builtin_return_address(0));
 	if (!area)
 		return NULL;
 	if (map_vm_area(area, prot, &pages)) {
 		vunmap(area->addr);
 		return NULL;
 	}
 	return area->addr;
 }
 EXPORT_SYMBOL(vmap);
 static void *__vmalloc_node(unsigned long size, unsigned long align,
 			    gfp_t gfp_mask, pgprot_t prot,
 			    int node, void *caller);
 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 				 pgprot_t prot, int node, void *caller)
 {
 	struct page **pages;
 	unsigned int nr_pages, array_size, i;
 	gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
 	nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
 	array_size = (nr_pages * sizeof(struct page *));
 	area->nr_pages = nr_pages;
 	/* Please note that the recursion is strictly bounded. */
 	if (array_size > PAGE_SIZE) {
 		pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
 				PAGE_KERNEL, node, caller);
 		area->flags |= VM_VPAGES;
 	} else {
 		pages = kmalloc_node(array_size, nested_gfp, node);
 	}
 	area->pages = pages;
 	area->caller = caller;
 	if (!area->pages) {
 		remove_vm_area(area->addr);
 		kfree(area);
 		return NULL;
 	}
 	for (i = 0; i < area->nr_pages; i++) {
 		struct page *page;
 		if (node < 0)
 			page = alloc_page(gfp_mask);
 		else
 			page = alloc_pages_node(node, gfp_mask, 0);
 		if (unlikely(!page)) {
 			/* Successfully allocated i pages, free them in __vunmap() */
 			area->nr_pages = i;
 			goto fail;
 		}
 		area->pages[i] = page;
 	}
 	if (map_vm_area(area, prot, &pages))
 		goto fail;
 	return area->addr;
 fail:
 	vfree(area->addr);
 	return NULL;
 }
 void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
 {
 	void *addr = __vmalloc_area_node(area, gfp_mask, prot, -1,
 					 __builtin_return_address(0));
 	/*
 	 * A ref_count = 3 is needed because the vm_struct and vmap_area
 	 * structures allocated in the __get_vm_area_node() function contain
 	 * references to the virtual address of the vmalloc'ed block.
 	 */
 	kmemleak_alloc(addr, area->size - PAGE_SIZE, 3, gfp_mask);
 	return addr;
 }
 /**
  *	__vmalloc_node  -  allocate virtually contiguous memory
  *	@size:		allocation size
  *	@align:		desired alignment
  *	@gfp_mask:	flags for the page level allocator
  *	@prot:		protection mask for the allocated pages
  *	@node:		node to use for allocation or -1
  *	@caller:	caller's return address
  *
  *	Allocate enough pages to cover @size from the page level
  *	allocator with @gfp_mask flags.  Map them into contiguous
  *	kernel virtual space, using a pagetable protection of @prot.
  */
 static void *__vmalloc_node(unsigned long size, unsigned long align,
 			    gfp_t gfp_mask, pgprot_t prot,
 			    int node, void *caller)
 {
 	struct vm_struct *area;
 	void *addr;
 	unsigned long real_size = size;
 	size = PAGE_ALIGN(size);
 	if (!size || (size >> PAGE_SHIFT) > totalram_pages)
 		return NULL;
 	area = __get_vm_area_node(size, align, VM_ALLOC, VMALLOC_START,
 				  VMALLOC_END, node, gfp_mask, caller);
 	if (!area)
 		return NULL;
 	addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
 	/*
 	 * A ref_count = 3 is needed because the vm_struct and vmap_area
 	 * structures allocated in the __get_vm_area_node() function contain
 	 * references to the virtual address of the vmalloc'ed block.
 	 */
 	kmemleak_alloc(addr, real_size, 3, gfp_mask);
 	return addr;
 }
 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
 {
 	return __vmalloc_node(size, 1, gfp_mask, prot, -1,
 				__builtin_return_address(0));
 }
 EXPORT_SYMBOL(__vmalloc);
+static inline void *__vmalloc_node_flags(unsigned long size,
+					int node, gfp_t flags)
+{
+	return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
+					node, __builtin_return_address(0));
+}
 /**
  *	vmalloc  -  allocate virtually contiguous memory
  *	@size:		allocation size
  *	Allocate enough pages to cover @size from the page level
  *	allocator and map them into contiguous kernel virtual space.
  *
  *	For tight control over page level allocator and protection flags
  *	use __vmalloc() instead.
  */
 void *vmalloc(unsigned long size)
 {
-	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
+	return __vmalloc_node_flags(size, -1, GFP_KERNEL | __GFP_HIGHMEM);
-					-1, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc);
 /**
+ *	vzalloc - allocate virtually contiguous memory with zero fill
+ *	@size:	allocation size
+ *	Allocate enough pages to cover @size from the page level
+ *	allocator and map them into contiguous kernel virtual space.
+ *	The memory allocated is set to zero.
+ *
+ *	For tight control over page level allocator and protection flags
+ *	use __vmalloc() instead.
+ */
+void *vzalloc(unsigned long size)
+{
+	return __vmalloc_node_flags(size, -1,
+				GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
+}
+EXPORT_SYMBOL(vzalloc);
+/**
  * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  * @size: allocation size
  *
  * The resulting memory area is zeroed so it can be mapped to userspace
  * without leaking data.
  */
 void *vmalloc_user(unsigned long size)
 {
 	struct vm_struct *area;
 	void *ret;
 	ret = __vmalloc_node(size, SHMLBA,
 			     GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
 			     PAGE_KERNEL, -1, __builtin_return_address(0));
 	if (ret) {
 		area = find_vm_area(ret);
 		area->flags |= VM_USERMAP;
 	}
 	return ret;
 }
 EXPORT_SYMBOL(vmalloc_user);
 /**
  *	vmalloc_node  -  allocate memory on a specific node
  *	@size:		allocation size
  *	@node:		numa node
  *
  *	Allocate enough pages to cover @size from the page level
  *	allocator and map them into contiguous kernel virtual space.
  *
  *	For tight control over page level allocator and protection flags
  *	use __vmalloc() instead.
  */
 void *vmalloc_node(unsigned long size, int node)
 {
 	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
 					node, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc_node);
+/**
+ * vzalloc_node - allocate memory on a specific node with zero fill
+ * @size:	allocation size
+ * @node:	numa node
+ *
+ * Allocate enough pages to cover @size from the page level
+ * allocator and map them into contiguous kernel virtual space.
+ * The memory allocated is set to zero.
+ *
+ * For tight control over page level allocator and protection flags
+ * use __vmalloc_node() instead.
+ */
+void *vzalloc_node(unsigned long size, int node)
+{
+	return __vmalloc_node_flags(size, node,
+			 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
+}
+EXPORT_SYMBOL(vzalloc_node);
 #ifndef PAGE_KERNEL_EXEC
 # define PAGE_KERNEL_EXEC PAGE_KERNEL
 #endif
 /**
  *	vmalloc_exec  -  allocate virtually contiguous, executable memory
  *	@size:		allocation size
  *
  *	Kernel-internal function to allocate enough pages to cover @size
  *	the page level allocator and map them into contiguous and
  *	executable kernel virtual space.
  *
  *	For tight control over page level allocator and protection flags
  *	use __vmalloc() instead.
  */
 void *vmalloc_exec(unsigned long size)
 {
 	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
 			      -1, __builtin_return_address(0));
 }
 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
 #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
 #else
 #define GFP_VMALLOC32 GFP_KERNEL
 #endif
 /**
  *	vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
  *	@size:		allocation size
  *
  *	Allocate enough 32bit PA addressable pages to cover @size from the
  *	page level allocator and map them into contiguous kernel virtual space.
  */
 void *vmalloc_32(unsigned long size)
 {
 	return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
 			      -1, __builtin_return_address(0));
 }
 EXPORT_SYMBOL(vmalloc_32);
 /**
  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  *	@size:		allocation size
  *
  * The resulting memory area is 32bit addressable and zeroed so it can be
  * mapped to userspace without leaking data.
  */
 void *vmalloc_32_user(unsigned long size)
 {
 	struct vm_struct *area;
 	void *ret;
 	ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
 			     -1, __builtin_return_address(0));
 	if (ret) {
 		area = find_vm_area(ret);
 		area->flags |= VM_USERMAP;
 	}
 	return ret;
 }
 EXPORT_SYMBOL(vmalloc_32_user);
 /*
  * small helper routine , copy contents to buf from addr.
  * If the page is not present, fill zero.
  */
 static int aligned_vread(char *buf, char *addr, unsigned long count)
 {
 	struct page *p;
 	int copied = 0;
 	while (count) {
 		unsigned long offset, length;
 		offset = (unsigned long)addr & ~PAGE_MASK;
 		length = PAGE_SIZE - offset;
 		if (length > count)
 			length = count;
 		p = vmalloc_to_page(addr);
 		/*
 		 * To do safe access to this _mapped_ area, we need
 		 * lock. But adding lock here means that we need to add
 		 * overhead of vmalloc()/vfree() calles for this _debug_
 		 * interface, rarely used. Instead of that, we'll use
 		 * kmap() and get small overhead in this access function.
 		 */
 		if (p) {
 			/*
 			 * we can expect USER0 is not used (see vread/vwrite's
 			 * function description)
 			 */
 			void *map = kmap_atomic(p, KM_USER0);
 			memcpy(buf, map + offset, length);
 			kunmap_atomic(map, KM_USER0);
 		} else
 			memset(buf, 0, length);
 		addr += length;
 		buf += length;
 		copied += length;
 		count -= length;
 	}
 	return copied;
 }
 static int aligned_vwrite(char *buf, char *addr, unsigned long count)
 {
 	struct page *p;
 	int copied = 0;
 	while (count) {
 		unsigned long offset, length;
 		offset = (unsigned long)addr & ~PAGE_MASK;
 		length = PAGE_SIZE - offset;
 		if (length > count)
 			length = count;
 		p = vmalloc_to_page(addr);
 		/*
 		 * To do safe access to this _mapped_ area, we need
 		 * lock. But adding lock here means that we need to add
 		 * overhead of vmalloc()/vfree() calles for this _debug_
 		 * interface, rarely used. Instead of that, we'll use
 		 * kmap() and get small overhead in this access function.
 		 */
 		if (p) {
 			/*
 			 * we can expect USER0 is not used (see vread/vwrite's
 			 * function description)
 			 */
 			void *map = kmap_atomic(p, KM_USER0);
 			memcpy(map + offset, buf, length);
 			kunmap_atomic(map, KM_USER0);
 		}
 		addr += length;
 		buf += length;
 		copied += length;
 		count -= length;
 	}
 	return copied;
 }
 /**
  *	vread() -  read vmalloc area in a safe way.
  *	@buf:		buffer for reading data
  *	@addr:		vm address.
  *	@count:		number of bytes to be read.
  *
  *	Returns # of bytes which addr and buf should be increased.
  *	(same number to @count). Returns 0 if [addr...addr+count) doesn't
  *	includes any intersect with alive vmalloc area.
  *
  *	This function checks that addr is a valid vmalloc'ed area, and
  *	copy data from that area to a given buffer. If the given memory range
  *	of [addr...addr+count) includes some valid address, data is copied to
  *	proper area of @buf. If there are memory holes, they'll be zero-filled.
  *	IOREMAP area is treated as memory hole and no copy is done.
  *
  *	If [addr...addr+count) doesn't includes any intersects with alive
  *	vm_struct area, returns 0.
  *	@buf should be kernel's buffer. Because	this function uses KM_USER0,
  *	the caller should guarantee KM_USER0 is not used.
  *
  *	Note: In usual ops, vread() is never necessary because the caller
  *	should know vmalloc() area is valid and can use memcpy().
  *	This is for routines which have to access vmalloc area without
  *	any informaion, as /dev/kmem.
  *
  */
 long vread(char *buf, char *addr, unsigned long count)
 {
 	struct vm_struct *tmp;
 	char *vaddr, *buf_start = buf;
 	unsigned long buflen = count;
 	unsigned long n;
 	/* Don't allow overflow */
 	if ((unsigned long) addr + count < count)
 		count = -(unsigned long) addr;
 	read_lock(&vmlist_lock);
 	for (tmp = vmlist; count && tmp; tmp = tmp->next) {
 		vaddr = (char *) tmp->addr;
 		if (addr >= vaddr + tmp->size - PAGE_SIZE)
 			continue;
 		while (addr < vaddr) {
 			if (count == 0)
 				goto finished;
 			*buf = '\0';
 			buf++;
 			addr++;
 			count--;
 		}
 		n = vaddr + tmp->size - PAGE_SIZE - addr;
 		if (n > count)
 			n = count;
 		if (!(tmp->flags & VM_IOREMAP))
 			aligned_vread(buf, addr, n);
 		else /* IOREMAP area is treated as memory hole */
 			memset(buf, 0, n);
 		buf += n;
 		addr += n;
 		count -= n;
 	}
 finished:
 	read_unlock(&vmlist_lock);
 	if (buf == buf_start)
 		return 0;
 	/* zero-fill memory holes */
 	if (buf != buf_start + buflen)
 		memset(buf, 0, buflen - (buf - buf_start));
 	return buflen;
 }
 /**
  *	vwrite() -  write vmalloc area in a safe way.
  *	@buf:		buffer for source data
  *	@addr:		vm address.
  *	@count:		number of bytes to be read.
  *
  *	Returns # of bytes which addr and buf should be incresed.
  *	(same number to @count).
  *	If [addr...addr+count) doesn't includes any intersect with valid
  *	vmalloc area, returns 0.
  *
  *	This function checks that addr is a valid vmalloc'ed area, and
  *	copy data from a buffer to the given addr. If specified range of
  *	[addr...addr+count) includes some valid address, data is copied from
  *	proper area of @buf. If there are memory holes, no copy to hole.
  *	IOREMAP area is treated as memory hole and no copy is done.
  *
  *	If [addr...addr+count) doesn't includes any intersects with alive
  *	vm_struct area, returns 0.
  *	@buf should be kernel's buffer. Because	this function uses KM_USER0,
  *	the caller should guarantee KM_USER0 is not used.
  *
  *	Note: In usual ops, vwrite() is never necessary because the caller
  *	should know vmalloc() area is valid and can use memcpy().
  *	This is for routines which have to access vmalloc area without
  *	any informaion, as /dev/kmem.
  *
  *	The caller should guarantee KM_USER1 is not used.
  */
 long vwrite(char *buf, char *addr, unsigned long count)
 {
 	struct vm_struct *tmp;
 	char *vaddr;
 	unsigned long n, buflen;
 	int copied = 0;
 	/* Don't allow overflow */
 	if ((unsigned long) addr + count < count)
 		count = -(unsigned long) addr;
 	buflen = count;
 	read_lock(&vmlist_lock);
 	for (tmp = vmlist; count && tmp; tmp = tmp->next) {
 		vaddr = (char *) tmp->addr;
 		if (addr >= vaddr + tmp->size - PAGE_SIZE)
 			continue;
 		while (addr < vaddr) {
 			if (count == 0)
 				goto finished;
 			buf++;
 			addr++;
 			count--;
 		}
 		n = vaddr + tmp->size - PAGE_SIZE - addr;
 		if (n > count)
 			n = count;
 		if (!(tmp->flags & VM_IOREMAP)) {
 			aligned_vwrite(buf, addr, n);
 			copied++;
 		}
 		buf += n;
 		addr += n;
 		count -= n;
 	}
 finished:
 	read_unlock(&vmlist_lock);
 	if (!copied)
 		return 0;
 	return buflen;
 }
 /**
  *	remap_vmalloc_range  -  map vmalloc pages to userspace
  *	@vma:		vma to cover (map full range of vma)
  *	@addr:		vmalloc memory
  *	@pgoff:		number of pages into addr before first page to map
  *
  *	Returns:	0 for success, -Exxx on failure
  *
  *	This function checks that addr is a valid vmalloc'ed area, and
  *	that it is big enough to cover the vma. Will return failure if
  *	that criteria isn't met.
  *
  *	Similar to remap_pfn_range() (see mm/memory.c)
  */
 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
 						unsigned long pgoff)
 {
 	struct vm_struct *area;
 	unsigned long uaddr = vma->vm_start;
 	unsigned long usize = vma->vm_end - vma->vm_start;
 	if ((PAGE_SIZE-1) & (unsigned long)addr)
 		return -EINVAL;
 	area = find_vm_area(addr);
 	if (!area)
 		return -EINVAL;
 	if (!(area->flags & VM_USERMAP))
 		return -EINVAL;
 	if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
 		return -EINVAL;
 	addr += pgoff << PAGE_SHIFT;
 	do {
 		struct page *page = vmalloc_to_page(addr);
 		int ret;
 		ret = vm_insert_page(vma, uaddr, page);
 		if (ret)
 			return ret;
 		uaddr += PAGE_SIZE;
 		addr += PAGE_SIZE;
 		usize -= PAGE_SIZE;
 	} while (usize > 0);
 	/* Prevent "things" like memory migration? VM_flags need a cleanup... */
 	vma->vm_flags |= VM_RESERVED;
 	return 0;
 }
 EXPORT_SYMBOL(remap_vmalloc_range);
 /*
  * Implement a stub for vmalloc_sync_all() if the architecture chose not to
  * have one.
  */
 void  __attribute__((weak)) vmalloc_sync_all(void)
 {
 }
 static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
 {
 	/* apply_to_page_range() does all the hard work. */
 	return 0;
 }
 /**
  *	alloc_vm_area - allocate a range of kernel address space
  *	@size:		size of the area
  *
  *	Returns:	NULL on failure, vm_struct on success
  *
  *	This function reserves a range of kernel address space, and
  *	allocates pagetables to map that range.  No actual mappings
  *	are created.  If the kernel address space is not shared
  *	between processes, it syncs the pagetable across all
  *	processes.
  */
 struct vm_struct *alloc_vm_area(size_t size)
 {
 	struct vm_struct *area;
 	area = get_vm_area_caller(size, VM_IOREMAP,
 				__builtin_return_address(0));
 	if (area == NULL)
 		return NULL;
 	/*
 	 * This ensures that page tables are constructed for this region
 	 * of kernel virtual address space and mapped into init_mm.
 	 */
 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
 				area->size, f, NULL)) {
 		free_vm_area(area);
 		return NULL;
 	}
 	/* Make sure the pagetables are constructed in process kernel
 	   mappings */
 	vmalloc_sync_all();
 	return area;
 }
 EXPORT_SYMBOL_GPL(alloc_vm_area);
 void free_vm_area(struct vm_struct *area)
 {
 	struct vm_struct *ret;
 	ret = remove_vm_area(area->addr);
 	BUG_ON(ret != area);
 	kfree(area);
 }
 EXPORT_SYMBOL_GPL(free_vm_area);
 #ifdef CONFIG_SMP
 static struct vmap_area *node_to_va(struct rb_node *n)
 {
 	return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
 }
 /**
  * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
  * @end: target address
  * @pnext: out arg for the next vmap_area
  * @pprev: out arg for the previous vmap_area
  *
  * Returns: %true if either or both of next and prev are found,
  *	    %false if no vmap_area exists
  *
  * Find vmap_areas end addresses of which enclose @end.  ie. if not
  * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
  */
 static bool pvm_find_next_prev(unsigned long end,
 			       struct vmap_area **pnext,
 			       struct vmap_area **pprev)
 {
 	struct rb_node *n = vmap_area_root.rb_node;
 	struct vmap_area *va = NULL;
 	while (n) {
 		va = rb_entry(n, struct vmap_area, rb_node);
 		if (end < va->va_end)
 			n = n->rb_left;
 		else if (end > va->va_end)
 			n = n->rb_right;
 		else
 			break;
 	}
 	if (!va)
 		return false;
 	if (va->va_end > end) {
 		*pnext = va;
 		*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
 	} else {
 		*pprev = va;
 		*pnext = node_to_va(rb_next(&(*pprev)->rb_node));
 	}
 	return true;
 }
 /**
  * pvm_determine_end - find the highest aligned address between two vmap_areas
  * @pnext: in/out arg for the next vmap_area
  * @pprev: in/out arg for the previous vmap_area
  * @align: alignment
  *
  * Returns: determined end address
  *
  * Find the highest aligned address between *@pnext and *@pprev below
  * VMALLOC_END.  *@pnext and *@pprev are adjusted so that the aligned
  * down address is between the end addresses of the two vmap_areas.
  *
  * Please note that the address returned by this function may fall
  * inside *@pnext vmap_area.  The caller is responsible for checking
  * that.
  */
 static unsigned long pvm_determine_end(struct vmap_area **pnext,
 				       struct vmap_area **pprev,
 				       unsigned long align)
 {
 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
 	unsigned long addr;
 	if (*pnext)
 		addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
 	else
 		addr = vmalloc_end;
 	while (*pprev && (*pprev)->va_end > addr) {
 		*pnext = *pprev;
 		*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
 	}
 	return addr;
 }
 /**
  * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
  * @offsets: array containing offset of each area
  * @sizes: array containing size of each area
  * @nr_vms: the number of areas to allocate
  * @align: alignment, all entries in @offsets and @sizes must be aligned to this
  * @gfp_mask: allocation mask
  *
  * Returns: kmalloc'd vm_struct pointer array pointing to allocated
  *	    vm_structs on success, %NULL on failure
  *
  * Percpu allocator wants to use congruent vm areas so that it can
  * maintain the offsets among percpu areas.  This function allocates
  * congruent vmalloc areas for it.  These areas tend to be scattered
  * pretty far, distance between two areas easily going up to
  * gigabytes.  To avoid interacting with regular vmallocs, these areas
  * are allocated from top.
  *
  * Despite its complicated look, this allocator is rather simple.  It
  * does everything top-down and scans areas from the end looking for
  * matching slot.  While scanning, if any of the areas overlaps with
  * existing vmap_area, the base address is pulled down to fit the
  * area.  Scanning is repeated till all the areas fit and then all
  * necessary data structres are inserted and the result is returned.
  */
 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
 				     const size_t *sizes, int nr_vms,
 				     size_t align, gfp_t gfp_mask)
 {
 	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
 	struct vmap_area **vas, *prev, *next;
 	struct vm_struct **vms;
 	int area, area2, last_area, term_area;
 	unsigned long base, start, end, last_end;
 	bool purged = false;
 	gfp_mask &= GFP_RECLAIM_MASK;
 	/* verify parameters and allocate data structures */
 	BUG_ON(align & ~PAGE_MASK || !is_power_of_2(align));
 	for (last_area = 0, area = 0; area < nr_vms; area++) {
 		start = offsets[area];
 		end = start + sizes[area];
 		/* is everything aligned properly? */
 		BUG_ON(!IS_ALIGNED(offsets[area], align));
 		BUG_ON(!IS_ALIGNED(sizes[area], align));
 		/* detect the area with the highest address */
 		if (start > offsets[last_area])
 			last_area = area;
 		for (area2 = 0; area2 < nr_vms; area2++) {
 			unsigned long start2 = offsets[area2];
 			unsigned long end2 = start2 + sizes[area2];
 			if (area2 == area)
 				continue;
 			BUG_ON(start2 >= start && start2 < end);
 			BUG_ON(end2 <= end && end2 > start);
 		}
 	}
 	last_end = offsets[last_area] + sizes[last_area];
 	if (vmalloc_end - vmalloc_start < last_end) {
 		WARN_ON(true);
 		return NULL;
 	}
 	vms = kzalloc(sizeof(vms[0]) * nr_vms, gfp_mask);
 	vas = kzalloc(sizeof(vas[0]) * nr_vms, gfp_mask);
 	if (!vas || !vms)
 		goto err_free;
 	for (area = 0; area < nr_vms; area++) {
 		vas[area] = kzalloc(sizeof(struct vmap_area), gfp_mask);
 		vms[area] = kzalloc(sizeof(struct vm_struct), gfp_mask);
 		if (!vas[area] || !vms[area])
 			goto err_free;
 	}
 retry:
 	spin_lock(&vmap_area_lock);
 	/* start scanning - we scan from the top, begin with the last area */
 	area = term_area = last_area;
 	start = offsets[area];
 	end = start + sizes[area];
 	if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
 		base = vmalloc_end - last_end;
 		goto found;
 	}
 	base = pvm_determine_end(&next, &prev, align) - end;
 	while (true) {
 		BUG_ON(next && next->va_end <= base + end);
 		BUG_ON(prev && prev->va_end > base + end);
 		/*
 		 * base might have underflowed, add last_end before
 		 * comparing.
 		 */
 		if (base + last_end < vmalloc_start + last_end) {
 			spin_unlock(&vmap_area_lock);
 			if (!purged) {
 				purge_vmap_area_lazy();
 				purged = true;
 				goto retry;
 			}
 			goto err_free;
 		}
 		/*
 		 * If next overlaps, move base downwards so that it's
 		 * right below next and then recheck.
 		 */
 		if (next && next->va_start < base + end) {
 			base = pvm_determine_end(&next, &prev, align) - end;
 			term_area = area;
 			continue;
 		}
 		/*
 		 * If prev overlaps, shift down next and prev and move
 		 * base so that it's right below new next and then
 		 * recheck.
 		 */
 		if (prev && prev->va_end > base + start)  {
 			next = prev;
 			prev = node_to_va(rb_prev(&next->rb_node));
 			base = pvm_determine_end(&next, &prev, align) - end;
 			term_area = area;
 			continue;
 		}
 		/*
 		 * This area fits, move on to the previous one.  If
 		 * the previous one is the terminal one, we're done.
 		 */
 		area = (area + nr_vms - 1) % nr_vms;
 		if (area == term_area)
 			break;
 		start = offsets[area];
 		end = start + sizes[area];
 		pvm_find_next_prev(base + end, &next, &prev);
 	}
 found:
 	/* we've found a fitting base, insert all va's */
 	for (area = 0; area < nr_vms; area++) {
 		struct vmap_area *va = vas[area];
 		va->va_start = base + offsets[area];
 		va->va_end = va->va_start + sizes[area];
 		__insert_vmap_area(va);
 	}
 	vmap_area_pcpu_hole = base + offsets[last_area];
 	spin_unlock(&vmap_area_lock);
 	/* insert all vm's */
 	for (area = 0; area < nr_vms; area++)
 		insert_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
 				  pcpu_get_vm_areas);
 	kfree(vas);
 	return vms;
 err_free:
 	for (area = 0; area < nr_vms; area++) {
 		if (vas)
 			kfree(vas[area]);
 		if (vms)
 			kfree(vms[area]);
 	}
 	kfree(vas);
 	kfree(vms);
 	return NULL;
 }
 /**
  * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
  * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
  * @nr_vms: the number of allocated areas
  *
  * Free vm_structs and the array allocated by pcpu_get_vm_areas().
  */
 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
 {
 	int i;
 	for (i = 0; i < nr_vms; i++)
 		free_vm_area(vms[i]);
 	kfree(vms);
 }
 #endif	/* CONFIG_SMP */
 #ifdef CONFIG_PROC_FS
 static void *s_start(struct seq_file *m, loff_t *pos)
 	__acquires(&vmlist_lock)
 {
 	loff_t n = *pos;
 	struct vm_struct *v;
 	read_lock(&vmlist_lock);
 	v = vmlist;
 	while (n > 0 && v) {
 		n--;
 		v = v->next;
 	}
 	if (!n)
 		return v;
 	return NULL;
 }
 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
 {
 	struct vm_struct *v = p;
 	++*pos;
 	return v->next;
 }
 static void s_stop(struct seq_file *m, void *p)
 	__releases(&vmlist_lock)
 {
 	read_unlock(&vmlist_lock);
 }
 static void show_numa_info(struct seq_file *m, struct vm_struct *v)
 {
 	if (NUMA_BUILD) {
 		unsigned int nr, *counters = m->private;
 		if (!counters)
 			return;
 		memset(counters, 0, nr_node_ids * sizeof(unsigned int));
 		for (nr = 0; nr < v->nr_pages; nr++)
 			counters[page_to_nid(v->pages[nr])]++;
 		for_each_node_state(nr, N_HIGH_MEMORY)
 			if (counters[nr])
 				seq_printf(m, " N%u=%u", nr, counters[nr]);
 	}
 }
 static int s_show(struct seq_file *m, void *p)
 {
 	struct vm_struct *v = p;
 	seq_printf(m, "0x%p-0x%p %7ld",
 		v->addr, v->addr + v->size, v->size);
 	if (v->caller) {
 		char buff[KSYM_SYMBOL_LEN];
 		seq_putc(m, ' ');
 		sprint_symbol(buff, (unsigned long)v->caller);
 		seq_puts(m, buff);
 	}
 	if (v->nr_pages)
 		seq_printf(m, " pages=%d", v->nr_pages);
 	if (v->phys_addr)
 		seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
 	if (v->flags & VM_IOREMAP)
 		seq_printf(m, " ioremap");
 	if (v->flags & VM_ALLOC)
 		seq_printf(m, " vmalloc");
 	if (v->flags & VM_MAP)
 		seq_printf(m, " vmap");
 	if (v->flags & VM_USERMAP)
 		seq_printf(m, " user");
 	if (v->flags & VM_VPAGES)
 		seq_printf(m, " vpages");
 	show_numa_info(m, v);
 	seq_putc(m, '\n');
 	return 0;
 }
 static const struct seq_operations vmalloc_op = {
 	.start = s_start,
 	.next = s_next,
 	.stop = s_stop,
 	.show = s_show,
 };
 static int vmalloc_open(struct inode *inode, struct file *file)
 {
 	unsigned int *ptr = NULL;
 	int ret;
 	if (NUMA_BUILD) {
 		ptr = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL);
 		if (ptr == NULL)
 			return -ENOMEM;
 	}
 	ret = seq_open(file, &vmalloc_op);
 	if (!ret) {
 		struct seq_file *m = file->private_data;
 		m->private = ptr;
 	} else
 		kfree(ptr);
 	return ret;
 }
 static const struct file_operations proc_vmalloc_operations = {
 	.open		= vmalloc_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
 	.release	= seq_release_private,
 };
 static int __init proc_vmalloc_init(void)
 {
 	proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
 	return 0;
 }
 module_init(proc_vmalloc_init);
 #endif