Blame view

mm/fremap.c 6.11 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
  /*
   *   linux/mm/fremap.c
   * 
   * Explicit pagetable population and nonlinear (random) mappings support.
   *
   * started by Ingo Molnar, Copyright (C) 2002, 2003
   */
  
  #include <linux/mm.h>
  #include <linux/swap.h>
  #include <linux/file.h>
  #include <linux/mman.h>
  #include <linux/pagemap.h>
  #include <linux/swapops.h>
  #include <linux/rmap.h>
  #include <linux/module.h>
  #include <linux/syscalls.h>
  
  #include <asm/mmu_context.h>
  #include <asm/cacheflush.h>
  #include <asm/tlbflush.h>
861f2fb8e   Hugh Dickins   [PATCH] mm: zap_p...
22
  static int zap_pte(struct mm_struct *mm, struct vm_area_struct *vma,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
23
24
25
  			unsigned long addr, pte_t *ptep)
  {
  	pte_t pte = *ptep;
861f2fb8e   Hugh Dickins   [PATCH] mm: zap_p...
26
  	struct page *page = NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
27

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
28
  	if (pte_present(pte)) {
6aab341e0   Linus Torvalds   mm: re-architect ...
29
  		flush_cache_page(vma, addr, pte_pfn(pte));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
30
  		pte = ptep_clear_flush(vma, addr, ptep);
6aab341e0   Linus Torvalds   mm: re-architect ...
31
32
33
34
35
36
  		page = vm_normal_page(vma, addr, pte);
  		if (page) {
  			if (pte_dirty(pte))
  				set_page_dirty(page);
  			page_remove_rmap(page);
  			page_cache_release(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
37
38
39
40
  		}
  	} else {
  		if (!pte_file(pte))
  			free_swap_and_cache(pte_to_swp_entry(pte));
9888a1cae   Zachary Amsden   [PATCH] paravirt:...
41
  		pte_clear_not_present_full(mm, addr, ptep, 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
42
  	}
861f2fb8e   Hugh Dickins   [PATCH] mm: zap_p...
43
  	return !!page;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
44
45
46
47
48
49
50
51
52
53
54
55
56
  }
  
  /*
   * Install a file page to a given virtual memory address, release any
   * previously existing mapping.
   */
  int install_page(struct mm_struct *mm, struct vm_area_struct *vma,
  		unsigned long addr, struct page *page, pgprot_t prot)
  {
  	struct inode *inode;
  	pgoff_t size;
  	int err = -ENOMEM;
  	pte_t *pte;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
57
  	pte_t pte_val;
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
58
  	spinlock_t *ptl;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
59

c9cfcddfd   Linus Torvalds   VM: add common he...
60
  	pte = get_locked_pte(mm, addr, &ptl);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
61
  	if (!pte)
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
62
  		goto out;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
63
64
65
66
67
68
69
70
71
  
  	/*
  	 * This page may have been truncated. Tell the
  	 * caller about it.
  	 */
  	err = -EINVAL;
  	inode = vma->vm_file->f_mapping->host;
  	size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  	if (!page->mapping || page->index >= size)
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
72
  		goto unlock;
f5154a98a   Hugh Dickins   [PATCH] Don't map...
73
74
  	err = -ENOMEM;
  	if (page_mapcount(page) > INT_MAX/2)
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
75
  		goto unlock;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
76

861f2fb8e   Hugh Dickins   [PATCH] mm: zap_p...
77
78
  	if (pte_none(*pte) || !zap_pte(mm, vma, addr, pte))
  		inc_mm_counter(mm, file_rss);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
79

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
80
  	flush_icache_page(vma, page);
e88dd6c11   Peter Zijlstra   [PATCH] mm: small...
81
82
  	pte_val = mk_pte(page, prot);
  	set_pte_at(mm, addr, pte, pte_val);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
83
  	page_add_file_rmap(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
84
  	update_mmu_cache(vma, addr, pte_val);
668e0d8f1   Hugh Dickins   [PATCH] fix updat...
85
  	lazy_mmu_prot_update(pte_val);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
86
  	err = 0;
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
87
88
89
  unlock:
  	pte_unmap_unlock(pte, ptl);
  out:
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
90
91
92
  	return err;
  }
  EXPORT_SYMBOL(install_page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
93
94
95
96
97
98
99
100
101
  /*
   * Install a file pte to a given virtual memory address, release any
   * previously existing mapping.
   */
  int install_file_pte(struct mm_struct *mm, struct vm_area_struct *vma,
  		unsigned long addr, unsigned long pgoff, pgprot_t prot)
  {
  	int err = -ENOMEM;
  	pte_t *pte;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
102
  	pte_t pte_val;
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
103
  	spinlock_t *ptl;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
104

c9cfcddfd   Linus Torvalds   VM: add common he...
105
  	pte = get_locked_pte(mm, addr, &ptl);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
106
  	if (!pte)
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
107
  		goto out;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
108

365e9c87a   Hugh Dickins   [PATCH] mm: updat...
109
110
  	if (!pte_none(*pte) && zap_pte(mm, vma, addr, pte)) {
  		update_hiwater_rss(mm);
861f2fb8e   Hugh Dickins   [PATCH] mm: zap_p...
111
  		dec_mm_counter(mm, file_rss);
365e9c87a   Hugh Dickins   [PATCH] mm: updat...
112
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
113
114
115
  
  	set_pte_at(mm, addr, pte, pgoff_to_pte(pgoff));
  	pte_val = *pte;
668e0d8f1   Hugh Dickins   [PATCH] fix updat...
116
117
118
119
120
121
122
  	/*
  	 * We don't need to run update_mmu_cache() here because the "file pte"
  	 * being installed by install_file_pte() is not a real pte - it's a
  	 * non-present entry (like a swap entry), noting what file offset should
  	 * be mapped there when there's a fault (in a non-linear vma where
  	 * that's not obvious).
  	 */
c74df32c7   Hugh Dickins   [PATCH] mm: ptd_a...
123
124
125
  	pte_unmap_unlock(pte, ptl);
  	err = 0;
  out:
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
126
127
  	return err;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
  /***
   * sys_remap_file_pages - remap arbitrary pages of a shared backing store
   *                        file within an existing vma.
   * @start: start of the remapped virtual memory range
   * @size: size of the remapped virtual memory range
   * @prot: new protection bits of the range
   * @pgoff: to be mapped page of the backing store file
   * @flags: 0 or MAP_NONBLOCKED - the later will cause no IO.
   *
   * this syscall works purely via pagetables, so it's the most efficient
   * way to map the same (large) file into a given virtual window. Unlike
   * mmap()/mremap() it does not create any new vmas. The new mappings are
   * also safe across swapout.
   *
   * NOTE: the 'prot' parameter right now is ignored, and the vma's default
   * protection is used. Arbitrary protections might be implemented in the
   * future.
   */
  asmlinkage long sys_remap_file_pages(unsigned long start, unsigned long size,
  	unsigned long __prot, unsigned long pgoff, unsigned long flags)
  {
  	struct mm_struct *mm = current->mm;
  	struct address_space *mapping;
  	unsigned long end = start + size;
  	struct vm_area_struct *vma;
  	int err = -EINVAL;
  	int has_write_lock = 0;
  
  	if (__prot)
  		return err;
  	/*
  	 * Sanitize the syscall parameters:
  	 */
  	start = start & PAGE_MASK;
  	size = size & PAGE_MASK;
  
  	/* Does the address range wrap, or is the span zero-sized? */
  	if (start + size <= start)
  		return err;
  
  	/* Can we represent this offset inside this architecture's pte's? */
  #if PTE_FILE_MAX_BITS < BITS_PER_LONG
  	if (pgoff + (size >> PAGE_SHIFT) >= (1UL << PTE_FILE_MAX_BITS))
  		return err;
  #endif
  
  	/* We need down_write() to change vma->vm_flags. */
  	down_read(&mm->mmap_sem);
   retry:
  	vma = find_vma(mm, start);
  
  	/*
  	 * Make sure the vma is shared, that it supports prefaulting,
  	 * and that the remapped range is valid and fully within
  	 * the single existing vma.  vm_private_data is used as a
101d2be76   Hugh Dickins   [PATCH] unpaged: ...
183
  	 * swapout cursor in a VM_NONLINEAR vma.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
184
185
  	 */
  	if (vma && (vma->vm_flags & VM_SHARED) &&
101d2be76   Hugh Dickins   [PATCH] unpaged: ...
186
  		(!vma->vm_private_data || (vma->vm_flags & VM_NONLINEAR)) &&
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
  		vma->vm_ops && vma->vm_ops->populate &&
  			end > start && start >= vma->vm_start &&
  				end <= vma->vm_end) {
  
  		/* Must set VM_NONLINEAR before any pages are populated. */
  		if (pgoff != linear_page_index(vma, start) &&
  		    !(vma->vm_flags & VM_NONLINEAR)) {
  			if (!has_write_lock) {
  				up_read(&mm->mmap_sem);
  				down_write(&mm->mmap_sem);
  				has_write_lock = 1;
  				goto retry;
  			}
  			mapping = vma->vm_file->f_mapping;
  			spin_lock(&mapping->i_mmap_lock);
  			flush_dcache_mmap_lock(mapping);
  			vma->vm_flags |= VM_NONLINEAR;
  			vma_prio_tree_remove(vma, &mapping->i_mmap);
  			vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
  			flush_dcache_mmap_unlock(mapping);
  			spin_unlock(&mapping->i_mmap_lock);
  		}
  
  		err = vma->vm_ops->populate(vma, start, size,
  					    vma->vm_page_prot,
  					    pgoff, flags & MAP_NONBLOCK);
  
  		/*
  		 * We can't clear VM_NONLINEAR because we'd have to do
  		 * it after ->populate completes, and that would prevent
  		 * downgrading the lock.  (Locks can't be upgraded).
  		 */
  	}
  	if (likely(!has_write_lock))
  		up_read(&mm->mmap_sem);
  	else
  		up_write(&mm->mmap_sem);
  
  	return err;
  }