Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit ce7184bdbd38d920fb515266fbbdc585ad2e5493

Authored by Alex Shi
Committed by Konrad Rzeszutek Wilk
1 parent 593d0a3e9f
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

xen: fix logical error in tlb flushing 

While TLB_FLUSH_ALL gets passed as 'end' argument to
flush_tlb_others(), the Xen code was made to check its 'start'
parameter. That may give a incorrect op.cmd to MMUEXT_INVLPG_MULTI
instead of MMUEXT_TLB_FLUSH_MULTI. Then it causes some page can not
be flushed from TLB.

This patch fixed this issue.

Reported-by: Jan Beulich <jbeulich@suse.com>
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Jan Beulich <jbeulich@suse.com>
Tested-by: Yongjie Ren <yongjie.ren@intel.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

Showing 1 changed file with 1 additions and 1 deletions Inline Diff

1 /* 1 /*
2 * Xen mmu operations 2 * Xen mmu operations
3 * 3 *
4 * This file contains the various mmu fetch and update operations. 4 * This file contains the various mmu fetch and update operations.
5 * The most important job they must perform is the mapping between the 5 * The most important job they must perform is the mapping between the
6 * domain's pfn and the overall machine mfns. 6 * domain's pfn and the overall machine mfns.
7 * 7 *
8 * Xen allows guests to directly update the pagetable, in a controlled 8 * Xen allows guests to directly update the pagetable, in a controlled
9 * fashion. In other words, the guest modifies the same pagetable 9 * fashion. In other words, the guest modifies the same pagetable
10 * that the CPU actually uses, which eliminates the overhead of having 10 * that the CPU actually uses, which eliminates the overhead of having
11 * a separate shadow pagetable. 11 * a separate shadow pagetable.
12 * 12 *
13 * In order to allow this, it falls on the guest domain to map its 13 * In order to allow this, it falls on the guest domain to map its
14 * notion of a "physical" pfn - which is just a domain-local linear 14 * notion of a "physical" pfn - which is just a domain-local linear
15 * address - into a real "machine address" which the CPU's MMU can 15 * address - into a real "machine address" which the CPU's MMU can
16 * use. 16 * use.
17 * 17 *
18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be 18 * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be
19 * inserted directly into the pagetable. When creating a new 19 * inserted directly into the pagetable. When creating a new
20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely, 20 * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely,
21 * when reading the content back with __(pgd|pmd|pte)_val, it converts 21 * when reading the content back with __(pgd|pmd|pte)_val, it converts
22 * the mfn back into a pfn. 22 * the mfn back into a pfn.
23 * 23 *
24 * The other constraint is that all pages which make up a pagetable 24 * The other constraint is that all pages which make up a pagetable
25 * must be mapped read-only in the guest. This prevents uncontrolled 25 * must be mapped read-only in the guest. This prevents uncontrolled
26 * guest updates to the pagetable. Xen strictly enforces this, and 26 * guest updates to the pagetable. Xen strictly enforces this, and
27 * will disallow any pagetable update which will end up mapping a 27 * will disallow any pagetable update which will end up mapping a
28 * pagetable page RW, and will disallow using any writable page as a 28 * pagetable page RW, and will disallow using any writable page as a
29 * pagetable. 29 * pagetable.
30 * 30 *
31 * Naively, when loading %cr3 with the base of a new pagetable, Xen 31 * Naively, when loading %cr3 with the base of a new pagetable, Xen
32 * would need to validate the whole pagetable before going on. 32 * would need to validate the whole pagetable before going on.
33 * Naturally, this is quite slow. The solution is to "pin" a 33 * Naturally, this is quite slow. The solution is to "pin" a
34 * pagetable, which enforces all the constraints on the pagetable even 34 * pagetable, which enforces all the constraints on the pagetable even
35 * when it is not actively in use. This menas that Xen can be assured 35 * when it is not actively in use. This menas that Xen can be assured
36 * that it is still valid when you do load it into %cr3, and doesn't 36 * that it is still valid when you do load it into %cr3, and doesn't
37 * need to revalidate it. 37 * need to revalidate it.
38 * 38 *
39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 39 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
40 */ 40 */
41 #include <linux/sched.h> 41 #include <linux/sched.h>
42 #include <linux/highmem.h> 42 #include <linux/highmem.h>
43 #include <linux/debugfs.h> 43 #include <linux/debugfs.h>
44 #include <linux/bug.h> 44 #include <linux/bug.h>
45 #include <linux/vmalloc.h> 45 #include <linux/vmalloc.h>
46 #include <linux/module.h> 46 #include <linux/module.h>
47 #include <linux/gfp.h> 47 #include <linux/gfp.h>
48 #include <linux/memblock.h> 48 #include <linux/memblock.h>
49 #include <linux/seq_file.h> 49 #include <linux/seq_file.h>
50 50
51 #include <trace/events/xen.h> 51 #include <trace/events/xen.h>
52 52
53 #include <asm/pgtable.h> 53 #include <asm/pgtable.h>
54 #include <asm/tlbflush.h> 54 #include <asm/tlbflush.h>
55 #include <asm/fixmap.h> 55 #include <asm/fixmap.h>
56 #include <asm/mmu_context.h> 56 #include <asm/mmu_context.h>
57 #include <asm/setup.h> 57 #include <asm/setup.h>
58 #include <asm/paravirt.h> 58 #include <asm/paravirt.h>
59 #include <asm/e820.h> 59 #include <asm/e820.h>
60 #include <asm/linkage.h> 60 #include <asm/linkage.h>
61 #include <asm/page.h> 61 #include <asm/page.h>
62 #include <asm/init.h> 62 #include <asm/init.h>
63 #include <asm/pat.h> 63 #include <asm/pat.h>
64 #include <asm/smp.h> 64 #include <asm/smp.h>
65 65
66 #include <asm/xen/hypercall.h> 66 #include <asm/xen/hypercall.h>
67 #include <asm/xen/hypervisor.h> 67 #include <asm/xen/hypervisor.h>
68 68
69 #include <xen/xen.h> 69 #include <xen/xen.h>
70 #include <xen/page.h> 70 #include <xen/page.h>
71 #include <xen/interface/xen.h> 71 #include <xen/interface/xen.h>
72 #include <xen/interface/hvm/hvm_op.h> 72 #include <xen/interface/hvm/hvm_op.h>
73 #include <xen/interface/version.h> 73 #include <xen/interface/version.h>
74 #include <xen/interface/memory.h> 74 #include <xen/interface/memory.h>
75 #include <xen/hvc-console.h> 75 #include <xen/hvc-console.h>
76 76
77 #include "multicalls.h" 77 #include "multicalls.h"
78 #include "mmu.h" 78 #include "mmu.h"
79 #include "debugfs.h" 79 #include "debugfs.h"
80 80
81 /* 81 /*
82 * Protects atomic reservation decrease/increase against concurrent increases. 82 * Protects atomic reservation decrease/increase against concurrent increases.
83 * Also protects non-atomic updates of current_pages and balloon lists. 83 * Also protects non-atomic updates of current_pages and balloon lists.
84 */ 84 */
85 DEFINE_SPINLOCK(xen_reservation_lock); 85 DEFINE_SPINLOCK(xen_reservation_lock);
86 86
87 /* 87 /*
88 * Identity map, in addition to plain kernel map. This needs to be 88 * Identity map, in addition to plain kernel map. This needs to be
89 * large enough to allocate page table pages to allocate the rest. 89 * large enough to allocate page table pages to allocate the rest.
90 * Each page can map 2MB. 90 * Each page can map 2MB.
91 */ 91 */
92 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4) 92 #define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4)
93 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES); 93 static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES);
94 94
95 #ifdef CONFIG_X86_64 95 #ifdef CONFIG_X86_64
96 /* l3 pud for userspace vsyscall mapping */ 96 /* l3 pud for userspace vsyscall mapping */
97 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; 97 static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss;
98 #endif /* CONFIG_X86_64 */ 98 #endif /* CONFIG_X86_64 */
99 99
100 /* 100 /*
101 * Note about cr3 (pagetable base) values: 101 * Note about cr3 (pagetable base) values:
102 * 102 *
103 * xen_cr3 contains the current logical cr3 value; it contains the 103 * xen_cr3 contains the current logical cr3 value; it contains the
104 * last set cr3. This may not be the current effective cr3, because 104 * last set cr3. This may not be the current effective cr3, because
105 * its update may be being lazily deferred. However, a vcpu looking 105 * its update may be being lazily deferred. However, a vcpu looking
106 * at its own cr3 can use this value knowing that it everything will 106 * at its own cr3 can use this value knowing that it everything will
107 * be self-consistent. 107 * be self-consistent.
108 * 108 *
109 * xen_current_cr3 contains the actual vcpu cr3; it is set once the 109 * xen_current_cr3 contains the actual vcpu cr3; it is set once the
110 * hypercall to set the vcpu cr3 is complete (so it may be a little 110 * hypercall to set the vcpu cr3 is complete (so it may be a little
111 * out of date, but it will never be set early). If one vcpu is 111 * out of date, but it will never be set early). If one vcpu is
112 * looking at another vcpu's cr3 value, it should use this variable. 112 * looking at another vcpu's cr3 value, it should use this variable.
113 */ 113 */
114 DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ 114 DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */
115 DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ 115 DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */
116 116
117 117
118 /* 118 /*
119 * Just beyond the highest usermode address. STACK_TOP_MAX has a 119 * Just beyond the highest usermode address. STACK_TOP_MAX has a
120 * redzone above it, so round it up to a PGD boundary. 120 * redzone above it, so round it up to a PGD boundary.
121 */ 121 */
122 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK) 122 #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
123 123
124 unsigned long arbitrary_virt_to_mfn(void *vaddr) 124 unsigned long arbitrary_virt_to_mfn(void *vaddr)
125 { 125 {
126 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); 126 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr);
127 127
128 return PFN_DOWN(maddr.maddr); 128 return PFN_DOWN(maddr.maddr);
129 } 129 }
130 130
131 xmaddr_t arbitrary_virt_to_machine(void *vaddr) 131 xmaddr_t arbitrary_virt_to_machine(void *vaddr)
132 { 132 {
133 unsigned long address = (unsigned long)vaddr; 133 unsigned long address = (unsigned long)vaddr;
134 unsigned int level; 134 unsigned int level;
135 pte_t *pte; 135 pte_t *pte;
136 unsigned offset; 136 unsigned offset;
137 137
138 /* 138 /*
139 * if the PFN is in the linear mapped vaddr range, we can just use 139 * if the PFN is in the linear mapped vaddr range, we can just use
140 * the (quick) virt_to_machine() p2m lookup 140 * the (quick) virt_to_machine() p2m lookup
141 */ 141 */
142 if (virt_addr_valid(vaddr)) 142 if (virt_addr_valid(vaddr))
143 return virt_to_machine(vaddr); 143 return virt_to_machine(vaddr);
144 144
145 /* otherwise we have to do a (slower) full page-table walk */ 145 /* otherwise we have to do a (slower) full page-table walk */
146 146
147 pte = lookup_address(address, &level); 147 pte = lookup_address(address, &level);
148 BUG_ON(pte == NULL); 148 BUG_ON(pte == NULL);
149 offset = address & ~PAGE_MASK; 149 offset = address & ~PAGE_MASK;
150 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset); 150 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
151 } 151 }
152 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); 152 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine);
153 153
154 void make_lowmem_page_readonly(void *vaddr) 154 void make_lowmem_page_readonly(void *vaddr)
155 { 155 {
156 pte_t *pte, ptev; 156 pte_t *pte, ptev;
157 unsigned long address = (unsigned long)vaddr; 157 unsigned long address = (unsigned long)vaddr;
158 unsigned int level; 158 unsigned int level;
159 159
160 pte = lookup_address(address, &level); 160 pte = lookup_address(address, &level);
161 if (pte == NULL) 161 if (pte == NULL)
162 return; /* vaddr missing */ 162 return; /* vaddr missing */
163 163
164 ptev = pte_wrprotect(*pte); 164 ptev = pte_wrprotect(*pte);
165 165
166 if (HYPERVISOR_update_va_mapping(address, ptev, 0)) 166 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
167 BUG(); 167 BUG();
168 } 168 }
169 169
170 void make_lowmem_page_readwrite(void *vaddr) 170 void make_lowmem_page_readwrite(void *vaddr)
171 { 171 {
172 pte_t *pte, ptev; 172 pte_t *pte, ptev;
173 unsigned long address = (unsigned long)vaddr; 173 unsigned long address = (unsigned long)vaddr;
174 unsigned int level; 174 unsigned int level;
175 175
176 pte = lookup_address(address, &level); 176 pte = lookup_address(address, &level);
177 if (pte == NULL) 177 if (pte == NULL)
178 return; /* vaddr missing */ 178 return; /* vaddr missing */
179 179
180 ptev = pte_mkwrite(*pte); 180 ptev = pte_mkwrite(*pte);
181 181
182 if (HYPERVISOR_update_va_mapping(address, ptev, 0)) 182 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
183 BUG(); 183 BUG();
184 } 184 }
185 185
186 186
187 static bool xen_page_pinned(void *ptr) 187 static bool xen_page_pinned(void *ptr)
188 { 188 {
189 struct page *page = virt_to_page(ptr); 189 struct page *page = virt_to_page(ptr);
190 190
191 return PagePinned(page); 191 return PagePinned(page);
192 } 192 }
193 193
194 void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid) 194 void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid)
195 { 195 {
196 struct multicall_space mcs; 196 struct multicall_space mcs;
197 struct mmu_update *u; 197 struct mmu_update *u;
198 198
199 trace_xen_mmu_set_domain_pte(ptep, pteval, domid); 199 trace_xen_mmu_set_domain_pte(ptep, pteval, domid);
200 200
201 mcs = xen_mc_entry(sizeof(*u)); 201 mcs = xen_mc_entry(sizeof(*u));
202 u = mcs.args; 202 u = mcs.args;
203 203
204 /* ptep might be kmapped when using 32-bit HIGHPTE */ 204 /* ptep might be kmapped when using 32-bit HIGHPTE */
205 u->ptr = virt_to_machine(ptep).maddr; 205 u->ptr = virt_to_machine(ptep).maddr;
206 u->val = pte_val_ma(pteval); 206 u->val = pte_val_ma(pteval);
207 207
208 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid); 208 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid);
209 209
210 xen_mc_issue(PARAVIRT_LAZY_MMU); 210 xen_mc_issue(PARAVIRT_LAZY_MMU);
211 } 211 }
212 EXPORT_SYMBOL_GPL(xen_set_domain_pte); 212 EXPORT_SYMBOL_GPL(xen_set_domain_pte);
213 213
214 static void xen_extend_mmu_update(const struct mmu_update *update) 214 static void xen_extend_mmu_update(const struct mmu_update *update)
215 { 215 {
216 struct multicall_space mcs; 216 struct multicall_space mcs;
217 struct mmu_update *u; 217 struct mmu_update *u;
218 218
219 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u)); 219 mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
220 220
221 if (mcs.mc != NULL) { 221 if (mcs.mc != NULL) {
222 mcs.mc->args[1]++; 222 mcs.mc->args[1]++;
223 } else { 223 } else {
224 mcs = __xen_mc_entry(sizeof(*u)); 224 mcs = __xen_mc_entry(sizeof(*u));
225 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); 225 MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
226 } 226 }
227 227
228 u = mcs.args; 228 u = mcs.args;
229 *u = *update; 229 *u = *update;
230 } 230 }
231 231
232 static void xen_extend_mmuext_op(const struct mmuext_op *op) 232 static void xen_extend_mmuext_op(const struct mmuext_op *op)
233 { 233 {
234 struct multicall_space mcs; 234 struct multicall_space mcs;
235 struct mmuext_op *u; 235 struct mmuext_op *u;
236 236
237 mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u)); 237 mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u));
238 238
239 if (mcs.mc != NULL) { 239 if (mcs.mc != NULL) {
240 mcs.mc->args[1]++; 240 mcs.mc->args[1]++;
241 } else { 241 } else {
242 mcs = __xen_mc_entry(sizeof(*u)); 242 mcs = __xen_mc_entry(sizeof(*u));
243 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); 243 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
244 } 244 }
245 245
246 u = mcs.args; 246 u = mcs.args;
247 *u = *op; 247 *u = *op;
248 } 248 }
249 249
250 static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val) 250 static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
251 { 251 {
252 struct mmu_update u; 252 struct mmu_update u;
253 253
254 preempt_disable(); 254 preempt_disable();
255 255
256 xen_mc_batch(); 256 xen_mc_batch();
257 257
258 /* ptr may be ioremapped for 64-bit pagetable setup */ 258 /* ptr may be ioremapped for 64-bit pagetable setup */
259 u.ptr = arbitrary_virt_to_machine(ptr).maddr; 259 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
260 u.val = pmd_val_ma(val); 260 u.val = pmd_val_ma(val);
261 xen_extend_mmu_update(&u); 261 xen_extend_mmu_update(&u);
262 262
263 xen_mc_issue(PARAVIRT_LAZY_MMU); 263 xen_mc_issue(PARAVIRT_LAZY_MMU);
264 264
265 preempt_enable(); 265 preempt_enable();
266 } 266 }
267 267
268 static void xen_set_pmd(pmd_t *ptr, pmd_t val) 268 static void xen_set_pmd(pmd_t *ptr, pmd_t val)
269 { 269 {
270 trace_xen_mmu_set_pmd(ptr, val); 270 trace_xen_mmu_set_pmd(ptr, val);
271 271
272 /* If page is not pinned, we can just update the entry 272 /* If page is not pinned, we can just update the entry
273 directly */ 273 directly */
274 if (!xen_page_pinned(ptr)) { 274 if (!xen_page_pinned(ptr)) {
275 *ptr = val; 275 *ptr = val;
276 return; 276 return;
277 } 277 }
278 278
279 xen_set_pmd_hyper(ptr, val); 279 xen_set_pmd_hyper(ptr, val);
280 } 280 }
281 281
282 /* 282 /*
283 * Associate a virtual page frame with a given physical page frame 283 * Associate a virtual page frame with a given physical page frame
284 * and protection flags for that frame. 284 * and protection flags for that frame.
285 */ 285 */
286 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags) 286 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
287 { 287 {
288 set_pte_vaddr(vaddr, mfn_pte(mfn, flags)); 288 set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
289 } 289 }
290 290
291 static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) 291 static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval)
292 { 292 {
293 struct mmu_update u; 293 struct mmu_update u;
294 294
295 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU) 295 if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU)
296 return false; 296 return false;
297 297
298 xen_mc_batch(); 298 xen_mc_batch();
299 299
300 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; 300 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
301 u.val = pte_val_ma(pteval); 301 u.val = pte_val_ma(pteval);
302 xen_extend_mmu_update(&u); 302 xen_extend_mmu_update(&u);
303 303
304 xen_mc_issue(PARAVIRT_LAZY_MMU); 304 xen_mc_issue(PARAVIRT_LAZY_MMU);
305 305
306 return true; 306 return true;
307 } 307 }
308 308
309 static inline void __xen_set_pte(pte_t *ptep, pte_t pteval) 309 static inline void __xen_set_pte(pte_t *ptep, pte_t pteval)
310 { 310 {
311 if (!xen_batched_set_pte(ptep, pteval)) { 311 if (!xen_batched_set_pte(ptep, pteval)) {
312 /* 312 /*
313 * Could call native_set_pte() here and trap and 313 * Could call native_set_pte() here and trap and
314 * emulate the PTE write but with 32-bit guests this 314 * emulate the PTE write but with 32-bit guests this
315 * needs two traps (one for each of the two 32-bit 315 * needs two traps (one for each of the two 32-bit
316 * words in the PTE) so do one hypercall directly 316 * words in the PTE) so do one hypercall directly
317 * instead. 317 * instead.
318 */ 318 */
319 struct mmu_update u; 319 struct mmu_update u;
320 320
321 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; 321 u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE;
322 u.val = pte_val_ma(pteval); 322 u.val = pte_val_ma(pteval);
323 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF); 323 HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF);
324 } 324 }
325 } 325 }
326 326
327 static void xen_set_pte(pte_t *ptep, pte_t pteval) 327 static void xen_set_pte(pte_t *ptep, pte_t pteval)
328 { 328 {
329 trace_xen_mmu_set_pte(ptep, pteval); 329 trace_xen_mmu_set_pte(ptep, pteval);
330 __xen_set_pte(ptep, pteval); 330 __xen_set_pte(ptep, pteval);
331 } 331 }
332 332
333 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr, 333 static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
334 pte_t *ptep, pte_t pteval) 334 pte_t *ptep, pte_t pteval)
335 { 335 {
336 trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval); 336 trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval);
337 __xen_set_pte(ptep, pteval); 337 __xen_set_pte(ptep, pteval);
338 } 338 }
339 339
340 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, 340 pte_t xen_ptep_modify_prot_start(struct mm_struct *mm,
341 unsigned long addr, pte_t *ptep) 341 unsigned long addr, pte_t *ptep)
342 { 342 {
343 /* Just return the pte as-is. We preserve the bits on commit */ 343 /* Just return the pte as-is. We preserve the bits on commit */
344 trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep); 344 trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep);
345 return *ptep; 345 return *ptep;
346 } 346 }
347 347
348 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, 348 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
349 pte_t *ptep, pte_t pte) 349 pte_t *ptep, pte_t pte)
350 { 350 {
351 struct mmu_update u; 351 struct mmu_update u;
352 352
353 trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte); 353 trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte);
354 xen_mc_batch(); 354 xen_mc_batch();
355 355
356 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD; 356 u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
357 u.val = pte_val_ma(pte); 357 u.val = pte_val_ma(pte);
358 xen_extend_mmu_update(&u); 358 xen_extend_mmu_update(&u);
359 359
360 xen_mc_issue(PARAVIRT_LAZY_MMU); 360 xen_mc_issue(PARAVIRT_LAZY_MMU);
361 } 361 }
362 362
363 /* Assume pteval_t is equivalent to all the other *val_t types. */ 363 /* Assume pteval_t is equivalent to all the other *val_t types. */
364 static pteval_t pte_mfn_to_pfn(pteval_t val) 364 static pteval_t pte_mfn_to_pfn(pteval_t val)
365 { 365 {
366 if (val & _PAGE_PRESENT) { 366 if (val & _PAGE_PRESENT) {
367 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; 367 unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
368 unsigned long pfn = mfn_to_pfn(mfn); 368 unsigned long pfn = mfn_to_pfn(mfn);
369 369
370 pteval_t flags = val & PTE_FLAGS_MASK; 370 pteval_t flags = val & PTE_FLAGS_MASK;
371 if (unlikely(pfn == ~0)) 371 if (unlikely(pfn == ~0))
372 val = flags & ~_PAGE_PRESENT; 372 val = flags & ~_PAGE_PRESENT;
373 else 373 else
374 val = ((pteval_t)pfn << PAGE_SHIFT) | flags; 374 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
375 } 375 }
376 376
377 return val; 377 return val;
378 } 378 }
379 379
380 static pteval_t pte_pfn_to_mfn(pteval_t val) 380 static pteval_t pte_pfn_to_mfn(pteval_t val)
381 { 381 {
382 if (val & _PAGE_PRESENT) { 382 if (val & _PAGE_PRESENT) {
383 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; 383 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
384 pteval_t flags = val & PTE_FLAGS_MASK; 384 pteval_t flags = val & PTE_FLAGS_MASK;
385 unsigned long mfn; 385 unsigned long mfn;
386 386
387 if (!xen_feature(XENFEAT_auto_translated_physmap)) 387 if (!xen_feature(XENFEAT_auto_translated_physmap))
388 mfn = get_phys_to_machine(pfn); 388 mfn = get_phys_to_machine(pfn);
389 else 389 else
390 mfn = pfn; 390 mfn = pfn;
391 /* 391 /*
392 * If there's no mfn for the pfn, then just create an 392 * If there's no mfn for the pfn, then just create an
393 * empty non-present pte. Unfortunately this loses 393 * empty non-present pte. Unfortunately this loses
394 * information about the original pfn, so 394 * information about the original pfn, so
395 * pte_mfn_to_pfn is asymmetric. 395 * pte_mfn_to_pfn is asymmetric.
396 */ 396 */
397 if (unlikely(mfn == INVALID_P2M_ENTRY)) { 397 if (unlikely(mfn == INVALID_P2M_ENTRY)) {
398 mfn = 0; 398 mfn = 0;
399 flags = 0; 399 flags = 0;
400 } else { 400 } else {
401 /* 401 /*
402 * Paramount to do this test _after_ the 402 * Paramount to do this test _after_ the
403 * INVALID_P2M_ENTRY as INVALID_P2M_ENTRY & 403 * INVALID_P2M_ENTRY as INVALID_P2M_ENTRY &
404 * IDENTITY_FRAME_BIT resolves to true. 404 * IDENTITY_FRAME_BIT resolves to true.
405 */ 405 */
406 mfn &= ~FOREIGN_FRAME_BIT; 406 mfn &= ~FOREIGN_FRAME_BIT;
407 if (mfn & IDENTITY_FRAME_BIT) { 407 if (mfn & IDENTITY_FRAME_BIT) {
408 mfn &= ~IDENTITY_FRAME_BIT; 408 mfn &= ~IDENTITY_FRAME_BIT;
409 flags |= _PAGE_IOMAP; 409 flags |= _PAGE_IOMAP;
410 } 410 }
411 } 411 }
412 val = ((pteval_t)mfn << PAGE_SHIFT) | flags; 412 val = ((pteval_t)mfn << PAGE_SHIFT) | flags;
413 } 413 }
414 414
415 return val; 415 return val;
416 } 416 }
417 417
418 static pteval_t iomap_pte(pteval_t val) 418 static pteval_t iomap_pte(pteval_t val)
419 { 419 {
420 if (val & _PAGE_PRESENT) { 420 if (val & _PAGE_PRESENT) {
421 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; 421 unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
422 pteval_t flags = val & PTE_FLAGS_MASK; 422 pteval_t flags = val & PTE_FLAGS_MASK;
423 423
424 /* We assume the pte frame number is a MFN, so 424 /* We assume the pte frame number is a MFN, so
425 just use it as-is. */ 425 just use it as-is. */
426 val = ((pteval_t)pfn << PAGE_SHIFT) | flags; 426 val = ((pteval_t)pfn << PAGE_SHIFT) | flags;
427 } 427 }
428 428
429 return val; 429 return val;
430 } 430 }
431 431
432 static pteval_t xen_pte_val(pte_t pte) 432 static pteval_t xen_pte_val(pte_t pte)
433 { 433 {
434 pteval_t pteval = pte.pte; 434 pteval_t pteval = pte.pte;
435 #if 0 435 #if 0
436 /* If this is a WC pte, convert back from Xen WC to Linux WC */ 436 /* If this is a WC pte, convert back from Xen WC to Linux WC */
437 if ((pteval & (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)) == _PAGE_PAT) { 437 if ((pteval & (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)) == _PAGE_PAT) {
438 WARN_ON(!pat_enabled); 438 WARN_ON(!pat_enabled);
439 pteval = (pteval & ~_PAGE_PAT) | _PAGE_PWT; 439 pteval = (pteval & ~_PAGE_PAT) | _PAGE_PWT;
440 } 440 }
441 #endif 441 #endif
442 if (xen_initial_domain() && (pteval & _PAGE_IOMAP)) 442 if (xen_initial_domain() && (pteval & _PAGE_IOMAP))
443 return pteval; 443 return pteval;
444 444
445 return pte_mfn_to_pfn(pteval); 445 return pte_mfn_to_pfn(pteval);
446 } 446 }
447 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val); 447 PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
448 448
449 static pgdval_t xen_pgd_val(pgd_t pgd) 449 static pgdval_t xen_pgd_val(pgd_t pgd)
450 { 450 {
451 return pte_mfn_to_pfn(pgd.pgd); 451 return pte_mfn_to_pfn(pgd.pgd);
452 } 452 }
453 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val); 453 PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
454 454
455 /* 455 /*
456 * Xen's PAT setup is part of its ABI, though I assume entries 6 & 7 456 * Xen's PAT setup is part of its ABI, though I assume entries 6 & 7
457 * are reserved for now, to correspond to the Intel-reserved PAT 457 * are reserved for now, to correspond to the Intel-reserved PAT
458 * types. 458 * types.
459 * 459 *
460 * We expect Linux's PAT set as follows: 460 * We expect Linux's PAT set as follows:
461 * 461 *
462 * Idx PTE flags Linux Xen Default 462 * Idx PTE flags Linux Xen Default
463 * 0 WB WB WB 463 * 0 WB WB WB
464 * 1 PWT WC WT WT 464 * 1 PWT WC WT WT
465 * 2 PCD UC- UC- UC- 465 * 2 PCD UC- UC- UC-
466 * 3 PCD PWT UC UC UC 466 * 3 PCD PWT UC UC UC
467 * 4 PAT WB WC WB 467 * 4 PAT WB WC WB
468 * 5 PAT PWT WC WP WT 468 * 5 PAT PWT WC WP WT
469 * 6 PAT PCD UC- UC UC- 469 * 6 PAT PCD UC- UC UC-
470 * 7 PAT PCD PWT UC UC UC 470 * 7 PAT PCD PWT UC UC UC
471 */ 471 */
472 472
473 void xen_set_pat(u64 pat) 473 void xen_set_pat(u64 pat)
474 { 474 {
475 /* We expect Linux to use a PAT setting of 475 /* We expect Linux to use a PAT setting of
476 * UC UC- WC WB (ignoring the PAT flag) */ 476 * UC UC- WC WB (ignoring the PAT flag) */
477 WARN_ON(pat != 0x0007010600070106ull); 477 WARN_ON(pat != 0x0007010600070106ull);
478 } 478 }
479 479
480 static pte_t xen_make_pte(pteval_t pte) 480 static pte_t xen_make_pte(pteval_t pte)
481 { 481 {
482 phys_addr_t addr = (pte & PTE_PFN_MASK); 482 phys_addr_t addr = (pte & PTE_PFN_MASK);
483 #if 0 483 #if 0
484 /* If Linux is trying to set a WC pte, then map to the Xen WC. 484 /* If Linux is trying to set a WC pte, then map to the Xen WC.
485 * If _PAGE_PAT is set, then it probably means it is really 485 * If _PAGE_PAT is set, then it probably means it is really
486 * _PAGE_PSE, so avoid fiddling with the PAT mapping and hope 486 * _PAGE_PSE, so avoid fiddling with the PAT mapping and hope
487 * things work out OK... 487 * things work out OK...
488 * 488 *
489 * (We should never see kernel mappings with _PAGE_PSE set, 489 * (We should never see kernel mappings with _PAGE_PSE set,
490 * but we could see hugetlbfs mappings, I think.). 490 * but we could see hugetlbfs mappings, I think.).
491 */ 491 */
492 if (pat_enabled && !WARN_ON(pte & _PAGE_PAT)) { 492 if (pat_enabled && !WARN_ON(pte & _PAGE_PAT)) {
493 if ((pte & (_PAGE_PCD | _PAGE_PWT)) == _PAGE_PWT) 493 if ((pte & (_PAGE_PCD | _PAGE_PWT)) == _PAGE_PWT)
494 pte = (pte & ~(_PAGE_PCD | _PAGE_PWT)) | _PAGE_PAT; 494 pte = (pte & ~(_PAGE_PCD | _PAGE_PWT)) | _PAGE_PAT;
495 } 495 }
496 #endif 496 #endif
497 /* 497 /*
498 * Unprivileged domains are allowed to do IOMAPpings for 498 * Unprivileged domains are allowed to do IOMAPpings for
499 * PCI passthrough, but not map ISA space. The ISA 499 * PCI passthrough, but not map ISA space. The ISA
500 * mappings are just dummy local mappings to keep other 500 * mappings are just dummy local mappings to keep other
501 * parts of the kernel happy. 501 * parts of the kernel happy.
502 */ 502 */
503 if (unlikely(pte & _PAGE_IOMAP) && 503 if (unlikely(pte & _PAGE_IOMAP) &&
504 (xen_initial_domain() || addr >= ISA_END_ADDRESS)) { 504 (xen_initial_domain() || addr >= ISA_END_ADDRESS)) {
505 pte = iomap_pte(pte); 505 pte = iomap_pte(pte);
506 } else { 506 } else {
507 pte &= ~_PAGE_IOMAP; 507 pte &= ~_PAGE_IOMAP;
508 pte = pte_pfn_to_mfn(pte); 508 pte = pte_pfn_to_mfn(pte);
509 } 509 }
510 510
511 return native_make_pte(pte); 511 return native_make_pte(pte);
512 } 512 }
513 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte); 513 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
514 514
515 static pgd_t xen_make_pgd(pgdval_t pgd) 515 static pgd_t xen_make_pgd(pgdval_t pgd)
516 { 516 {
517 pgd = pte_pfn_to_mfn(pgd); 517 pgd = pte_pfn_to_mfn(pgd);
518 return native_make_pgd(pgd); 518 return native_make_pgd(pgd);
519 } 519 }
520 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd); 520 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
521 521
522 static pmdval_t xen_pmd_val(pmd_t pmd) 522 static pmdval_t xen_pmd_val(pmd_t pmd)
523 { 523 {
524 return pte_mfn_to_pfn(pmd.pmd); 524 return pte_mfn_to_pfn(pmd.pmd);
525 } 525 }
526 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val); 526 PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
527 527
528 static void xen_set_pud_hyper(pud_t *ptr, pud_t val) 528 static void xen_set_pud_hyper(pud_t *ptr, pud_t val)
529 { 529 {
530 struct mmu_update u; 530 struct mmu_update u;
531 531
532 preempt_disable(); 532 preempt_disable();
533 533
534 xen_mc_batch(); 534 xen_mc_batch();
535 535
536 /* ptr may be ioremapped for 64-bit pagetable setup */ 536 /* ptr may be ioremapped for 64-bit pagetable setup */
537 u.ptr = arbitrary_virt_to_machine(ptr).maddr; 537 u.ptr = arbitrary_virt_to_machine(ptr).maddr;
538 u.val = pud_val_ma(val); 538 u.val = pud_val_ma(val);
539 xen_extend_mmu_update(&u); 539 xen_extend_mmu_update(&u);
540 540
541 xen_mc_issue(PARAVIRT_LAZY_MMU); 541 xen_mc_issue(PARAVIRT_LAZY_MMU);
542 542
543 preempt_enable(); 543 preempt_enable();
544 } 544 }
545 545
546 static void xen_set_pud(pud_t *ptr, pud_t val) 546 static void xen_set_pud(pud_t *ptr, pud_t val)
547 { 547 {
548 trace_xen_mmu_set_pud(ptr, val); 548 trace_xen_mmu_set_pud(ptr, val);
549 549
550 /* If page is not pinned, we can just update the entry 550 /* If page is not pinned, we can just update the entry
551 directly */ 551 directly */
552 if (!xen_page_pinned(ptr)) { 552 if (!xen_page_pinned(ptr)) {
553 *ptr = val; 553 *ptr = val;
554 return; 554 return;
555 } 555 }
556 556
557 xen_set_pud_hyper(ptr, val); 557 xen_set_pud_hyper(ptr, val);
558 } 558 }
559 559
560 #ifdef CONFIG_X86_PAE 560 #ifdef CONFIG_X86_PAE
561 static void xen_set_pte_atomic(pte_t *ptep, pte_t pte) 561 static void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
562 { 562 {
563 trace_xen_mmu_set_pte_atomic(ptep, pte); 563 trace_xen_mmu_set_pte_atomic(ptep, pte);
564 set_64bit((u64 *)ptep, native_pte_val(pte)); 564 set_64bit((u64 *)ptep, native_pte_val(pte));
565 } 565 }
566 566
567 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 567 static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
568 { 568 {
569 trace_xen_mmu_pte_clear(mm, addr, ptep); 569 trace_xen_mmu_pte_clear(mm, addr, ptep);
570 if (!xen_batched_set_pte(ptep, native_make_pte(0))) 570 if (!xen_batched_set_pte(ptep, native_make_pte(0)))
571 native_pte_clear(mm, addr, ptep); 571 native_pte_clear(mm, addr, ptep);
572 } 572 }
573 573
574 static void xen_pmd_clear(pmd_t *pmdp) 574 static void xen_pmd_clear(pmd_t *pmdp)
575 { 575 {
576 trace_xen_mmu_pmd_clear(pmdp); 576 trace_xen_mmu_pmd_clear(pmdp);
577 set_pmd(pmdp, __pmd(0)); 577 set_pmd(pmdp, __pmd(0));
578 } 578 }
579 #endif /* CONFIG_X86_PAE */ 579 #endif /* CONFIG_X86_PAE */
580 580
581 static pmd_t xen_make_pmd(pmdval_t pmd) 581 static pmd_t xen_make_pmd(pmdval_t pmd)
582 { 582 {
583 pmd = pte_pfn_to_mfn(pmd); 583 pmd = pte_pfn_to_mfn(pmd);
584 return native_make_pmd(pmd); 584 return native_make_pmd(pmd);
585 } 585 }
586 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd); 586 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
587 587
588 #if PAGETABLE_LEVELS == 4 588 #if PAGETABLE_LEVELS == 4
589 static pudval_t xen_pud_val(pud_t pud) 589 static pudval_t xen_pud_val(pud_t pud)
590 { 590 {
591 return pte_mfn_to_pfn(pud.pud); 591 return pte_mfn_to_pfn(pud.pud);
592 } 592 }
593 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val); 593 PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
594 594
595 static pud_t xen_make_pud(pudval_t pud) 595 static pud_t xen_make_pud(pudval_t pud)
596 { 596 {
597 pud = pte_pfn_to_mfn(pud); 597 pud = pte_pfn_to_mfn(pud);
598 598
599 return native_make_pud(pud); 599 return native_make_pud(pud);
600 } 600 }
601 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud); 601 PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
602 602
603 static pgd_t *xen_get_user_pgd(pgd_t *pgd) 603 static pgd_t *xen_get_user_pgd(pgd_t *pgd)
604 { 604 {
605 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK); 605 pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
606 unsigned offset = pgd - pgd_page; 606 unsigned offset = pgd - pgd_page;
607 pgd_t *user_ptr = NULL; 607 pgd_t *user_ptr = NULL;
608 608
609 if (offset < pgd_index(USER_LIMIT)) { 609 if (offset < pgd_index(USER_LIMIT)) {
610 struct page *page = virt_to_page(pgd_page); 610 struct page *page = virt_to_page(pgd_page);
611 user_ptr = (pgd_t *)page->private; 611 user_ptr = (pgd_t *)page->private;
612 if (user_ptr) 612 if (user_ptr)
613 user_ptr += offset; 613 user_ptr += offset;
614 } 614 }
615 615
616 return user_ptr; 616 return user_ptr;
617 } 617 }
618 618
619 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val) 619 static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
620 { 620 {
621 struct mmu_update u; 621 struct mmu_update u;
622 622
623 u.ptr = virt_to_machine(ptr).maddr; 623 u.ptr = virt_to_machine(ptr).maddr;
624 u.val = pgd_val_ma(val); 624 u.val = pgd_val_ma(val);
625 xen_extend_mmu_update(&u); 625 xen_extend_mmu_update(&u);
626 } 626 }
627 627
628 /* 628 /*
629 * Raw hypercall-based set_pgd, intended for in early boot before 629 * Raw hypercall-based set_pgd, intended for in early boot before
630 * there's a page structure. This implies: 630 * there's a page structure. This implies:
631 * 1. The only existing pagetable is the kernel's 631 * 1. The only existing pagetable is the kernel's
632 * 2. It is always pinned 632 * 2. It is always pinned
633 * 3. It has no user pagetable attached to it 633 * 3. It has no user pagetable attached to it
634 */ 634 */
635 static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val) 635 static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
636 { 636 {
637 preempt_disable(); 637 preempt_disable();
638 638
639 xen_mc_batch(); 639 xen_mc_batch();
640 640
641 __xen_set_pgd_hyper(ptr, val); 641 __xen_set_pgd_hyper(ptr, val);
642 642
643 xen_mc_issue(PARAVIRT_LAZY_MMU); 643 xen_mc_issue(PARAVIRT_LAZY_MMU);
644 644
645 preempt_enable(); 645 preempt_enable();
646 } 646 }
647 647
648 static void xen_set_pgd(pgd_t *ptr, pgd_t val) 648 static void xen_set_pgd(pgd_t *ptr, pgd_t val)
649 { 649 {
650 pgd_t *user_ptr = xen_get_user_pgd(ptr); 650 pgd_t *user_ptr = xen_get_user_pgd(ptr);
651 651
652 trace_xen_mmu_set_pgd(ptr, user_ptr, val); 652 trace_xen_mmu_set_pgd(ptr, user_ptr, val);
653 653
654 /* If page is not pinned, we can just update the entry 654 /* If page is not pinned, we can just update the entry
655 directly */ 655 directly */
656 if (!xen_page_pinned(ptr)) { 656 if (!xen_page_pinned(ptr)) {
657 *ptr = val; 657 *ptr = val;
658 if (user_ptr) { 658 if (user_ptr) {
659 WARN_ON(xen_page_pinned(user_ptr)); 659 WARN_ON(xen_page_pinned(user_ptr));
660 *user_ptr = val; 660 *user_ptr = val;
661 } 661 }
662 return; 662 return;
663 } 663 }
664 664
665 /* If it's pinned, then we can at least batch the kernel and 665 /* If it's pinned, then we can at least batch the kernel and
666 user updates together. */ 666 user updates together. */
667 xen_mc_batch(); 667 xen_mc_batch();
668 668
669 __xen_set_pgd_hyper(ptr, val); 669 __xen_set_pgd_hyper(ptr, val);
670 if (user_ptr) 670 if (user_ptr)
671 __xen_set_pgd_hyper(user_ptr, val); 671 __xen_set_pgd_hyper(user_ptr, val);
672 672
673 xen_mc_issue(PARAVIRT_LAZY_MMU); 673 xen_mc_issue(PARAVIRT_LAZY_MMU);
674 } 674 }
675 #endif /* PAGETABLE_LEVELS == 4 */ 675 #endif /* PAGETABLE_LEVELS == 4 */
676 676
677 /* 677 /*
678 * (Yet another) pagetable walker. This one is intended for pinning a 678 * (Yet another) pagetable walker. This one is intended for pinning a
679 * pagetable. This means that it walks a pagetable and calls the 679 * pagetable. This means that it walks a pagetable and calls the
680 * callback function on each page it finds making up the page table, 680 * callback function on each page it finds making up the page table,
681 * at every level. It walks the entire pagetable, but it only bothers 681 * at every level. It walks the entire pagetable, but it only bothers
682 * pinning pte pages which are below limit. In the normal case this 682 * pinning pte pages which are below limit. In the normal case this
683 * will be STACK_TOP_MAX, but at boot we need to pin up to 683 * will be STACK_TOP_MAX, but at boot we need to pin up to
684 * FIXADDR_TOP. 684 * FIXADDR_TOP.
685 * 685 *
686 * For 32-bit the important bit is that we don't pin beyond there, 686 * For 32-bit the important bit is that we don't pin beyond there,
687 * because then we start getting into Xen's ptes. 687 * because then we start getting into Xen's ptes.
688 * 688 *
689 * For 64-bit, we must skip the Xen hole in the middle of the address 689 * For 64-bit, we must skip the Xen hole in the middle of the address
690 * space, just after the big x86-64 virtual hole. 690 * space, just after the big x86-64 virtual hole.
691 */ 691 */
692 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd, 692 static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd,
693 int (*func)(struct mm_struct *mm, struct page *, 693 int (*func)(struct mm_struct *mm, struct page *,
694 enum pt_level), 694 enum pt_level),
695 unsigned long limit) 695 unsigned long limit)
696 { 696 {
697 int flush = 0; 697 int flush = 0;
698 unsigned hole_low, hole_high; 698 unsigned hole_low, hole_high;
699 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit; 699 unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
700 unsigned pgdidx, pudidx, pmdidx; 700 unsigned pgdidx, pudidx, pmdidx;
701 701
702 /* The limit is the last byte to be touched */ 702 /* The limit is the last byte to be touched */
703 limit--; 703 limit--;
704 BUG_ON(limit >= FIXADDR_TOP); 704 BUG_ON(limit >= FIXADDR_TOP);
705 705
706 if (xen_feature(XENFEAT_auto_translated_physmap)) 706 if (xen_feature(XENFEAT_auto_translated_physmap))
707 return 0; 707 return 0;
708 708
709 /* 709 /*
710 * 64-bit has a great big hole in the middle of the address 710 * 64-bit has a great big hole in the middle of the address
711 * space, which contains the Xen mappings. On 32-bit these 711 * space, which contains the Xen mappings. On 32-bit these
712 * will end up making a zero-sized hole and so is a no-op. 712 * will end up making a zero-sized hole and so is a no-op.
713 */ 713 */
714 hole_low = pgd_index(USER_LIMIT); 714 hole_low = pgd_index(USER_LIMIT);
715 hole_high = pgd_index(PAGE_OFFSET); 715 hole_high = pgd_index(PAGE_OFFSET);
716 716
717 pgdidx_limit = pgd_index(limit); 717 pgdidx_limit = pgd_index(limit);
718 #if PTRS_PER_PUD > 1 718 #if PTRS_PER_PUD > 1
719 pudidx_limit = pud_index(limit); 719 pudidx_limit = pud_index(limit);
720 #else 720 #else
721 pudidx_limit = 0; 721 pudidx_limit = 0;
722 #endif 722 #endif
723 #if PTRS_PER_PMD > 1 723 #if PTRS_PER_PMD > 1
724 pmdidx_limit = pmd_index(limit); 724 pmdidx_limit = pmd_index(limit);
725 #else 725 #else
726 pmdidx_limit = 0; 726 pmdidx_limit = 0;
727 #endif 727 #endif
728 728
729 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) { 729 for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
730 pud_t *pud; 730 pud_t *pud;
731 731
732 if (pgdidx >= hole_low && pgdidx < hole_high) 732 if (pgdidx >= hole_low && pgdidx < hole_high)
733 continue; 733 continue;
734 734
735 if (!pgd_val(pgd[pgdidx])) 735 if (!pgd_val(pgd[pgdidx]))
736 continue; 736 continue;
737 737
738 pud = pud_offset(&pgd[pgdidx], 0); 738 pud = pud_offset(&pgd[pgdidx], 0);
739 739
740 if (PTRS_PER_PUD > 1) /* not folded */ 740 if (PTRS_PER_PUD > 1) /* not folded */
741 flush |= (*func)(mm, virt_to_page(pud), PT_PUD); 741 flush |= (*func)(mm, virt_to_page(pud), PT_PUD);
742 742
743 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) { 743 for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
744 pmd_t *pmd; 744 pmd_t *pmd;
745 745
746 if (pgdidx == pgdidx_limit && 746 if (pgdidx == pgdidx_limit &&
747 pudidx > pudidx_limit) 747 pudidx > pudidx_limit)
748 goto out; 748 goto out;
749 749
750 if (pud_none(pud[pudidx])) 750 if (pud_none(pud[pudidx]))
751 continue; 751 continue;
752 752
753 pmd = pmd_offset(&pud[pudidx], 0); 753 pmd = pmd_offset(&pud[pudidx], 0);
754 754
755 if (PTRS_PER_PMD > 1) /* not folded */ 755 if (PTRS_PER_PMD > 1) /* not folded */
756 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD); 756 flush |= (*func)(mm, virt_to_page(pmd), PT_PMD);
757 757
758 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) { 758 for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
759 struct page *pte; 759 struct page *pte;
760 760
761 if (pgdidx == pgdidx_limit && 761 if (pgdidx == pgdidx_limit &&
762 pudidx == pudidx_limit && 762 pudidx == pudidx_limit &&
763 pmdidx > pmdidx_limit) 763 pmdidx > pmdidx_limit)
764 goto out; 764 goto out;
765 765
766 if (pmd_none(pmd[pmdidx])) 766 if (pmd_none(pmd[pmdidx]))
767 continue; 767 continue;
768 768
769 pte = pmd_page(pmd[pmdidx]); 769 pte = pmd_page(pmd[pmdidx]);
770 flush |= (*func)(mm, pte, PT_PTE); 770 flush |= (*func)(mm, pte, PT_PTE);
771 } 771 }
772 } 772 }
773 } 773 }
774 774
775 out: 775 out:
776 /* Do the top level last, so that the callbacks can use it as 776 /* Do the top level last, so that the callbacks can use it as
777 a cue to do final things like tlb flushes. */ 777 a cue to do final things like tlb flushes. */
778 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD); 778 flush |= (*func)(mm, virt_to_page(pgd), PT_PGD);
779 779
780 return flush; 780 return flush;
781 } 781 }
782 782
783 static int xen_pgd_walk(struct mm_struct *mm, 783 static int xen_pgd_walk(struct mm_struct *mm,
784 int (*func)(struct mm_struct *mm, struct page *, 784 int (*func)(struct mm_struct *mm, struct page *,
785 enum pt_level), 785 enum pt_level),
786 unsigned long limit) 786 unsigned long limit)
787 { 787 {
788 return __xen_pgd_walk(mm, mm->pgd, func, limit); 788 return __xen_pgd_walk(mm, mm->pgd, func, limit);
789 } 789 }
790 790
791 /* If we're using split pte locks, then take the page's lock and 791 /* If we're using split pte locks, then take the page's lock and
792 return a pointer to it. Otherwise return NULL. */ 792 return a pointer to it. Otherwise return NULL. */
793 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm) 793 static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
794 { 794 {
795 spinlock_t *ptl = NULL; 795 spinlock_t *ptl = NULL;
796 796
797 #if USE_SPLIT_PTLOCKS 797 #if USE_SPLIT_PTLOCKS
798 ptl = __pte_lockptr(page); 798 ptl = __pte_lockptr(page);
799 spin_lock_nest_lock(ptl, &mm->page_table_lock); 799 spin_lock_nest_lock(ptl, &mm->page_table_lock);
800 #endif 800 #endif
801 801
802 return ptl; 802 return ptl;
803 } 803 }
804 804
805 static void xen_pte_unlock(void *v) 805 static void xen_pte_unlock(void *v)
806 { 806 {
807 spinlock_t *ptl = v; 807 spinlock_t *ptl = v;
808 spin_unlock(ptl); 808 spin_unlock(ptl);
809 } 809 }
810 810
811 static void xen_do_pin(unsigned level, unsigned long pfn) 811 static void xen_do_pin(unsigned level, unsigned long pfn)
812 { 812 {
813 struct mmuext_op op; 813 struct mmuext_op op;
814 814
815 op.cmd = level; 815 op.cmd = level;
816 op.arg1.mfn = pfn_to_mfn(pfn); 816 op.arg1.mfn = pfn_to_mfn(pfn);
817 817
818 xen_extend_mmuext_op(&op); 818 xen_extend_mmuext_op(&op);
819 } 819 }
820 820
821 static int xen_pin_page(struct mm_struct *mm, struct page *page, 821 static int xen_pin_page(struct mm_struct *mm, struct page *page,
822 enum pt_level level) 822 enum pt_level level)
823 { 823 {
824 unsigned pgfl = TestSetPagePinned(page); 824 unsigned pgfl = TestSetPagePinned(page);
825 int flush; 825 int flush;
826 826
827 if (pgfl) 827 if (pgfl)
828 flush = 0; /* already pinned */ 828 flush = 0; /* already pinned */
829 else if (PageHighMem(page)) 829 else if (PageHighMem(page))
830 /* kmaps need flushing if we found an unpinned 830 /* kmaps need flushing if we found an unpinned
831 highpage */ 831 highpage */
832 flush = 1; 832 flush = 1;
833 else { 833 else {
834 void *pt = lowmem_page_address(page); 834 void *pt = lowmem_page_address(page);
835 unsigned long pfn = page_to_pfn(page); 835 unsigned long pfn = page_to_pfn(page);
836 struct multicall_space mcs = __xen_mc_entry(0); 836 struct multicall_space mcs = __xen_mc_entry(0);
837 spinlock_t *ptl; 837 spinlock_t *ptl;
838 838
839 flush = 0; 839 flush = 0;
840 840
841 /* 841 /*
842 * We need to hold the pagetable lock between the time 842 * We need to hold the pagetable lock between the time
843 * we make the pagetable RO and when we actually pin 843 * we make the pagetable RO and when we actually pin
844 * it. If we don't, then other users may come in and 844 * it. If we don't, then other users may come in and
845 * attempt to update the pagetable by writing it, 845 * attempt to update the pagetable by writing it,
846 * which will fail because the memory is RO but not 846 * which will fail because the memory is RO but not
847 * pinned, so Xen won't do the trap'n'emulate. 847 * pinned, so Xen won't do the trap'n'emulate.
848 * 848 *
849 * If we're using split pte locks, we can't hold the 849 * If we're using split pte locks, we can't hold the
850 * entire pagetable's worth of locks during the 850 * entire pagetable's worth of locks during the
851 * traverse, because we may wrap the preempt count (8 851 * traverse, because we may wrap the preempt count (8
852 * bits). The solution is to mark RO and pin each PTE 852 * bits). The solution is to mark RO and pin each PTE
853 * page while holding the lock. This means the number 853 * page while holding the lock. This means the number
854 * of locks we end up holding is never more than a 854 * of locks we end up holding is never more than a
855 * batch size (~32 entries, at present). 855 * batch size (~32 entries, at present).
856 * 856 *
857 * If we're not using split pte locks, we needn't pin 857 * If we're not using split pte locks, we needn't pin
858 * the PTE pages independently, because we're 858 * the PTE pages independently, because we're
859 * protected by the overall pagetable lock. 859 * protected by the overall pagetable lock.
860 */ 860 */
861 ptl = NULL; 861 ptl = NULL;
862 if (level == PT_PTE) 862 if (level == PT_PTE)
863 ptl = xen_pte_lock(page, mm); 863 ptl = xen_pte_lock(page, mm);
864 864
865 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, 865 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
866 pfn_pte(pfn, PAGE_KERNEL_RO), 866 pfn_pte(pfn, PAGE_KERNEL_RO),
867 level == PT_PGD ? UVMF_TLB_FLUSH : 0); 867 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
868 868
869 if (ptl) { 869 if (ptl) {
870 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn); 870 xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn);
871 871
872 /* Queue a deferred unlock for when this batch 872 /* Queue a deferred unlock for when this batch
873 is completed. */ 873 is completed. */
874 xen_mc_callback(xen_pte_unlock, ptl); 874 xen_mc_callback(xen_pte_unlock, ptl);
875 } 875 }
876 } 876 }
877 877
878 return flush; 878 return flush;
879 } 879 }
880 880
881 /* This is called just after a mm has been created, but it has not 881 /* This is called just after a mm has been created, but it has not
882 been used yet. We need to make sure that its pagetable is all 882 been used yet. We need to make sure that its pagetable is all
883 read-only, and can be pinned. */ 883 read-only, and can be pinned. */
884 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd) 884 static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd)
885 { 885 {
886 trace_xen_mmu_pgd_pin(mm, pgd); 886 trace_xen_mmu_pgd_pin(mm, pgd);
887 887
888 xen_mc_batch(); 888 xen_mc_batch();
889 889
890 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) { 890 if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) {
891 /* re-enable interrupts for flushing */ 891 /* re-enable interrupts for flushing */
892 xen_mc_issue(0); 892 xen_mc_issue(0);
893 893
894 kmap_flush_unused(); 894 kmap_flush_unused();
895 895
896 xen_mc_batch(); 896 xen_mc_batch();
897 } 897 }
898 898
899 #ifdef CONFIG_X86_64 899 #ifdef CONFIG_X86_64
900 { 900 {
901 pgd_t *user_pgd = xen_get_user_pgd(pgd); 901 pgd_t *user_pgd = xen_get_user_pgd(pgd);
902 902
903 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd))); 903 xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
904 904
905 if (user_pgd) { 905 if (user_pgd) {
906 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD); 906 xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD);
907 xen_do_pin(MMUEXT_PIN_L4_TABLE, 907 xen_do_pin(MMUEXT_PIN_L4_TABLE,
908 PFN_DOWN(__pa(user_pgd))); 908 PFN_DOWN(__pa(user_pgd)));
909 } 909 }
910 } 910 }
911 #else /* CONFIG_X86_32 */ 911 #else /* CONFIG_X86_32 */
912 #ifdef CONFIG_X86_PAE 912 #ifdef CONFIG_X86_PAE
913 /* Need to make sure unshared kernel PMD is pinnable */ 913 /* Need to make sure unshared kernel PMD is pinnable */
914 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), 914 xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
915 PT_PMD); 915 PT_PMD);
916 #endif 916 #endif
917 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd))); 917 xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
918 #endif /* CONFIG_X86_64 */ 918 #endif /* CONFIG_X86_64 */
919 xen_mc_issue(0); 919 xen_mc_issue(0);
920 } 920 }
921 921
922 static void xen_pgd_pin(struct mm_struct *mm) 922 static void xen_pgd_pin(struct mm_struct *mm)
923 { 923 {
924 __xen_pgd_pin(mm, mm->pgd); 924 __xen_pgd_pin(mm, mm->pgd);
925 } 925 }
926 926
927 /* 927 /*
928 * On save, we need to pin all pagetables to make sure they get their 928 * On save, we need to pin all pagetables to make sure they get their
929 * mfns turned into pfns. Search the list for any unpinned pgds and pin 929 * mfns turned into pfns. Search the list for any unpinned pgds and pin
930 * them (unpinned pgds are not currently in use, probably because the 930 * them (unpinned pgds are not currently in use, probably because the
931 * process is under construction or destruction). 931 * process is under construction or destruction).
932 * 932 *
933 * Expected to be called in stop_machine() ("equivalent to taking 933 * Expected to be called in stop_machine() ("equivalent to taking
934 * every spinlock in the system"), so the locking doesn't really 934 * every spinlock in the system"), so the locking doesn't really
935 * matter all that much. 935 * matter all that much.
936 */ 936 */
937 void xen_mm_pin_all(void) 937 void xen_mm_pin_all(void)
938 { 938 {
939 struct page *page; 939 struct page *page;
940 940
941 spin_lock(&pgd_lock); 941 spin_lock(&pgd_lock);
942 942
943 list_for_each_entry(page, &pgd_list, lru) { 943 list_for_each_entry(page, &pgd_list, lru) {
944 if (!PagePinned(page)) { 944 if (!PagePinned(page)) {
945 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); 945 __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page));
946 SetPageSavePinned(page); 946 SetPageSavePinned(page);
947 } 947 }
948 } 948 }
949 949
950 spin_unlock(&pgd_lock); 950 spin_unlock(&pgd_lock);
951 } 951 }
952 952
953 /* 953 /*
954 * The init_mm pagetable is really pinned as soon as its created, but 954 * The init_mm pagetable is really pinned as soon as its created, but
955 * that's before we have page structures to store the bits. So do all 955 * that's before we have page structures to store the bits. So do all
956 * the book-keeping now. 956 * the book-keeping now.
957 */ 957 */
958 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page, 958 static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page,
959 enum pt_level level) 959 enum pt_level level)
960 { 960 {
961 SetPagePinned(page); 961 SetPagePinned(page);
962 return 0; 962 return 0;
963 } 963 }
964 964
965 static void __init xen_mark_init_mm_pinned(void) 965 static void __init xen_mark_init_mm_pinned(void)
966 { 966 {
967 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP); 967 xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP);
968 } 968 }
969 969
970 static int xen_unpin_page(struct mm_struct *mm, struct page *page, 970 static int xen_unpin_page(struct mm_struct *mm, struct page *page,
971 enum pt_level level) 971 enum pt_level level)
972 { 972 {
973 unsigned pgfl = TestClearPagePinned(page); 973 unsigned pgfl = TestClearPagePinned(page);
974 974
975 if (pgfl && !PageHighMem(page)) { 975 if (pgfl && !PageHighMem(page)) {
976 void *pt = lowmem_page_address(page); 976 void *pt = lowmem_page_address(page);
977 unsigned long pfn = page_to_pfn(page); 977 unsigned long pfn = page_to_pfn(page);
978 spinlock_t *ptl = NULL; 978 spinlock_t *ptl = NULL;
979 struct multicall_space mcs; 979 struct multicall_space mcs;
980 980
981 /* 981 /*
982 * Do the converse to pin_page. If we're using split 982 * Do the converse to pin_page. If we're using split
983 * pte locks, we must be holding the lock for while 983 * pte locks, we must be holding the lock for while
984 * the pte page is unpinned but still RO to prevent 984 * the pte page is unpinned but still RO to prevent
985 * concurrent updates from seeing it in this 985 * concurrent updates from seeing it in this
986 * partially-pinned state. 986 * partially-pinned state.
987 */ 987 */
988 if (level == PT_PTE) { 988 if (level == PT_PTE) {
989 ptl = xen_pte_lock(page, mm); 989 ptl = xen_pte_lock(page, mm);
990 990
991 if (ptl) 991 if (ptl)
992 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn); 992 xen_do_pin(MMUEXT_UNPIN_TABLE, pfn);
993 } 993 }
994 994
995 mcs = __xen_mc_entry(0); 995 mcs = __xen_mc_entry(0);
996 996
997 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, 997 MULTI_update_va_mapping(mcs.mc, (unsigned long)pt,
998 pfn_pte(pfn, PAGE_KERNEL), 998 pfn_pte(pfn, PAGE_KERNEL),
999 level == PT_PGD ? UVMF_TLB_FLUSH : 0); 999 level == PT_PGD ? UVMF_TLB_FLUSH : 0);
1000 1000
1001 if (ptl) { 1001 if (ptl) {
1002 /* unlock when batch completed */ 1002 /* unlock when batch completed */
1003 xen_mc_callback(xen_pte_unlock, ptl); 1003 xen_mc_callback(xen_pte_unlock, ptl);
1004 } 1004 }
1005 } 1005 }
1006 1006
1007 return 0; /* never need to flush on unpin */ 1007 return 0; /* never need to flush on unpin */
1008 } 1008 }
1009 1009
1010 /* Release a pagetables pages back as normal RW */ 1010 /* Release a pagetables pages back as normal RW */
1011 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd) 1011 static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd)
1012 { 1012 {
1013 trace_xen_mmu_pgd_unpin(mm, pgd); 1013 trace_xen_mmu_pgd_unpin(mm, pgd);
1014 1014
1015 xen_mc_batch(); 1015 xen_mc_batch();
1016 1016
1017 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); 1017 xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1018 1018
1019 #ifdef CONFIG_X86_64 1019 #ifdef CONFIG_X86_64
1020 { 1020 {
1021 pgd_t *user_pgd = xen_get_user_pgd(pgd); 1021 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1022 1022
1023 if (user_pgd) { 1023 if (user_pgd) {
1024 xen_do_pin(MMUEXT_UNPIN_TABLE, 1024 xen_do_pin(MMUEXT_UNPIN_TABLE,
1025 PFN_DOWN(__pa(user_pgd))); 1025 PFN_DOWN(__pa(user_pgd)));
1026 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD); 1026 xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD);
1027 } 1027 }
1028 } 1028 }
1029 #endif 1029 #endif
1030 1030
1031 #ifdef CONFIG_X86_PAE 1031 #ifdef CONFIG_X86_PAE
1032 /* Need to make sure unshared kernel PMD is unpinned */ 1032 /* Need to make sure unshared kernel PMD is unpinned */
1033 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), 1033 xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]),
1034 PT_PMD); 1034 PT_PMD);
1035 #endif 1035 #endif
1036 1036
1037 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT); 1037 __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT);
1038 1038
1039 xen_mc_issue(0); 1039 xen_mc_issue(0);
1040 } 1040 }
1041 1041
1042 static void xen_pgd_unpin(struct mm_struct *mm) 1042 static void xen_pgd_unpin(struct mm_struct *mm)
1043 { 1043 {
1044 __xen_pgd_unpin(mm, mm->pgd); 1044 __xen_pgd_unpin(mm, mm->pgd);
1045 } 1045 }
1046 1046
1047 /* 1047 /*
1048 * On resume, undo any pinning done at save, so that the rest of the 1048 * On resume, undo any pinning done at save, so that the rest of the
1049 * kernel doesn't see any unexpected pinned pagetables. 1049 * kernel doesn't see any unexpected pinned pagetables.
1050 */ 1050 */
1051 void xen_mm_unpin_all(void) 1051 void xen_mm_unpin_all(void)
1052 { 1052 {
1053 struct page *page; 1053 struct page *page;
1054 1054
1055 spin_lock(&pgd_lock); 1055 spin_lock(&pgd_lock);
1056 1056
1057 list_for_each_entry(page, &pgd_list, lru) { 1057 list_for_each_entry(page, &pgd_list, lru) {
1058 if (PageSavePinned(page)) { 1058 if (PageSavePinned(page)) {
1059 BUG_ON(!PagePinned(page)); 1059 BUG_ON(!PagePinned(page));
1060 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); 1060 __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page));
1061 ClearPageSavePinned(page); 1061 ClearPageSavePinned(page);
1062 } 1062 }
1063 } 1063 }
1064 1064
1065 spin_unlock(&pgd_lock); 1065 spin_unlock(&pgd_lock);
1066 } 1066 }
1067 1067
1068 static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next) 1068 static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next)
1069 { 1069 {
1070 spin_lock(&next->page_table_lock); 1070 spin_lock(&next->page_table_lock);
1071 xen_pgd_pin(next); 1071 xen_pgd_pin(next);
1072 spin_unlock(&next->page_table_lock); 1072 spin_unlock(&next->page_table_lock);
1073 } 1073 }
1074 1074
1075 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) 1075 static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
1076 { 1076 {
1077 spin_lock(&mm->page_table_lock); 1077 spin_lock(&mm->page_table_lock);
1078 xen_pgd_pin(mm); 1078 xen_pgd_pin(mm);
1079 spin_unlock(&mm->page_table_lock); 1079 spin_unlock(&mm->page_table_lock);
1080 } 1080 }
1081 1081
1082 1082
1083 #ifdef CONFIG_SMP 1083 #ifdef CONFIG_SMP
1084 /* Another cpu may still have their %cr3 pointing at the pagetable, so 1084 /* Another cpu may still have their %cr3 pointing at the pagetable, so
1085 we need to repoint it somewhere else before we can unpin it. */ 1085 we need to repoint it somewhere else before we can unpin it. */
1086 static void drop_other_mm_ref(void *info) 1086 static void drop_other_mm_ref(void *info)
1087 { 1087 {
1088 struct mm_struct *mm = info; 1088 struct mm_struct *mm = info;
1089 struct mm_struct *active_mm; 1089 struct mm_struct *active_mm;
1090 1090
1091 active_mm = this_cpu_read(cpu_tlbstate.active_mm); 1091 active_mm = this_cpu_read(cpu_tlbstate.active_mm);
1092 1092
1093 if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) 1093 if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
1094 leave_mm(smp_processor_id()); 1094 leave_mm(smp_processor_id());
1095 1095
1096 /* If this cpu still has a stale cr3 reference, then make sure 1096 /* If this cpu still has a stale cr3 reference, then make sure
1097 it has been flushed. */ 1097 it has been flushed. */
1098 if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd)) 1098 if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
1099 load_cr3(swapper_pg_dir); 1099 load_cr3(swapper_pg_dir);
1100 } 1100 }
1101 1101
1102 static void xen_drop_mm_ref(struct mm_struct *mm) 1102 static void xen_drop_mm_ref(struct mm_struct *mm)
1103 { 1103 {
1104 cpumask_var_t mask; 1104 cpumask_var_t mask;
1105 unsigned cpu; 1105 unsigned cpu;
1106 1106
1107 if (current->active_mm == mm) { 1107 if (current->active_mm == mm) {
1108 if (current->mm == mm) 1108 if (current->mm == mm)
1109 load_cr3(swapper_pg_dir); 1109 load_cr3(swapper_pg_dir);
1110 else 1110 else
1111 leave_mm(smp_processor_id()); 1111 leave_mm(smp_processor_id());
1112 } 1112 }
1113 1113
1114 /* Get the "official" set of cpus referring to our pagetable. */ 1114 /* Get the "official" set of cpus referring to our pagetable. */
1115 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) { 1115 if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) {
1116 for_each_online_cpu(cpu) { 1116 for_each_online_cpu(cpu) {
1117 if (!cpumask_test_cpu(cpu, mm_cpumask(mm)) 1117 if (!cpumask_test_cpu(cpu, mm_cpumask(mm))
1118 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd)) 1118 && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd))
1119 continue; 1119 continue;
1120 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1); 1120 smp_call_function_single(cpu, drop_other_mm_ref, mm, 1);
1121 } 1121 }
1122 return; 1122 return;
1123 } 1123 }
1124 cpumask_copy(mask, mm_cpumask(mm)); 1124 cpumask_copy(mask, mm_cpumask(mm));
1125 1125
1126 /* It's possible that a vcpu may have a stale reference to our 1126 /* It's possible that a vcpu may have a stale reference to our
1127 cr3, because its in lazy mode, and it hasn't yet flushed 1127 cr3, because its in lazy mode, and it hasn't yet flushed
1128 its set of pending hypercalls yet. In this case, we can 1128 its set of pending hypercalls yet. In this case, we can
1129 look at its actual current cr3 value, and force it to flush 1129 look at its actual current cr3 value, and force it to flush
1130 if needed. */ 1130 if needed. */
1131 for_each_online_cpu(cpu) { 1131 for_each_online_cpu(cpu) {
1132 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd)) 1132 if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd))
1133 cpumask_set_cpu(cpu, mask); 1133 cpumask_set_cpu(cpu, mask);
1134 } 1134 }
1135 1135
1136 if (!cpumask_empty(mask)) 1136 if (!cpumask_empty(mask))
1137 smp_call_function_many(mask, drop_other_mm_ref, mm, 1); 1137 smp_call_function_many(mask, drop_other_mm_ref, mm, 1);
1138 free_cpumask_var(mask); 1138 free_cpumask_var(mask);
1139 } 1139 }
1140 #else 1140 #else
1141 static void xen_drop_mm_ref(struct mm_struct *mm) 1141 static void xen_drop_mm_ref(struct mm_struct *mm)
1142 { 1142 {
1143 if (current->active_mm == mm) 1143 if (current->active_mm == mm)
1144 load_cr3(swapper_pg_dir); 1144 load_cr3(swapper_pg_dir);
1145 } 1145 }
1146 #endif 1146 #endif
1147 1147
1148 /* 1148 /*
1149 * While a process runs, Xen pins its pagetables, which means that the 1149 * While a process runs, Xen pins its pagetables, which means that the
1150 * hypervisor forces it to be read-only, and it controls all updates 1150 * hypervisor forces it to be read-only, and it controls all updates
1151 * to it. This means that all pagetable updates have to go via the 1151 * to it. This means that all pagetable updates have to go via the
1152 * hypervisor, which is moderately expensive. 1152 * hypervisor, which is moderately expensive.
1153 * 1153 *
1154 * Since we're pulling the pagetable down, we switch to use init_mm, 1154 * Since we're pulling the pagetable down, we switch to use init_mm,
1155 * unpin old process pagetable and mark it all read-write, which 1155 * unpin old process pagetable and mark it all read-write, which
1156 * allows further operations on it to be simple memory accesses. 1156 * allows further operations on it to be simple memory accesses.
1157 * 1157 *
1158 * The only subtle point is that another CPU may be still using the 1158 * The only subtle point is that another CPU may be still using the
1159 * pagetable because of lazy tlb flushing. This means we need need to 1159 * pagetable because of lazy tlb flushing. This means we need need to
1160 * switch all CPUs off this pagetable before we can unpin it. 1160 * switch all CPUs off this pagetable before we can unpin it.
1161 */ 1161 */
1162 static void xen_exit_mmap(struct mm_struct *mm) 1162 static void xen_exit_mmap(struct mm_struct *mm)
1163 { 1163 {
1164 get_cpu(); /* make sure we don't move around */ 1164 get_cpu(); /* make sure we don't move around */
1165 xen_drop_mm_ref(mm); 1165 xen_drop_mm_ref(mm);
1166 put_cpu(); 1166 put_cpu();
1167 1167
1168 spin_lock(&mm->page_table_lock); 1168 spin_lock(&mm->page_table_lock);
1169 1169
1170 /* pgd may not be pinned in the error exit path of execve */ 1170 /* pgd may not be pinned in the error exit path of execve */
1171 if (xen_page_pinned(mm->pgd)) 1171 if (xen_page_pinned(mm->pgd))
1172 xen_pgd_unpin(mm); 1172 xen_pgd_unpin(mm);
1173 1173
1174 spin_unlock(&mm->page_table_lock); 1174 spin_unlock(&mm->page_table_lock);
1175 } 1175 }
1176 1176
1177 static void __init xen_pagetable_setup_start(pgd_t *base) 1177 static void __init xen_pagetable_setup_start(pgd_t *base)
1178 { 1178 {
1179 } 1179 }
1180 1180
1181 static __init void xen_mapping_pagetable_reserve(u64 start, u64 end) 1181 static __init void xen_mapping_pagetable_reserve(u64 start, u64 end)
1182 { 1182 {
1183 /* reserve the range used */ 1183 /* reserve the range used */
1184 native_pagetable_reserve(start, end); 1184 native_pagetable_reserve(start, end);
1185 1185
1186 /* set as RW the rest */ 1186 /* set as RW the rest */
1187 printk(KERN_DEBUG "xen: setting RW the range %llx - %llx\n", end, 1187 printk(KERN_DEBUG "xen: setting RW the range %llx - %llx\n", end,
1188 PFN_PHYS(pgt_buf_top)); 1188 PFN_PHYS(pgt_buf_top));
1189 while (end < PFN_PHYS(pgt_buf_top)) { 1189 while (end < PFN_PHYS(pgt_buf_top)) {
1190 make_lowmem_page_readwrite(__va(end)); 1190 make_lowmem_page_readwrite(__va(end));
1191 end += PAGE_SIZE; 1191 end += PAGE_SIZE;
1192 } 1192 }
1193 } 1193 }
1194 1194
1195 static void xen_post_allocator_init(void); 1195 static void xen_post_allocator_init(void);
1196 1196
1197 static void __init xen_pagetable_setup_done(pgd_t *base) 1197 static void __init xen_pagetable_setup_done(pgd_t *base)
1198 { 1198 {
1199 xen_setup_shared_info(); 1199 xen_setup_shared_info();
1200 xen_post_allocator_init(); 1200 xen_post_allocator_init();
1201 } 1201 }
1202 1202
1203 static void xen_write_cr2(unsigned long cr2) 1203 static void xen_write_cr2(unsigned long cr2)
1204 { 1204 {
1205 this_cpu_read(xen_vcpu)->arch.cr2 = cr2; 1205 this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
1206 } 1206 }
1207 1207
1208 static unsigned long xen_read_cr2(void) 1208 static unsigned long xen_read_cr2(void)
1209 { 1209 {
1210 return this_cpu_read(xen_vcpu)->arch.cr2; 1210 return this_cpu_read(xen_vcpu)->arch.cr2;
1211 } 1211 }
1212 1212
1213 unsigned long xen_read_cr2_direct(void) 1213 unsigned long xen_read_cr2_direct(void)
1214 { 1214 {
1215 return this_cpu_read(xen_vcpu_info.arch.cr2); 1215 return this_cpu_read(xen_vcpu_info.arch.cr2);
1216 } 1216 }
1217 1217
1218 static void xen_flush_tlb(void) 1218 static void xen_flush_tlb(void)
1219 { 1219 {
1220 struct mmuext_op *op; 1220 struct mmuext_op *op;
1221 struct multicall_space mcs; 1221 struct multicall_space mcs;
1222 1222
1223 trace_xen_mmu_flush_tlb(0); 1223 trace_xen_mmu_flush_tlb(0);
1224 1224
1225 preempt_disable(); 1225 preempt_disable();
1226 1226
1227 mcs = xen_mc_entry(sizeof(*op)); 1227 mcs = xen_mc_entry(sizeof(*op));
1228 1228
1229 op = mcs.args; 1229 op = mcs.args;
1230 op->cmd = MMUEXT_TLB_FLUSH_LOCAL; 1230 op->cmd = MMUEXT_TLB_FLUSH_LOCAL;
1231 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 1231 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1232 1232
1233 xen_mc_issue(PARAVIRT_LAZY_MMU); 1233 xen_mc_issue(PARAVIRT_LAZY_MMU);
1234 1234
1235 preempt_enable(); 1235 preempt_enable();
1236 } 1236 }
1237 1237
1238 static void xen_flush_tlb_single(unsigned long addr) 1238 static void xen_flush_tlb_single(unsigned long addr)
1239 { 1239 {
1240 struct mmuext_op *op; 1240 struct mmuext_op *op;
1241 struct multicall_space mcs; 1241 struct multicall_space mcs;
1242 1242
1243 trace_xen_mmu_flush_tlb_single(addr); 1243 trace_xen_mmu_flush_tlb_single(addr);
1244 1244
1245 preempt_disable(); 1245 preempt_disable();
1246 1246
1247 mcs = xen_mc_entry(sizeof(*op)); 1247 mcs = xen_mc_entry(sizeof(*op));
1248 op = mcs.args; 1248 op = mcs.args;
1249 op->cmd = MMUEXT_INVLPG_LOCAL; 1249 op->cmd = MMUEXT_INVLPG_LOCAL;
1250 op->arg1.linear_addr = addr & PAGE_MASK; 1250 op->arg1.linear_addr = addr & PAGE_MASK;
1251 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 1251 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
1252 1252
1253 xen_mc_issue(PARAVIRT_LAZY_MMU); 1253 xen_mc_issue(PARAVIRT_LAZY_MMU);
1254 1254
1255 preempt_enable(); 1255 preempt_enable();
1256 } 1256 }
1257 1257
1258 static void xen_flush_tlb_others(const struct cpumask *cpus, 1258 static void xen_flush_tlb_others(const struct cpumask *cpus,
1259 struct mm_struct *mm, unsigned long start, 1259 struct mm_struct *mm, unsigned long start,
1260 unsigned long end) 1260 unsigned long end)
1261 { 1261 {
1262 struct { 1262 struct {
1263 struct mmuext_op op; 1263 struct mmuext_op op;
1264 #ifdef CONFIG_SMP 1264 #ifdef CONFIG_SMP
1265 DECLARE_BITMAP(mask, num_processors); 1265 DECLARE_BITMAP(mask, num_processors);
1266 #else 1266 #else
1267 DECLARE_BITMAP(mask, NR_CPUS); 1267 DECLARE_BITMAP(mask, NR_CPUS);
1268 #endif 1268 #endif
1269 } *args; 1269 } *args;
1270 struct multicall_space mcs; 1270 struct multicall_space mcs;
1271 1271
1272 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end); 1272 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1273 1273
1274 if (cpumask_empty(cpus)) 1274 if (cpumask_empty(cpus))
1275 return; /* nothing to do */ 1275 return; /* nothing to do */
1276 1276
1277 mcs = xen_mc_entry(sizeof(*args)); 1277 mcs = xen_mc_entry(sizeof(*args));
1278 args = mcs.args; 1278 args = mcs.args;
1279 args->op.arg2.vcpumask = to_cpumask(args->mask); 1279 args->op.arg2.vcpumask = to_cpumask(args->mask);
1280 1280
1281 /* Remove us, and any offline CPUS. */ 1281 /* Remove us, and any offline CPUS. */
1282 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask); 1282 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1283 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask)); 1283 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1284 1284
1285 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; 1285 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1286 if (start != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) { 1286 if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1287 args->op.cmd = MMUEXT_INVLPG_MULTI; 1287 args->op.cmd = MMUEXT_INVLPG_MULTI;
1288 args->op.arg1.linear_addr = start; 1288 args->op.arg1.linear_addr = start;
1289 } 1289 }
1290 1290
1291 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); 1291 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
1292 1292
1293 xen_mc_issue(PARAVIRT_LAZY_MMU); 1293 xen_mc_issue(PARAVIRT_LAZY_MMU);
1294 } 1294 }
1295 1295
1296 static unsigned long xen_read_cr3(void) 1296 static unsigned long xen_read_cr3(void)
1297 { 1297 {
1298 return this_cpu_read(xen_cr3); 1298 return this_cpu_read(xen_cr3);
1299 } 1299 }
1300 1300
1301 static void set_current_cr3(void *v) 1301 static void set_current_cr3(void *v)
1302 { 1302 {
1303 this_cpu_write(xen_current_cr3, (unsigned long)v); 1303 this_cpu_write(xen_current_cr3, (unsigned long)v);
1304 } 1304 }
1305 1305
1306 static void __xen_write_cr3(bool kernel, unsigned long cr3) 1306 static void __xen_write_cr3(bool kernel, unsigned long cr3)
1307 { 1307 {
1308 struct mmuext_op op; 1308 struct mmuext_op op;
1309 unsigned long mfn; 1309 unsigned long mfn;
1310 1310
1311 trace_xen_mmu_write_cr3(kernel, cr3); 1311 trace_xen_mmu_write_cr3(kernel, cr3);
1312 1312
1313 if (cr3) 1313 if (cr3)
1314 mfn = pfn_to_mfn(PFN_DOWN(cr3)); 1314 mfn = pfn_to_mfn(PFN_DOWN(cr3));
1315 else 1315 else
1316 mfn = 0; 1316 mfn = 0;
1317 1317
1318 WARN_ON(mfn == 0 && kernel); 1318 WARN_ON(mfn == 0 && kernel);
1319 1319
1320 op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR; 1320 op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR;
1321 op.arg1.mfn = mfn; 1321 op.arg1.mfn = mfn;
1322 1322
1323 xen_extend_mmuext_op(&op); 1323 xen_extend_mmuext_op(&op);
1324 1324
1325 if (kernel) { 1325 if (kernel) {
1326 this_cpu_write(xen_cr3, cr3); 1326 this_cpu_write(xen_cr3, cr3);
1327 1327
1328 /* Update xen_current_cr3 once the batch has actually 1328 /* Update xen_current_cr3 once the batch has actually
1329 been submitted. */ 1329 been submitted. */
1330 xen_mc_callback(set_current_cr3, (void *)cr3); 1330 xen_mc_callback(set_current_cr3, (void *)cr3);
1331 } 1331 }
1332 } 1332 }
1333 1333
1334 static void xen_write_cr3(unsigned long cr3) 1334 static void xen_write_cr3(unsigned long cr3)
1335 { 1335 {
1336 BUG_ON(preemptible()); 1336 BUG_ON(preemptible());
1337 1337
1338 xen_mc_batch(); /* disables interrupts */ 1338 xen_mc_batch(); /* disables interrupts */
1339 1339
1340 /* Update while interrupts are disabled, so its atomic with 1340 /* Update while interrupts are disabled, so its atomic with
1341 respect to ipis */ 1341 respect to ipis */
1342 this_cpu_write(xen_cr3, cr3); 1342 this_cpu_write(xen_cr3, cr3);
1343 1343
1344 __xen_write_cr3(true, cr3); 1344 __xen_write_cr3(true, cr3);
1345 1345
1346 #ifdef CONFIG_X86_64 1346 #ifdef CONFIG_X86_64
1347 { 1347 {
1348 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3)); 1348 pgd_t *user_pgd = xen_get_user_pgd(__va(cr3));
1349 if (user_pgd) 1349 if (user_pgd)
1350 __xen_write_cr3(false, __pa(user_pgd)); 1350 __xen_write_cr3(false, __pa(user_pgd));
1351 else 1351 else
1352 __xen_write_cr3(false, 0); 1352 __xen_write_cr3(false, 0);
1353 } 1353 }
1354 #endif 1354 #endif
1355 1355
1356 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */ 1356 xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */
1357 } 1357 }
1358 1358
1359 static int xen_pgd_alloc(struct mm_struct *mm) 1359 static int xen_pgd_alloc(struct mm_struct *mm)
1360 { 1360 {
1361 pgd_t *pgd = mm->pgd; 1361 pgd_t *pgd = mm->pgd;
1362 int ret = 0; 1362 int ret = 0;
1363 1363
1364 BUG_ON(PagePinned(virt_to_page(pgd))); 1364 BUG_ON(PagePinned(virt_to_page(pgd)));
1365 1365
1366 #ifdef CONFIG_X86_64 1366 #ifdef CONFIG_X86_64
1367 { 1367 {
1368 struct page *page = virt_to_page(pgd); 1368 struct page *page = virt_to_page(pgd);
1369 pgd_t *user_pgd; 1369 pgd_t *user_pgd;
1370 1370
1371 BUG_ON(page->private != 0); 1371 BUG_ON(page->private != 0);
1372 1372
1373 ret = -ENOMEM; 1373 ret = -ENOMEM;
1374 1374
1375 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 1375 user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
1376 page->private = (unsigned long)user_pgd; 1376 page->private = (unsigned long)user_pgd;
1377 1377
1378 if (user_pgd != NULL) { 1378 if (user_pgd != NULL) {
1379 user_pgd[pgd_index(VSYSCALL_START)] = 1379 user_pgd[pgd_index(VSYSCALL_START)] =
1380 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE); 1380 __pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE);
1381 ret = 0; 1381 ret = 0;
1382 } 1382 }
1383 1383
1384 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd)))); 1384 BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd))));
1385 } 1385 }
1386 #endif 1386 #endif
1387 1387
1388 return ret; 1388 return ret;
1389 } 1389 }
1390 1390
1391 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd) 1391 static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd)
1392 { 1392 {
1393 #ifdef CONFIG_X86_64 1393 #ifdef CONFIG_X86_64
1394 pgd_t *user_pgd = xen_get_user_pgd(pgd); 1394 pgd_t *user_pgd = xen_get_user_pgd(pgd);
1395 1395
1396 if (user_pgd) 1396 if (user_pgd)
1397 free_page((unsigned long)user_pgd); 1397 free_page((unsigned long)user_pgd);
1398 #endif 1398 #endif
1399 } 1399 }
1400 1400
1401 #ifdef CONFIG_X86_32 1401 #ifdef CONFIG_X86_32
1402 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) 1402 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1403 { 1403 {
1404 /* If there's an existing pte, then don't allow _PAGE_RW to be set */ 1404 /* If there's an existing pte, then don't allow _PAGE_RW to be set */
1405 if (pte_val_ma(*ptep) & _PAGE_PRESENT) 1405 if (pte_val_ma(*ptep) & _PAGE_PRESENT)
1406 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & 1406 pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) &
1407 pte_val_ma(pte)); 1407 pte_val_ma(pte));
1408 1408
1409 return pte; 1409 return pte;
1410 } 1410 }
1411 #else /* CONFIG_X86_64 */ 1411 #else /* CONFIG_X86_64 */
1412 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) 1412 static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte)
1413 { 1413 {
1414 unsigned long pfn = pte_pfn(pte); 1414 unsigned long pfn = pte_pfn(pte);
1415 1415
1416 /* 1416 /*
1417 * If the new pfn is within the range of the newly allocated 1417 * If the new pfn is within the range of the newly allocated
1418 * kernel pagetable, and it isn't being mapped into an 1418 * kernel pagetable, and it isn't being mapped into an
1419 * early_ioremap fixmap slot as a freshly allocated page, make sure 1419 * early_ioremap fixmap slot as a freshly allocated page, make sure
1420 * it is RO. 1420 * it is RO.
1421 */ 1421 */
1422 if (((!is_early_ioremap_ptep(ptep) && 1422 if (((!is_early_ioremap_ptep(ptep) &&
1423 pfn >= pgt_buf_start && pfn < pgt_buf_top)) || 1423 pfn >= pgt_buf_start && pfn < pgt_buf_top)) ||
1424 (is_early_ioremap_ptep(ptep) && pfn != (pgt_buf_end - 1))) 1424 (is_early_ioremap_ptep(ptep) && pfn != (pgt_buf_end - 1)))
1425 pte = pte_wrprotect(pte); 1425 pte = pte_wrprotect(pte);
1426 1426
1427 return pte; 1427 return pte;
1428 } 1428 }
1429 #endif /* CONFIG_X86_64 */ 1429 #endif /* CONFIG_X86_64 */
1430 1430
1431 /* 1431 /*
1432 * Init-time set_pte while constructing initial pagetables, which 1432 * Init-time set_pte while constructing initial pagetables, which
1433 * doesn't allow RO page table pages to be remapped RW. 1433 * doesn't allow RO page table pages to be remapped RW.
1434 * 1434 *
1435 * If there is no MFN for this PFN then this page is initially 1435 * If there is no MFN for this PFN then this page is initially
1436 * ballooned out so clear the PTE (as in decrease_reservation() in 1436 * ballooned out so clear the PTE (as in decrease_reservation() in
1437 * drivers/xen/balloon.c). 1437 * drivers/xen/balloon.c).
1438 * 1438 *
1439 * Many of these PTE updates are done on unpinned and writable pages 1439 * Many of these PTE updates are done on unpinned and writable pages
1440 * and doing a hypercall for these is unnecessary and expensive. At 1440 * and doing a hypercall for these is unnecessary and expensive. At
1441 * this point it is not possible to tell if a page is pinned or not, 1441 * this point it is not possible to tell if a page is pinned or not,
1442 * so always write the PTE directly and rely on Xen trapping and 1442 * so always write the PTE directly and rely on Xen trapping and
1443 * emulating any updates as necessary. 1443 * emulating any updates as necessary.
1444 */ 1444 */
1445 static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) 1445 static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1446 { 1446 {
1447 if (pte_mfn(pte) != INVALID_P2M_ENTRY) 1447 if (pte_mfn(pte) != INVALID_P2M_ENTRY)
1448 pte = mask_rw_pte(ptep, pte); 1448 pte = mask_rw_pte(ptep, pte);
1449 else 1449 else
1450 pte = __pte_ma(0); 1450 pte = __pte_ma(0);
1451 1451
1452 native_set_pte(ptep, pte); 1452 native_set_pte(ptep, pte);
1453 } 1453 }
1454 1454
1455 static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn) 1455 static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1456 { 1456 {
1457 struct mmuext_op op; 1457 struct mmuext_op op;
1458 op.cmd = cmd; 1458 op.cmd = cmd;
1459 op.arg1.mfn = pfn_to_mfn(pfn); 1459 op.arg1.mfn = pfn_to_mfn(pfn);
1460 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) 1460 if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF))
1461 BUG(); 1461 BUG();
1462 } 1462 }
1463 1463
1464 /* Early in boot, while setting up the initial pagetable, assume 1464 /* Early in boot, while setting up the initial pagetable, assume
1465 everything is pinned. */ 1465 everything is pinned. */
1466 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn) 1466 static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn)
1467 { 1467 {
1468 #ifdef CONFIG_FLATMEM 1468 #ifdef CONFIG_FLATMEM
1469 BUG_ON(mem_map); /* should only be used early */ 1469 BUG_ON(mem_map); /* should only be used early */
1470 #endif 1470 #endif
1471 make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); 1471 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1472 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); 1472 pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1473 } 1473 }
1474 1474
1475 /* Used for pmd and pud */ 1475 /* Used for pmd and pud */
1476 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn) 1476 static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn)
1477 { 1477 {
1478 #ifdef CONFIG_FLATMEM 1478 #ifdef CONFIG_FLATMEM
1479 BUG_ON(mem_map); /* should only be used early */ 1479 BUG_ON(mem_map); /* should only be used early */
1480 #endif 1480 #endif
1481 make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); 1481 make_lowmem_page_readonly(__va(PFN_PHYS(pfn)));
1482 } 1482 }
1483 1483
1484 /* Early release_pte assumes that all pts are pinned, since there's 1484 /* Early release_pte assumes that all pts are pinned, since there's
1485 only init_mm and anything attached to that is pinned. */ 1485 only init_mm and anything attached to that is pinned. */
1486 static void __init xen_release_pte_init(unsigned long pfn) 1486 static void __init xen_release_pte_init(unsigned long pfn)
1487 { 1487 {
1488 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn); 1488 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1489 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); 1489 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1490 } 1490 }
1491 1491
1492 static void __init xen_release_pmd_init(unsigned long pfn) 1492 static void __init xen_release_pmd_init(unsigned long pfn)
1493 { 1493 {
1494 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); 1494 make_lowmem_page_readwrite(__va(PFN_PHYS(pfn)));
1495 } 1495 }
1496 1496
1497 static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn) 1497 static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1498 { 1498 {
1499 struct multicall_space mcs; 1499 struct multicall_space mcs;
1500 struct mmuext_op *op; 1500 struct mmuext_op *op;
1501 1501
1502 mcs = __xen_mc_entry(sizeof(*op)); 1502 mcs = __xen_mc_entry(sizeof(*op));
1503 op = mcs.args; 1503 op = mcs.args;
1504 op->cmd = cmd; 1504 op->cmd = cmd;
1505 op->arg1.mfn = pfn_to_mfn(pfn); 1505 op->arg1.mfn = pfn_to_mfn(pfn);
1506 1506
1507 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); 1507 MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
1508 } 1508 }
1509 1509
1510 static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot) 1510 static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot)
1511 { 1511 {
1512 struct multicall_space mcs; 1512 struct multicall_space mcs;
1513 unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT); 1513 unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT);
1514 1514
1515 mcs = __xen_mc_entry(0); 1515 mcs = __xen_mc_entry(0);
1516 MULTI_update_va_mapping(mcs.mc, (unsigned long)addr, 1516 MULTI_update_va_mapping(mcs.mc, (unsigned long)addr,
1517 pfn_pte(pfn, prot), 0); 1517 pfn_pte(pfn, prot), 0);
1518 } 1518 }
1519 1519
1520 /* This needs to make sure the new pte page is pinned iff its being 1520 /* This needs to make sure the new pte page is pinned iff its being
1521 attached to a pinned pagetable. */ 1521 attached to a pinned pagetable. */
1522 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, 1522 static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn,
1523 unsigned level) 1523 unsigned level)
1524 { 1524 {
1525 bool pinned = PagePinned(virt_to_page(mm->pgd)); 1525 bool pinned = PagePinned(virt_to_page(mm->pgd));
1526 1526
1527 trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned); 1527 trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned);
1528 1528
1529 if (pinned) { 1529 if (pinned) {
1530 struct page *page = pfn_to_page(pfn); 1530 struct page *page = pfn_to_page(pfn);
1531 1531
1532 SetPagePinned(page); 1532 SetPagePinned(page);
1533 1533
1534 if (!PageHighMem(page)) { 1534 if (!PageHighMem(page)) {
1535 xen_mc_batch(); 1535 xen_mc_batch();
1536 1536
1537 __set_pfn_prot(pfn, PAGE_KERNEL_RO); 1537 __set_pfn_prot(pfn, PAGE_KERNEL_RO);
1538 1538
1539 if (level == PT_PTE && USE_SPLIT_PTLOCKS) 1539 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1540 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); 1540 __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn);
1541 1541
1542 xen_mc_issue(PARAVIRT_LAZY_MMU); 1542 xen_mc_issue(PARAVIRT_LAZY_MMU);
1543 } else { 1543 } else {
1544 /* make sure there are no stray mappings of 1544 /* make sure there are no stray mappings of
1545 this page */ 1545 this page */
1546 kmap_flush_unused(); 1546 kmap_flush_unused();
1547 } 1547 }
1548 } 1548 }
1549 } 1549 }
1550 1550
1551 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn) 1551 static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn)
1552 { 1552 {
1553 xen_alloc_ptpage(mm, pfn, PT_PTE); 1553 xen_alloc_ptpage(mm, pfn, PT_PTE);
1554 } 1554 }
1555 1555
1556 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn) 1556 static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
1557 { 1557 {
1558 xen_alloc_ptpage(mm, pfn, PT_PMD); 1558 xen_alloc_ptpage(mm, pfn, PT_PMD);
1559 } 1559 }
1560 1560
1561 /* This should never happen until we're OK to use struct page */ 1561 /* This should never happen until we're OK to use struct page */
1562 static inline void xen_release_ptpage(unsigned long pfn, unsigned level) 1562 static inline void xen_release_ptpage(unsigned long pfn, unsigned level)
1563 { 1563 {
1564 struct page *page = pfn_to_page(pfn); 1564 struct page *page = pfn_to_page(pfn);
1565 bool pinned = PagePinned(page); 1565 bool pinned = PagePinned(page);
1566 1566
1567 trace_xen_mmu_release_ptpage(pfn, level, pinned); 1567 trace_xen_mmu_release_ptpage(pfn, level, pinned);
1568 1568
1569 if (pinned) { 1569 if (pinned) {
1570 if (!PageHighMem(page)) { 1570 if (!PageHighMem(page)) {
1571 xen_mc_batch(); 1571 xen_mc_batch();
1572 1572
1573 if (level == PT_PTE && USE_SPLIT_PTLOCKS) 1573 if (level == PT_PTE && USE_SPLIT_PTLOCKS)
1574 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn); 1574 __pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn);
1575 1575
1576 __set_pfn_prot(pfn, PAGE_KERNEL); 1576 __set_pfn_prot(pfn, PAGE_KERNEL);
1577 1577
1578 xen_mc_issue(PARAVIRT_LAZY_MMU); 1578 xen_mc_issue(PARAVIRT_LAZY_MMU);
1579 } 1579 }
1580 ClearPagePinned(page); 1580 ClearPagePinned(page);
1581 } 1581 }
1582 } 1582 }
1583 1583
1584 static void xen_release_pte(unsigned long pfn) 1584 static void xen_release_pte(unsigned long pfn)
1585 { 1585 {
1586 xen_release_ptpage(pfn, PT_PTE); 1586 xen_release_ptpage(pfn, PT_PTE);
1587 } 1587 }
1588 1588
1589 static void xen_release_pmd(unsigned long pfn) 1589 static void xen_release_pmd(unsigned long pfn)
1590 { 1590 {
1591 xen_release_ptpage(pfn, PT_PMD); 1591 xen_release_ptpage(pfn, PT_PMD);
1592 } 1592 }
1593 1593
1594 #if PAGETABLE_LEVELS == 4 1594 #if PAGETABLE_LEVELS == 4
1595 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn) 1595 static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn)
1596 { 1596 {
1597 xen_alloc_ptpage(mm, pfn, PT_PUD); 1597 xen_alloc_ptpage(mm, pfn, PT_PUD);
1598 } 1598 }
1599 1599
1600 static void xen_release_pud(unsigned long pfn) 1600 static void xen_release_pud(unsigned long pfn)
1601 { 1601 {
1602 xen_release_ptpage(pfn, PT_PUD); 1602 xen_release_ptpage(pfn, PT_PUD);
1603 } 1603 }
1604 #endif 1604 #endif
1605 1605
1606 void __init xen_reserve_top(void) 1606 void __init xen_reserve_top(void)
1607 { 1607 {
1608 #ifdef CONFIG_X86_32 1608 #ifdef CONFIG_X86_32
1609 unsigned long top = HYPERVISOR_VIRT_START; 1609 unsigned long top = HYPERVISOR_VIRT_START;
1610 struct xen_platform_parameters pp; 1610 struct xen_platform_parameters pp;
1611 1611
1612 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0) 1612 if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0)
1613 top = pp.virt_start; 1613 top = pp.virt_start;
1614 1614
1615 reserve_top_address(-top); 1615 reserve_top_address(-top);
1616 #endif /* CONFIG_X86_32 */ 1616 #endif /* CONFIG_X86_32 */
1617 } 1617 }
1618 1618
1619 /* 1619 /*
1620 * Like __va(), but returns address in the kernel mapping (which is 1620 * Like __va(), but returns address in the kernel mapping (which is
1621 * all we have until the physical memory mapping has been set up. 1621 * all we have until the physical memory mapping has been set up.
1622 */ 1622 */
1623 static void *__ka(phys_addr_t paddr) 1623 static void *__ka(phys_addr_t paddr)
1624 { 1624 {
1625 #ifdef CONFIG_X86_64 1625 #ifdef CONFIG_X86_64
1626 return (void *)(paddr + __START_KERNEL_map); 1626 return (void *)(paddr + __START_KERNEL_map);
1627 #else 1627 #else
1628 return __va(paddr); 1628 return __va(paddr);
1629 #endif 1629 #endif
1630 } 1630 }
1631 1631
1632 /* Convert a machine address to physical address */ 1632 /* Convert a machine address to physical address */
1633 static unsigned long m2p(phys_addr_t maddr) 1633 static unsigned long m2p(phys_addr_t maddr)
1634 { 1634 {
1635 phys_addr_t paddr; 1635 phys_addr_t paddr;
1636 1636
1637 maddr &= PTE_PFN_MASK; 1637 maddr &= PTE_PFN_MASK;
1638 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT; 1638 paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT;
1639 1639
1640 return paddr; 1640 return paddr;
1641 } 1641 }
1642 1642
1643 /* Convert a machine address to kernel virtual */ 1643 /* Convert a machine address to kernel virtual */
1644 static void *m2v(phys_addr_t maddr) 1644 static void *m2v(phys_addr_t maddr)
1645 { 1645 {
1646 return __ka(m2p(maddr)); 1646 return __ka(m2p(maddr));
1647 } 1647 }
1648 1648
1649 /* Set the page permissions on an identity-mapped pages */ 1649 /* Set the page permissions on an identity-mapped pages */
1650 static void set_page_prot(void *addr, pgprot_t prot) 1650 static void set_page_prot(void *addr, pgprot_t prot)
1651 { 1651 {
1652 unsigned long pfn = __pa(addr) >> PAGE_SHIFT; 1652 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1653 pte_t pte = pfn_pte(pfn, prot); 1653 pte_t pte = pfn_pte(pfn, prot);
1654 1654
1655 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0)) 1655 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
1656 BUG(); 1656 BUG();
1657 } 1657 }
1658 1658
1659 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) 1659 static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1660 { 1660 {
1661 unsigned pmdidx, pteidx; 1661 unsigned pmdidx, pteidx;
1662 unsigned ident_pte; 1662 unsigned ident_pte;
1663 unsigned long pfn; 1663 unsigned long pfn;
1664 1664
1665 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES, 1665 level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES,
1666 PAGE_SIZE); 1666 PAGE_SIZE);
1667 1667
1668 ident_pte = 0; 1668 ident_pte = 0;
1669 pfn = 0; 1669 pfn = 0;
1670 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) { 1670 for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) {
1671 pte_t *pte_page; 1671 pte_t *pte_page;
1672 1672
1673 /* Reuse or allocate a page of ptes */ 1673 /* Reuse or allocate a page of ptes */
1674 if (pmd_present(pmd[pmdidx])) 1674 if (pmd_present(pmd[pmdidx]))
1675 pte_page = m2v(pmd[pmdidx].pmd); 1675 pte_page = m2v(pmd[pmdidx].pmd);
1676 else { 1676 else {
1677 /* Check for free pte pages */ 1677 /* Check for free pte pages */
1678 if (ident_pte == LEVEL1_IDENT_ENTRIES) 1678 if (ident_pte == LEVEL1_IDENT_ENTRIES)
1679 break; 1679 break;
1680 1680
1681 pte_page = &level1_ident_pgt[ident_pte]; 1681 pte_page = &level1_ident_pgt[ident_pte];
1682 ident_pte += PTRS_PER_PTE; 1682 ident_pte += PTRS_PER_PTE;
1683 1683
1684 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE); 1684 pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE);
1685 } 1685 }
1686 1686
1687 /* Install mappings */ 1687 /* Install mappings */
1688 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { 1688 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1689 pte_t pte; 1689 pte_t pte;
1690 1690
1691 #ifdef CONFIG_X86_32 1691 #ifdef CONFIG_X86_32
1692 if (pfn > max_pfn_mapped) 1692 if (pfn > max_pfn_mapped)
1693 max_pfn_mapped = pfn; 1693 max_pfn_mapped = pfn;
1694 #endif 1694 #endif
1695 1695
1696 if (!pte_none(pte_page[pteidx])) 1696 if (!pte_none(pte_page[pteidx]))
1697 continue; 1697 continue;
1698 1698
1699 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC); 1699 pte = pfn_pte(pfn, PAGE_KERNEL_EXEC);
1700 pte_page[pteidx] = pte; 1700 pte_page[pteidx] = pte;
1701 } 1701 }
1702 } 1702 }
1703 1703
1704 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE) 1704 for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
1705 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO); 1705 set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO);
1706 1706
1707 set_page_prot(pmd, PAGE_KERNEL_RO); 1707 set_page_prot(pmd, PAGE_KERNEL_RO);
1708 } 1708 }
1709 1709
1710 void __init xen_setup_machphys_mapping(void) 1710 void __init xen_setup_machphys_mapping(void)
1711 { 1711 {
1712 struct xen_machphys_mapping mapping; 1712 struct xen_machphys_mapping mapping;
1713 1713
1714 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) { 1714 if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) {
1715 machine_to_phys_mapping = (unsigned long *)mapping.v_start; 1715 machine_to_phys_mapping = (unsigned long *)mapping.v_start;
1716 machine_to_phys_nr = mapping.max_mfn + 1; 1716 machine_to_phys_nr = mapping.max_mfn + 1;
1717 } else { 1717 } else {
1718 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES; 1718 machine_to_phys_nr = MACH2PHYS_NR_ENTRIES;
1719 } 1719 }
1720 #ifdef CONFIG_X86_32 1720 #ifdef CONFIG_X86_32
1721 WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1)) 1721 WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1))
1722 < machine_to_phys_mapping); 1722 < machine_to_phys_mapping);
1723 #endif 1723 #endif
1724 } 1724 }
1725 1725
1726 #ifdef CONFIG_X86_64 1726 #ifdef CONFIG_X86_64
1727 static void convert_pfn_mfn(void *v) 1727 static void convert_pfn_mfn(void *v)
1728 { 1728 {
1729 pte_t *pte = v; 1729 pte_t *pte = v;
1730 int i; 1730 int i;
1731 1731
1732 /* All levels are converted the same way, so just treat them 1732 /* All levels are converted the same way, so just treat them
1733 as ptes. */ 1733 as ptes. */
1734 for (i = 0; i < PTRS_PER_PTE; i++) 1734 for (i = 0; i < PTRS_PER_PTE; i++)
1735 pte[i] = xen_make_pte(pte[i].pte); 1735 pte[i] = xen_make_pte(pte[i].pte);
1736 } 1736 }
1737 1737
1738 /* 1738 /*
1739 * Set up the initial kernel pagetable. 1739 * Set up the initial kernel pagetable.
1740 * 1740 *
1741 * We can construct this by grafting the Xen provided pagetable into 1741 * We can construct this by grafting the Xen provided pagetable into
1742 * head_64.S's preconstructed pagetables. We copy the Xen L2's into 1742 * head_64.S's preconstructed pagetables. We copy the Xen L2's into
1743 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This 1743 * level2_ident_pgt, level2_kernel_pgt and level2_fixmap_pgt. This
1744 * means that only the kernel has a physical mapping to start with - 1744 * means that only the kernel has a physical mapping to start with -
1745 * but that's enough to get __va working. We need to fill in the rest 1745 * but that's enough to get __va working. We need to fill in the rest
1746 * of the physical mapping once some sort of allocator has been set 1746 * of the physical mapping once some sort of allocator has been set
1747 * up. 1747 * up.
1748 */ 1748 */
1749 pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, 1749 pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
1750 unsigned long max_pfn) 1750 unsigned long max_pfn)
1751 { 1751 {
1752 pud_t *l3; 1752 pud_t *l3;
1753 pmd_t *l2; 1753 pmd_t *l2;
1754 1754
1755 /* max_pfn_mapped is the last pfn mapped in the initial memory 1755 /* max_pfn_mapped is the last pfn mapped in the initial memory
1756 * mappings. Considering that on Xen after the kernel mappings we 1756 * mappings. Considering that on Xen after the kernel mappings we
1757 * have the mappings of some pages that don't exist in pfn space, we 1757 * have the mappings of some pages that don't exist in pfn space, we
1758 * set max_pfn_mapped to the last real pfn mapped. */ 1758 * set max_pfn_mapped to the last real pfn mapped. */
1759 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); 1759 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list));
1760 1760
1761 /* Zap identity mapping */ 1761 /* Zap identity mapping */
1762 init_level4_pgt[0] = __pgd(0); 1762 init_level4_pgt[0] = __pgd(0);
1763 1763
1764 /* Pre-constructed entries are in pfn, so convert to mfn */ 1764 /* Pre-constructed entries are in pfn, so convert to mfn */
1765 convert_pfn_mfn(init_level4_pgt); 1765 convert_pfn_mfn(init_level4_pgt);
1766 convert_pfn_mfn(level3_ident_pgt); 1766 convert_pfn_mfn(level3_ident_pgt);
1767 convert_pfn_mfn(level3_kernel_pgt); 1767 convert_pfn_mfn(level3_kernel_pgt);
1768 1768
1769 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd); 1769 l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd);
1770 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud); 1770 l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud);
1771 1771
1772 memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1772 memcpy(level2_ident_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1773 memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1773 memcpy(level2_kernel_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1774 1774
1775 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd); 1775 l3 = m2v(pgd[pgd_index(__START_KERNEL_map + PMD_SIZE)].pgd);
1776 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud); 1776 l2 = m2v(l3[pud_index(__START_KERNEL_map + PMD_SIZE)].pud);
1777 memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD); 1777 memcpy(level2_fixmap_pgt, l2, sizeof(pmd_t) * PTRS_PER_PMD);
1778 1778
1779 /* Set up identity map */ 1779 /* Set up identity map */
1780 xen_map_identity_early(level2_ident_pgt, max_pfn); 1780 xen_map_identity_early(level2_ident_pgt, max_pfn);
1781 1781
1782 /* Make pagetable pieces RO */ 1782 /* Make pagetable pieces RO */
1783 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); 1783 set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
1784 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); 1784 set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
1785 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); 1785 set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
1786 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); 1786 set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
1787 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); 1787 set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
1788 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); 1788 set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
1789 1789
1790 /* Pin down new L4 */ 1790 /* Pin down new L4 */
1791 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE, 1791 pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
1792 PFN_DOWN(__pa_symbol(init_level4_pgt))); 1792 PFN_DOWN(__pa_symbol(init_level4_pgt)));
1793 1793
1794 /* Unpin Xen-provided one */ 1794 /* Unpin Xen-provided one */
1795 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); 1795 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1796 1796
1797 /* Switch over */ 1797 /* Switch over */
1798 pgd = init_level4_pgt; 1798 pgd = init_level4_pgt;
1799 1799
1800 /* 1800 /*
1801 * At this stage there can be no user pgd, and no page 1801 * At this stage there can be no user pgd, and no page
1802 * structure to attach it to, so make sure we just set kernel 1802 * structure to attach it to, so make sure we just set kernel
1803 * pgd. 1803 * pgd.
1804 */ 1804 */
1805 xen_mc_batch(); 1805 xen_mc_batch();
1806 __xen_write_cr3(true, __pa(pgd)); 1806 __xen_write_cr3(true, __pa(pgd));
1807 xen_mc_issue(PARAVIRT_LAZY_CPU); 1807 xen_mc_issue(PARAVIRT_LAZY_CPU);
1808 1808
1809 memblock_reserve(__pa(xen_start_info->pt_base), 1809 memblock_reserve(__pa(xen_start_info->pt_base),
1810 xen_start_info->nr_pt_frames * PAGE_SIZE); 1810 xen_start_info->nr_pt_frames * PAGE_SIZE);
1811 1811
1812 return pgd; 1812 return pgd;
1813 } 1813 }
1814 #else /* !CONFIG_X86_64 */ 1814 #else /* !CONFIG_X86_64 */
1815 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD); 1815 static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD);
1816 static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD); 1816 static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD);
1817 1817
1818 static void __init xen_write_cr3_init(unsigned long cr3) 1818 static void __init xen_write_cr3_init(unsigned long cr3)
1819 { 1819 {
1820 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir)); 1820 unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir));
1821 1821
1822 BUG_ON(read_cr3() != __pa(initial_page_table)); 1822 BUG_ON(read_cr3() != __pa(initial_page_table));
1823 BUG_ON(cr3 != __pa(swapper_pg_dir)); 1823 BUG_ON(cr3 != __pa(swapper_pg_dir));
1824 1824
1825 /* 1825 /*
1826 * We are switching to swapper_pg_dir for the first time (from 1826 * We are switching to swapper_pg_dir for the first time (from
1827 * initial_page_table) and therefore need to mark that page 1827 * initial_page_table) and therefore need to mark that page
1828 * read-only and then pin it. 1828 * read-only and then pin it.
1829 * 1829 *
1830 * Xen disallows sharing of kernel PMDs for PAE 1830 * Xen disallows sharing of kernel PMDs for PAE
1831 * guests. Therefore we must copy the kernel PMD from 1831 * guests. Therefore we must copy the kernel PMD from
1832 * initial_page_table into a new kernel PMD to be used in 1832 * initial_page_table into a new kernel PMD to be used in
1833 * swapper_pg_dir. 1833 * swapper_pg_dir.
1834 */ 1834 */
1835 swapper_kernel_pmd = 1835 swapper_kernel_pmd =
1836 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); 1836 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1837 memcpy(swapper_kernel_pmd, initial_kernel_pmd, 1837 memcpy(swapper_kernel_pmd, initial_kernel_pmd,
1838 sizeof(pmd_t) * PTRS_PER_PMD); 1838 sizeof(pmd_t) * PTRS_PER_PMD);
1839 swapper_pg_dir[KERNEL_PGD_BOUNDARY] = 1839 swapper_pg_dir[KERNEL_PGD_BOUNDARY] =
1840 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT); 1840 __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT);
1841 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO); 1841 set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO);
1842 1842
1843 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO); 1843 set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
1844 xen_write_cr3(cr3); 1844 xen_write_cr3(cr3);
1845 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn); 1845 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn);
1846 1846
1847 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, 1847 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE,
1848 PFN_DOWN(__pa(initial_page_table))); 1848 PFN_DOWN(__pa(initial_page_table)));
1849 set_page_prot(initial_page_table, PAGE_KERNEL); 1849 set_page_prot(initial_page_table, PAGE_KERNEL);
1850 set_page_prot(initial_kernel_pmd, PAGE_KERNEL); 1850 set_page_prot(initial_kernel_pmd, PAGE_KERNEL);
1851 1851
1852 pv_mmu_ops.write_cr3 = &xen_write_cr3; 1852 pv_mmu_ops.write_cr3 = &xen_write_cr3;
1853 } 1853 }
1854 1854
1855 pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd, 1855 pgd_t * __init xen_setup_kernel_pagetable(pgd_t *pgd,
1856 unsigned long max_pfn) 1856 unsigned long max_pfn)
1857 { 1857 {
1858 pmd_t *kernel_pmd; 1858 pmd_t *kernel_pmd;
1859 1859
1860 initial_kernel_pmd = 1860 initial_kernel_pmd =
1861 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); 1861 extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE);
1862 1862
1863 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) + 1863 max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) +
1864 xen_start_info->nr_pt_frames * PAGE_SIZE + 1864 xen_start_info->nr_pt_frames * PAGE_SIZE +
1865 512*1024); 1865 512*1024);
1866 1866
1867 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); 1867 kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd);
1868 memcpy(initial_kernel_pmd, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD); 1868 memcpy(initial_kernel_pmd, kernel_pmd, sizeof(pmd_t) * PTRS_PER_PMD);
1869 1869
1870 xen_map_identity_early(initial_kernel_pmd, max_pfn); 1870 xen_map_identity_early(initial_kernel_pmd, max_pfn);
1871 1871
1872 memcpy(initial_page_table, pgd, sizeof(pgd_t) * PTRS_PER_PGD); 1872 memcpy(initial_page_table, pgd, sizeof(pgd_t) * PTRS_PER_PGD);
1873 initial_page_table[KERNEL_PGD_BOUNDARY] = 1873 initial_page_table[KERNEL_PGD_BOUNDARY] =
1874 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT); 1874 __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT);
1875 1875
1876 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO); 1876 set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO);
1877 set_page_prot(initial_page_table, PAGE_KERNEL_RO); 1877 set_page_prot(initial_page_table, PAGE_KERNEL_RO);
1878 set_page_prot(empty_zero_page, PAGE_KERNEL_RO); 1878 set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
1879 1879
1880 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); 1880 pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
1881 1881
1882 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, 1882 pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE,
1883 PFN_DOWN(__pa(initial_page_table))); 1883 PFN_DOWN(__pa(initial_page_table)));
1884 xen_write_cr3(__pa(initial_page_table)); 1884 xen_write_cr3(__pa(initial_page_table));
1885 1885
1886 memblock_reserve(__pa(xen_start_info->pt_base), 1886 memblock_reserve(__pa(xen_start_info->pt_base),
1887 xen_start_info->nr_pt_frames * PAGE_SIZE); 1887 xen_start_info->nr_pt_frames * PAGE_SIZE);
1888 1888
1889 return initial_page_table; 1889 return initial_page_table;
1890 } 1890 }
1891 #endif /* CONFIG_X86_64 */ 1891 #endif /* CONFIG_X86_64 */
1892 1892
1893 static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss; 1893 static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss;
1894 1894
1895 static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) 1895 static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot)
1896 { 1896 {
1897 pte_t pte; 1897 pte_t pte;
1898 1898
1899 phys >>= PAGE_SHIFT; 1899 phys >>= PAGE_SHIFT;
1900 1900
1901 switch (idx) { 1901 switch (idx) {
1902 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: 1902 case FIX_BTMAP_END ... FIX_BTMAP_BEGIN:
1903 #ifdef CONFIG_X86_F00F_BUG 1903 #ifdef CONFIG_X86_F00F_BUG
1904 case FIX_F00F_IDT: 1904 case FIX_F00F_IDT:
1905 #endif 1905 #endif
1906 #ifdef CONFIG_X86_32 1906 #ifdef CONFIG_X86_32
1907 case FIX_WP_TEST: 1907 case FIX_WP_TEST:
1908 case FIX_VDSO: 1908 case FIX_VDSO:
1909 # ifdef CONFIG_HIGHMEM 1909 # ifdef CONFIG_HIGHMEM
1910 case FIX_KMAP_BEGIN ... FIX_KMAP_END: 1910 case FIX_KMAP_BEGIN ... FIX_KMAP_END:
1911 # endif 1911 # endif
1912 #else 1912 #else
1913 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE: 1913 case VSYSCALL_LAST_PAGE ... VSYSCALL_FIRST_PAGE:
1914 case VVAR_PAGE: 1914 case VVAR_PAGE:
1915 #endif 1915 #endif
1916 case FIX_TEXT_POKE0: 1916 case FIX_TEXT_POKE0:
1917 case FIX_TEXT_POKE1: 1917 case FIX_TEXT_POKE1:
1918 /* All local page mappings */ 1918 /* All local page mappings */
1919 pte = pfn_pte(phys, prot); 1919 pte = pfn_pte(phys, prot);
1920 break; 1920 break;
1921 1921
1922 #ifdef CONFIG_X86_LOCAL_APIC 1922 #ifdef CONFIG_X86_LOCAL_APIC
1923 case FIX_APIC_BASE: /* maps dummy local APIC */ 1923 case FIX_APIC_BASE: /* maps dummy local APIC */
1924 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL); 1924 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
1925 break; 1925 break;
1926 #endif 1926 #endif
1927 1927
1928 #ifdef CONFIG_X86_IO_APIC 1928 #ifdef CONFIG_X86_IO_APIC
1929 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END: 1929 case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END:
1930 /* 1930 /*
1931 * We just don't map the IO APIC - all access is via 1931 * We just don't map the IO APIC - all access is via
1932 * hypercalls. Keep the address in the pte for reference. 1932 * hypercalls. Keep the address in the pte for reference.
1933 */ 1933 */
1934 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL); 1934 pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL);
1935 break; 1935 break;
1936 #endif 1936 #endif
1937 1937
1938 case FIX_PARAVIRT_BOOTMAP: 1938 case FIX_PARAVIRT_BOOTMAP:
1939 /* This is an MFN, but it isn't an IO mapping from the 1939 /* This is an MFN, but it isn't an IO mapping from the
1940 IO domain */ 1940 IO domain */
1941 pte = mfn_pte(phys, prot); 1941 pte = mfn_pte(phys, prot);
1942 break; 1942 break;
1943 1943
1944 default: 1944 default:
1945 /* By default, set_fixmap is used for hardware mappings */ 1945 /* By default, set_fixmap is used for hardware mappings */
1946 pte = mfn_pte(phys, __pgprot(pgprot_val(prot) | _PAGE_IOMAP)); 1946 pte = mfn_pte(phys, __pgprot(pgprot_val(prot) | _PAGE_IOMAP));
1947 break; 1947 break;
1948 } 1948 }
1949 1949
1950 __native_set_fixmap(idx, pte); 1950 __native_set_fixmap(idx, pte);
1951 1951
1952 #ifdef CONFIG_X86_64 1952 #ifdef CONFIG_X86_64
1953 /* Replicate changes to map the vsyscall page into the user 1953 /* Replicate changes to map the vsyscall page into the user
1954 pagetable vsyscall mapping. */ 1954 pagetable vsyscall mapping. */
1955 if ((idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) || 1955 if ((idx >= VSYSCALL_LAST_PAGE && idx <= VSYSCALL_FIRST_PAGE) ||
1956 idx == VVAR_PAGE) { 1956 idx == VVAR_PAGE) {
1957 unsigned long vaddr = __fix_to_virt(idx); 1957 unsigned long vaddr = __fix_to_virt(idx);
1958 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte); 1958 set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte);
1959 } 1959 }
1960 #endif 1960 #endif
1961 } 1961 }
1962 1962
1963 static void __init xen_post_allocator_init(void) 1963 static void __init xen_post_allocator_init(void)
1964 { 1964 {
1965 pv_mmu_ops.set_pte = xen_set_pte; 1965 pv_mmu_ops.set_pte = xen_set_pte;
1966 pv_mmu_ops.set_pmd = xen_set_pmd; 1966 pv_mmu_ops.set_pmd = xen_set_pmd;
1967 pv_mmu_ops.set_pud = xen_set_pud; 1967 pv_mmu_ops.set_pud = xen_set_pud;
1968 #if PAGETABLE_LEVELS == 4 1968 #if PAGETABLE_LEVELS == 4
1969 pv_mmu_ops.set_pgd = xen_set_pgd; 1969 pv_mmu_ops.set_pgd = xen_set_pgd;
1970 #endif 1970 #endif
1971 1971
1972 /* This will work as long as patching hasn't happened yet 1972 /* This will work as long as patching hasn't happened yet
1973 (which it hasn't) */ 1973 (which it hasn't) */
1974 pv_mmu_ops.alloc_pte = xen_alloc_pte; 1974 pv_mmu_ops.alloc_pte = xen_alloc_pte;
1975 pv_mmu_ops.alloc_pmd = xen_alloc_pmd; 1975 pv_mmu_ops.alloc_pmd = xen_alloc_pmd;
1976 pv_mmu_ops.release_pte = xen_release_pte; 1976 pv_mmu_ops.release_pte = xen_release_pte;
1977 pv_mmu_ops.release_pmd = xen_release_pmd; 1977 pv_mmu_ops.release_pmd = xen_release_pmd;
1978 #if PAGETABLE_LEVELS == 4 1978 #if PAGETABLE_LEVELS == 4
1979 pv_mmu_ops.alloc_pud = xen_alloc_pud; 1979 pv_mmu_ops.alloc_pud = xen_alloc_pud;
1980 pv_mmu_ops.release_pud = xen_release_pud; 1980 pv_mmu_ops.release_pud = xen_release_pud;
1981 #endif 1981 #endif
1982 1982
1983 #ifdef CONFIG_X86_64 1983 #ifdef CONFIG_X86_64
1984 SetPagePinned(virt_to_page(level3_user_vsyscall)); 1984 SetPagePinned(virt_to_page(level3_user_vsyscall));
1985 #endif 1985 #endif
1986 xen_mark_init_mm_pinned(); 1986 xen_mark_init_mm_pinned();
1987 } 1987 }
1988 1988
1989 static void xen_leave_lazy_mmu(void) 1989 static void xen_leave_lazy_mmu(void)
1990 { 1990 {
1991 preempt_disable(); 1991 preempt_disable();
1992 xen_mc_flush(); 1992 xen_mc_flush();
1993 paravirt_leave_lazy_mmu(); 1993 paravirt_leave_lazy_mmu();
1994 preempt_enable(); 1994 preempt_enable();
1995 } 1995 }
1996 1996
1997 static const struct pv_mmu_ops xen_mmu_ops __initconst = { 1997 static const struct pv_mmu_ops xen_mmu_ops __initconst = {
1998 .read_cr2 = xen_read_cr2, 1998 .read_cr2 = xen_read_cr2,
1999 .write_cr2 = xen_write_cr2, 1999 .write_cr2 = xen_write_cr2,
2000 2000
2001 .read_cr3 = xen_read_cr3, 2001 .read_cr3 = xen_read_cr3,
2002 #ifdef CONFIG_X86_32 2002 #ifdef CONFIG_X86_32
2003 .write_cr3 = xen_write_cr3_init, 2003 .write_cr3 = xen_write_cr3_init,
2004 #else 2004 #else
2005 .write_cr3 = xen_write_cr3, 2005 .write_cr3 = xen_write_cr3,
2006 #endif 2006 #endif
2007 2007
2008 .flush_tlb_user = xen_flush_tlb, 2008 .flush_tlb_user = xen_flush_tlb,
2009 .flush_tlb_kernel = xen_flush_tlb, 2009 .flush_tlb_kernel = xen_flush_tlb,
2010 .flush_tlb_single = xen_flush_tlb_single, 2010 .flush_tlb_single = xen_flush_tlb_single,
2011 .flush_tlb_others = xen_flush_tlb_others, 2011 .flush_tlb_others = xen_flush_tlb_others,
2012 2012
2013 .pte_update = paravirt_nop, 2013 .pte_update = paravirt_nop,
2014 .pte_update_defer = paravirt_nop, 2014 .pte_update_defer = paravirt_nop,
2015 2015
2016 .pgd_alloc = xen_pgd_alloc, 2016 .pgd_alloc = xen_pgd_alloc,
2017 .pgd_free = xen_pgd_free, 2017 .pgd_free = xen_pgd_free,
2018 2018
2019 .alloc_pte = xen_alloc_pte_init, 2019 .alloc_pte = xen_alloc_pte_init,
2020 .release_pte = xen_release_pte_init, 2020 .release_pte = xen_release_pte_init,
2021 .alloc_pmd = xen_alloc_pmd_init, 2021 .alloc_pmd = xen_alloc_pmd_init,
2022 .release_pmd = xen_release_pmd_init, 2022 .release_pmd = xen_release_pmd_init,
2023 2023
2024 .set_pte = xen_set_pte_init, 2024 .set_pte = xen_set_pte_init,
2025 .set_pte_at = xen_set_pte_at, 2025 .set_pte_at = xen_set_pte_at,
2026 .set_pmd = xen_set_pmd_hyper, 2026 .set_pmd = xen_set_pmd_hyper,
2027 2027
2028 .ptep_modify_prot_start = __ptep_modify_prot_start, 2028 .ptep_modify_prot_start = __ptep_modify_prot_start,
2029 .ptep_modify_prot_commit = __ptep_modify_prot_commit, 2029 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
2030 2030
2031 .pte_val = PV_CALLEE_SAVE(xen_pte_val), 2031 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
2032 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), 2032 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
2033 2033
2034 .make_pte = PV_CALLEE_SAVE(xen_make_pte), 2034 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
2035 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), 2035 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
2036 2036
2037 #ifdef CONFIG_X86_PAE 2037 #ifdef CONFIG_X86_PAE
2038 .set_pte_atomic = xen_set_pte_atomic, 2038 .set_pte_atomic = xen_set_pte_atomic,
2039 .pte_clear = xen_pte_clear, 2039 .pte_clear = xen_pte_clear,
2040 .pmd_clear = xen_pmd_clear, 2040 .pmd_clear = xen_pmd_clear,
2041 #endif /* CONFIG_X86_PAE */ 2041 #endif /* CONFIG_X86_PAE */
2042 .set_pud = xen_set_pud_hyper, 2042 .set_pud = xen_set_pud_hyper,
2043 2043
2044 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd), 2044 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
2045 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), 2045 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
2046 2046
2047 #if PAGETABLE_LEVELS == 4 2047 #if PAGETABLE_LEVELS == 4
2048 .pud_val = PV_CALLEE_SAVE(xen_pud_val), 2048 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
2049 .make_pud = PV_CALLEE_SAVE(xen_make_pud), 2049 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
2050 .set_pgd = xen_set_pgd_hyper, 2050 .set_pgd = xen_set_pgd_hyper,
2051 2051
2052 .alloc_pud = xen_alloc_pmd_init, 2052 .alloc_pud = xen_alloc_pmd_init,
2053 .release_pud = xen_release_pmd_init, 2053 .release_pud = xen_release_pmd_init,
2054 #endif /* PAGETABLE_LEVELS == 4 */ 2054 #endif /* PAGETABLE_LEVELS == 4 */
2055 2055
2056 .activate_mm = xen_activate_mm, 2056 .activate_mm = xen_activate_mm,
2057 .dup_mmap = xen_dup_mmap, 2057 .dup_mmap = xen_dup_mmap,
2058 .exit_mmap = xen_exit_mmap, 2058 .exit_mmap = xen_exit_mmap,
2059 2059
2060 .lazy_mode = { 2060 .lazy_mode = {
2061 .enter = paravirt_enter_lazy_mmu, 2061 .enter = paravirt_enter_lazy_mmu,
2062 .leave = xen_leave_lazy_mmu, 2062 .leave = xen_leave_lazy_mmu,
2063 }, 2063 },
2064 2064
2065 .set_fixmap = xen_set_fixmap, 2065 .set_fixmap = xen_set_fixmap,
2066 }; 2066 };
2067 2067
2068 void __init xen_init_mmu_ops(void) 2068 void __init xen_init_mmu_ops(void)
2069 { 2069 {
2070 x86_init.mapping.pagetable_reserve = xen_mapping_pagetable_reserve; 2070 x86_init.mapping.pagetable_reserve = xen_mapping_pagetable_reserve;
2071 x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start; 2071 x86_init.paging.pagetable_setup_start = xen_pagetable_setup_start;
2072 x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done; 2072 x86_init.paging.pagetable_setup_done = xen_pagetable_setup_done;
2073 pv_mmu_ops = xen_mmu_ops; 2073 pv_mmu_ops = xen_mmu_ops;
2074 2074
2075 memset(dummy_mapping, 0xff, PAGE_SIZE); 2075 memset(dummy_mapping, 0xff, PAGE_SIZE);
2076 } 2076 }
2077 2077
2078 /* Protected by xen_reservation_lock. */ 2078 /* Protected by xen_reservation_lock. */
2079 #define MAX_CONTIG_ORDER 9 /* 2MB */ 2079 #define MAX_CONTIG_ORDER 9 /* 2MB */
2080 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER]; 2080 static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER];
2081 2081
2082 #define VOID_PTE (mfn_pte(0, __pgprot(0))) 2082 #define VOID_PTE (mfn_pte(0, __pgprot(0)))
2083 static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order, 2083 static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order,
2084 unsigned long *in_frames, 2084 unsigned long *in_frames,
2085 unsigned long *out_frames) 2085 unsigned long *out_frames)
2086 { 2086 {
2087 int i; 2087 int i;
2088 struct multicall_space mcs; 2088 struct multicall_space mcs;
2089 2089
2090 xen_mc_batch(); 2090 xen_mc_batch();
2091 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) { 2091 for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) {
2092 mcs = __xen_mc_entry(0); 2092 mcs = __xen_mc_entry(0);
2093 2093
2094 if (in_frames) 2094 if (in_frames)
2095 in_frames[i] = virt_to_mfn(vaddr); 2095 in_frames[i] = virt_to_mfn(vaddr);
2096 2096
2097 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0); 2097 MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0);
2098 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY); 2098 __set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY);
2099 2099
2100 if (out_frames) 2100 if (out_frames)
2101 out_frames[i] = virt_to_pfn(vaddr); 2101 out_frames[i] = virt_to_pfn(vaddr);
2102 } 2102 }
2103 xen_mc_issue(0); 2103 xen_mc_issue(0);
2104 } 2104 }
2105 2105
2106 /* 2106 /*
2107 * Update the pfn-to-mfn mappings for a virtual address range, either to 2107 * Update the pfn-to-mfn mappings for a virtual address range, either to
2108 * point to an array of mfns, or contiguously from a single starting 2108 * point to an array of mfns, or contiguously from a single starting
2109 * mfn. 2109 * mfn.
2110 */ 2110 */
2111 static void xen_remap_exchanged_ptes(unsigned long vaddr, int order, 2111 static void xen_remap_exchanged_ptes(unsigned long vaddr, int order,
2112 unsigned long *mfns, 2112 unsigned long *mfns,
2113 unsigned long first_mfn) 2113 unsigned long first_mfn)
2114 { 2114 {
2115 unsigned i, limit; 2115 unsigned i, limit;
2116 unsigned long mfn; 2116 unsigned long mfn;
2117 2117
2118 xen_mc_batch(); 2118 xen_mc_batch();
2119 2119
2120 limit = 1u << order; 2120 limit = 1u << order;
2121 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) { 2121 for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) {
2122 struct multicall_space mcs; 2122 struct multicall_space mcs;
2123 unsigned flags; 2123 unsigned flags;
2124 2124
2125 mcs = __xen_mc_entry(0); 2125 mcs = __xen_mc_entry(0);
2126 if (mfns) 2126 if (mfns)
2127 mfn = mfns[i]; 2127 mfn = mfns[i];
2128 else 2128 else
2129 mfn = first_mfn + i; 2129 mfn = first_mfn + i;
2130 2130
2131 if (i < (limit - 1)) 2131 if (i < (limit - 1))
2132 flags = 0; 2132 flags = 0;
2133 else { 2133 else {
2134 if (order == 0) 2134 if (order == 0)
2135 flags = UVMF_INVLPG | UVMF_ALL; 2135 flags = UVMF_INVLPG | UVMF_ALL;
2136 else 2136 else
2137 flags = UVMF_TLB_FLUSH | UVMF_ALL; 2137 flags = UVMF_TLB_FLUSH | UVMF_ALL;
2138 } 2138 }
2139 2139
2140 MULTI_update_va_mapping(mcs.mc, vaddr, 2140 MULTI_update_va_mapping(mcs.mc, vaddr,
2141 mfn_pte(mfn, PAGE_KERNEL), flags); 2141 mfn_pte(mfn, PAGE_KERNEL), flags);
2142 2142
2143 set_phys_to_machine(virt_to_pfn(vaddr), mfn); 2143 set_phys_to_machine(virt_to_pfn(vaddr), mfn);
2144 } 2144 }
2145 2145
2146 xen_mc_issue(0); 2146 xen_mc_issue(0);
2147 } 2147 }
2148 2148
2149 /* 2149 /*
2150 * Perform the hypercall to exchange a region of our pfns to point to 2150 * Perform the hypercall to exchange a region of our pfns to point to
2151 * memory with the required contiguous alignment. Takes the pfns as 2151 * memory with the required contiguous alignment. Takes the pfns as
2152 * input, and populates mfns as output. 2152 * input, and populates mfns as output.
2153 * 2153 *
2154 * Returns a success code indicating whether the hypervisor was able to 2154 * Returns a success code indicating whether the hypervisor was able to
2155 * satisfy the request or not. 2155 * satisfy the request or not.
2156 */ 2156 */
2157 static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in, 2157 static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in,
2158 unsigned long *pfns_in, 2158 unsigned long *pfns_in,
2159 unsigned long extents_out, 2159 unsigned long extents_out,
2160 unsigned int order_out, 2160 unsigned int order_out,
2161 unsigned long *mfns_out, 2161 unsigned long *mfns_out,
2162 unsigned int address_bits) 2162 unsigned int address_bits)
2163 { 2163 {
2164 long rc; 2164 long rc;
2165 int success; 2165 int success;
2166 2166
2167 struct xen_memory_exchange exchange = { 2167 struct xen_memory_exchange exchange = {
2168 .in = { 2168 .in = {
2169 .nr_extents = extents_in, 2169 .nr_extents = extents_in,
2170 .extent_order = order_in, 2170 .extent_order = order_in,
2171 .extent_start = pfns_in, 2171 .extent_start = pfns_in,
2172 .domid = DOMID_SELF 2172 .domid = DOMID_SELF
2173 }, 2173 },
2174 .out = { 2174 .out = {
2175 .nr_extents = extents_out, 2175 .nr_extents = extents_out,
2176 .extent_order = order_out, 2176 .extent_order = order_out,
2177 .extent_start = mfns_out, 2177 .extent_start = mfns_out,
2178 .address_bits = address_bits, 2178 .address_bits = address_bits,
2179 .domid = DOMID_SELF 2179 .domid = DOMID_SELF
2180 } 2180 }
2181 }; 2181 };
2182 2182
2183 BUG_ON(extents_in << order_in != extents_out << order_out); 2183 BUG_ON(extents_in << order_in != extents_out << order_out);
2184 2184
2185 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange); 2185 rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange);
2186 success = (exchange.nr_exchanged == extents_in); 2186 success = (exchange.nr_exchanged == extents_in);
2187 2187
2188 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0))); 2188 BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0)));
2189 BUG_ON(success && (rc != 0)); 2189 BUG_ON(success && (rc != 0));
2190 2190
2191 return success; 2191 return success;
2192 } 2192 }
2193 2193
2194 int xen_create_contiguous_region(unsigned long vstart, unsigned int order, 2194 int xen_create_contiguous_region(unsigned long vstart, unsigned int order,
2195 unsigned int address_bits) 2195 unsigned int address_bits)
2196 { 2196 {
2197 unsigned long *in_frames = discontig_frames, out_frame; 2197 unsigned long *in_frames = discontig_frames, out_frame;
2198 unsigned long flags; 2198 unsigned long flags;
2199 int success; 2199 int success;
2200 2200
2201 /* 2201 /*
2202 * Currently an auto-translated guest will not perform I/O, nor will 2202 * Currently an auto-translated guest will not perform I/O, nor will
2203 * it require PAE page directories below 4GB. Therefore any calls to 2203 * it require PAE page directories below 4GB. Therefore any calls to
2204 * this function are redundant and can be ignored. 2204 * this function are redundant and can be ignored.
2205 */ 2205 */
2206 2206
2207 if (xen_feature(XENFEAT_auto_translated_physmap)) 2207 if (xen_feature(XENFEAT_auto_translated_physmap))
2208 return 0; 2208 return 0;
2209 2209
2210 if (unlikely(order > MAX_CONTIG_ORDER)) 2210 if (unlikely(order > MAX_CONTIG_ORDER))
2211 return -ENOMEM; 2211 return -ENOMEM;
2212 2212
2213 memset((void *) vstart, 0, PAGE_SIZE << order); 2213 memset((void *) vstart, 0, PAGE_SIZE << order);
2214 2214
2215 spin_lock_irqsave(&xen_reservation_lock, flags); 2215 spin_lock_irqsave(&xen_reservation_lock, flags);
2216 2216
2217 /* 1. Zap current PTEs, remembering MFNs. */ 2217 /* 1. Zap current PTEs, remembering MFNs. */
2218 xen_zap_pfn_range(vstart, order, in_frames, NULL); 2218 xen_zap_pfn_range(vstart, order, in_frames, NULL);
2219 2219
2220 /* 2. Get a new contiguous memory extent. */ 2220 /* 2. Get a new contiguous memory extent. */
2221 out_frame = virt_to_pfn(vstart); 2221 out_frame = virt_to_pfn(vstart);
2222 success = xen_exchange_memory(1UL << order, 0, in_frames, 2222 success = xen_exchange_memory(1UL << order, 0, in_frames,
2223 1, order, &out_frame, 2223 1, order, &out_frame,
2224 address_bits); 2224 address_bits);
2225 2225
2226 /* 3. Map the new extent in place of old pages. */ 2226 /* 3. Map the new extent in place of old pages. */
2227 if (success) 2227 if (success)
2228 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame); 2228 xen_remap_exchanged_ptes(vstart, order, NULL, out_frame);
2229 else 2229 else
2230 xen_remap_exchanged_ptes(vstart, order, in_frames, 0); 2230 xen_remap_exchanged_ptes(vstart, order, in_frames, 0);
2231 2231
2232 spin_unlock_irqrestore(&xen_reservation_lock, flags); 2232 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2233 2233
2234 return success ? 0 : -ENOMEM; 2234 return success ? 0 : -ENOMEM;
2235 } 2235 }
2236 EXPORT_SYMBOL_GPL(xen_create_contiguous_region); 2236 EXPORT_SYMBOL_GPL(xen_create_contiguous_region);
2237 2237
2238 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order) 2238 void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
2239 { 2239 {
2240 unsigned long *out_frames = discontig_frames, in_frame; 2240 unsigned long *out_frames = discontig_frames, in_frame;
2241 unsigned long flags; 2241 unsigned long flags;
2242 int success; 2242 int success;
2243 2243
2244 if (xen_feature(XENFEAT_auto_translated_physmap)) 2244 if (xen_feature(XENFEAT_auto_translated_physmap))
2245 return; 2245 return;
2246 2246
2247 if (unlikely(order > MAX_CONTIG_ORDER)) 2247 if (unlikely(order > MAX_CONTIG_ORDER))
2248 return; 2248 return;
2249 2249
2250 memset((void *) vstart, 0, PAGE_SIZE << order); 2250 memset((void *) vstart, 0, PAGE_SIZE << order);
2251 2251
2252 spin_lock_irqsave(&xen_reservation_lock, flags); 2252 spin_lock_irqsave(&xen_reservation_lock, flags);
2253 2253
2254 /* 1. Find start MFN of contiguous extent. */ 2254 /* 1. Find start MFN of contiguous extent. */
2255 in_frame = virt_to_mfn(vstart); 2255 in_frame = virt_to_mfn(vstart);
2256 2256
2257 /* 2. Zap current PTEs. */ 2257 /* 2. Zap current PTEs. */
2258 xen_zap_pfn_range(vstart, order, NULL, out_frames); 2258 xen_zap_pfn_range(vstart, order, NULL, out_frames);
2259 2259
2260 /* 3. Do the exchange for non-contiguous MFNs. */ 2260 /* 3. Do the exchange for non-contiguous MFNs. */
2261 success = xen_exchange_memory(1, order, &in_frame, 1UL << order, 2261 success = xen_exchange_memory(1, order, &in_frame, 1UL << order,
2262 0, out_frames, 0); 2262 0, out_frames, 0);
2263 2263
2264 /* 4. Map new pages in place of old pages. */ 2264 /* 4. Map new pages in place of old pages. */
2265 if (success) 2265 if (success)
2266 xen_remap_exchanged_ptes(vstart, order, out_frames, 0); 2266 xen_remap_exchanged_ptes(vstart, order, out_frames, 0);
2267 else 2267 else
2268 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame); 2268 xen_remap_exchanged_ptes(vstart, order, NULL, in_frame);
2269 2269
2270 spin_unlock_irqrestore(&xen_reservation_lock, flags); 2270 spin_unlock_irqrestore(&xen_reservation_lock, flags);
2271 } 2271 }
2272 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); 2272 EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
2273 2273
2274 #ifdef CONFIG_XEN_PVHVM 2274 #ifdef CONFIG_XEN_PVHVM
2275 static void xen_hvm_exit_mmap(struct mm_struct *mm) 2275 static void xen_hvm_exit_mmap(struct mm_struct *mm)
2276 { 2276 {
2277 struct xen_hvm_pagetable_dying a; 2277 struct xen_hvm_pagetable_dying a;
2278 int rc; 2278 int rc;
2279 2279
2280 a.domid = DOMID_SELF; 2280 a.domid = DOMID_SELF;
2281 a.gpa = __pa(mm->pgd); 2281 a.gpa = __pa(mm->pgd);
2282 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a); 2282 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2283 WARN_ON_ONCE(rc < 0); 2283 WARN_ON_ONCE(rc < 0);
2284 } 2284 }
2285 2285
2286 static int is_pagetable_dying_supported(void) 2286 static int is_pagetable_dying_supported(void)
2287 { 2287 {
2288 struct xen_hvm_pagetable_dying a; 2288 struct xen_hvm_pagetable_dying a;
2289 int rc = 0; 2289 int rc = 0;
2290 2290
2291 a.domid = DOMID_SELF; 2291 a.domid = DOMID_SELF;
2292 a.gpa = 0x00; 2292 a.gpa = 0x00;
2293 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a); 2293 rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a);
2294 if (rc < 0) { 2294 if (rc < 0) {
2295 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n"); 2295 printk(KERN_DEBUG "HVMOP_pagetable_dying not supported\n");
2296 return 0; 2296 return 0;
2297 } 2297 }
2298 return 1; 2298 return 1;
2299 } 2299 }
2300 2300
2301 void __init xen_hvm_init_mmu_ops(void) 2301 void __init xen_hvm_init_mmu_ops(void)
2302 { 2302 {
2303 if (is_pagetable_dying_supported()) 2303 if (is_pagetable_dying_supported())
2304 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap; 2304 pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
2305 } 2305 }
2306 #endif 2306 #endif
2307 2307
2308 #define REMAP_BATCH_SIZE 16 2308 #define REMAP_BATCH_SIZE 16
2309 2309
2310 struct remap_data { 2310 struct remap_data {
2311 unsigned long mfn; 2311 unsigned long mfn;
2312 pgprot_t prot; 2312 pgprot_t prot;
2313 struct mmu_update *mmu_update; 2313 struct mmu_update *mmu_update;
2314 }; 2314 };
2315 2315
2316 static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token, 2316 static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token,
2317 unsigned long addr, void *data) 2317 unsigned long addr, void *data)
2318 { 2318 {
2319 struct remap_data *rmd = data; 2319 struct remap_data *rmd = data;
2320 pte_t pte = pte_mkspecial(pfn_pte(rmd->mfn++, rmd->prot)); 2320 pte_t pte = pte_mkspecial(pfn_pte(rmd->mfn++, rmd->prot));
2321 2321
2322 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; 2322 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr;
2323 rmd->mmu_update->val = pte_val_ma(pte); 2323 rmd->mmu_update->val = pte_val_ma(pte);
2324 rmd->mmu_update++; 2324 rmd->mmu_update++;
2325 2325
2326 return 0; 2326 return 0;
2327 } 2327 }
2328 2328
2329 int xen_remap_domain_mfn_range(struct vm_area_struct *vma, 2329 int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
2330 unsigned long addr, 2330 unsigned long addr,
2331 unsigned long mfn, int nr, 2331 unsigned long mfn, int nr,
2332 pgprot_t prot, unsigned domid) 2332 pgprot_t prot, unsigned domid)
2333 { 2333 {
2334 struct remap_data rmd; 2334 struct remap_data rmd;
2335 struct mmu_update mmu_update[REMAP_BATCH_SIZE]; 2335 struct mmu_update mmu_update[REMAP_BATCH_SIZE];
2336 int batch; 2336 int batch;
2337 unsigned long range; 2337 unsigned long range;
2338 int err = 0; 2338 int err = 0;
2339 2339
2340 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP); 2340 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP);
2341 2341
2342 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) == 2342 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) ==
2343 (VM_PFNMAP | VM_RESERVED | VM_IO))); 2343 (VM_PFNMAP | VM_RESERVED | VM_IO)));
2344 2344
2345 rmd.mfn = mfn; 2345 rmd.mfn = mfn;
2346 rmd.prot = prot; 2346 rmd.prot = prot;
2347 2347
2348 while (nr) { 2348 while (nr) {
2349 batch = min(REMAP_BATCH_SIZE, nr); 2349 batch = min(REMAP_BATCH_SIZE, nr);
2350 range = (unsigned long)batch << PAGE_SHIFT; 2350 range = (unsigned long)batch << PAGE_SHIFT;
2351 2351
2352 rmd.mmu_update = mmu_update; 2352 rmd.mmu_update = mmu_update;
2353 err = apply_to_page_range(vma->vm_mm, addr, range, 2353 err = apply_to_page_range(vma->vm_mm, addr, range,
2354 remap_area_mfn_pte_fn, &rmd); 2354 remap_area_mfn_pte_fn, &rmd);
2355 if (err) 2355 if (err)
2356 goto out; 2356 goto out;
2357 2357
2358 err = -EFAULT; 2358 err = -EFAULT;
2359 if (HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid) < 0) 2359 if (HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid) < 0)
2360 goto out; 2360 goto out;
2361 2361
2362 nr -= batch; 2362 nr -= batch;
2363 addr += range; 2363 addr += range;
2364 } 2364 }
2365 2365
2366 err = 0; 2366 err = 0;
2367 out: 2367 out:
2368 2368
2369 flush_tlb_all(); 2369 flush_tlb_all();
2370 2370
2371 return err; 2371 return err;
2372 } 2372 }
2373 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range); 2373 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range);
2374 2374