Blame view
arch/x86/xen/mmu.c
74.5 KB
3b827c1b3 xen: virtual mmu |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
/* * Xen mmu operations * * This file contains the various mmu fetch and update operations. * The most important job they must perform is the mapping between the * domain's pfn and the overall machine mfns. * * Xen allows guests to directly update the pagetable, in a controlled * fashion. In other words, the guest modifies the same pagetable * that the CPU actually uses, which eliminates the overhead of having * a separate shadow pagetable. * * In order to allow this, it falls on the guest domain to map its * notion of a "physical" pfn - which is just a domain-local linear * address - into a real "machine address" which the CPU's MMU can * use. * * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be * inserted directly into the pagetable. When creating a new * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely, * when reading the content back with __(pgd|pmd|pte)_val, it converts * the mfn back into a pfn. * * The other constraint is that all pages which make up a pagetable * must be mapped read-only in the guest. This prevents uncontrolled * guest updates to the pagetable. Xen strictly enforces this, and * will disallow any pagetable update which will end up mapping a * pagetable page RW, and will disallow using any writable page as a * pagetable. * * Naively, when loading %cr3 with the base of a new pagetable, Xen * would need to validate the whole pagetable before going on. * Naturally, this is quite slow. The solution is to "pin" a * pagetable, which enforces all the constraints on the pagetable even * when it is not actively in use. This menas that Xen can be assured * that it is still valid when you do load it into %cr3, and doesn't * need to revalidate it. * * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 */ |
f120f13ea xen: Add support ... |
41 |
#include <linux/sched.h> |
f4f97b3ea xen: Complete pag... |
42 |
#include <linux/highmem.h> |
994025cab xen: add debugfs ... |
43 |
#include <linux/debugfs.h> |
3b827c1b3 xen: virtual mmu |
44 |
#include <linux/bug.h> |
d2cb21455 xen/mmu: inhibit ... |
45 |
#include <linux/vmalloc.h> |
44408ad73 xen: use header f... |
46 |
#include <linux/module.h> |
5a0e3ad6a include cleanup: ... |
47 |
#include <linux/gfp.h> |
a9ce6bc15 x86, memblock: Re... |
48 |
#include <linux/memblock.h> |
2222e71bd xen/debugfs: Add ... |
49 |
#include <linux/seq_file.h> |
34b6f01a7 xen pv-on-hvm: ad... |
50 |
#include <linux/crash_dump.h> |
3b827c1b3 xen: virtual mmu |
51 |
|
847088079 xen/trace: add mm... |
52 |
#include <trace/events/xen.h> |
3b827c1b3 xen: virtual mmu |
53 54 |
#include <asm/pgtable.h> #include <asm/tlbflush.h> |
5deb30d19 xen: rework pgd_w... |
55 |
#include <asm/fixmap.h> |
3b827c1b3 xen: virtual mmu |
56 |
#include <asm/mmu_context.h> |
319f3ba52 xen: move remaini... |
57 |
#include <asm/setup.h> |
f4f97b3ea xen: Complete pag... |
58 |
#include <asm/paravirt.h> |
7347b4082 xen: Allow unpriv... |
59 |
#include <asm/e820.h> |
cbcd79c2e x86: use __page_a... |
60 |
#include <asm/linkage.h> |
08bbc9da9 xen: Add xen_crea... |
61 |
#include <asm/page.h> |
fef5ba797 xen: Cope with un... |
62 |
#include <asm/init.h> |
41f2e4771 xen: add support ... |
63 |
#include <asm/pat.h> |
900cba888 xen: support CONF... |
64 |
#include <asm/smp.h> |
3b827c1b3 xen: virtual mmu |
65 66 |
#include <asm/xen/hypercall.h> |
f4f97b3ea xen: Complete pag... |
67 |
#include <asm/xen/hypervisor.h> |
3b827c1b3 xen: virtual mmu |
68 |
|
c0011dbfc xen: use _PAGE_IO... |
69 |
#include <xen/xen.h> |
3b827c1b3 xen: virtual mmu |
70 71 |
#include <xen/page.h> #include <xen/interface/xen.h> |
591510010 x86: Call HVMOP_p... |
72 |
#include <xen/interface/hvm/hvm_op.h> |
319f3ba52 xen: move remaini... |
73 |
#include <xen/interface/version.h> |
c0011dbfc xen: use _PAGE_IO... |
74 |
#include <xen/interface/memory.h> |
319f3ba52 xen: move remaini... |
75 |
#include <xen/hvc-console.h> |
3b827c1b3 xen: virtual mmu |
76 |
|
f4f97b3ea xen: Complete pag... |
77 |
#include "multicalls.h" |
3b827c1b3 xen: virtual mmu |
78 |
#include "mmu.h" |
994025cab xen: add debugfs ... |
79 |
#include "debugfs.h" |
19001c8c5 xen: Rename the b... |
80 81 |
/* * Protects atomic reservation decrease/increase against concurrent increases. |
06f521d5d xen/balloon: Remo... |
82 |
* Also protects non-atomic updates of current_pages and balloon lists. |
19001c8c5 xen: Rename the b... |
83 84 |
*/ DEFINE_SPINLOCK(xen_reservation_lock); |
caaf9ecf1 xen/mmu: For 64-b... |
85 |
#ifdef CONFIG_X86_32 |
319f3ba52 xen: move remaini... |
86 87 88 89 90 |
/* * Identity map, in addition to plain kernel map. This needs to be * large enough to allocate page table pages to allocate the rest. * Each page can map 2MB. */ |
764f0138b xen: allocate lev... |
91 92 |
#define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4) static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES); |
caaf9ecf1 xen/mmu: For 64-b... |
93 |
#endif |
319f3ba52 xen: move remaini... |
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
#ifdef CONFIG_X86_64 /* l3 pud for userspace vsyscall mapping */ static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; #endif /* CONFIG_X86_64 */ /* * Note about cr3 (pagetable base) values: * * xen_cr3 contains the current logical cr3 value; it contains the * last set cr3. This may not be the current effective cr3, because * its update may be being lazily deferred. However, a vcpu looking * at its own cr3 can use this value knowing that it everything will * be self-consistent. * * xen_current_cr3 contains the actual vcpu cr3; it is set once the * hypercall to set the vcpu cr3 is complete (so it may be a little * out of date, but it will never be set early). If one vcpu is * looking at another vcpu's cr3 value, it should use this variable. */ DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ |
04414baab xen: check pre-al... |
115 |
static phys_addr_t xen_pt_base, xen_pt_size __initdata; |
319f3ba52 xen: move remaini... |
116 |
|
d6182fbf0 xen64: allocate a... |
117 118 119 120 121 |
/* * Just beyond the highest usermode address. STACK_TOP_MAX has a * redzone above it, so round it up to a PGD boundary. */ #define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK) |
9976b39b5 xen: deal with vi... |
122 123 124 125 126 127 |
unsigned long arbitrary_virt_to_mfn(void *vaddr) { xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); return PFN_DOWN(maddr.maddr); } |
ce803e705 xen64: use arbitr... |
128 |
xmaddr_t arbitrary_virt_to_machine(void *vaddr) |
3b827c1b3 xen: virtual mmu |
129 |
{ |
ce803e705 xen64: use arbitr... |
130 |
unsigned long address = (unsigned long)vaddr; |
da7bfc50f x86: sparse warni... |
131 |
unsigned int level; |
9f32d21c9 xen: fix Xen domU... |
132 133 |
pte_t *pte; unsigned offset; |
3b827c1b3 xen: virtual mmu |
134 |
|
9f32d21c9 xen: fix Xen domU... |
135 136 137 138 139 140 141 142 |
/* * if the PFN is in the linear mapped vaddr range, we can just use * the (quick) virt_to_machine() p2m lookup */ if (virt_addr_valid(vaddr)) return virt_to_machine(vaddr); /* otherwise we have to do a (slower) full page-table walk */ |
3b827c1b3 xen: virtual mmu |
143 |
|
9f32d21c9 xen: fix Xen domU... |
144 145 146 |
pte = lookup_address(address, &level); BUG_ON(pte == NULL); offset = address & ~PAGE_MASK; |
ebd879e39 xen: fix truncati... |
147 |
return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset); |
3b827c1b3 xen: virtual mmu |
148 |
} |
de23be5f3 xen: export arbit... |
149 |
EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); |
3b827c1b3 xen: virtual mmu |
150 151 152 153 154 |
void make_lowmem_page_readonly(void *vaddr) { pte_t *pte, ptev; unsigned long address = (unsigned long)vaddr; |
da7bfc50f x86: sparse warni... |
155 |
unsigned int level; |
3b827c1b3 xen: virtual mmu |
156 |
|
f0646e43a x86: return the p... |
157 |
pte = lookup_address(address, &level); |
fef5ba797 xen: Cope with un... |
158 159 |
if (pte == NULL) return; /* vaddr missing */ |
3b827c1b3 xen: virtual mmu |
160 161 162 163 164 165 166 167 168 169 170 |
ptev = pte_wrprotect(*pte); if (HYPERVISOR_update_va_mapping(address, ptev, 0)) BUG(); } void make_lowmem_page_readwrite(void *vaddr) { pte_t *pte, ptev; unsigned long address = (unsigned long)vaddr; |
da7bfc50f x86: sparse warni... |
171 |
unsigned int level; |
3b827c1b3 xen: virtual mmu |
172 |
|
f0646e43a x86: return the p... |
173 |
pte = lookup_address(address, &level); |
fef5ba797 xen: Cope with un... |
174 175 |
if (pte == NULL) return; /* vaddr missing */ |
3b827c1b3 xen: virtual mmu |
176 177 178 179 180 181 |
ptev = pte_mkwrite(*pte); if (HYPERVISOR_update_va_mapping(address, ptev, 0)) BUG(); } |
7708ad64a xen: add xen_ pre... |
182 |
static bool xen_page_pinned(void *ptr) |
e2426cf85 xen: avoid hyperc... |
183 184 185 186 187 |
{ struct page *page = virt_to_page(ptr); return PagePinned(page); } |
eba3ff8b9 xen: add xen_set_... |
188 |
void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid) |
c0011dbfc xen: use _PAGE_IO... |
189 190 191 |
{ struct multicall_space mcs; struct mmu_update *u; |
847088079 xen/trace: add mm... |
192 |
trace_xen_mmu_set_domain_pte(ptep, pteval, domid); |
c0011dbfc xen: use _PAGE_IO... |
193 194 195 196 |
mcs = xen_mc_entry(sizeof(*u)); u = mcs.args; /* ptep might be kmapped when using 32-bit HIGHPTE */ |
d5108316b xen: use normal v... |
197 |
u->ptr = virt_to_machine(ptep).maddr; |
c0011dbfc xen: use _PAGE_IO... |
198 |
u->val = pte_val_ma(pteval); |
eba3ff8b9 xen: add xen_set_... |
199 |
MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid); |
c0011dbfc xen: use _PAGE_IO... |
200 201 202 |
xen_mc_issue(PARAVIRT_LAZY_MMU); } |
eba3ff8b9 xen: add xen_set_... |
203 |
EXPORT_SYMBOL_GPL(xen_set_domain_pte); |
7708ad64a xen: add xen_ pre... |
204 |
static void xen_extend_mmu_update(const struct mmu_update *update) |
3b827c1b3 xen: virtual mmu |
205 |
{ |
d66bf8fcf xen: lazy-mmu ope... |
206 207 |
struct multicall_space mcs; struct mmu_update *u; |
3b827c1b3 xen: virtual mmu |
208 |
|
400d34944 xen: add mechanis... |
209 |
mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u)); |
994025cab xen: add debugfs ... |
210 |
if (mcs.mc != NULL) { |
400d34944 xen: add mechanis... |
211 |
mcs.mc->args[1]++; |
994025cab xen: add debugfs ... |
212 |
} else { |
400d34944 xen: add mechanis... |
213 214 215 |
mcs = __xen_mc_entry(sizeof(*u)); MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); } |
d66bf8fcf xen: lazy-mmu ope... |
216 |
|
d66bf8fcf xen: lazy-mmu ope... |
217 |
u = mcs.args; |
400d34944 xen: add mechanis... |
218 219 |
*u = *update; } |
dcf7435cf xen/mmu: use exte... |
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 |
static void xen_extend_mmuext_op(const struct mmuext_op *op) { struct multicall_space mcs; struct mmuext_op *u; mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u)); if (mcs.mc != NULL) { mcs.mc->args[1]++; } else { mcs = __xen_mc_entry(sizeof(*u)); MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); } u = mcs.args; *u = *op; } |
4c13629f8 xen: make a pile ... |
237 |
static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val) |
400d34944 xen: add mechanis... |
238 239 240 241 242 243 |
{ struct mmu_update u; preempt_disable(); xen_mc_batch(); |
ce803e705 xen64: use arbitr... |
244 245 |
/* ptr may be ioremapped for 64-bit pagetable setup */ u.ptr = arbitrary_virt_to_machine(ptr).maddr; |
400d34944 xen: add mechanis... |
246 |
u.val = pmd_val_ma(val); |
7708ad64a xen: add xen_ pre... |
247 |
xen_extend_mmu_update(&u); |
d66bf8fcf xen: lazy-mmu ope... |
248 249 250 251 |
xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); |
3b827c1b3 xen: virtual mmu |
252 |
} |
4c13629f8 xen: make a pile ... |
253 |
static void xen_set_pmd(pmd_t *ptr, pmd_t val) |
e2426cf85 xen: avoid hyperc... |
254 |
{ |
847088079 xen/trace: add mm... |
255 |
trace_xen_mmu_set_pmd(ptr, val); |
e2426cf85 xen: avoid hyperc... |
256 257 |
/* If page is not pinned, we can just update the entry directly */ |
7708ad64a xen: add xen_ pre... |
258 |
if (!xen_page_pinned(ptr)) { |
e2426cf85 xen: avoid hyperc... |
259 260 261 262 263 264 |
*ptr = val; return; } xen_set_pmd_hyper(ptr, val); } |
3b827c1b3 xen: virtual mmu |
265 266 267 268 269 270 |
/* * Associate a virtual page frame with a given physical page frame * and protection flags for that frame. */ void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags) { |
836fe2f29 xen: use set_pte_... |
271 |
set_pte_vaddr(vaddr, mfn_pte(mfn, flags)); |
3b827c1b3 xen: virtual mmu |
272 |
} |
4a35c13cb xen: condense eve... |
273 |
static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) |
3b827c1b3 xen: virtual mmu |
274 |
{ |
4a35c13cb xen: condense eve... |
275 |
struct mmu_update u; |
c0011dbfc xen: use _PAGE_IO... |
276 |
|
4a35c13cb xen: condense eve... |
277 278 |
if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU) return false; |
994025cab xen: add debugfs ... |
279 |
|
4a35c13cb xen: condense eve... |
280 |
xen_mc_batch(); |
d66bf8fcf xen: lazy-mmu ope... |
281 |
|
4a35c13cb xen: condense eve... |
282 283 284 |
u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; u.val = pte_val_ma(pteval); xen_extend_mmu_update(&u); |
a99ac5e86 xen: use mmu_upda... |
285 |
|
4a35c13cb xen: condense eve... |
286 |
xen_mc_issue(PARAVIRT_LAZY_MMU); |
2bd50036b xen: allow set_pt... |
287 |
|
4a35c13cb xen: condense eve... |
288 289 |
return true; } |
847088079 xen/trace: add mm... |
290 |
static inline void __xen_set_pte(pte_t *ptep, pte_t pteval) |
4a35c13cb xen: condense eve... |
291 |
{ |
d095d43e7 xen/mm: do direct... |
292 293 294 295 296 297 298 299 300 301 302 303 304 305 |
if (!xen_batched_set_pte(ptep, pteval)) { /* * Could call native_set_pte() here and trap and * emulate the PTE write but with 32-bit guests this * needs two traps (one for each of the two 32-bit * words in the PTE) so do one hypercall directly * instead. */ struct mmu_update u; u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; u.val = pte_val_ma(pteval); HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF); } |
3b827c1b3 xen: virtual mmu |
306 |
} |
847088079 xen/trace: add mm... |
307 308 309 310 311 |
static void xen_set_pte(pte_t *ptep, pte_t pteval) { trace_xen_mmu_set_pte(ptep, pteval); __xen_set_pte(ptep, pteval); } |
4c13629f8 xen: make a pile ... |
312 |
static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr, |
4a35c13cb xen: condense eve... |
313 314 |
pte_t *ptep, pte_t pteval) { |
847088079 xen/trace: add mm... |
315 316 |
trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval); __xen_set_pte(ptep, pteval); |
3b827c1b3 xen: virtual mmu |
317 |
} |
f63c2f248 xen: whitespace/c... |
318 319 |
pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
947a69c90 xen: unify pte op... |
320 |
{ |
e57778a1e xen: implement pt... |
321 |
/* Just return the pte as-is. We preserve the bits on commit */ |
847088079 xen/trace: add mm... |
322 |
trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep); |
e57778a1e xen: implement pt... |
323 324 325 326 327 328 |
return *ptep; } void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { |
400d34944 xen: add mechanis... |
329 |
struct mmu_update u; |
e57778a1e xen: implement pt... |
330 |
|
847088079 xen/trace: add mm... |
331 |
trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte); |
400d34944 xen: add mechanis... |
332 |
xen_mc_batch(); |
947a69c90 xen: unify pte op... |
333 |
|
d5108316b xen: use normal v... |
334 |
u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD; |
400d34944 xen: add mechanis... |
335 |
u.val = pte_val_ma(pte); |
7708ad64a xen: add xen_ pre... |
336 |
xen_extend_mmu_update(&u); |
947a69c90 xen: unify pte op... |
337 |
|
e57778a1e xen: implement pt... |
338 |
xen_mc_issue(PARAVIRT_LAZY_MMU); |
947a69c90 xen: unify pte op... |
339 |
} |
ebb9cfe20 xen: don't drop N... |
340 341 |
/* Assume pteval_t is equivalent to all the other *val_t types. */ static pteval_t pte_mfn_to_pfn(pteval_t val) |
947a69c90 xen: unify pte op... |
342 |
{ |
5926f87fd Revert "xen: prop... |
343 |
if (val & _PAGE_PRESENT) { |
59438c9fc x86: rename PTE_M... |
344 |
unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; |
b7e5ffe5d xen/pte: Fix cras... |
345 |
unsigned long pfn = mfn_to_pfn(mfn); |
77be1fabd x86: add PTE_FLAG... |
346 |
pteval_t flags = val & PTE_FLAGS_MASK; |
b7e5ffe5d xen/pte: Fix cras... |
347 348 349 350 |
if (unlikely(pfn == ~0)) val = flags & ~_PAGE_PRESENT; else val = ((pteval_t)pfn << PAGE_SHIFT) | flags; |
ebb9cfe20 xen: don't drop N... |
351 |
} |
947a69c90 xen: unify pte op... |
352 |
|
ebb9cfe20 xen: don't drop N... |
353 |
return val; |
947a69c90 xen: unify pte op... |
354 |
} |
ebb9cfe20 xen: don't drop N... |
355 |
static pteval_t pte_pfn_to_mfn(pteval_t val) |
947a69c90 xen: unify pte op... |
356 |
{ |
5926f87fd Revert "xen: prop... |
357 |
if (val & _PAGE_PRESENT) { |
59438c9fc x86: rename PTE_M... |
358 |
unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; |
77be1fabd x86: add PTE_FLAG... |
359 |
pteval_t flags = val & PTE_FLAGS_MASK; |
fb38923ea xen/mmu: Set _PAG... |
360 |
unsigned long mfn; |
cfd8951e0 xen: don't map mi... |
361 |
|
fb38923ea xen/mmu: Set _PAG... |
362 |
if (!xen_feature(XENFEAT_auto_translated_physmap)) |
0aad56898 xen: Hide get_phy... |
363 |
mfn = __pfn_to_mfn(pfn); |
fb38923ea xen/mmu: Set _PAG... |
364 365 |
else mfn = pfn; |
cfd8951e0 xen: don't map mi... |
366 367 368 369 370 371 372 373 374 |
/* * If there's no mfn for the pfn, then just create an * empty non-present pte. Unfortunately this loses * information about the original pfn, so * pte_mfn_to_pfn is asymmetric. */ if (unlikely(mfn == INVALID_P2M_ENTRY)) { mfn = 0; flags = 0; |
7f2f88224 x86/xen: do not u... |
375 376 |
} else mfn &= ~(FOREIGN_FRAME_BIT | IDENTITY_FRAME_BIT); |
cfd8951e0 xen: don't map mi... |
377 |
val = ((pteval_t)mfn << PAGE_SHIFT) | flags; |
947a69c90 xen: unify pte op... |
378 |
} |
ebb9cfe20 xen: don't drop N... |
379 |
return val; |
947a69c90 xen: unify pte op... |
380 |
} |
a2e7f0e3a x86, asmlinkage, ... |
381 |
__visible pteval_t xen_pte_val(pte_t pte) |
947a69c90 xen: unify pte op... |
382 |
{ |
41f2e4771 xen: add support ... |
383 |
pteval_t pteval = pte.pte; |
47591df50 xen: Support Xen ... |
384 |
|
41f2e4771 xen: add support ... |
385 |
return pte_mfn_to_pfn(pteval); |
947a69c90 xen: unify pte op... |
386 |
} |
da5de7c22 x86/paravirt: use... |
387 |
PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val); |
947a69c90 xen: unify pte op... |
388 |
|
a2e7f0e3a x86, asmlinkage, ... |
389 |
__visible pgdval_t xen_pgd_val(pgd_t pgd) |
947a69c90 xen: unify pte op... |
390 |
{ |
ebb9cfe20 xen: don't drop N... |
391 |
return pte_mfn_to_pfn(pgd.pgd); |
947a69c90 xen: unify pte op... |
392 |
} |
da5de7c22 x86/paravirt: use... |
393 |
PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val); |
947a69c90 xen: unify pte op... |
394 |
|
a2e7f0e3a x86, asmlinkage, ... |
395 |
__visible pte_t xen_make_pte(pteval_t pte) |
947a69c90 xen: unify pte op... |
396 |
{ |
7f2f88224 x86/xen: do not u... |
397 |
pte = pte_pfn_to_mfn(pte); |
c0011dbfc xen: use _PAGE_IO... |
398 |
|
ebb9cfe20 xen: don't drop N... |
399 |
return native_make_pte(pte); |
947a69c90 xen: unify pte op... |
400 |
} |
da5de7c22 x86/paravirt: use... |
401 |
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte); |
947a69c90 xen: unify pte op... |
402 |
|
a2e7f0e3a x86, asmlinkage, ... |
403 |
__visible pgd_t xen_make_pgd(pgdval_t pgd) |
947a69c90 xen: unify pte op... |
404 |
{ |
ebb9cfe20 xen: don't drop N... |
405 406 |
pgd = pte_pfn_to_mfn(pgd); return native_make_pgd(pgd); |
947a69c90 xen: unify pte op... |
407 |
} |
da5de7c22 x86/paravirt: use... |
408 |
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd); |
947a69c90 xen: unify pte op... |
409 |
|
a2e7f0e3a x86, asmlinkage, ... |
410 |
__visible pmdval_t xen_pmd_val(pmd_t pmd) |
947a69c90 xen: unify pte op... |
411 |
{ |
ebb9cfe20 xen: don't drop N... |
412 |
return pte_mfn_to_pfn(pmd.pmd); |
947a69c90 xen: unify pte op... |
413 |
} |
da5de7c22 x86/paravirt: use... |
414 |
PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val); |
284991439 xen: remove suppo... |
415 |
|
4c13629f8 xen: make a pile ... |
416 |
static void xen_set_pud_hyper(pud_t *ptr, pud_t val) |
f4f97b3ea xen: Complete pag... |
417 |
{ |
400d34944 xen: add mechanis... |
418 |
struct mmu_update u; |
f4f97b3ea xen: Complete pag... |
419 |
|
d66bf8fcf xen: lazy-mmu ope... |
420 |
preempt_disable(); |
400d34944 xen: add mechanis... |
421 |
xen_mc_batch(); |
ce803e705 xen64: use arbitr... |
422 423 |
/* ptr may be ioremapped for 64-bit pagetable setup */ u.ptr = arbitrary_virt_to_machine(ptr).maddr; |
400d34944 xen: add mechanis... |
424 |
u.val = pud_val_ma(val); |
7708ad64a xen: add xen_ pre... |
425 |
xen_extend_mmu_update(&u); |
d66bf8fcf xen: lazy-mmu ope... |
426 427 428 429 |
xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); |
f4f97b3ea xen: Complete pag... |
430 |
} |
4c13629f8 xen: make a pile ... |
431 |
static void xen_set_pud(pud_t *ptr, pud_t val) |
e2426cf85 xen: avoid hyperc... |
432 |
{ |
847088079 xen/trace: add mm... |
433 |
trace_xen_mmu_set_pud(ptr, val); |
e2426cf85 xen: avoid hyperc... |
434 435 |
/* If page is not pinned, we can just update the entry directly */ |
7708ad64a xen: add xen_ pre... |
436 |
if (!xen_page_pinned(ptr)) { |
e2426cf85 xen: avoid hyperc... |
437 438 439 440 441 442 |
*ptr = val; return; } xen_set_pud_hyper(ptr, val); } |
f6e587325 xen64: add extra ... |
443 |
#ifdef CONFIG_X86_PAE |
4c13629f8 xen: make a pile ... |
444 |
static void xen_set_pte_atomic(pte_t *ptep, pte_t pte) |
3b827c1b3 xen: virtual mmu |
445 |
{ |
847088079 xen/trace: add mm... |
446 |
trace_xen_mmu_set_pte_atomic(ptep, pte); |
f6e587325 xen64: add extra ... |
447 |
set_64bit((u64 *)ptep, native_pte_val(pte)); |
3b827c1b3 xen: virtual mmu |
448 |
} |
4c13629f8 xen: make a pile ... |
449 |
static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) |
3b827c1b3 xen: virtual mmu |
450 |
{ |
847088079 xen/trace: add mm... |
451 |
trace_xen_mmu_pte_clear(mm, addr, ptep); |
4a35c13cb xen: condense eve... |
452 453 |
if (!xen_batched_set_pte(ptep, native_make_pte(0))) native_pte_clear(mm, addr, ptep); |
3b827c1b3 xen: virtual mmu |
454 |
} |
4c13629f8 xen: make a pile ... |
455 |
static void xen_pmd_clear(pmd_t *pmdp) |
3b827c1b3 xen: virtual mmu |
456 |
{ |
847088079 xen/trace: add mm... |
457 |
trace_xen_mmu_pmd_clear(pmdp); |
e2426cf85 xen: avoid hyperc... |
458 |
set_pmd(pmdp, __pmd(0)); |
3b827c1b3 xen: virtual mmu |
459 |
} |
f6e587325 xen64: add extra ... |
460 |
#endif /* CONFIG_X86_PAE */ |
3b827c1b3 xen: virtual mmu |
461 |
|
a2e7f0e3a x86, asmlinkage, ... |
462 |
__visible pmd_t xen_make_pmd(pmdval_t pmd) |
3b827c1b3 xen: virtual mmu |
463 |
{ |
ebb9cfe20 xen: don't drop N... |
464 |
pmd = pte_pfn_to_mfn(pmd); |
947a69c90 xen: unify pte op... |
465 |
return native_make_pmd(pmd); |
3b827c1b3 xen: virtual mmu |
466 |
} |
da5de7c22 x86/paravirt: use... |
467 |
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd); |
3b827c1b3 xen: virtual mmu |
468 |
|
982333683 x86: expose numbe... |
469 |
#if CONFIG_PGTABLE_LEVELS == 4 |
a2e7f0e3a x86, asmlinkage, ... |
470 |
__visible pudval_t xen_pud_val(pud_t pud) |
f6e587325 xen64: add extra ... |
471 472 473 |
{ return pte_mfn_to_pfn(pud.pud); } |
da5de7c22 x86/paravirt: use... |
474 |
PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val); |
f6e587325 xen64: add extra ... |
475 |
|
a2e7f0e3a x86, asmlinkage, ... |
476 |
__visible pud_t xen_make_pud(pudval_t pud) |
f6e587325 xen64: add extra ... |
477 478 479 480 481 |
{ pud = pte_pfn_to_mfn(pud); return native_make_pud(pud); } |
da5de7c22 x86/paravirt: use... |
482 |
PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud); |
f6e587325 xen64: add extra ... |
483 |
|
4c13629f8 xen: make a pile ... |
484 |
static pgd_t *xen_get_user_pgd(pgd_t *pgd) |
f6e587325 xen64: add extra ... |
485 |
{ |
d6182fbf0 xen64: allocate a... |
486 487 488 |
pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK); unsigned offset = pgd - pgd_page; pgd_t *user_ptr = NULL; |
f6e587325 xen64: add extra ... |
489 |
|
d6182fbf0 xen64: allocate a... |
490 491 492 493 494 495 |
if (offset < pgd_index(USER_LIMIT)) { struct page *page = virt_to_page(pgd_page); user_ptr = (pgd_t *)page->private; if (user_ptr) user_ptr += offset; } |
f6e587325 xen64: add extra ... |
496 |
|
d6182fbf0 xen64: allocate a... |
497 498 499 500 501 502 |
return user_ptr; } static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val) { struct mmu_update u; |
f6e587325 xen64: add extra ... |
503 504 505 |
u.ptr = virt_to_machine(ptr).maddr; u.val = pgd_val_ma(val); |
7708ad64a xen: add xen_ pre... |
506 |
xen_extend_mmu_update(&u); |
d6182fbf0 xen64: allocate a... |
507 508 509 510 511 512 513 514 515 |
} /* * Raw hypercall-based set_pgd, intended for in early boot before * there's a page structure. This implies: * 1. The only existing pagetable is the kernel's * 2. It is always pinned * 3. It has no user pagetable attached to it */ |
4c13629f8 xen: make a pile ... |
516 |
static void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val) |
d6182fbf0 xen64: allocate a... |
517 518 519 520 521 522 |
{ preempt_disable(); xen_mc_batch(); __xen_set_pgd_hyper(ptr, val); |
f6e587325 xen64: add extra ... |
523 524 525 526 527 |
xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); } |
4c13629f8 xen: make a pile ... |
528 |
static void xen_set_pgd(pgd_t *ptr, pgd_t val) |
f6e587325 xen64: add extra ... |
529 |
{ |
d6182fbf0 xen64: allocate a... |
530 |
pgd_t *user_ptr = xen_get_user_pgd(ptr); |
847088079 xen/trace: add mm... |
531 |
trace_xen_mmu_set_pgd(ptr, user_ptr, val); |
f6e587325 xen64: add extra ... |
532 533 |
/* If page is not pinned, we can just update the entry directly */ |
7708ad64a xen: add xen_ pre... |
534 |
if (!xen_page_pinned(ptr)) { |
f6e587325 xen64: add extra ... |
535 |
*ptr = val; |
d6182fbf0 xen64: allocate a... |
536 |
if (user_ptr) { |
7708ad64a xen: add xen_ pre... |
537 |
WARN_ON(xen_page_pinned(user_ptr)); |
d6182fbf0 xen64: allocate a... |
538 539 |
*user_ptr = val; } |
f6e587325 xen64: add extra ... |
540 541 |
return; } |
d6182fbf0 xen64: allocate a... |
542 543 544 545 546 547 548 549 550 |
/* If it's pinned, then we can at least batch the kernel and user updates together. */ xen_mc_batch(); __xen_set_pgd_hyper(ptr, val); if (user_ptr) __xen_set_pgd_hyper(user_ptr, val); xen_mc_issue(PARAVIRT_LAZY_MMU); |
f6e587325 xen64: add extra ... |
551 |
} |
982333683 x86: expose numbe... |
552 |
#endif /* CONFIG_PGTABLE_LEVELS == 4 */ |
f6e587325 xen64: add extra ... |
553 |
|
f4f97b3ea xen: Complete pag... |
554 |
/* |
5deb30d19 xen: rework pgd_w... |
555 556 557 558 559 560 561 562 563 564 565 566 567 568 |
* (Yet another) pagetable walker. This one is intended for pinning a * pagetable. This means that it walks a pagetable and calls the * callback function on each page it finds making up the page table, * at every level. It walks the entire pagetable, but it only bothers * pinning pte pages which are below limit. In the normal case this * will be STACK_TOP_MAX, but at boot we need to pin up to * FIXADDR_TOP. * * For 32-bit the important bit is that we don't pin beyond there, * because then we start getting into Xen's ptes. * * For 64-bit, we must skip the Xen hole in the middle of the address * space, just after the big x86-64 virtual hole. */ |
86bbc2c23 xen: pin correct ... |
569 570 571 572 |
static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd, int (*func)(struct mm_struct *mm, struct page *, enum pt_level), unsigned long limit) |
3b827c1b3 xen: virtual mmu |
573 |
{ |
f4f97b3ea xen: Complete pag... |
574 |
int flush = 0; |
5deb30d19 xen: rework pgd_w... |
575 576 577 |
unsigned hole_low, hole_high; unsigned pgdidx_limit, pudidx_limit, pmdidx_limit; unsigned pgdidx, pudidx, pmdidx; |
f4f97b3ea xen: Complete pag... |
578 |
|
5deb30d19 xen: rework pgd_w... |
579 580 581 |
/* The limit is the last byte to be touched */ limit--; BUG_ON(limit >= FIXADDR_TOP); |
3b827c1b3 xen: virtual mmu |
582 583 |
if (xen_feature(XENFEAT_auto_translated_physmap)) |
f4f97b3ea xen: Complete pag... |
584 |
return 0; |
5deb30d19 xen: rework pgd_w... |
585 586 587 588 589 |
/* * 64-bit has a great big hole in the middle of the address * space, which contains the Xen mappings. On 32-bit these * will end up making a zero-sized hole and so is a no-op. */ |
d6182fbf0 xen64: allocate a... |
590 |
hole_low = pgd_index(USER_LIMIT); |
5deb30d19 xen: rework pgd_w... |
591 592 593 594 595 596 597 598 599 600 601 602 603 |
hole_high = pgd_index(PAGE_OFFSET); pgdidx_limit = pgd_index(limit); #if PTRS_PER_PUD > 1 pudidx_limit = pud_index(limit); #else pudidx_limit = 0; #endif #if PTRS_PER_PMD > 1 pmdidx_limit = pmd_index(limit); #else pmdidx_limit = 0; #endif |
5deb30d19 xen: rework pgd_w... |
604 |
for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) { |
f4f97b3ea xen: Complete pag... |
605 |
pud_t *pud; |
3b827c1b3 xen: virtual mmu |
606 |
|
5deb30d19 xen: rework pgd_w... |
607 608 |
if (pgdidx >= hole_low && pgdidx < hole_high) continue; |
f4f97b3ea xen: Complete pag... |
609 |
|
5deb30d19 xen: rework pgd_w... |
610 |
if (!pgd_val(pgd[pgdidx])) |
3b827c1b3 xen: virtual mmu |
611 |
continue; |
f4f97b3ea xen: Complete pag... |
612 |
|
5deb30d19 xen: rework pgd_w... |
613 |
pud = pud_offset(&pgd[pgdidx], 0); |
3b827c1b3 xen: virtual mmu |
614 615 |
if (PTRS_PER_PUD > 1) /* not folded */ |
eefb47f6a xen: use spin_loc... |
616 |
flush |= (*func)(mm, virt_to_page(pud), PT_PUD); |
f4f97b3ea xen: Complete pag... |
617 |
|
5deb30d19 xen: rework pgd_w... |
618 |
for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) { |
f4f97b3ea xen: Complete pag... |
619 |
pmd_t *pmd; |
f4f97b3ea xen: Complete pag... |
620 |
|
5deb30d19 xen: rework pgd_w... |
621 622 623 |
if (pgdidx == pgdidx_limit && pudidx > pudidx_limit) goto out; |
3b827c1b3 xen: virtual mmu |
624 |
|
5deb30d19 xen: rework pgd_w... |
625 |
if (pud_none(pud[pudidx])) |
3b827c1b3 xen: virtual mmu |
626 |
continue; |
f4f97b3ea xen: Complete pag... |
627 |
|
5deb30d19 xen: rework pgd_w... |
628 |
pmd = pmd_offset(&pud[pudidx], 0); |
3b827c1b3 xen: virtual mmu |
629 630 |
if (PTRS_PER_PMD > 1) /* not folded */ |
eefb47f6a xen: use spin_loc... |
631 |
flush |= (*func)(mm, virt_to_page(pmd), PT_PMD); |
f4f97b3ea xen: Complete pag... |
632 |
|
5deb30d19 xen: rework pgd_w... |
633 634 635 636 637 638 639 |
for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) { struct page *pte; if (pgdidx == pgdidx_limit && pudidx == pudidx_limit && pmdidx > pmdidx_limit) goto out; |
3b827c1b3 xen: virtual mmu |
640 |
|
5deb30d19 xen: rework pgd_w... |
641 |
if (pmd_none(pmd[pmdidx])) |
3b827c1b3 xen: virtual mmu |
642 |
continue; |
5deb30d19 xen: rework pgd_w... |
643 |
pte = pmd_page(pmd[pmdidx]); |
eefb47f6a xen: use spin_loc... |
644 |
flush |= (*func)(mm, pte, PT_PTE); |
3b827c1b3 xen: virtual mmu |
645 646 647 |
} } } |
11ad93e59 xen: clarify lock... |
648 |
|
5deb30d19 xen: rework pgd_w... |
649 |
out: |
11ad93e59 xen: clarify lock... |
650 651 |
/* Do the top level last, so that the callbacks can use it as a cue to do final things like tlb flushes. */ |
eefb47f6a xen: use spin_loc... |
652 |
flush |= (*func)(mm, virt_to_page(pgd), PT_PGD); |
f4f97b3ea xen: Complete pag... |
653 654 |
return flush; |
3b827c1b3 xen: virtual mmu |
655 |
} |
86bbc2c23 xen: pin correct ... |
656 657 658 659 660 661 662 |
static int xen_pgd_walk(struct mm_struct *mm, int (*func)(struct mm_struct *mm, struct page *, enum pt_level), unsigned long limit) { return __xen_pgd_walk(mm, mm->pgd, func, limit); } |
7708ad64a xen: add xen_ pre... |
663 664 |
/* If we're using split pte locks, then take the page's lock and return a pointer to it. Otherwise return NULL. */ |
eefb47f6a xen: use spin_loc... |
665 |
static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm) |
74260714c xen: lock pte pag... |
666 667 |
{ spinlock_t *ptl = NULL; |
57c1ffcef mm: rename USE_SP... |
668 |
#if USE_SPLIT_PTE_PTLOCKS |
49076ec2c mm: dynamically a... |
669 |
ptl = ptlock_ptr(page); |
eefb47f6a xen: use spin_loc... |
670 |
spin_lock_nest_lock(ptl, &mm->page_table_lock); |
74260714c xen: lock pte pag... |
671 672 673 674 |
#endif return ptl; } |
7708ad64a xen: add xen_ pre... |
675 |
static void xen_pte_unlock(void *v) |
74260714c xen: lock pte pag... |
676 677 678 679 680 681 682 |
{ spinlock_t *ptl = v; spin_unlock(ptl); } static void xen_do_pin(unsigned level, unsigned long pfn) { |
dcf7435cf xen/mmu: use exte... |
683 |
struct mmuext_op op; |
74260714c xen: lock pte pag... |
684 |
|
dcf7435cf xen/mmu: use exte... |
685 686 687 688 |
op.cmd = level; op.arg1.mfn = pfn_to_mfn(pfn); xen_extend_mmuext_op(&op); |
74260714c xen: lock pte pag... |
689 |
} |
eefb47f6a xen: use spin_loc... |
690 691 |
static int xen_pin_page(struct mm_struct *mm, struct page *page, enum pt_level level) |
f4f97b3ea xen: Complete pag... |
692 |
{ |
d60cd46bb pageflags: use pr... |
693 |
unsigned pgfl = TestSetPagePinned(page); |
f4f97b3ea xen: Complete pag... |
694 695 696 697 698 699 700 701 702 703 704 705 |
int flush; if (pgfl) flush = 0; /* already pinned */ else if (PageHighMem(page)) /* kmaps need flushing if we found an unpinned highpage */ flush = 1; else { void *pt = lowmem_page_address(page); unsigned long pfn = page_to_pfn(page); struct multicall_space mcs = __xen_mc_entry(0); |
74260714c xen: lock pte pag... |
706 |
spinlock_t *ptl; |
f4f97b3ea xen: Complete pag... |
707 708 |
flush = 0; |
11ad93e59 xen: clarify lock... |
709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 |
/* * We need to hold the pagetable lock between the time * we make the pagetable RO and when we actually pin * it. If we don't, then other users may come in and * attempt to update the pagetable by writing it, * which will fail because the memory is RO but not * pinned, so Xen won't do the trap'n'emulate. * * If we're using split pte locks, we can't hold the * entire pagetable's worth of locks during the * traverse, because we may wrap the preempt count (8 * bits). The solution is to mark RO and pin each PTE * page while holding the lock. This means the number * of locks we end up holding is never more than a * batch size (~32 entries, at present). * * If we're not using split pte locks, we needn't pin * the PTE pages independently, because we're * protected by the overall pagetable lock. */ |
74260714c xen: lock pte pag... |
729 730 |
ptl = NULL; if (level == PT_PTE) |
eefb47f6a xen: use spin_loc... |
731 |
ptl = xen_pte_lock(page, mm); |
74260714c xen: lock pte pag... |
732 |
|
f4f97b3ea xen: Complete pag... |
733 734 |
MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, pfn_pte(pfn, PAGE_KERNEL_RO), |
74260714c xen: lock pte pag... |
735 |
level == PT_PGD ? UVMF_TLB_FLUSH : 0); |
11ad93e59 xen: clarify lock... |
736 |
if (ptl) { |
74260714c xen: lock pte pag... |
737 |
xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn); |
74260714c xen: lock pte pag... |
738 739 |
/* Queue a deferred unlock for when this batch is completed. */ |
7708ad64a xen: add xen_ pre... |
740 |
xen_mc_callback(xen_pte_unlock, ptl); |
74260714c xen: lock pte pag... |
741 |
} |
f4f97b3ea xen: Complete pag... |
742 743 744 745 |
} return flush; } |
3b827c1b3 xen: virtual mmu |
746 |
|
f4f97b3ea xen: Complete pag... |
747 748 749 |
/* This is called just after a mm has been created, but it has not been used yet. We need to make sure that its pagetable is all read-only, and can be pinned. */ |
eefb47f6a xen: use spin_loc... |
750 |
static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd) |
3b827c1b3 xen: virtual mmu |
751 |
{ |
5f94fb5b8 xen/trace: add xe... |
752 |
trace_xen_mmu_pgd_pin(mm, pgd); |
f4f97b3ea xen: Complete pag... |
753 |
xen_mc_batch(); |
3b827c1b3 xen: virtual mmu |
754 |
|
86bbc2c23 xen: pin correct ... |
755 |
if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) { |
d05fdf316 xen: make sure st... |
756 |
/* re-enable interrupts for flushing */ |
f87e4cac4 xen: SMP guest su... |
757 |
xen_mc_issue(0); |
d05fdf316 xen: make sure st... |
758 |
|
f4f97b3ea xen: Complete pag... |
759 |
kmap_flush_unused(); |
d05fdf316 xen: make sure st... |
760 |
|
f87e4cac4 xen: SMP guest su... |
761 762 |
xen_mc_batch(); } |
f4f97b3ea xen: Complete pag... |
763 |
|
d6182fbf0 xen64: allocate a... |
764 765 766 767 768 769 770 |
#ifdef CONFIG_X86_64 { pgd_t *user_pgd = xen_get_user_pgd(pgd); xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd))); if (user_pgd) { |
eefb47f6a xen: use spin_loc... |
771 |
xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD); |
f63c2f248 xen: whitespace/c... |
772 773 |
xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd))); |
d6182fbf0 xen64: allocate a... |
774 775 776 |
} } #else /* CONFIG_X86_32 */ |
5deb30d19 xen: rework pgd_w... |
777 778 |
#ifdef CONFIG_X86_PAE /* Need to make sure unshared kernel PMD is pinnable */ |
47cb2ed9d x86, xen: fix use... |
779 |
xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), |
eefb47f6a xen: use spin_loc... |
780 |
PT_PMD); |
5deb30d19 xen: rework pgd_w... |
781 |
#endif |
284991439 xen: remove suppo... |
782 |
xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd))); |
d6182fbf0 xen64: allocate a... |
783 |
#endif /* CONFIG_X86_64 */ |
f4f97b3ea xen: Complete pag... |
784 |
xen_mc_issue(0); |
3b827c1b3 xen: virtual mmu |
785 |
} |
eefb47f6a xen: use spin_loc... |
786 787 788 789 |
static void xen_pgd_pin(struct mm_struct *mm) { __xen_pgd_pin(mm, mm->pgd); } |
0e91398f2 xen: implement sa... |
790 791 792 793 794 |
/* * On save, we need to pin all pagetables to make sure they get their * mfns turned into pfns. Search the list for any unpinned pgds and pin * them (unpinned pgds are not currently in use, probably because the * process is under construction or destruction). |
eefb47f6a xen: use spin_loc... |
795 796 797 798 |
* * Expected to be called in stop_machine() ("equivalent to taking * every spinlock in the system"), so the locking doesn't really * matter all that much. |
0e91398f2 xen: implement sa... |
799 800 801 |
*/ void xen_mm_pin_all(void) { |
0e91398f2 xen: implement sa... |
802 |
struct page *page; |
74260714c xen: lock pte pag... |
803 |
|
a79e53d85 x86/mm: Fix pgd_l... |
804 |
spin_lock(&pgd_lock); |
f4f97b3ea xen: Complete pag... |
805 |
|
0e91398f2 xen: implement sa... |
806 807 |
list_for_each_entry(page, &pgd_list, lru) { if (!PagePinned(page)) { |
eefb47f6a xen: use spin_loc... |
808 |
__xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); |
0e91398f2 xen: implement sa... |
809 810 811 |
SetPageSavePinned(page); } } |
a79e53d85 x86/mm: Fix pgd_l... |
812 |
spin_unlock(&pgd_lock); |
3b827c1b3 xen: virtual mmu |
813 |
} |
c1f2f09ef pvops-64: call pa... |
814 815 816 817 818 |
/* * The init_mm pagetable is really pinned as soon as its created, but * that's before we have page structures to store the bits. So do all * the book-keeping now. */ |
3f508953d arch/x86/xen/mmu:... |
819 |
static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page, |
eefb47f6a xen: use spin_loc... |
820 |
enum pt_level level) |
3b827c1b3 xen: virtual mmu |
821 |
{ |
f4f97b3ea xen: Complete pag... |
822 823 824 |
SetPagePinned(page); return 0; } |
3b827c1b3 xen: virtual mmu |
825 |
|
b96229b50 xen/mmu: some ear... |
826 |
static void __init xen_mark_init_mm_pinned(void) |
f4f97b3ea xen: Complete pag... |
827 |
{ |
eefb47f6a xen: use spin_loc... |
828 |
xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP); |
f4f97b3ea xen: Complete pag... |
829 |
} |
3b827c1b3 xen: virtual mmu |
830 |
|
eefb47f6a xen: use spin_loc... |
831 832 |
static int xen_unpin_page(struct mm_struct *mm, struct page *page, enum pt_level level) |
f4f97b3ea xen: Complete pag... |
833 |
{ |
d60cd46bb pageflags: use pr... |
834 |
unsigned pgfl = TestClearPagePinned(page); |
3b827c1b3 xen: virtual mmu |
835 |
|
f4f97b3ea xen: Complete pag... |
836 837 838 |
if (pgfl && !PageHighMem(page)) { void *pt = lowmem_page_address(page); unsigned long pfn = page_to_pfn(page); |
74260714c xen: lock pte pag... |
839 840 |
spinlock_t *ptl = NULL; struct multicall_space mcs; |
11ad93e59 xen: clarify lock... |
841 842 843 844 845 846 847 |
/* * Do the converse to pin_page. If we're using split * pte locks, we must be holding the lock for while * the pte page is unpinned but still RO to prevent * concurrent updates from seeing it in this * partially-pinned state. */ |
74260714c xen: lock pte pag... |
848 |
if (level == PT_PTE) { |
eefb47f6a xen: use spin_loc... |
849 |
ptl = xen_pte_lock(page, mm); |
74260714c xen: lock pte pag... |
850 |
|
11ad93e59 xen: clarify lock... |
851 852 |
if (ptl) xen_do_pin(MMUEXT_UNPIN_TABLE, pfn); |
74260714c xen: lock pte pag... |
853 854 855 |
} mcs = __xen_mc_entry(0); |
f4f97b3ea xen: Complete pag... |
856 857 858 |
MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, pfn_pte(pfn, PAGE_KERNEL), |
74260714c xen: lock pte pag... |
859 860 861 862 |
level == PT_PGD ? UVMF_TLB_FLUSH : 0); if (ptl) { /* unlock when batch completed */ |
7708ad64a xen: add xen_ pre... |
863 |
xen_mc_callback(xen_pte_unlock, ptl); |
74260714c xen: lock pte pag... |
864 |
} |
f4f97b3ea xen: Complete pag... |
865 866 867 |
} return 0; /* never need to flush on unpin */ |
3b827c1b3 xen: virtual mmu |
868 |
} |
f4f97b3ea xen: Complete pag... |
869 |
/* Release a pagetables pages back as normal RW */ |
eefb47f6a xen: use spin_loc... |
870 |
static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd) |
f4f97b3ea xen: Complete pag... |
871 |
{ |
5f94fb5b8 xen/trace: add xe... |
872 |
trace_xen_mmu_pgd_unpin(mm, pgd); |
f4f97b3ea xen: Complete pag... |
873 |
xen_mc_batch(); |
74260714c xen: lock pte pag... |
874 |
xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); |
f4f97b3ea xen: Complete pag... |
875 |
|
d6182fbf0 xen64: allocate a... |
876 877 878 879 880 |
#ifdef CONFIG_X86_64 { pgd_t *user_pgd = xen_get_user_pgd(pgd); if (user_pgd) { |
f63c2f248 xen: whitespace/c... |
881 882 |
xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd))); |
eefb47f6a xen: use spin_loc... |
883 |
xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD); |
d6182fbf0 xen64: allocate a... |
884 885 886 |
} } #endif |
5deb30d19 xen: rework pgd_w... |
887 888 |
#ifdef CONFIG_X86_PAE /* Need to make sure unshared kernel PMD is unpinned */ |
47cb2ed9d x86, xen: fix use... |
889 |
xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), |
eefb47f6a xen: use spin_loc... |
890 |
PT_PMD); |
5deb30d19 xen: rework pgd_w... |
891 |
#endif |
d6182fbf0 xen64: allocate a... |
892 |
|
86bbc2c23 xen: pin correct ... |
893 |
__xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT); |
f4f97b3ea xen: Complete pag... |
894 895 896 |
xen_mc_issue(0); } |
3b827c1b3 xen: virtual mmu |
897 |
|
eefb47f6a xen: use spin_loc... |
898 899 900 901 |
static void xen_pgd_unpin(struct mm_struct *mm) { __xen_pgd_unpin(mm, mm->pgd); } |
0e91398f2 xen: implement sa... |
902 903 904 905 906 907 |
/* * On resume, undo any pinning done at save, so that the rest of the * kernel doesn't see any unexpected pinned pagetables. */ void xen_mm_unpin_all(void) { |
0e91398f2 xen: implement sa... |
908 |
struct page *page; |
a79e53d85 x86/mm: Fix pgd_l... |
909 |
spin_lock(&pgd_lock); |
0e91398f2 xen: implement sa... |
910 911 912 913 |
list_for_each_entry(page, &pgd_list, lru) { if (PageSavePinned(page)) { BUG_ON(!PagePinned(page)); |
eefb47f6a xen: use spin_loc... |
914 |
__xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); |
0e91398f2 xen: implement sa... |
915 916 917 |
ClearPageSavePinned(page); } } |
a79e53d85 x86/mm: Fix pgd_l... |
918 |
spin_unlock(&pgd_lock); |
0e91398f2 xen: implement sa... |
919 |
} |
4c13629f8 xen: make a pile ... |
920 |
static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next) |
3b827c1b3 xen: virtual mmu |
921 |
{ |
f4f97b3ea xen: Complete pag... |
922 |
spin_lock(&next->page_table_lock); |
eefb47f6a xen: use spin_loc... |
923 |
xen_pgd_pin(next); |
f4f97b3ea xen: Complete pag... |
924 |
spin_unlock(&next->page_table_lock); |
3b827c1b3 xen: virtual mmu |
925 |
} |
4c13629f8 xen: make a pile ... |
926 |
static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) |
3b827c1b3 xen: virtual mmu |
927 |
{ |
f4f97b3ea xen: Complete pag... |
928 |
spin_lock(&mm->page_table_lock); |
eefb47f6a xen: use spin_loc... |
929 |
xen_pgd_pin(mm); |
f4f97b3ea xen: Complete pag... |
930 |
spin_unlock(&mm->page_table_lock); |
3b827c1b3 xen: virtual mmu |
931 |
} |
3b827c1b3 xen: virtual mmu |
932 |
|
f87e4cac4 xen: SMP guest su... |
933 934 935 936 937 938 |
#ifdef CONFIG_SMP /* Another cpu may still have their %cr3 pointing at the pagetable, so we need to repoint it somewhere else before we can unpin it. */ static void drop_other_mm_ref(void *info) { struct mm_struct *mm = info; |
ce87b3d32 xen64: get active... |
939 |
struct mm_struct *active_mm; |
3b827c1b3 xen: virtual mmu |
940 |
|
2113f4691 xen: use this_cpu... |
941 |
active_mm = this_cpu_read(cpu_tlbstate.active_mm); |
ce87b3d32 xen64: get active... |
942 |
|
2113f4691 xen: use this_cpu... |
943 |
if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) |
f87e4cac4 xen: SMP guest su... |
944 |
leave_mm(smp_processor_id()); |
9f79991d4 xen: deal with st... |
945 946 947 |
/* If this cpu still has a stale cr3 reference, then make sure it has been flushed. */ |
2113f4691 xen: use this_cpu... |
948 |
if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd)) |
9f79991d4 xen: deal with st... |
949 |
load_cr3(swapper_pg_dir); |
f87e4cac4 xen: SMP guest su... |
950 |
} |
3b827c1b3 xen: virtual mmu |
951 |
|
7708ad64a xen: add xen_ pre... |
952 |
static void xen_drop_mm_ref(struct mm_struct *mm) |
f87e4cac4 xen: SMP guest su... |
953 |
{ |
e4d98207e x86: xen: use smp... |
954 |
cpumask_var_t mask; |
9f79991d4 xen: deal with st... |
955 |
unsigned cpu; |
f87e4cac4 xen: SMP guest su... |
956 957 958 959 960 |
if (current->active_mm == mm) { if (current->mm == mm) load_cr3(swapper_pg_dir); else leave_mm(smp_processor_id()); |
9f79991d4 xen: deal with st... |
961 962 963 |
} /* Get the "official" set of cpus referring to our pagetable. */ |
e4d98207e x86: xen: use smp... |
964 965 |
if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) { for_each_online_cpu(cpu) { |
78f1c4d6b cpumask: use mm_c... |
966 |
if (!cpumask_test_cpu(cpu, mm_cpumask(mm)) |
e4d98207e x86: xen: use smp... |
967 968 969 970 971 972 |
&& per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd)) continue; smp_call_function_single(cpu, drop_other_mm_ref, mm, 1); } return; } |
78f1c4d6b cpumask: use mm_c... |
973 |
cpumask_copy(mask, mm_cpumask(mm)); |
9f79991d4 xen: deal with st... |
974 975 976 977 978 979 980 981 |
/* It's possible that a vcpu may have a stale reference to our cr3, because its in lazy mode, and it hasn't yet flushed its set of pending hypercalls yet. In this case, we can look at its actual current cr3 value, and force it to flush if needed. */ for_each_online_cpu(cpu) { if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd)) |
e4d98207e x86: xen: use smp... |
982 |
cpumask_set_cpu(cpu, mask); |
3b827c1b3 xen: virtual mmu |
983 |
} |
e4d98207e x86: xen: use smp... |
984 985 986 |
if (!cpumask_empty(mask)) smp_call_function_many(mask, drop_other_mm_ref, mm, 1); free_cpumask_var(mask); |
f87e4cac4 xen: SMP guest su... |
987 988 |
} #else |
7708ad64a xen: add xen_ pre... |
989 |
static void xen_drop_mm_ref(struct mm_struct *mm) |
f87e4cac4 xen: SMP guest su... |
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 |
{ if (current->active_mm == mm) load_cr3(swapper_pg_dir); } #endif /* * While a process runs, Xen pins its pagetables, which means that the * hypervisor forces it to be read-only, and it controls all updates * to it. This means that all pagetable updates have to go via the * hypervisor, which is moderately expensive. * * Since we're pulling the pagetable down, we switch to use init_mm, * unpin old process pagetable and mark it all read-write, which * allows further operations on it to be simple memory accesses. * * The only subtle point is that another CPU may be still using the * pagetable because of lazy tlb flushing. This means we need need to * switch all CPUs off this pagetable before we can unpin it. */ |
4c13629f8 xen: make a pile ... |
1010 |
static void xen_exit_mmap(struct mm_struct *mm) |
f87e4cac4 xen: SMP guest su... |
1011 1012 |
{ get_cpu(); /* make sure we don't move around */ |
7708ad64a xen: add xen_ pre... |
1013 |
xen_drop_mm_ref(mm); |
f87e4cac4 xen: SMP guest su... |
1014 |
put_cpu(); |
3b827c1b3 xen: virtual mmu |
1015 |
|
f120f13ea xen: Add support ... |
1016 |
spin_lock(&mm->page_table_lock); |
df912ea4a xen: execve's err... |
1017 1018 |
/* pgd may not be pinned in the error exit path of execve */ |
7708ad64a xen: add xen_ pre... |
1019 |
if (xen_page_pinned(mm->pgd)) |
eefb47f6a xen: use spin_loc... |
1020 |
xen_pgd_unpin(mm); |
74260714c xen: lock pte pag... |
1021 |
|
f120f13ea xen: Add support ... |
1022 |
spin_unlock(&mm->page_table_lock); |
3b827c1b3 xen: virtual mmu |
1023 |
} |
994025cab xen: add debugfs ... |
1024 |
|
c71128872 x86: xen: Cleanup... |
1025 |
static void xen_post_allocator_init(void); |
70e611995 xen: move p2m lis... |
1026 1027 1028 1029 1030 1031 1032 1033 1034 |
static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn) { struct mmuext_op op; op.cmd = cmd; op.arg1.mfn = pfn_to_mfn(pfn); if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) BUG(); } |
7f9140626 xen/mmu: Copy and... |
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 |
#ifdef CONFIG_X86_64 static void __init xen_cleanhighmap(unsigned long vaddr, unsigned long vaddr_end) { unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1; pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr); /* NOTE: The loop is more greedy than the cleanup_highmap variant. * We include the PMD passed in on _both_ boundaries. */ for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PAGE_SIZE)); pmd++, vaddr += PMD_SIZE) { if (pmd_none(*pmd)) continue; if (vaddr < (unsigned long) _text || vaddr > kernel_end) set_pmd(pmd, __pmd(0)); } /* In case we did something silly, we should crash in this function * instead of somewhere later and be confusing. */ xen_mc_flush(); } |
054954eb0 xen: switch to li... |
1055 |
|
8f5b0c639 xen: eliminate sc... |
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 |
/* * Make a page range writeable and free it. */ static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size) { void *vaddr = __va(paddr); void *vaddr_end = vaddr + size; for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) make_lowmem_page_readwrite(vaddr); memblock_free(paddr, size); } |
70e611995 xen: move p2m lis... |
1069 |
static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin) |
8f5b0c639 xen: eliminate sc... |
1070 1071 |
{ unsigned long pa = __pa(pgtbl) & PHYSICAL_PAGE_MASK; |
70e611995 xen: move p2m lis... |
1072 1073 |
if (unpin) pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(pa)); |
8f5b0c639 xen: eliminate sc... |
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 |
ClearPagePinned(virt_to_page(__va(pa))); xen_free_ro_pages(pa, PAGE_SIZE); } /* * Since it is well isolated we can (and since it is perhaps large we should) * also free the page tables mapping the initial P->M table. */ static void __init xen_cleanmfnmap(unsigned long vaddr) { unsigned long va = vaddr & PMD_MASK; unsigned long pa; pgd_t *pgd = pgd_offset_k(va); pud_t *pud_page = pud_offset(pgd, 0); pud_t *pud; pmd_t *pmd; pte_t *pte; unsigned int i; |
70e611995 xen: move p2m lis... |
1092 |
bool unpin; |
8f5b0c639 xen: eliminate sc... |
1093 |
|
70e611995 xen: move p2m lis... |
1094 |
unpin = (vaddr == 2 * PGDIR_SIZE); |
8f5b0c639 xen: eliminate sc... |
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 |
set_pgd(pgd, __pgd(0)); do { pud = pud_page + pud_index(va); if (pud_none(*pud)) { va += PUD_SIZE; } else if (pud_large(*pud)) { pa = pud_val(*pud) & PHYSICAL_PAGE_MASK; xen_free_ro_pages(pa, PUD_SIZE); va += PUD_SIZE; } else { pmd = pmd_offset(pud, va); if (pmd_large(*pmd)) { pa = pmd_val(*pmd) & PHYSICAL_PAGE_MASK; xen_free_ro_pages(pa, PMD_SIZE); } else if (!pmd_none(*pmd)) { pte = pte_offset_kernel(pmd, va); |
70e611995 xen: move p2m lis... |
1111 |
set_pmd(pmd, __pmd(0)); |
8f5b0c639 xen: eliminate sc... |
1112 1113 1114 1115 1116 1117 |
for (i = 0; i < PTRS_PER_PTE; ++i) { if (pte_none(pte[i])) break; pa = pte_pfn(pte[i]) << PAGE_SHIFT; xen_free_ro_pages(pa, PAGE_SIZE); } |
70e611995 xen: move p2m lis... |
1118 |
xen_cleanmfnmap_free_pgtbl(pte, unpin); |
8f5b0c639 xen: eliminate sc... |
1119 1120 1121 1122 |
} va += PMD_SIZE; if (pmd_index(va)) continue; |
70e611995 xen: move p2m lis... |
1123 1124 |
set_pud(pud, __pud(0)); xen_cleanmfnmap_free_pgtbl(pmd, unpin); |
8f5b0c639 xen: eliminate sc... |
1125 1126 1127 |
} } while (pud_index(va) || pmd_index(va)); |
70e611995 xen: move p2m lis... |
1128 |
xen_cleanmfnmap_free_pgtbl(pud_page, unpin); |
8f5b0c639 xen: eliminate sc... |
1129 |
} |
054954eb0 xen: switch to li... |
1130 |
static void __init xen_pagetable_p2m_free(void) |
319f3ba52 xen: move remaini... |
1131 |
{ |
7f9140626 xen/mmu: Copy and... |
1132 1133 |
unsigned long size; unsigned long addr; |
32df75cd1 xen/mmu/p2m: Refa... |
1134 1135 |
size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); |
b621e157b xen/mmu: Cleanup ... |
1136 |
/* No memory or already called. */ |
054954eb0 xen: switch to li... |
1137 |
if ((unsigned long)xen_p2m_addr == xen_start_info->mfn_list) |
32df75cd1 xen/mmu/p2m: Refa... |
1138 |
return; |
7f9140626 xen/mmu: Copy and... |
1139 |
|
b621e157b xen/mmu: Cleanup ... |
1140 1141 |
/* using __ka address and sticking INVALID_P2M_ENTRY! */ memset((void *)xen_start_info->mfn_list, 0xff, size); |
b621e157b xen/mmu: Cleanup ... |
1142 |
addr = xen_start_info->mfn_list; |
8f5b0c639 xen: eliminate sc... |
1143 1144 1145 1146 1147 1148 1149 1150 |
/* * We could be in __ka space. * We roundup to the PMD, which means that if anybody at this stage is * using the __ka address of xen_start_info or * xen_start_info->shared_info they are in going to crash. Fortunatly * we have already revectored in xen_setup_kernel_pagetable and in * xen_setup_shared_info. */ |
b621e157b xen/mmu: Cleanup ... |
1151 |
size = roundup(size, PMD_SIZE); |
b621e157b xen/mmu: Cleanup ... |
1152 |
|
8f5b0c639 xen: eliminate sc... |
1153 1154 1155 1156 1157 1158 1159 1160 |
if (addr >= __START_KERNEL_map) { xen_cleanhighmap(addr, addr + size); size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); memblock_free(__pa(addr), size); } else { xen_cleanmfnmap(addr); } |
70e611995 xen: move p2m lis... |
1161 1162 1163 1164 1165 1166 |
} static void __init xen_pagetable_cleanhighmap(void) { unsigned long size; unsigned long addr; |
b621e157b xen/mmu: Cleanup ... |
1167 |
|
3aca7fbc8 xen/mmu: Remove f... |
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 |
/* At this stage, cleanup_highmap has already cleaned __ka space * from _brk_limit way up to the max_pfn_mapped (which is the end of * the ramdisk). We continue on, erasing PMD entries that point to page * tables - do note that they are accessible at this stage via __va. * For good measure we also round up to the PMD - which means that if * anybody is using __ka address to the initial boot-stack - and try * to use it - they are going to crash. The xen_start_info has been * taken care of already in xen_setup_kernel_pagetable. */ addr = xen_start_info->pt_base; size = roundup(xen_start_info->nr_pt_frames * PAGE_SIZE, PMD_SIZE); xen_cleanhighmap(addr, addr + size); xen_start_info->pt_base = (unsigned long)__va(__pa(xen_start_info->pt_base)); #ifdef DEBUG /* This is superflous and is not neccessary, but you know what * lets do it. The MODULES_VADDR -> MODULES_END should be clear of * anything at this stage. */ xen_cleanhighmap(MODULES_VADDR, roundup(MODULES_VADDR, PUD_SIZE) - 1); #endif |
32df75cd1 xen/mmu/p2m: Refa... |
1187 1188 |
} #endif |
054954eb0 xen: switch to li... |
1189 |
static void __init xen_pagetable_p2m_setup(void) |
32df75cd1 xen/mmu/p2m: Refa... |
1190 |
{ |
054954eb0 xen: switch to li... |
1191 1192 1193 1194 |
if (xen_feature(XENFEAT_auto_translated_physmap)) return; xen_vmalloc_p2m_tree(); |
32df75cd1 xen/mmu/p2m: Refa... |
1195 |
#ifdef CONFIG_X86_64 |
054954eb0 xen: switch to li... |
1196 |
xen_pagetable_p2m_free(); |
70e611995 xen: move p2m lis... |
1197 1198 |
xen_pagetable_cleanhighmap(); |
7f9140626 xen/mmu: Copy and... |
1199 |
#endif |
054954eb0 xen: switch to li... |
1200 1201 1202 1203 1204 1205 1206 |
/* And revector! Bye bye old array */ xen_start_info->mfn_list = (unsigned long)xen_p2m_addr; } static void __init xen_pagetable_init(void) { paging_init(); |
cdfa0badf xen: switch to po... |
1207 |
xen_post_allocator_init(); |
054954eb0 xen: switch to li... |
1208 1209 |
xen_pagetable_p2m_setup(); |
2c185687a x86/xen: delay co... |
1210 1211 |
/* Allocate and initialize top and mid mfn levels for p2m structure */ xen_build_mfn_list_list(); |
1f3ac86b4 xen: Delay remapp... |
1212 1213 1214 |
/* Remap memory freed due to conflicts with E820 map */ if (!xen_feature(XENFEAT_auto_translated_physmap)) xen_remap_memory(); |
2c185687a x86/xen: delay co... |
1215 |
xen_setup_shared_info(); |
319f3ba52 xen: move remaini... |
1216 |
} |
319f3ba52 xen: move remaini... |
1217 1218 |
static void xen_write_cr2(unsigned long cr2) { |
2113f4691 xen: use this_cpu... |
1219 |
this_cpu_read(xen_vcpu)->arch.cr2 = cr2; |
319f3ba52 xen: move remaini... |
1220 1221 1222 1223 |
} static unsigned long xen_read_cr2(void) { |
2113f4691 xen: use this_cpu... |
1224 |
return this_cpu_read(xen_vcpu)->arch.cr2; |
319f3ba52 xen: move remaini... |
1225 1226 1227 1228 |
} unsigned long xen_read_cr2_direct(void) { |
2113f4691 xen: use this_cpu... |
1229 |
return this_cpu_read(xen_vcpu_info.arch.cr2); |
319f3ba52 xen: move remaini... |
1230 |
} |
95a7d7689 xen/mmu: Use Xen ... |
1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 |
void xen_flush_tlb_all(void) { struct mmuext_op *op; struct multicall_space mcs; trace_xen_mmu_flush_tlb_all(0); preempt_disable(); mcs = xen_mc_entry(sizeof(*op)); op = mcs.args; op->cmd = MMUEXT_TLB_FLUSH_ALL; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); } |
319f3ba52 xen: move remaini... |
1250 1251 1252 1253 |
static void xen_flush_tlb(void) { struct mmuext_op *op; struct multicall_space mcs; |
c8eed1719 xen/trace: add tl... |
1254 |
trace_xen_mmu_flush_tlb(0); |
319f3ba52 xen: move remaini... |
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 |
preempt_disable(); mcs = xen_mc_entry(sizeof(*op)); op = mcs.args; op->cmd = MMUEXT_TLB_FLUSH_LOCAL; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); } static void xen_flush_tlb_single(unsigned long addr) { struct mmuext_op *op; struct multicall_space mcs; |
c8eed1719 xen/trace: add tl... |
1272 |
trace_xen_mmu_flush_tlb_single(addr); |
319f3ba52 xen: move remaini... |
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 |
preempt_disable(); mcs = xen_mc_entry(sizeof(*op)); op = mcs.args; op->cmd = MMUEXT_INVLPG_LOCAL; op->arg1.linear_addr = addr & PAGE_MASK; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); xen_mc_issue(PARAVIRT_LAZY_MMU); preempt_enable(); } static void xen_flush_tlb_others(const struct cpumask *cpus, |
e7b52ffd4 x86/flush_tlb: tr... |
1287 1288 |
struct mm_struct *mm, unsigned long start, unsigned long end) |
319f3ba52 xen: move remaini... |
1289 1290 1291 |
{ struct { struct mmuext_op op; |
32dd11942 xen/mmu: Fix for ... |
1292 |
#ifdef CONFIG_SMP |
900cba888 xen: support CONF... |
1293 |
DECLARE_BITMAP(mask, num_processors); |
32dd11942 xen/mmu: Fix for ... |
1294 1295 1296 |
#else DECLARE_BITMAP(mask, NR_CPUS); #endif |
319f3ba52 xen: move remaini... |
1297 1298 |
} *args; struct multicall_space mcs; |
e7b52ffd4 x86/flush_tlb: tr... |
1299 |
trace_xen_mmu_flush_tlb_others(cpus, mm, start, end); |
c8eed1719 xen/trace: add tl... |
1300 |
|
e3f8a74e3 xen/mmu: weaken f... |
1301 1302 |
if (cpumask_empty(cpus)) return; /* nothing to do */ |
319f3ba52 xen: move remaini... |
1303 1304 1305 1306 1307 1308 1309 1310 |
mcs = xen_mc_entry(sizeof(*args)); args = mcs.args; args->op.arg2.vcpumask = to_cpumask(args->mask); /* Remove us, and any offline CPUS. */ cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask)); |
319f3ba52 xen: move remaini... |
1311 |
|
e7b52ffd4 x86/flush_tlb: tr... |
1312 |
args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; |
ce7184bdb xen: fix logical ... |
1313 |
if (end != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) { |
319f3ba52 xen: move remaini... |
1314 |
args->op.cmd = MMUEXT_INVLPG_MULTI; |
e7b52ffd4 x86/flush_tlb: tr... |
1315 |
args->op.arg1.linear_addr = start; |
319f3ba52 xen: move remaini... |
1316 1317 1318 |
} MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); |
319f3ba52 xen: move remaini... |
1319 1320 1321 1322 1323 |
xen_mc_issue(PARAVIRT_LAZY_MMU); } static unsigned long xen_read_cr3(void) { |
2113f4691 xen: use this_cpu... |
1324 |
return this_cpu_read(xen_cr3); |
319f3ba52 xen: move remaini... |
1325 1326 1327 1328 |
} static void set_current_cr3(void *v) { |
2113f4691 xen: use this_cpu... |
1329 |
this_cpu_write(xen_current_cr3, (unsigned long)v); |
319f3ba52 xen: move remaini... |
1330 1331 1332 1333 |
} static void __xen_write_cr3(bool kernel, unsigned long cr3) { |
dcf7435cf xen/mmu: use exte... |
1334 |
struct mmuext_op op; |
319f3ba52 xen: move remaini... |
1335 |
unsigned long mfn; |
c8eed1719 xen/trace: add tl... |
1336 |
trace_xen_mmu_write_cr3(kernel, cr3); |
319f3ba52 xen: move remaini... |
1337 1338 1339 1340 1341 1342 |
if (cr3) mfn = pfn_to_mfn(PFN_DOWN(cr3)); else mfn = 0; WARN_ON(mfn == 0 && kernel); |
dcf7435cf xen/mmu: use exte... |
1343 1344 |
op.cmd = kernel ? MMUEXT_NEW_BASEPTR : MMUEXT_NEW_USER_BASEPTR; op.arg1.mfn = mfn; |
319f3ba52 xen: move remaini... |
1345 |
|
dcf7435cf xen/mmu: use exte... |
1346 |
xen_extend_mmuext_op(&op); |
319f3ba52 xen: move remaini... |
1347 1348 |
if (kernel) { |
2113f4691 xen: use this_cpu... |
1349 |
this_cpu_write(xen_cr3, cr3); |
319f3ba52 xen: move remaini... |
1350 1351 1352 1353 1354 1355 |
/* Update xen_current_cr3 once the batch has actually been submitted. */ xen_mc_callback(set_current_cr3, (void *)cr3); } } |
319f3ba52 xen: move remaini... |
1356 1357 1358 1359 1360 1361 1362 1363 |
static void xen_write_cr3(unsigned long cr3) { BUG_ON(preemptible()); xen_mc_batch(); /* disables interrupts */ /* Update while interrupts are disabled, so its atomic with respect to ipis */ |
2113f4691 xen: use this_cpu... |
1364 |
this_cpu_write(xen_cr3, cr3); |
319f3ba52 xen: move remaini... |
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 |
__xen_write_cr3(true, cr3); #ifdef CONFIG_X86_64 { pgd_t *user_pgd = xen_get_user_pgd(__va(cr3)); if (user_pgd) __xen_write_cr3(false, __pa(user_pgd)); else __xen_write_cr3(false, 0); } #endif xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */ } |
0cc9129d7 x86-64, xen, mmu:... |
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 |
#ifdef CONFIG_X86_64 /* * At the start of the day - when Xen launches a guest, it has already * built pagetables for the guest. We diligently look over them * in xen_setup_kernel_pagetable and graft as appropiate them in the * init_level4_pgt and its friends. Then when we are happy we load * the new init_level4_pgt - and continue on. * * The generic code starts (start_kernel) and 'init_mem_mapping' sets * up the rest of the pagetables. When it has completed it loads the cr3. * N.B. that baremetal would start at 'start_kernel' (and the early * #PF handler would create bootstrap pagetables) - so we are running * with the same assumptions as what to do when write_cr3 is executed * at this point. * * Since there are no user-page tables at all, we have two variants * of xen_write_cr3 - the early bootup (this one), and the late one * (xen_write_cr3). The reason we have to do that is that in 64-bit * the Linux kernel and user-space are both in ring 3 while the * hypervisor is in ring 0. */ static void __init xen_write_cr3_init(unsigned long cr3) { BUG_ON(preemptible()); xen_mc_batch(); /* disables interrupts */ /* Update while interrupts are disabled, so its atomic with respect to ipis */ this_cpu_write(xen_cr3, cr3); __xen_write_cr3(true, cr3); xen_mc_issue(PARAVIRT_LAZY_CPU); /* interrupts restored */ |
0cc9129d7 x86-64, xen, mmu:... |
1414 1415 |
} #endif |
319f3ba52 xen: move remaini... |
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 |
static int xen_pgd_alloc(struct mm_struct *mm) { pgd_t *pgd = mm->pgd; int ret = 0; BUG_ON(PagePinned(virt_to_page(pgd))); #ifdef CONFIG_X86_64 { struct page *page = virt_to_page(pgd); pgd_t *user_pgd; BUG_ON(page->private != 0); ret = -ENOMEM; user_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); page->private = (unsigned long)user_pgd; if (user_pgd != NULL) { |
1ad83c858 x86_64,vsyscall: ... |
1436 |
#ifdef CONFIG_X86_VSYSCALL_EMULATION |
f40c33009 x86, vdso: Move t... |
1437 |
user_pgd[pgd_index(VSYSCALL_ADDR)] = |
319f3ba52 xen: move remaini... |
1438 |
__pgd(__pa(level3_user_vsyscall) | _PAGE_TABLE); |
1ad83c858 x86_64,vsyscall: ... |
1439 |
#endif |
319f3ba52 xen: move remaini... |
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 |
ret = 0; } BUG_ON(PagePinned(virt_to_page(xen_get_user_pgd(pgd)))); } #endif return ret; } static void xen_pgd_free(struct mm_struct *mm, pgd_t *pgd) { #ifdef CONFIG_X86_64 pgd_t *user_pgd = xen_get_user_pgd(pgd); if (user_pgd) free_page((unsigned long)user_pgd); #endif } |
ee176455e xen: mask_rw_pte:... |
1459 |
#ifdef CONFIG_X86_32 |
3f508953d arch/x86/xen/mmu:... |
1460 |
static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) |
1f4f93150 xen: fix 32-bit b... |
1461 1462 1463 1464 1465 |
{ /* If there's an existing pte, then don't allow _PAGE_RW to be set */ if (pte_val_ma(*ptep) & _PAGE_PRESENT) pte = __pte_ma(((pte_val_ma(*ptep) & _PAGE_RW) | ~_PAGE_RW) & pte_val_ma(pte)); |
ee176455e xen: mask_rw_pte:... |
1466 1467 1468 1469 |
return pte; } #else /* CONFIG_X86_64 */ |
3f508953d arch/x86/xen/mmu:... |
1470 |
static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) |
ee176455e xen: mask_rw_pte:... |
1471 |
{ |
8f5b0c639 xen: eliminate sc... |
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 |
unsigned long pfn; if (xen_feature(XENFEAT_writable_page_tables) || xen_feature(XENFEAT_auto_translated_physmap) || xen_start_info->mfn_list >= __START_KERNEL_map) return pte; /* * Pages belonging to the initial p2m list mapped outside the default * address range must be mapped read-only. This region contains the * page tables for mapping the p2m list, too, and page tables MUST be * mapped read-only. */ pfn = pte_pfn(pte); if (pfn >= xen_start_info->first_p2m_pfn && pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames) pte = __pte_ma(pte_val_ma(pte) & ~_PAGE_RW); |
1f4f93150 xen: fix 32-bit b... |
1489 1490 |
return pte; } |
ee176455e xen: mask_rw_pte:... |
1491 |
#endif /* CONFIG_X86_64 */ |
1f4f93150 xen: fix 32-bit b... |
1492 |
|
d095d43e7 xen/mm: do direct... |
1493 1494 1495 1496 |
/* * Init-time set_pte while constructing initial pagetables, which * doesn't allow RO page table pages to be remapped RW. * |
66a27dde9 xen/mm: zero PTEs... |
1497 1498 1499 1500 |
* If there is no MFN for this PFN then this page is initially * ballooned out so clear the PTE (as in decrease_reservation() in * drivers/xen/balloon.c). * |
d095d43e7 xen/mm: do direct... |
1501 1502 1503 1504 1505 1506 |
* Many of these PTE updates are done on unpinned and writable pages * and doing a hypercall for these is unnecessary and expensive. At * this point it is not possible to tell if a page is pinned or not, * so always write the PTE directly and rely on Xen trapping and * emulating any updates as necessary. */ |
3f508953d arch/x86/xen/mmu:... |
1507 |
static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) |
1f4f93150 xen: fix 32-bit b... |
1508 |
{ |
66a27dde9 xen/mm: zero PTEs... |
1509 1510 1511 1512 |
if (pte_mfn(pte) != INVALID_P2M_ENTRY) pte = mask_rw_pte(ptep, pte); else pte = __pte_ma(0); |
1f4f93150 xen: fix 32-bit b... |
1513 |
|
d095d43e7 xen/mm: do direct... |
1514 |
native_set_pte(ptep, pte); |
1f4f93150 xen: fix 32-bit b... |
1515 |
} |
319f3ba52 xen: move remaini... |
1516 1517 1518 |
/* Early in boot, while setting up the initial pagetable, assume everything is pinned. */ |
3f508953d arch/x86/xen/mmu:... |
1519 |
static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn) |
319f3ba52 xen: move remaini... |
1520 1521 1522 1523 1524 |
{ #ifdef CONFIG_FLATMEM BUG_ON(mem_map); /* should only be used early */ #endif make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); |
b96229b50 xen/mmu: some ear... |
1525 1526 1527 1528 |
pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); } /* Used for pmd and pud */ |
3f508953d arch/x86/xen/mmu:... |
1529 |
static void __init xen_alloc_pmd_init(struct mm_struct *mm, unsigned long pfn) |
b96229b50 xen/mmu: some ear... |
1530 1531 1532 1533 1534 |
{ #ifdef CONFIG_FLATMEM BUG_ON(mem_map); /* should only be used early */ #endif make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); |
319f3ba52 xen: move remaini... |
1535 1536 1537 1538 |
} /* Early release_pte assumes that all pts are pinned, since there's only init_mm and anything attached to that is pinned. */ |
3f508953d arch/x86/xen/mmu:... |
1539 |
static void __init xen_release_pte_init(unsigned long pfn) |
319f3ba52 xen: move remaini... |
1540 |
{ |
b96229b50 xen/mmu: some ear... |
1541 |
pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn); |
319f3ba52 xen: move remaini... |
1542 1543 |
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); } |
3f508953d arch/x86/xen/mmu:... |
1544 |
static void __init xen_release_pmd_init(unsigned long pfn) |
319f3ba52 xen: move remaini... |
1545 |
{ |
b96229b50 xen/mmu: some ear... |
1546 |
make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); |
319f3ba52 xen: move remaini... |
1547 |
} |
bc7fe1d97 xen/mmu: tune pgt... |
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 |
static inline void __pin_pagetable_pfn(unsigned cmd, unsigned long pfn) { struct multicall_space mcs; struct mmuext_op *op; mcs = __xen_mc_entry(sizeof(*op)); op = mcs.args; op->cmd = cmd; op->arg1.mfn = pfn_to_mfn(pfn); MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); } static inline void __set_pfn_prot(unsigned long pfn, pgprot_t prot) { struct multicall_space mcs; unsigned long addr = (unsigned long)__va(pfn << PAGE_SHIFT); mcs = __xen_mc_entry(0); MULTI_update_va_mapping(mcs.mc, (unsigned long)addr, pfn_pte(pfn, prot), 0); } |
319f3ba52 xen: move remaini... |
1570 1571 |
/* This needs to make sure the new pte page is pinned iff its being attached to a pinned pagetable. */ |
bc7fe1d97 xen/mmu: tune pgt... |
1572 1573 |
static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, unsigned level) |
319f3ba52 xen: move remaini... |
1574 |
{ |
bc7fe1d97 xen/mmu: tune pgt... |
1575 |
bool pinned = PagePinned(virt_to_page(mm->pgd)); |
c2ba050d2 xen/trace: add pt... |
1576 |
trace_xen_mmu_alloc_ptpage(mm, pfn, level, pinned); |
319f3ba52 xen: move remaini... |
1577 |
|
c2ba050d2 xen/trace: add pt... |
1578 |
if (pinned) { |
bc7fe1d97 xen/mmu: tune pgt... |
1579 |
struct page *page = pfn_to_page(pfn); |
319f3ba52 xen: move remaini... |
1580 |
|
319f3ba52 xen: move remaini... |
1581 |
SetPagePinned(page); |
319f3ba52 xen: move remaini... |
1582 |
if (!PageHighMem(page)) { |
bc7fe1d97 xen/mmu: tune pgt... |
1583 1584 1585 |
xen_mc_batch(); __set_pfn_prot(pfn, PAGE_KERNEL_RO); |
57c1ffcef mm: rename USE_SP... |
1586 |
if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS) |
bc7fe1d97 xen/mmu: tune pgt... |
1587 1588 1589 |
__pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); xen_mc_issue(PARAVIRT_LAZY_MMU); |
319f3ba52 xen: move remaini... |
1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 |
} else { /* make sure there are no stray mappings of this page */ kmap_flush_unused(); } } } static void xen_alloc_pte(struct mm_struct *mm, unsigned long pfn) { xen_alloc_ptpage(mm, pfn, PT_PTE); } static void xen_alloc_pmd(struct mm_struct *mm, unsigned long pfn) { xen_alloc_ptpage(mm, pfn, PT_PMD); } /* This should never happen until we're OK to use struct page */ |
bc7fe1d97 xen/mmu: tune pgt... |
1609 |
static inline void xen_release_ptpage(unsigned long pfn, unsigned level) |
319f3ba52 xen: move remaini... |
1610 1611 |
{ struct page *page = pfn_to_page(pfn); |
c2ba050d2 xen/trace: add pt... |
1612 |
bool pinned = PagePinned(page); |
319f3ba52 xen: move remaini... |
1613 |
|
c2ba050d2 xen/trace: add pt... |
1614 |
trace_xen_mmu_release_ptpage(pfn, level, pinned); |
319f3ba52 xen: move remaini... |
1615 |
|
c2ba050d2 xen/trace: add pt... |
1616 |
if (pinned) { |
319f3ba52 xen: move remaini... |
1617 |
if (!PageHighMem(page)) { |
bc7fe1d97 xen/mmu: tune pgt... |
1618 |
xen_mc_batch(); |
57c1ffcef mm: rename USE_SP... |
1619 |
if (level == PT_PTE && USE_SPLIT_PTE_PTLOCKS) |
bc7fe1d97 xen/mmu: tune pgt... |
1620 1621 1622 1623 1624 |
__pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, pfn); __set_pfn_prot(pfn, PAGE_KERNEL); xen_mc_issue(PARAVIRT_LAZY_MMU); |
319f3ba52 xen: move remaini... |
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 |
} ClearPagePinned(page); } } static void xen_release_pte(unsigned long pfn) { xen_release_ptpage(pfn, PT_PTE); } static void xen_release_pmd(unsigned long pfn) { xen_release_ptpage(pfn, PT_PMD); } |
982333683 x86: expose numbe... |
1639 |
#if CONFIG_PGTABLE_LEVELS == 4 |
319f3ba52 xen: move remaini... |
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 |
static void xen_alloc_pud(struct mm_struct *mm, unsigned long pfn) { xen_alloc_ptpage(mm, pfn, PT_PUD); } static void xen_release_pud(unsigned long pfn) { xen_release_ptpage(pfn, PT_PUD); } #endif void __init xen_reserve_top(void) { #ifdef CONFIG_X86_32 unsigned long top = HYPERVISOR_VIRT_START; struct xen_platform_parameters pp; if (HYPERVISOR_xen_version(XENVER_platform_parameters, &pp) == 0) top = pp.virt_start; reserve_top_address(-top); #endif /* CONFIG_X86_32 */ } /* * Like __va(), but returns address in the kernel mapping (which is * all we have until the physical memory mapping has been set up. */ |
bf9d834a9 x86/xen: add some... |
1668 |
static void * __init __ka(phys_addr_t paddr) |
319f3ba52 xen: move remaini... |
1669 1670 1671 1672 1673 1674 1675 1676 1677 |
{ #ifdef CONFIG_X86_64 return (void *)(paddr + __START_KERNEL_map); #else return __va(paddr); #endif } /* Convert a machine address to physical address */ |
bf9d834a9 x86/xen: add some... |
1678 |
static unsigned long __init m2p(phys_addr_t maddr) |
319f3ba52 xen: move remaini... |
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 |
{ phys_addr_t paddr; maddr &= PTE_PFN_MASK; paddr = mfn_to_pfn(maddr >> PAGE_SHIFT) << PAGE_SHIFT; return paddr; } /* Convert a machine address to kernel virtual */ |
bf9d834a9 x86/xen: add some... |
1689 |
static void * __init m2v(phys_addr_t maddr) |
319f3ba52 xen: move remaini... |
1690 1691 1692 |
{ return __ka(m2p(maddr)); } |
4ec5387cc xen: add the dire... |
1693 |
/* Set the page permissions on an identity-mapped pages */ |
bf9d834a9 x86/xen: add some... |
1694 1695 |
static void __init set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags) |
319f3ba52 xen: move remaini... |
1696 1697 1698 |
{ unsigned long pfn = __pa(addr) >> PAGE_SHIFT; pte_t pte = pfn_pte(pfn, prot); |
4e44e44b0 xen/pvh: MMU chan... |
1699 1700 1701 |
/* For PVH no need to set R/O or R/W to pin them or unpin them. */ if (xen_feature(XENFEAT_auto_translated_physmap)) return; |
b22227944 xen/mmu: On early... |
1702 |
if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags)) |
319f3ba52 xen: move remaini... |
1703 1704 |
BUG(); } |
bf9d834a9 x86/xen: add some... |
1705 |
static void __init set_page_prot(void *addr, pgprot_t prot) |
b22227944 xen/mmu: On early... |
1706 1707 1708 |
{ return set_page_prot_flags(addr, prot, UVMF_NONE); } |
caaf9ecf1 xen/mmu: For 64-b... |
1709 |
#ifdef CONFIG_X86_32 |
3f508953d arch/x86/xen/mmu:... |
1710 |
static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) |
319f3ba52 xen: move remaini... |
1711 1712 1713 1714 |
{ unsigned pmdidx, pteidx; unsigned ident_pte; unsigned long pfn; |
764f0138b xen: allocate lev... |
1715 1716 |
level1_ident_pgt = extend_brk(sizeof(pte_t) * LEVEL1_IDENT_ENTRIES, PAGE_SIZE); |
319f3ba52 xen: move remaini... |
1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 |
ident_pte = 0; pfn = 0; for (pmdidx = 0; pmdidx < PTRS_PER_PMD && pfn < max_pfn; pmdidx++) { pte_t *pte_page; /* Reuse or allocate a page of ptes */ if (pmd_present(pmd[pmdidx])) pte_page = m2v(pmd[pmdidx].pmd); else { /* Check for free pte pages */ |
764f0138b xen: allocate lev... |
1727 |
if (ident_pte == LEVEL1_IDENT_ENTRIES) |
319f3ba52 xen: move remaini... |
1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 |
break; pte_page = &level1_ident_pgt[ident_pte]; ident_pte += PTRS_PER_PTE; pmd[pmdidx] = __pmd(__pa(pte_page) | _PAGE_TABLE); } /* Install mappings */ for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { pte_t pte; |
a91d92875 xen: partially re... |
1739 1740 |
if (pfn > max_pfn_mapped) max_pfn_mapped = pfn; |
a91d92875 xen: partially re... |
1741 |
|
319f3ba52 xen: move remaini... |
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 |
if (!pte_none(pte_page[pteidx])) continue; pte = pfn_pte(pfn, PAGE_KERNEL_EXEC); pte_page[pteidx] = pte; } } for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE) set_page_prot(&level1_ident_pgt[pteidx], PAGE_KERNEL_RO); set_page_prot(pmd, PAGE_KERNEL_RO); } |
caaf9ecf1 xen/mmu: For 64-b... |
1755 |
#endif |
7e77506a5 xen: implement XE... |
1756 1757 1758 |
void __init xen_setup_machphys_mapping(void) { struct xen_machphys_mapping mapping; |
7e77506a5 xen: implement XE... |
1759 1760 1761 |
if (HYPERVISOR_memory_op(XENMEM_machphys_mapping, &mapping) == 0) { machine_to_phys_mapping = (unsigned long *)mapping.v_start; |
ccbcdf7cf xen/x86: replace ... |
1762 |
machine_to_phys_nr = mapping.max_mfn + 1; |
7e77506a5 xen: implement XE... |
1763 |
} else { |
ccbcdf7cf xen/x86: replace ... |
1764 |
machine_to_phys_nr = MACH2PHYS_NR_ENTRIES; |
7e77506a5 xen: implement XE... |
1765 |
} |
ccbcdf7cf xen/x86: replace ... |
1766 |
#ifdef CONFIG_X86_32 |
61cca2fab xen/i386: follow-... |
1767 1768 |
WARN_ON((machine_to_phys_mapping + (machine_to_phys_nr - 1)) < machine_to_phys_mapping); |
ccbcdf7cf xen/x86: replace ... |
1769 |
#endif |
7e77506a5 xen: implement XE... |
1770 |
} |
319f3ba52 xen: move remaini... |
1771 |
#ifdef CONFIG_X86_64 |
bf9d834a9 x86/xen: add some... |
1772 |
static void __init convert_pfn_mfn(void *v) |
319f3ba52 xen: move remaini... |
1773 1774 1775 1776 1777 1778 1779 1780 1781 |
{ pte_t *pte = v; int i; /* All levels are converted the same way, so just treat them as ptes. */ for (i = 0; i < PTRS_PER_PTE; i++) pte[i] = xen_make_pte(pte[i].pte); } |
488f046df xen/mmu: Recycle ... |
1782 1783 1784 1785 |
static void __init check_pt_base(unsigned long *pt_base, unsigned long *pt_end, unsigned long addr) { if (*pt_base == PFN_DOWN(__pa(addr))) { |
b22227944 xen/mmu: On early... |
1786 |
set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG); |
488f046df xen/mmu: Recycle ... |
1787 1788 1789 1790 |
clear_page((void *)addr); (*pt_base)++; } if (*pt_end == PFN_DOWN(__pa(addr))) { |
b22227944 xen/mmu: On early... |
1791 |
set_page_prot_flags((void *)addr, PAGE_KERNEL, UVMF_INVLPG); |
488f046df xen/mmu: Recycle ... |
1792 1793 1794 1795 |
clear_page((void *)addr); (*pt_end)--; } } |
319f3ba52 xen: move remaini... |
1796 |
/* |
0d2eb44f6 x86: Fix common m... |
1797 |
* Set up the initial kernel pagetable. |
319f3ba52 xen: move remaini... |
1798 1799 1800 |
* * We can construct this by grafting the Xen provided pagetable into * head_64.S's preconstructed pagetables. We copy the Xen L2's into |
0b5a50635 x86/xen: don't co... |
1801 1802 1803 1804 1805 |
* level2_ident_pgt, and level2_kernel_pgt. This means that only the * kernel has a physical mapping to start with - but that's enough to * get __va working. We need to fill in the rest of the physical * mapping once some sort of allocator has been set up. NOTE: for * PVH, the page tables are native. |
319f3ba52 xen: move remaini... |
1806 |
*/ |
3699aad04 xen/mmu: The xen_... |
1807 |
void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) |
319f3ba52 xen: move remaini... |
1808 1809 1810 |
{ pud_t *l3; pmd_t *l2; |
488f046df xen/mmu: Recycle ... |
1811 1812 1813 |
unsigned long addr[3]; unsigned long pt_base, pt_end; unsigned i; |
319f3ba52 xen: move remaini... |
1814 |
|
14988a4d3 xen: set max_pfn_... |
1815 1816 1817 1818 |
/* max_pfn_mapped is the last pfn mapped in the initial memory * mappings. Considering that on Xen after the kernel mappings we * have the mappings of some pages that don't exist in pfn space, we * set max_pfn_mapped to the last real pfn mapped. */ |
8f5b0c639 xen: eliminate sc... |
1819 1820 1821 1822 |
if (xen_start_info->mfn_list < __START_KERNEL_map) max_pfn_mapped = xen_start_info->first_p2m_pfn; else max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); |
14988a4d3 xen: set max_pfn_... |
1823 |
|
488f046df xen/mmu: Recycle ... |
1824 1825 |
pt_base = PFN_DOWN(__pa(xen_start_info->pt_base)); pt_end = pt_base + xen_start_info->nr_pt_frames; |
319f3ba52 xen: move remaini... |
1826 1827 |
/* Zap identity mapping */ init_level4_pgt[0] = __pgd(0); |
4e44e44b0 xen/pvh: MMU chan... |
1828 1829 1830 1831 1832 1833 1834 1835 1836 |
if (!xen_feature(XENFEAT_auto_translated_physmap)) { /* Pre-constructed entries are in pfn, so convert to mfn */ /* L4[272] -> level3_ident_pgt * L4[511] -> level3_kernel_pgt */ convert_pfn_mfn(init_level4_pgt); /* L3_i[0] -> level2_ident_pgt */ convert_pfn_mfn(level3_ident_pgt); /* L3_k[510] -> level2_kernel_pgt |
0b5a50635 x86/xen: don't co... |
1837 |
* L3_k[511] -> level2_fixmap_pgt */ |
4e44e44b0 xen/pvh: MMU chan... |
1838 |
convert_pfn_mfn(level3_kernel_pgt); |
0b5a50635 x86/xen: don't co... |
1839 1840 1841 |
/* L3_k[511][506] -> level1_fixmap_pgt */ convert_pfn_mfn(level2_fixmap_pgt); |
4e44e44b0 xen/pvh: MMU chan... |
1842 |
} |
4fac153a7 xen/mmu: Provide ... |
1843 |
/* We get [511][511] and have Xen's version of level2_kernel_pgt */ |
319f3ba52 xen: move remaini... |
1844 1845 |
l3 = m2v(pgd[pgd_index(__START_KERNEL_map)].pgd); l2 = m2v(l3[pud_index(__START_KERNEL_map)].pud); |
488f046df xen/mmu: Recycle ... |
1846 1847 1848 |
addr[0] = (unsigned long)pgd; addr[1] = (unsigned long)l3; addr[2] = (unsigned long)l2; |
4fac153a7 xen/mmu: Provide ... |
1849 |
/* Graft it onto L4[272][0]. Note that we creating an aliasing problem: |
0b5a50635 x86/xen: don't co... |
1850 |
* Both L4[272][0] and L4[511][510] have entries that point to the same |
4fac153a7 xen/mmu: Provide ... |
1851 1852 1853 1854 |
* L2 (PMD) tables. Meaning that if you modify it in __va space * it will be also modified in the __ka space! (But if you just * modify the PMD table to point to other PTE's or none, then you * are OK - which is what cleanup_highmap does) */ |
ae895ed78 xen/mmu: use copy... |
1855 |
copy_page(level2_ident_pgt, l2); |
0b5a50635 x86/xen: don't co... |
1856 |
/* Graft it onto L4[511][510] */ |
ae895ed78 xen/mmu: use copy... |
1857 |
copy_page(level2_kernel_pgt, l2); |
319f3ba52 xen: move remaini... |
1858 |
|
8f5b0c639 xen: eliminate sc... |
1859 1860 1861 1862 |
/* Copy the initial P->M table mappings if necessary. */ i = pgd_index(xen_start_info->mfn_list); if (i && i < pgd_index(__START_KERNEL_map)) init_level4_pgt[i] = ((pgd_t *)xen_start_info->pt_base)[i]; |
4e44e44b0 xen/pvh: MMU chan... |
1863 1864 1865 1866 1867 1868 1869 1870 1871 |
if (!xen_feature(XENFEAT_auto_translated_physmap)) { /* Make pagetable pieces RO */ set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO); set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO); set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO); set_page_prot(level2_ident_pgt, PAGE_KERNEL_RO); set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO); set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO); |
0b5a50635 x86/xen: don't co... |
1872 |
set_page_prot(level1_fixmap_pgt, PAGE_KERNEL_RO); |
4e44e44b0 xen/pvh: MMU chan... |
1873 1874 1875 1876 1877 1878 1879 |
/* Pin down new L4 */ pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa_symbol(init_level4_pgt))); /* Unpin Xen-provided one */ pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); |
319f3ba52 xen: move remaini... |
1880 |
|
4e44e44b0 xen/pvh: MMU chan... |
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 |
/* * At this stage there can be no user pgd, and no page * structure to attach it to, so make sure we just set kernel * pgd. */ xen_mc_batch(); __xen_write_cr3(true, __pa(init_level4_pgt)); xen_mc_issue(PARAVIRT_LAZY_CPU); } else native_write_cr3(__pa(init_level4_pgt)); |
319f3ba52 xen: move remaini... |
1891 |
|
488f046df xen/mmu: Recycle ... |
1892 1893 1894 1895 1896 1897 1898 1899 |
/* We can't that easily rip out L3 and L2, as the Xen pagetables are * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for * the initial domain. For guests using the toolstack, they are in: * [L4], [L3], [L2], [L1], [L1], order .. So for dom0 we can only * rip out the [L4] (pgd), but for guests we shave off three pages. */ for (i = 0; i < ARRAY_SIZE(addr); i++) check_pt_base(&pt_base, &pt_end, addr[i]); |
319f3ba52 xen: move remaini... |
1900 |
|
488f046df xen/mmu: Recycle ... |
1901 |
/* Our (by three pages) smaller Xen pagetable that we are using */ |
04414baab xen: check pre-al... |
1902 1903 1904 |
xen_pt_base = PFN_PHYS(pt_base); xen_pt_size = (pt_end - pt_base) * PAGE_SIZE; memblock_reserve(xen_pt_base, xen_pt_size); |
70e611995 xen: move p2m lis... |
1905 |
|
7f9140626 xen/mmu: Copy and... |
1906 1907 |
/* Revector the xen_start_info */ xen_start_info = (struct start_info *)__va(__pa(xen_start_info)); |
319f3ba52 xen: move remaini... |
1908 |
} |
70e611995 xen: move p2m lis... |
1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 |
/* * Read a value from a physical address. */ static unsigned long __init xen_read_phys_ulong(phys_addr_t addr) { unsigned long *vaddr; unsigned long val; vaddr = early_memremap_ro(addr, sizeof(val)); val = *vaddr; early_memunmap(vaddr, sizeof(val)); return val; } /* * Translate a virtual address to a physical one without relying on mapped * page tables. */ static phys_addr_t __init xen_early_virt_to_phys(unsigned long vaddr) { phys_addr_t pa; pgd_t pgd; pud_t pud; pmd_t pmd; pte_t pte; pa = read_cr3(); pgd = native_make_pgd(xen_read_phys_ulong(pa + pgd_index(vaddr) * sizeof(pgd))); if (!pgd_present(pgd)) return 0; pa = pgd_val(pgd) & PTE_PFN_MASK; pud = native_make_pud(xen_read_phys_ulong(pa + pud_index(vaddr) * sizeof(pud))); if (!pud_present(pud)) return 0; pa = pud_pfn(pud) << PAGE_SHIFT; if (pud_large(pud)) return pa + (vaddr & ~PUD_MASK); pmd = native_make_pmd(xen_read_phys_ulong(pa + pmd_index(vaddr) * sizeof(pmd))); if (!pmd_present(pmd)) return 0; pa = pmd_pfn(pmd) << PAGE_SHIFT; if (pmd_large(pmd)) return pa + (vaddr & ~PMD_MASK); pte = native_make_pte(xen_read_phys_ulong(pa + pte_index(vaddr) * sizeof(pte))); if (!pte_present(pte)) return 0; pa = pte_pfn(pte) << PAGE_SHIFT; return pa | (vaddr & ~PAGE_MASK); } /* * Find a new area for the hypervisor supplied p2m list and relocate the p2m to * this area. */ void __init xen_relocate_p2m(void) { phys_addr_t size, new_area, pt_phys, pmd_phys, pud_phys; unsigned long p2m_pfn, p2m_pfn_end, n_frames, pfn, pfn_end; int n_pte, n_pt, n_pmd, n_pud, idx_pte, idx_pt, idx_pmd, idx_pud; pte_t *pt; pmd_t *pmd; pud_t *pud; pgd_t *pgd; unsigned long *new_p2m; size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); n_pte = roundup(size, PAGE_SIZE) >> PAGE_SHIFT; n_pt = roundup(size, PMD_SIZE) >> PMD_SHIFT; n_pmd = roundup(size, PUD_SIZE) >> PUD_SHIFT; n_pud = roundup(size, PGDIR_SIZE) >> PGDIR_SHIFT; n_frames = n_pte + n_pt + n_pmd + n_pud; new_area = xen_find_free_area(PFN_PHYS(n_frames)); if (!new_area) { xen_raw_console_write("Can't find new memory area for p2m needed due to E820 map conflict "); BUG(); } /* * Setup the page tables for addressing the new p2m list. * We have asked the hypervisor to map the p2m list at the user address * PUD_SIZE. It may have done so, or it may have used a kernel space * address depending on the Xen version. * To avoid any possible virtual address collision, just use * 2 * PUD_SIZE for the new area. */ pud_phys = new_area; pmd_phys = pud_phys + PFN_PHYS(n_pud); pt_phys = pmd_phys + PFN_PHYS(n_pmd); p2m_pfn = PFN_DOWN(pt_phys) + n_pt; pgd = __va(read_cr3()); new_p2m = (unsigned long *)(2 * PGDIR_SIZE); for (idx_pud = 0; idx_pud < n_pud; idx_pud++) { pud = early_memremap(pud_phys, PAGE_SIZE); clear_page(pud); for (idx_pmd = 0; idx_pmd < min(n_pmd, PTRS_PER_PUD); idx_pmd++) { pmd = early_memremap(pmd_phys, PAGE_SIZE); clear_page(pmd); for (idx_pt = 0; idx_pt < min(n_pt, PTRS_PER_PMD); idx_pt++) { pt = early_memremap(pt_phys, PAGE_SIZE); clear_page(pt); for (idx_pte = 0; idx_pte < min(n_pte, PTRS_PER_PTE); idx_pte++) { set_pte(pt + idx_pte, pfn_pte(p2m_pfn, PAGE_KERNEL)); p2m_pfn++; } n_pte -= PTRS_PER_PTE; early_memunmap(pt, PAGE_SIZE); make_lowmem_page_readonly(__va(pt_phys)); pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, PFN_DOWN(pt_phys)); set_pmd(pmd + idx_pt, __pmd(_PAGE_TABLE | pt_phys)); pt_phys += PAGE_SIZE; } n_pt -= PTRS_PER_PMD; early_memunmap(pmd, PAGE_SIZE); make_lowmem_page_readonly(__va(pmd_phys)); pin_pagetable_pfn(MMUEXT_PIN_L2_TABLE, PFN_DOWN(pmd_phys)); set_pud(pud + idx_pmd, __pud(_PAGE_TABLE | pmd_phys)); pmd_phys += PAGE_SIZE; } n_pmd -= PTRS_PER_PUD; early_memunmap(pud, PAGE_SIZE); make_lowmem_page_readonly(__va(pud_phys)); pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(pud_phys)); set_pgd(pgd + 2 + idx_pud, __pgd(_PAGE_TABLE | pud_phys)); pud_phys += PAGE_SIZE; } /* Now copy the old p2m info to the new area. */ memcpy(new_p2m, xen_p2m_addr, size); xen_p2m_addr = new_p2m; /* Release the old p2m list and set new list info. */ p2m_pfn = PFN_DOWN(xen_early_virt_to_phys(xen_start_info->mfn_list)); BUG_ON(!p2m_pfn); p2m_pfn_end = p2m_pfn + PFN_DOWN(size); if (xen_start_info->mfn_list < __START_KERNEL_map) { pfn = xen_start_info->first_p2m_pfn; pfn_end = xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames; set_pgd(pgd + 1, __pgd(0)); } else { pfn = p2m_pfn; pfn_end = p2m_pfn_end; } memblock_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn)); while (pfn < pfn_end) { if (pfn == p2m_pfn) { pfn = p2m_pfn_end; continue; } make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); pfn++; } xen_start_info->mfn_list = (unsigned long)xen_p2m_addr; xen_start_info->first_p2m_pfn = PFN_DOWN(new_area); xen_start_info->nr_p2m_frames = n_frames; } |
319f3ba52 xen: move remaini... |
2088 |
#else /* !CONFIG_X86_64 */ |
5b5c1af10 xen: x86/32: perf... |
2089 2090 |
static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD); static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD); |
3f508953d arch/x86/xen/mmu:... |
2091 |
static void __init xen_write_cr3_init(unsigned long cr3) |
5b5c1af10 xen: x86/32: perf... |
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 |
{ unsigned long pfn = PFN_DOWN(__pa(swapper_pg_dir)); BUG_ON(read_cr3() != __pa(initial_page_table)); BUG_ON(cr3 != __pa(swapper_pg_dir)); /* * We are switching to swapper_pg_dir for the first time (from * initial_page_table) and therefore need to mark that page * read-only and then pin it. * * Xen disallows sharing of kernel PMDs for PAE * guests. Therefore we must copy the kernel PMD from * initial_page_table into a new kernel PMD to be used in * swapper_pg_dir. */ swapper_kernel_pmd = extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); |
ae895ed78 xen/mmu: use copy... |
2110 |
copy_page(swapper_kernel_pmd, initial_kernel_pmd); |
5b5c1af10 xen: x86/32: perf... |
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 |
swapper_pg_dir[KERNEL_PGD_BOUNDARY] = __pgd(__pa(swapper_kernel_pmd) | _PAGE_PRESENT); set_page_prot(swapper_kernel_pmd, PAGE_KERNEL_RO); set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO); xen_write_cr3(cr3); pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, pfn); pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(initial_page_table))); set_page_prot(initial_page_table, PAGE_KERNEL); set_page_prot(initial_kernel_pmd, PAGE_KERNEL); pv_mmu_ops.write_cr3 = &xen_write_cr3; } |
319f3ba52 xen: move remaini... |
2126 |
|
70e611995 xen: move p2m lis... |
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 |
/* * For 32 bit domains xen_start_info->pt_base is the pgd address which might be * not the first page table in the page table pool. * Iterate through the initial page tables to find the real page table base. */ static phys_addr_t xen_find_pt_base(pmd_t *pmd) { phys_addr_t pt_base, paddr; unsigned pmdidx; pt_base = min(__pa(xen_start_info->pt_base), __pa(pmd)); for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) if (pmd_present(pmd[pmdidx]) && !pmd_large(pmd[pmdidx])) { paddr = m2p(pmd[pmdidx].pmd); pt_base = min(pt_base, paddr); } return pt_base; } |
3699aad04 xen/mmu: The xen_... |
2147 |
void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) |
319f3ba52 xen: move remaini... |
2148 2149 |
{ pmd_t *kernel_pmd; |
70e611995 xen: move p2m lis... |
2150 2151 2152 2153 |
kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); xen_pt_base = xen_find_pt_base(kernel_pmd); xen_pt_size = xen_start_info->nr_pt_frames * PAGE_SIZE; |
5b5c1af10 xen: x86/32: perf... |
2154 2155 |
initial_kernel_pmd = extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); |
f0991802b xen: use early_br... |
2156 |
|
70e611995 xen: move p2m lis... |
2157 |
max_pfn_mapped = PFN_DOWN(xen_pt_base + xen_pt_size + 512 * 1024); |
319f3ba52 xen: move remaini... |
2158 |
|
ae895ed78 xen/mmu: use copy... |
2159 |
copy_page(initial_kernel_pmd, kernel_pmd); |
319f3ba52 xen: move remaini... |
2160 |
|
5b5c1af10 xen: x86/32: perf... |
2161 |
xen_map_identity_early(initial_kernel_pmd, max_pfn); |
319f3ba52 xen: move remaini... |
2162 |
|
ae895ed78 xen/mmu: use copy... |
2163 |
copy_page(initial_page_table, pgd); |
5b5c1af10 xen: x86/32: perf... |
2164 2165 |
initial_page_table[KERNEL_PGD_BOUNDARY] = __pgd(__pa(initial_kernel_pmd) | _PAGE_PRESENT); |
319f3ba52 xen: move remaini... |
2166 |
|
5b5c1af10 xen: x86/32: perf... |
2167 2168 |
set_page_prot(initial_kernel_pmd, PAGE_KERNEL_RO); set_page_prot(initial_page_table, PAGE_KERNEL_RO); |
319f3ba52 xen: move remaini... |
2169 2170 2171 |
set_page_prot(empty_zero_page, PAGE_KERNEL_RO); pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); |
5b5c1af10 xen: x86/32: perf... |
2172 2173 2174 |
pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(initial_page_table))); xen_write_cr3(__pa(initial_page_table)); |
319f3ba52 xen: move remaini... |
2175 |
|
04414baab xen: check pre-al... |
2176 |
memblock_reserve(xen_pt_base, xen_pt_size); |
319f3ba52 xen: move remaini... |
2177 2178 |
} #endif /* CONFIG_X86_64 */ |
6c2681c86 xen: add explicit... |
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 |
void __init xen_reserve_special_pages(void) { phys_addr_t paddr; memblock_reserve(__pa(xen_start_info), PAGE_SIZE); if (xen_start_info->store_mfn) { paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->store_mfn)); memblock_reserve(paddr, PAGE_SIZE); } if (!xen_initial_domain()) { paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->console.domU.mfn)); memblock_reserve(paddr, PAGE_SIZE); } } |
04414baab xen: check pre-al... |
2193 2194 2195 2196 2197 2198 2199 2200 |
void __init xen_pt_check_e820(void) { if (xen_is_e820_reserved(xen_pt_base, xen_pt_size)) { xen_raw_console_write("Xen hypervisor allocated page table memory conflicts with E820 map "); BUG(); } } |
98511f353 xen: map a dummy ... |
2201 |
static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss; |
3b3809ac5 x86: fix set_fixm... |
2202 |
static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) |
319f3ba52 xen: move remaini... |
2203 2204 2205 2206 2207 2208 2209 |
{ pte_t pte; phys >>= PAGE_SHIFT; switch (idx) { case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: |
4eefbe792 x86: Use a read-o... |
2210 |
case FIX_RO_IDT: |
319f3ba52 xen: move remaini... |
2211 2212 |
#ifdef CONFIG_X86_32 case FIX_WP_TEST: |
319f3ba52 xen: move remaini... |
2213 2214 2215 |
# ifdef CONFIG_HIGHMEM case FIX_KMAP_BEGIN ... FIX_KMAP_END: # endif |
1ad83c858 x86_64,vsyscall: ... |
2216 |
#elif defined(CONFIG_X86_VSYSCALL_EMULATION) |
f40c33009 x86, vdso: Move t... |
2217 |
case VSYSCALL_PAGE: |
319f3ba52 xen: move remaini... |
2218 |
#endif |
3ecb1b7df xen: add FIX_TEXT... |
2219 2220 2221 |
case FIX_TEXT_POKE0: case FIX_TEXT_POKE1: /* All local page mappings */ |
319f3ba52 xen: move remaini... |
2222 2223 |
pte = pfn_pte(phys, prot); break; |
98511f353 xen: map a dummy ... |
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 |
#ifdef CONFIG_X86_LOCAL_APIC case FIX_APIC_BASE: /* maps dummy local APIC */ pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL); break; #endif #ifdef CONFIG_X86_IO_APIC case FIX_IO_APIC_BASE_0 ... FIX_IO_APIC_BASE_END: /* * We just don't map the IO APIC - all access is via * hypercalls. Keep the address in the pte for reference. */ |
27abd14bd Revert "xen/x86: ... |
2236 |
pte = pfn_pte(PFN_DOWN(__pa(dummy_mapping)), PAGE_KERNEL); |
98511f353 xen: map a dummy ... |
2237 2238 |
break; #endif |
c0011dbfc xen: use _PAGE_IO... |
2239 2240 2241 |
case FIX_PARAVIRT_BOOTMAP: /* This is an MFN, but it isn't an IO mapping from the IO domain */ |
319f3ba52 xen: move remaini... |
2242 2243 |
pte = mfn_pte(phys, prot); break; |
c0011dbfc xen: use _PAGE_IO... |
2244 2245 2246 |
default: /* By default, set_fixmap is used for hardware mappings */ |
7f2f88224 x86/xen: do not u... |
2247 |
pte = mfn_pte(phys, prot); |
c0011dbfc xen: use _PAGE_IO... |
2248 |
break; |
319f3ba52 xen: move remaini... |
2249 2250 2251 |
} __native_set_fixmap(idx, pte); |
1ad83c858 x86_64,vsyscall: ... |
2252 |
#ifdef CONFIG_X86_VSYSCALL_EMULATION |
319f3ba52 xen: move remaini... |
2253 2254 |
/* Replicate changes to map the vsyscall page into the user pagetable vsyscall mapping. */ |
f40c33009 x86, vdso: Move t... |
2255 |
if (idx == VSYSCALL_PAGE) { |
319f3ba52 xen: move remaini... |
2256 2257 2258 2259 2260 |
unsigned long vaddr = __fix_to_virt(idx); set_pte_vaddr_pud(level3_user_vsyscall, vaddr, pte); } #endif } |
3f508953d arch/x86/xen/mmu:... |
2261 |
static void __init xen_post_allocator_init(void) |
319f3ba52 xen: move remaini... |
2262 |
{ |
4e44e44b0 xen/pvh: MMU chan... |
2263 2264 |
if (xen_feature(XENFEAT_auto_translated_physmap)) return; |
319f3ba52 xen: move remaini... |
2265 2266 2267 |
pv_mmu_ops.set_pte = xen_set_pte; pv_mmu_ops.set_pmd = xen_set_pmd; pv_mmu_ops.set_pud = xen_set_pud; |
982333683 x86: expose numbe... |
2268 |
#if CONFIG_PGTABLE_LEVELS == 4 |
319f3ba52 xen: move remaini... |
2269 2270 2271 2272 2273 2274 2275 2276 2277 |
pv_mmu_ops.set_pgd = xen_set_pgd; #endif /* This will work as long as patching hasn't happened yet (which it hasn't) */ pv_mmu_ops.alloc_pte = xen_alloc_pte; pv_mmu_ops.alloc_pmd = xen_alloc_pmd; pv_mmu_ops.release_pte = xen_release_pte; pv_mmu_ops.release_pmd = xen_release_pmd; |
982333683 x86: expose numbe... |
2278 |
#if CONFIG_PGTABLE_LEVELS == 4 |
319f3ba52 xen: move remaini... |
2279 2280 2281 2282 2283 |
pv_mmu_ops.alloc_pud = xen_alloc_pud; pv_mmu_ops.release_pud = xen_release_pud; #endif #ifdef CONFIG_X86_64 |
d3eb2c89e xen/mmu: Move the... |
2284 |
pv_mmu_ops.write_cr3 = &xen_write_cr3; |
319f3ba52 xen: move remaini... |
2285 2286 2287 2288 |
SetPagePinned(virt_to_page(level3_user_vsyscall)); #endif xen_mark_init_mm_pinned(); } |
b407fc57b x86/paravirt: flu... |
2289 2290 |
static void xen_leave_lazy_mmu(void) { |
5caecb943 xen: disable pree... |
2291 |
preempt_disable(); |
b407fc57b x86/paravirt: flu... |
2292 2293 |
xen_mc_flush(); paravirt_leave_lazy_mmu(); |
5caecb943 xen: disable pree... |
2294 |
preempt_enable(); |
b407fc57b x86/paravirt: flu... |
2295 |
} |
319f3ba52 xen: move remaini... |
2296 |
|
3f508953d arch/x86/xen/mmu:... |
2297 |
static const struct pv_mmu_ops xen_mmu_ops __initconst = { |
319f3ba52 xen: move remaini... |
2298 2299 2300 2301 |
.read_cr2 = xen_read_cr2, .write_cr2 = xen_write_cr2, .read_cr3 = xen_read_cr3, |
5b5c1af10 xen: x86/32: perf... |
2302 |
.write_cr3 = xen_write_cr3_init, |
319f3ba52 xen: move remaini... |
2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 |
.flush_tlb_user = xen_flush_tlb, .flush_tlb_kernel = xen_flush_tlb, .flush_tlb_single = xen_flush_tlb_single, .flush_tlb_others = xen_flush_tlb_others, .pte_update = paravirt_nop, .pte_update_defer = paravirt_nop, .pgd_alloc = xen_pgd_alloc, .pgd_free = xen_pgd_free, .alloc_pte = xen_alloc_pte_init, .release_pte = xen_release_pte_init, |
b96229b50 xen/mmu: some ear... |
2317 |
.alloc_pmd = xen_alloc_pmd_init, |
b96229b50 xen/mmu: some ear... |
2318 |
.release_pmd = xen_release_pmd_init, |
319f3ba52 xen: move remaini... |
2319 |
|
319f3ba52 xen: move remaini... |
2320 |
.set_pte = xen_set_pte_init, |
319f3ba52 xen: move remaini... |
2321 2322 2323 2324 2325 |
.set_pte_at = xen_set_pte_at, .set_pmd = xen_set_pmd_hyper, .ptep_modify_prot_start = __ptep_modify_prot_start, .ptep_modify_prot_commit = __ptep_modify_prot_commit, |
da5de7c22 x86/paravirt: use... |
2326 2327 |
.pte_val = PV_CALLEE_SAVE(xen_pte_val), .pgd_val = PV_CALLEE_SAVE(xen_pgd_val), |
319f3ba52 xen: move remaini... |
2328 |
|
da5de7c22 x86/paravirt: use... |
2329 2330 |
.make_pte = PV_CALLEE_SAVE(xen_make_pte), .make_pgd = PV_CALLEE_SAVE(xen_make_pgd), |
319f3ba52 xen: move remaini... |
2331 2332 2333 |
#ifdef CONFIG_X86_PAE .set_pte_atomic = xen_set_pte_atomic, |
319f3ba52 xen: move remaini... |
2334 2335 2336 2337 |
.pte_clear = xen_pte_clear, .pmd_clear = xen_pmd_clear, #endif /* CONFIG_X86_PAE */ .set_pud = xen_set_pud_hyper, |
da5de7c22 x86/paravirt: use... |
2338 2339 |
.make_pmd = PV_CALLEE_SAVE(xen_make_pmd), .pmd_val = PV_CALLEE_SAVE(xen_pmd_val), |
319f3ba52 xen: move remaini... |
2340 |
|
982333683 x86: expose numbe... |
2341 |
#if CONFIG_PGTABLE_LEVELS == 4 |
da5de7c22 x86/paravirt: use... |
2342 2343 |
.pud_val = PV_CALLEE_SAVE(xen_pud_val), .make_pud = PV_CALLEE_SAVE(xen_make_pud), |
319f3ba52 xen: move remaini... |
2344 |
.set_pgd = xen_set_pgd_hyper, |
b96229b50 xen/mmu: some ear... |
2345 2346 |
.alloc_pud = xen_alloc_pmd_init, .release_pud = xen_release_pmd_init, |
982333683 x86: expose numbe... |
2347 |
#endif /* CONFIG_PGTABLE_LEVELS == 4 */ |
319f3ba52 xen: move remaini... |
2348 2349 2350 2351 2352 2353 2354 |
.activate_mm = xen_activate_mm, .dup_mmap = xen_dup_mmap, .exit_mmap = xen_exit_mmap, .lazy_mode = { .enter = paravirt_enter_lazy_mmu, |
b407fc57b x86/paravirt: flu... |
2355 |
.leave = xen_leave_lazy_mmu, |
511ba86e1 x86, mm: Patch ou... |
2356 |
.flush = paravirt_flush_lazy_mmu, |
319f3ba52 xen: move remaini... |
2357 2358 2359 2360 |
}, .set_fixmap = xen_set_fixmap, }; |
030cb6c00 x86: Move paravir... |
2361 2362 |
void __init xen_init_mmu_ops(void) { |
7737b215a x86: Rename paget... |
2363 |
x86_init.paging.pagetable_init = xen_pagetable_init; |
76bcceff0 xen/pvh/mmu: Use ... |
2364 |
|
20f36e038 xen/x86/pvh: Use ... |
2365 |
if (xen_feature(XENFEAT_auto_translated_physmap)) |
76bcceff0 xen/pvh/mmu: Use ... |
2366 |
return; |
20f36e038 xen/x86/pvh: Use ... |
2367 |
|
030cb6c00 x86: Move paravir... |
2368 |
pv_mmu_ops = xen_mmu_ops; |
d2cb21455 xen/mmu: inhibit ... |
2369 |
|
98511f353 xen: map a dummy ... |
2370 |
memset(dummy_mapping, 0xff, PAGE_SIZE); |
030cb6c00 x86: Move paravir... |
2371 |
} |
319f3ba52 xen: move remaini... |
2372 |
|
08bbc9da9 xen: Add xen_crea... |
2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 |
/* Protected by xen_reservation_lock. */ #define MAX_CONTIG_ORDER 9 /* 2MB */ static unsigned long discontig_frames[1<<MAX_CONTIG_ORDER]; #define VOID_PTE (mfn_pte(0, __pgprot(0))) static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order, unsigned long *in_frames, unsigned long *out_frames) { int i; struct multicall_space mcs; xen_mc_batch(); for (i = 0; i < (1UL<<order); i++, vaddr += PAGE_SIZE) { mcs = __xen_mc_entry(0); if (in_frames) in_frames[i] = virt_to_mfn(vaddr); MULTI_update_va_mapping(mcs.mc, vaddr, VOID_PTE, 0); |
6eaa412f2 xen: Mark all ini... |
2393 |
__set_phys_to_machine(virt_to_pfn(vaddr), INVALID_P2M_ENTRY); |
08bbc9da9 xen: Add xen_crea... |
2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 |
if (out_frames) out_frames[i] = virt_to_pfn(vaddr); } xen_mc_issue(0); } /* * Update the pfn-to-mfn mappings for a virtual address range, either to * point to an array of mfns, or contiguously from a single starting * mfn. */ static void xen_remap_exchanged_ptes(unsigned long vaddr, int order, unsigned long *mfns, unsigned long first_mfn) { unsigned i, limit; unsigned long mfn; xen_mc_batch(); limit = 1u << order; for (i = 0; i < limit; i++, vaddr += PAGE_SIZE) { struct multicall_space mcs; unsigned flags; mcs = __xen_mc_entry(0); if (mfns) mfn = mfns[i]; else mfn = first_mfn + i; if (i < (limit - 1)) flags = 0; else { if (order == 0) flags = UVMF_INVLPG | UVMF_ALL; else flags = UVMF_TLB_FLUSH | UVMF_ALL; } MULTI_update_va_mapping(mcs.mc, vaddr, mfn_pte(mfn, PAGE_KERNEL), flags); set_phys_to_machine(virt_to_pfn(vaddr), mfn); } xen_mc_issue(0); } /* * Perform the hypercall to exchange a region of our pfns to point to * memory with the required contiguous alignment. Takes the pfns as * input, and populates mfns as output. * * Returns a success code indicating whether the hypervisor was able to * satisfy the request or not. */ static int xen_exchange_memory(unsigned long extents_in, unsigned int order_in, unsigned long *pfns_in, unsigned long extents_out, unsigned int order_out, unsigned long *mfns_out, unsigned int address_bits) { long rc; int success; struct xen_memory_exchange exchange = { .in = { .nr_extents = extents_in, .extent_order = order_in, .extent_start = pfns_in, .domid = DOMID_SELF }, .out = { .nr_extents = extents_out, .extent_order = order_out, .extent_start = mfns_out, .address_bits = address_bits, .domid = DOMID_SELF } }; BUG_ON(extents_in << order_in != extents_out << order_out); rc = HYPERVISOR_memory_op(XENMEM_exchange, &exchange); success = (exchange.nr_exchanged == extents_in); BUG_ON(!success && ((exchange.nr_exchanged != 0) || (rc == 0))); BUG_ON(success && (rc != 0)); return success; } |
1b65c4e5a swiotlb-xen: use ... |
2488 |
int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, |
69908907b xen: make xen_cre... |
2489 2490 |
unsigned int address_bits, dma_addr_t *dma_handle) |
08bbc9da9 xen: Add xen_crea... |
2491 2492 2493 2494 |
{ unsigned long *in_frames = discontig_frames, out_frame; unsigned long flags; int success; |
1b65c4e5a swiotlb-xen: use ... |
2495 |
unsigned long vstart = (unsigned long)phys_to_virt(pstart); |
08bbc9da9 xen: Add xen_crea... |
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 |
/* * Currently an auto-translated guest will not perform I/O, nor will * it require PAE page directories below 4GB. Therefore any calls to * this function are redundant and can be ignored. */ if (xen_feature(XENFEAT_auto_translated_physmap)) return 0; if (unlikely(order > MAX_CONTIG_ORDER)) return -ENOMEM; memset((void *) vstart, 0, PAGE_SIZE << order); |
08bbc9da9 xen: Add xen_crea... |
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 |
spin_lock_irqsave(&xen_reservation_lock, flags); /* 1. Zap current PTEs, remembering MFNs. */ xen_zap_pfn_range(vstart, order, in_frames, NULL); /* 2. Get a new contiguous memory extent. */ out_frame = virt_to_pfn(vstart); success = xen_exchange_memory(1UL << order, 0, in_frames, 1, order, &out_frame, address_bits); /* 3. Map the new extent in place of old pages. */ if (success) xen_remap_exchanged_ptes(vstart, order, NULL, out_frame); else xen_remap_exchanged_ptes(vstart, order, in_frames, 0); spin_unlock_irqrestore(&xen_reservation_lock, flags); |
69908907b xen: make xen_cre... |
2528 |
*dma_handle = virt_to_machine(vstart).maddr; |
08bbc9da9 xen: Add xen_crea... |
2529 2530 2531 |
return success ? 0 : -ENOMEM; } EXPORT_SYMBOL_GPL(xen_create_contiguous_region); |
1b65c4e5a swiotlb-xen: use ... |
2532 |
void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) |
08bbc9da9 xen: Add xen_crea... |
2533 2534 2535 2536 |
{ unsigned long *out_frames = discontig_frames, in_frame; unsigned long flags; int success; |
1b65c4e5a swiotlb-xen: use ... |
2537 |
unsigned long vstart; |
08bbc9da9 xen: Add xen_crea... |
2538 2539 2540 2541 2542 2543 |
if (xen_feature(XENFEAT_auto_translated_physmap)) return; if (unlikely(order > MAX_CONTIG_ORDER)) return; |
1b65c4e5a swiotlb-xen: use ... |
2544 |
vstart = (unsigned long)phys_to_virt(pstart); |
08bbc9da9 xen: Add xen_crea... |
2545 |
memset((void *) vstart, 0, PAGE_SIZE << order); |
08bbc9da9 xen: Add xen_crea... |
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 |
spin_lock_irqsave(&xen_reservation_lock, flags); /* 1. Find start MFN of contiguous extent. */ in_frame = virt_to_mfn(vstart); /* 2. Zap current PTEs. */ xen_zap_pfn_range(vstart, order, NULL, out_frames); /* 3. Do the exchange for non-contiguous MFNs. */ success = xen_exchange_memory(1, order, &in_frame, 1UL << order, 0, out_frames, 0); /* 4. Map new pages in place of old pages. */ if (success) xen_remap_exchanged_ptes(vstart, order, out_frames, 0); else xen_remap_exchanged_ptes(vstart, order, NULL, in_frame); spin_unlock_irqrestore(&xen_reservation_lock, flags); |
030cb6c00 x86: Move paravir... |
2565 |
} |
08bbc9da9 xen: Add xen_crea... |
2566 |
EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); |
319f3ba52 xen: move remaini... |
2567 |
|
ca65f9fc0 Introduce CONFIG_... |
2568 |
#ifdef CONFIG_XEN_PVHVM |
34b6f01a7 xen pv-on-hvm: ad... |
2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 |
#ifdef CONFIG_PROC_VMCORE /* * This function is used in two contexts: * - the kdump kernel has to check whether a pfn of the crashed kernel * was a ballooned page. vmcore is using this function to decide * whether to access a pfn of the crashed kernel. * - the kexec kernel has to check whether a pfn was ballooned by the * previous kernel. If the pfn is ballooned, handle it properly. * Returns 0 if the pfn is not backed by a RAM page, the caller may * handle the pfn special in this case. */ static int xen_oldmem_pfn_is_ram(unsigned long pfn) { struct xen_hvm_get_mem_type a = { .domid = DOMID_SELF, .pfn = pfn, }; int ram; if (HYPERVISOR_hvm_op(HVMOP_get_mem_type, &a)) return -ENXIO; switch (a.mem_type) { case HVMMEM_mmio_dm: ram = 0; break; case HVMMEM_ram_rw: case HVMMEM_ram_ro: default: ram = 1; break; } return ram; } #endif |
591510010 x86: Call HVMOP_p... |
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 |
static void xen_hvm_exit_mmap(struct mm_struct *mm) { struct xen_hvm_pagetable_dying a; int rc; a.domid = DOMID_SELF; a.gpa = __pa(mm->pgd); rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a); WARN_ON_ONCE(rc < 0); } static int is_pagetable_dying_supported(void) { struct xen_hvm_pagetable_dying a; int rc = 0; a.domid = DOMID_SELF; a.gpa = 0x00; rc = HYPERVISOR_hvm_op(HVMOP_pagetable_dying, &a); if (rc < 0) { printk(KERN_DEBUG "HVMOP_pagetable_dying not supported "); return 0; } return 1; } void __init xen_hvm_init_mmu_ops(void) { if (is_pagetable_dying_supported()) pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap; |
34b6f01a7 xen pv-on-hvm: ad... |
2636 2637 2638 |
#ifdef CONFIG_PROC_VMCORE register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram); #endif |
591510010 x86: Call HVMOP_p... |
2639 |
} |
ca65f9fc0 Introduce CONFIG_... |
2640 |
#endif |
591510010 x86: Call HVMOP_p... |
2641 |
|
de1ef2065 xen/privcmd: move... |
2642 2643 2644 |
#define REMAP_BATCH_SIZE 16 struct remap_data { |
4e8c0c8c4 xen/privcmd: impr... |
2645 2646 |
xen_pfn_t *mfn; bool contiguous; |
de1ef2065 xen/privcmd: move... |
2647 2648 2649 2650 2651 2652 2653 2654 |
pgprot_t prot; struct mmu_update *mmu_update; }; static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr, void *data) { struct remap_data *rmd = data; |
4e8c0c8c4 xen/privcmd: impr... |
2655 2656 2657 2658 2659 2660 2661 2662 |
pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot)); /* If we have a contigious range, just update the mfn itself, else update pointer to be "next mfn". */ if (rmd->contiguous) (*rmd->mfn)++; else rmd->mfn++; |
de1ef2065 xen/privcmd: move... |
2663 |
|
d5108316b xen: use normal v... |
2664 |
rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; |
de1ef2065 xen/privcmd: move... |
2665 2666 2667 2668 2669 |
rmd->mmu_update->val = pte_val_ma(pte); rmd->mmu_update++; return 0; } |
a13d7201d xen/privcmd: Furt... |
2670 |
static int do_remap_gfn(struct vm_area_struct *vma, |
4e8c0c8c4 xen/privcmd: impr... |
2671 |
unsigned long addr, |
a13d7201d xen/privcmd: Furt... |
2672 |
xen_pfn_t *gfn, int nr, |
4e8c0c8c4 xen/privcmd: impr... |
2673 2674 2675 |
int *err_ptr, pgprot_t prot, unsigned domid, struct page **pages) |
de1ef2065 xen/privcmd: move... |
2676 |
{ |
4e8c0c8c4 xen/privcmd: impr... |
2677 |
int err = 0; |
de1ef2065 xen/privcmd: move... |
2678 2679 |
struct remap_data rmd; struct mmu_update mmu_update[REMAP_BATCH_SIZE]; |
de1ef2065 xen/privcmd: move... |
2680 |
unsigned long range; |
4e8c0c8c4 xen/privcmd: impr... |
2681 |
int mapped = 0; |
de1ef2065 xen/privcmd: move... |
2682 |
|
314e51b98 mm: kill vma flag... |
2683 |
BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO))); |
de1ef2065 xen/privcmd: move... |
2684 |
|
77945ca73 x86/xen: map fore... |
2685 2686 2687 |
if (xen_feature(XENFEAT_auto_translated_physmap)) { #ifdef CONFIG_XEN_PVH /* We need to update the local page tables and the xen HAP */ |
a13d7201d xen/privcmd: Furt... |
2688 |
return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr, |
4e8c0c8c4 xen/privcmd: impr... |
2689 |
prot, domid, pages); |
77945ca73 x86/xen: map fore... |
2690 2691 2692 2693 |
#else return -EINVAL; #endif } |
a13d7201d xen/privcmd: Furt... |
2694 |
rmd.mfn = gfn; |
de1ef2065 xen/privcmd: move... |
2695 |
rmd.prot = prot; |
4e8c0c8c4 xen/privcmd: impr... |
2696 2697 2698 |
/* We use the err_ptr to indicate if there we are doing a contigious * mapping or a discontigious mapping. */ rmd.contiguous = !err_ptr; |
de1ef2065 xen/privcmd: move... |
2699 2700 |
while (nr) { |
4e8c0c8c4 xen/privcmd: impr... |
2701 2702 2703 2704 |
int index = 0; int done = 0; int batch = min(REMAP_BATCH_SIZE, nr); int batch_left = batch; |
de1ef2065 xen/privcmd: move... |
2705 2706 2707 2708 2709 2710 2711 |
range = (unsigned long)batch << PAGE_SHIFT; rmd.mmu_update = mmu_update; err = apply_to_page_range(vma->vm_mm, addr, range, remap_area_mfn_pte_fn, &rmd); if (err) goto out; |
4e8c0c8c4 xen/privcmd: impr... |
2712 2713 2714 2715 2716 2717 2718 2719 2720 |
/* We record the error for each page that gives an error, but * continue mapping until the whole set is done */ do { int i; err = HYPERVISOR_mmu_update(&mmu_update[index], batch_left, &done, domid); /* |
a13d7201d xen/privcmd: Furt... |
2721 2722 |
* @err_ptr may be the same buffer as @gfn, so * only clear it after each chunk of @gfn is |
4e8c0c8c4 xen/privcmd: impr... |
2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 |
* used. */ if (err_ptr) { for (i = index; i < index + done; i++) err_ptr[i] = 0; } if (err < 0) { if (!err_ptr) goto out; err_ptr[i] = err; done++; /* Skip failed frame. */ } else mapped += done; batch_left -= done; index += done; } while (batch_left); |
de1ef2065 xen/privcmd: move... |
2739 2740 2741 |
nr -= batch; addr += range; |
4e8c0c8c4 xen/privcmd: impr... |
2742 2743 |
if (err_ptr) err_ptr += batch; |
914beb9fc x86/xen: add resc... |
2744 |
cond_resched(); |
de1ef2065 xen/privcmd: move... |
2745 |
} |
de1ef2065 xen/privcmd: move... |
2746 |
out: |
95a7d7689 xen/mmu: Use Xen ... |
2747 |
xen_flush_tlb_all(); |
de1ef2065 xen/privcmd: move... |
2748 |
|
4e8c0c8c4 xen/privcmd: impr... |
2749 2750 |
return err < 0 ? err : mapped; } |
a13d7201d xen/privcmd: Furt... |
2751 |
int xen_remap_domain_gfn_range(struct vm_area_struct *vma, |
4e8c0c8c4 xen/privcmd: impr... |
2752 |
unsigned long addr, |
a13d7201d xen/privcmd: Furt... |
2753 |
xen_pfn_t gfn, int nr, |
4e8c0c8c4 xen/privcmd: impr... |
2754 2755 2756 |
pgprot_t prot, unsigned domid, struct page **pages) { |
a13d7201d xen/privcmd: Furt... |
2757 |
return do_remap_gfn(vma, addr, &gfn, nr, NULL, prot, domid, pages); |
de1ef2065 xen/privcmd: move... |
2758 |
} |
a13d7201d xen/privcmd: Furt... |
2759 |
EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range); |
9a032e393 xen: add pages pa... |
2760 |
|
a13d7201d xen/privcmd: Furt... |
2761 |
int xen_remap_domain_gfn_array(struct vm_area_struct *vma, |
4e8c0c8c4 xen/privcmd: impr... |
2762 |
unsigned long addr, |
a13d7201d xen/privcmd: Furt... |
2763 |
xen_pfn_t *gfn, int nr, |
4e8c0c8c4 xen/privcmd: impr... |
2764 2765 2766 2767 2768 2769 2770 2771 |
int *err_ptr, pgprot_t prot, unsigned domid, struct page **pages) { /* We BUG_ON because it's a programmer error to pass a NULL err_ptr, * and the consequences later is quite hard to detect what the actual * cause of "wrong memory was mapped in". */ BUG_ON(err_ptr == NULL); |
a13d7201d xen/privcmd: Furt... |
2772 |
return do_remap_gfn(vma, addr, gfn, nr, err_ptr, prot, domid, pages); |
4e8c0c8c4 xen/privcmd: impr... |
2773 |
} |
a13d7201d xen/privcmd: Furt... |
2774 |
EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array); |
4e8c0c8c4 xen/privcmd: impr... |
2775 |
|
9a032e393 xen: add pages pa... |
2776 |
/* Returns: 0 success */ |
a13d7201d xen/privcmd: Furt... |
2777 |
int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, |
9a032e393 xen: add pages pa... |
2778 2779 2780 2781 |
int numpgs, struct page **pages) { if (!pages || !xen_feature(XENFEAT_auto_translated_physmap)) return 0; |
77945ca73 x86/xen: map fore... |
2782 |
#ifdef CONFIG_XEN_PVH |
628c28eef xen: unify foreig... |
2783 |
return xen_xlate_unmap_gfn_range(vma, numpgs, pages); |
77945ca73 x86/xen: map fore... |
2784 |
#else |
9a032e393 xen: add pages pa... |
2785 |
return -EINVAL; |
77945ca73 x86/xen: map fore... |
2786 |
#endif |
9a032e393 xen: add pages pa... |
2787 |
} |
a13d7201d xen/privcmd: Furt... |
2788 |
EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); |