Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit 02512b2bd63385d1f34f6956860dedbfc9ac20d7

Authored by Paolo Bonzini 2015-01-30 16:41:38 +0800

Exists in ti-lsk-linux-4.1.y and in 10 other branches

Merge tag 'kvm-arm-fixes-3.19-2' of git://git.kernel.org/pub/scm/linux/kernel/gi…

…t/kvmarm/kvmarm into kvm-master

Second round of fixes for KVM/ARM for 3.19.

Fixes memory corruption issues on APM platforms and swapping issues on
DMA-coherent systems.

Showing 14 changed files Inline Diff

arch/arm/include/asm/kvm_emulate.h
arch/arm/include/asm/kvm_host.h
arch/arm/include/asm/kvm_mmu.h
arch/arm/kvm/arm.c
arch/arm/kvm/coproc.c
arch/arm/kvm/coproc.h
arch/arm/kvm/coproc_a15.c
arch/arm/kvm/coproc_a7.c
arch/arm/kvm/mmu.c
arch/arm/kvm/trace.h
arch/arm64/include/asm/kvm_emulate.h
arch/arm64/include/asm/kvm_host.h
arch/arm64/include/asm/kvm_mmu.h
arch/arm64/kvm/sys_regs.c

arch/arm/include/asm/kvm_emulate.h

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #ifndef __ARM_KVM_EMULATE_H__
 #define __ARM_KVM_EMULATE_H__
 #include <linux/kvm_host.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 #include <asm/kvm_arm.h>
 unsigned long *vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num);
 unsigned long *vcpu_spsr(struct kvm_vcpu *vcpu);
 bool kvm_condition_valid(struct kvm_vcpu *vcpu);
 void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr);
 void kvm_inject_undefined(struct kvm_vcpu *vcpu);
 void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
 void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcr = HCR_GUEST_MASK;
 }
+static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.hcr;
+}
+static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
+{
+	vcpu->arch.hcr = hcr;
+}
 static inline bool vcpu_mode_is_32bit(struct kvm_vcpu *vcpu)
 {
 	return 1;
 }
 static inline unsigned long *vcpu_pc(struct kvm_vcpu *vcpu)
 {
 	return &vcpu->arch.regs.usr_regs.ARM_pc;
 }
 static inline unsigned long *vcpu_cpsr(struct kvm_vcpu *vcpu)
 {
 	return &vcpu->arch.regs.usr_regs.ARM_cpsr;
 }
 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
 {
 	*vcpu_cpsr(vcpu) |= PSR_T_BIT;
 }
 static inline bool mode_has_spsr(struct kvm_vcpu *vcpu)
 {
 	unsigned long cpsr_mode = vcpu->arch.regs.usr_regs.ARM_cpsr & MODE_MASK;
 	return (cpsr_mode > USR_MODE && cpsr_mode < SYSTEM_MODE);
 }
 static inline bool vcpu_mode_priv(struct kvm_vcpu *vcpu)
 {
 	unsigned long cpsr_mode = vcpu->arch.regs.usr_regs.ARM_cpsr & MODE_MASK;
 	return cpsr_mode > USR_MODE;;
 }
 static inline u32 kvm_vcpu_get_hsr(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.fault.hsr;
 }
 static inline unsigned long kvm_vcpu_get_hfar(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.fault.hxfar;
 }
 static inline phys_addr_t kvm_vcpu_get_fault_ipa(struct kvm_vcpu *vcpu)
 {
 	return ((phys_addr_t)vcpu->arch.fault.hpfar & HPFAR_MASK) << 8;
 }
 static inline unsigned long kvm_vcpu_get_hyp_pc(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.fault.hyp_pc;
 }
 static inline bool kvm_vcpu_dabt_isvalid(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_ISV;
 }
 static inline bool kvm_vcpu_dabt_iswrite(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_WNR;
 }
 static inline bool kvm_vcpu_dabt_issext(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_SSE;
 }
 static inline int kvm_vcpu_dabt_get_rd(struct kvm_vcpu *vcpu)
 {
 	return (kvm_vcpu_get_hsr(vcpu) & HSR_SRT_MASK) >> HSR_SRT_SHIFT;
 }
 static inline bool kvm_vcpu_dabt_isextabt(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_EA;
 }
 static inline bool kvm_vcpu_dabt_iss1tw(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_DABT_S1PTW;
 }
 /* Get Access Size from a data abort */
 static inline int kvm_vcpu_dabt_get_as(struct kvm_vcpu *vcpu)
 {
 	switch ((kvm_vcpu_get_hsr(vcpu) >> 22) & 0x3) {
 	case 0:
 		return 1;
 	case 1:
 		return 2;
 	case 2:
 		return 4;
 	default:
 		kvm_err("Hardware is weird: SAS 0b11 is reserved\n");
 		return -EFAULT;
 	}
 }
 /* This one is not specific to Data Abort */
 static inline bool kvm_vcpu_trap_il_is32bit(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_IL;
 }
 static inline u8 kvm_vcpu_trap_get_class(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) >> HSR_EC_SHIFT;
 }
 static inline bool kvm_vcpu_trap_is_iabt(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_trap_get_class(vcpu) == HSR_EC_IABT;
 }
 static inline u8 kvm_vcpu_trap_get_fault(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_FSC;
 }
 static inline u8 kvm_vcpu_trap_get_fault_type(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_FSC_TYPE;
 }
 static inline u32 kvm_vcpu_hvc_get_imm(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & HSR_HVC_IMM_MASK;
 }
 static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.cp15[c0_MPIDR];
 }
 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
 {
 	*vcpu_cpsr(vcpu) |= PSR_E_BIT;
 }
 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
 {
 	return !!(*vcpu_cpsr(vcpu) & PSR_E_BIT);
 }
 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
 						    unsigned long data,
 						    unsigned int len)
 {
 	if (kvm_vcpu_is_be(vcpu)) {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return be16_to_cpu(data & 0xffff);
 		default:
 			return be32_to_cpu(data);
 		}
 	} else {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return le16_to_cpu(data & 0xffff);
 		default:
 			return le32_to_cpu(data);
 		}
 	}
 }
 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
 						    unsigned long data,
 						    unsigned int len)
 {
 	if (kvm_vcpu_is_be(vcpu)) {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return cpu_to_be16(data & 0xffff);
 		default:
 			return cpu_to_be32(data);
 		}
 	} else {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return cpu_to_le16(data & 0xffff);
 		default:
 			return cpu_to_le32(data);
 		}
 	}
 }
 #endif /* __ARM_KVM_EMULATE_H__ */

arch/arm/include/asm/kvm_host.h

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #ifndef __ARM_KVM_HOST_H__
 #define __ARM_KVM_HOST_H__
 #include <linux/types.h>
 #include <linux/kvm_types.h>
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 #include <asm/fpstate.h>
 #include <kvm/arm_arch_timer.h>
 #if defined(CONFIG_KVM_ARM_MAX_VCPUS)
 #define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
 #else
 #define KVM_MAX_VCPUS 0
 #endif
 #define KVM_USER_MEM_SLOTS 32
 #define KVM_PRIVATE_MEM_SLOTS 4
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
 #define KVM_HAVE_ONE_REG
 #define KVM_VCPU_MAX_FEATURES 2
 #include <kvm/arm_vgic.h>
 u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode);
 int __attribute_const__ kvm_target_cpu(void);
 int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
 void kvm_reset_coprocs(struct kvm_vcpu *vcpu);
 struct kvm_arch {
 	/* VTTBR value associated with below pgd and vmid */
 	u64    vttbr;
 	/* Timer */
 	struct arch_timer_kvm	timer;
 	/*
 	 * Anything that is not used directly from assembly code goes
 	 * here.
 	 */
 	/* The VMID generation used for the virt. memory system */
 	u64    vmid_gen;
 	u32    vmid;
 	/* Stage-2 page table */
 	pgd_t *pgd;
 	/* Interrupt controller */
 	struct vgic_dist	vgic;
 };
 #define KVM_NR_MEM_OBJS     40
 /*
  * We don't want allocation failures within the mmu code, so we preallocate
  * enough memory for a single page fault in a cache.
  */
 struct kvm_mmu_memory_cache {
 	int nobjs;
 	void *objects[KVM_NR_MEM_OBJS];
 };
 struct kvm_vcpu_fault_info {
 	u32 hsr;		/* Hyp Syndrome Register */
 	u32 hxfar;		/* Hyp Data/Inst. Fault Address Register */
 	u32 hpfar;		/* Hyp IPA Fault Address Register */
 	u32 hyp_pc;		/* PC when exception was taken from Hyp mode */
 };
 typedef struct vfp_hard_struct kvm_cpu_context_t;
 struct kvm_vcpu_arch {
 	struct kvm_regs regs;
 	int target; /* Processor target */
 	DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
 	/* System control coprocessor (cp15) */
 	u32 cp15[NR_CP15_REGS];
 	/* The CPU type we expose to the VM */
 	u32 midr;
 	/* HYP trapping configuration */
 	u32 hcr;
 	/* Interrupt related fields */
 	u32 irq_lines;		/* IRQ and FIQ levels */
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
 	/* Floating point registers (VFP and Advanced SIMD/NEON) */
 	struct vfp_hard_struct vfp_guest;
 	/* Host FP context */
 	kvm_cpu_context_t *host_cpu_context;
 	/* VGIC state */
 	struct vgic_cpu vgic_cpu;
 	struct arch_timer_cpu timer_cpu;
 	/*
 	 * Anything that is not used directly from assembly code goes
 	 * here.
 	 */
-	/* dcache set/way operation pending */
-	int last_pcpu;
-	cpumask_t require_dcache_flush;
 	/* Don't run the guest on this vcpu */
 	bool pause;
 	/* IO related fields */
 	struct kvm_decode mmio_decode;
 	/* Cache some mmu pages needed inside spinlock regions */
 	struct kvm_mmu_memory_cache mmu_page_cache;
 	/* Detect first run of a vcpu */
 	bool has_run_once;
 };
 struct kvm_vm_stat {
 	u32 remote_tlb_flush;
 };
 struct kvm_vcpu_stat {
 	u32 halt_wakeup;
 };
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
 u64 kvm_call_hyp(void *hypfn, ...);
 void force_vm_exit(const cpumask_t *mask);
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_unmap_hva_range(struct kvm *kvm,
 			unsigned long start, unsigned long end);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
 /* We do not have shadow page tables, hence the empty hooks */
 static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
 			      unsigned long end)
 {
 	return 0;
 }
 static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
 {
 	return 0;
 }
 static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
 							 unsigned long address)
 {
 }
 struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
 struct kvm_vcpu __percpu **kvm_get_running_vcpus(void);
 int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
 unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu);
 int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
 int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
 int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		int exception_index);
 static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
 				       phys_addr_t pgd_ptr,
 				       unsigned long hyp_stack_ptr,
 				       unsigned long vector_ptr)
 {
 	/*
 	 * Call initialization code, and switch to the full blown HYP
 	 * code. The init code doesn't need to preserve these
 	 * registers as r0-r3 are already callee saved according to
 	 * the AAPCS.
 	 * Note that we slightly misuse the prototype by casing the
 	 * stack pointer to a void *.
 	 *
 	 * We don't have enough registers to perform the full init in
 	 * one go.  Install the boot PGD first, and then install the
 	 * runtime PGD, stack pointer and vectors. The PGDs are always
 	 * passed as the third argument, in order to be passed into
 	 * r2-r3 to the init code (yes, this is compliant with the
 	 * PCS!).
 	 */
 	kvm_call_hyp(NULL, 0, boot_pgd_ptr);
 	kvm_call_hyp((void*)hyp_stack_ptr, vector_ptr, pgd_ptr);
 }
 static inline int kvm_arch_dev_ioctl_check_extension(long ext)
 {
 	return 0;
 }
 static inline void vgic_arch_setup(const struct vgic_params *vgic)
 {
 	BUG_ON(vgic->type != VGIC_V2);
 }
 int kvm_perf_init(void);
 int kvm_perf_teardown(void);
 static inline void kvm_arch_hardware_disable(void) {}
 static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 #endif /* __ARM_KVM_HOST_H__ */

arch/arm/include/asm/kvm_mmu.h

Diff comments View file @ 02512b2

1	/*	1	/*
2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University	2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
3	* Author: Christoffer Dall <c.dall@virtualopensystems.com>	3	* Author: Christoffer Dall <c.dall@virtualopensystems.com>
4	*	4	*
5	* This program is free software; you can redistribute it and/or modify	5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License, version 2, as	6	* it under the terms of the GNU General Public License, version 2, as
7	* published by the Free Software Foundation.	7	* published by the Free Software Foundation.
8	*	8	*
9	* This program is distributed in the hope that it will be useful,	9	* This program is distributed in the hope that it will be useful,
10	* but WITHOUT ANY WARRANTY; without even the implied warranty of	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	* GNU General Public License for more details.	12	* GNU General Public License for more details.
13	*	13	*
14	* You should have received a copy of the GNU General Public License	14	* You should have received a copy of the GNU General Public License
15	* along with this program; if not, write to the Free Software	15	* along with this program; if not, write to the Free Software
16	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.	16	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17	*/	17	*/
18		18
19	#ifndef __ARM_KVM_MMU_H__	19	#ifndef __ARM_KVM_MMU_H__
20	#define __ARM_KVM_MMU_H__	20	#define __ARM_KVM_MMU_H__
21		21
22	#include <asm/memory.h>	22	#include <asm/memory.h>
23	#include <asm/page.h>	23	#include <asm/page.h>
24		24
25	/*	25	/*
26	* We directly use the kernel VA for the HYP, as we can directly share	26	* We directly use the kernel VA for the HYP, as we can directly share
27	* the mapping (HTTBR "covers" TTBR1).	27	* the mapping (HTTBR "covers" TTBR1).
28	*/	28	*/
29	#define HYP_PAGE_OFFSET_MASK UL(~0)	29	#define HYP_PAGE_OFFSET_MASK UL(~0)
30	#define HYP_PAGE_OFFSET PAGE_OFFSET	30	#define HYP_PAGE_OFFSET PAGE_OFFSET
31	#define KERN_TO_HYP(kva) (kva)	31	#define KERN_TO_HYP(kva) (kva)
32		32
33	/*	33	/*
34	* Our virtual mapping for the boot-time MMU-enable code. Must be	34	* Our virtual mapping for the boot-time MMU-enable code. Must be
35	* shared across all the page-tables. Conveniently, we use the vectors	35	* shared across all the page-tables. Conveniently, we use the vectors
36	* page, where no kernel data will ever be shared with HYP.	36	* page, where no kernel data will ever be shared with HYP.
37	*/	37	*/
38	#define TRAMPOLINE_VA UL(CONFIG_VECTORS_BASE)	38	#define TRAMPOLINE_VA UL(CONFIG_VECTORS_BASE)
39		39
40	/*	40	/*
41	* KVM_MMU_CACHE_MIN_PAGES is the number of stage2 page table translation levels.	41	* KVM_MMU_CACHE_MIN_PAGES is the number of stage2 page table translation levels.
42	*/	42	*/
43	#define KVM_MMU_CACHE_MIN_PAGES 2	43	#define KVM_MMU_CACHE_MIN_PAGES 2
44		44
45	#ifndef __ASSEMBLY__	45	#ifndef __ASSEMBLY__
46		46
		47	#include <linux/highmem.h>
47	#include <asm/cacheflush.h>	48	#include <asm/cacheflush.h>
48	#include <asm/pgalloc.h>	49	#include <asm/pgalloc.h>
49		50
50	int create_hyp_mappings(void from, void to);	51	int create_hyp_mappings(void from, void to);
51	int create_hyp_io_mappings(void from, void to, phys_addr_t);	52	int create_hyp_io_mappings(void from, void to, phys_addr_t);
52	void free_boot_hyp_pgd(void);	53	void free_boot_hyp_pgd(void);
53	void free_hyp_pgds(void);	54	void free_hyp_pgds(void);
54		55
55	void stage2_unmap_vm(struct kvm *kvm);	56	void stage2_unmap_vm(struct kvm *kvm);
56	int kvm_alloc_stage2_pgd(struct kvm *kvm);	57	int kvm_alloc_stage2_pgd(struct kvm *kvm);
57	void kvm_free_stage2_pgd(struct kvm *kvm);	58	void kvm_free_stage2_pgd(struct kvm *kvm);
58	int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,	59	int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
59	phys_addr_t pa, unsigned long size, bool writable);	60	phys_addr_t pa, unsigned long size, bool writable);
60		61
61	int kvm_handle_guest_abort(struct kvm_vcpu vcpu, struct kvm_run run);	62	int kvm_handle_guest_abort(struct kvm_vcpu vcpu, struct kvm_run run);
62		63
63	void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);	64	void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
64		65
65	phys_addr_t kvm_mmu_get_httbr(void);	66	phys_addr_t kvm_mmu_get_httbr(void);
66	phys_addr_t kvm_mmu_get_boot_httbr(void);	67	phys_addr_t kvm_mmu_get_boot_httbr(void);
67	phys_addr_t kvm_get_idmap_vector(void);	68	phys_addr_t kvm_get_idmap_vector(void);
68	int kvm_mmu_init(void);	69	int kvm_mmu_init(void);
69	void kvm_clear_hyp_idmap(void);	70	void kvm_clear_hyp_idmap(void);
70		71
71	static inline void kvm_set_pmd(pmd_t *pmd, pmd_t new_pmd)	72	static inline void kvm_set_pmd(pmd_t *pmd, pmd_t new_pmd)
72	{	73	{
73	*pmd = new_pmd;	74	*pmd = new_pmd;
74	flush_pmd_entry(pmd);	75	flush_pmd_entry(pmd);
75	}	76	}
76		77
77	static inline void kvm_set_pte(pte_t *pte, pte_t new_pte)	78	static inline void kvm_set_pte(pte_t *pte, pte_t new_pte)
78	{	79	{
79	*pte = new_pte;	80	*pte = new_pte;
80	/*	81	/*
81	* flush_pmd_entry just takes a void pointer and cleans the necessary	82	* flush_pmd_entry just takes a void pointer and cleans the necessary
82	* cache entries, so we can reuse the function for ptes.	83	* cache entries, so we can reuse the function for ptes.
83	*/	84	*/
84	flush_pmd_entry(pte);	85	flush_pmd_entry(pte);
85	}	86	}
86		87
87	static inline void kvm_clean_pgd(pgd_t *pgd)	88	static inline void kvm_clean_pgd(pgd_t *pgd)
88	{	89	{
89	clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t));	90	clean_dcache_area(pgd, PTRS_PER_S2_PGD * sizeof(pgd_t));
90	}	91	}
91		92
92	static inline void kvm_clean_pmd(pmd_t *pmd)	93	static inline void kvm_clean_pmd(pmd_t *pmd)
93	{	94	{
94	clean_dcache_area(pmd, PTRS_PER_PMD * sizeof(pmd_t));	95	clean_dcache_area(pmd, PTRS_PER_PMD * sizeof(pmd_t));
95	}	96	}
96		97
97	static inline void kvm_clean_pmd_entry(pmd_t *pmd)	98	static inline void kvm_clean_pmd_entry(pmd_t *pmd)
98	{	99	{
99	clean_pmd_entry(pmd);	100	clean_pmd_entry(pmd);
100	}	101	}
101		102
102	static inline void kvm_clean_pte(pte_t *pte)	103	static inline void kvm_clean_pte(pte_t *pte)
103	{	104	{
104	clean_pte_table(pte);	105	clean_pte_table(pte);
105	}	106	}
106		107
107	static inline void kvm_set_s2pte_writable(pte_t *pte)	108	static inline void kvm_set_s2pte_writable(pte_t *pte)
108	{	109	{
109	pte_val(*pte) \|= L_PTE_S2_RDWR;	110	pte_val(*pte) \|= L_PTE_S2_RDWR;
110	}	111	}
111		112
112	static inline void kvm_set_s2pmd_writable(pmd_t *pmd)	113	static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
113	{	114	{
114	pmd_val(*pmd) \|= L_PMD_S2_RDWR;	115	pmd_val(*pmd) \|= L_PMD_S2_RDWR;
115	}	116	}
116		117
117	/* Open coded pd_addr_end that can deal with 64bit addresses /	118	/* Open coded pd_addr_end that can deal with 64bit addresses /
118	#define kvm_pgd_addr_end(addr, end) \	119	#define kvm_pgd_addr_end(addr, end) \
119	({ u64 __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \	120	({ u64 __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \
120	(__boundary - 1 < (end) - 1)? __boundary: (end); \	121	(__boundary - 1 < (end) - 1)? __boundary: (end); \
121	})	122	})
122		123
123	#define kvm_pud_addr_end(addr,end) (end)	124	#define kvm_pud_addr_end(addr,end) (end)
124		125
125	#define kvm_pmd_addr_end(addr, end) \	126	#define kvm_pmd_addr_end(addr, end) \
126	({ u64 __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \	127	({ u64 __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \
127	(__boundary - 1 < (end) - 1)? __boundary: (end); \	128	(__boundary - 1 < (end) - 1)? __boundary: (end); \
128	})	129	})
129		130
130	static inline bool kvm_page_empty(void *ptr)	131	static inline bool kvm_page_empty(void *ptr)
131	{	132	{
132	struct page *ptr_page = virt_to_page(ptr);	133	struct page *ptr_page = virt_to_page(ptr);
133	return page_count(ptr_page) == 1;	134	return page_count(ptr_page) == 1;
134	}	135	}
135		136
136		137
137	#define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)	138	#define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)
138	#define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp)	139	#define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp)
139	#define kvm_pud_table_empty(kvm, pudp) (0)	140	#define kvm_pud_table_empty(kvm, pudp) (0)
140		141
141	#define KVM_PREALLOC_LEVEL 0	142	#define KVM_PREALLOC_LEVEL 0
142		143
143	static inline int kvm_prealloc_hwpgd(struct kvm kvm, pgd_t pgd)	144	static inline int kvm_prealloc_hwpgd(struct kvm kvm, pgd_t pgd)
144	{	145	{
145	return 0;	146	return 0;
146	}	147	}
147		148
148	static inline void kvm_free_hwpgd(struct kvm *kvm) { }	149	static inline void kvm_free_hwpgd(struct kvm *kvm) { }
149		150
150	static inline void kvm_get_hwpgd(struct kvm kvm)	151	static inline void kvm_get_hwpgd(struct kvm kvm)
151	{	152	{
152	return kvm->arch.pgd;	153	return kvm->arch.pgd;
153	}	154	}
154		155
155	struct kvm;	156	struct kvm;
156		157
157	#define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))	158	#define kvm_flush_dcache_to_poc(a,l) __cpuc_flush_dcache_area((a), (l))
158		159
159	static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)	160	static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
160	{	161	{
161	return (vcpu->arch.cp15[c1_SCTLR] & 0b101) == 0b101;	162	return (vcpu->arch.cp15[c1_SCTLR] & 0b101) == 0b101;
162	}	163	}
163		164
164	static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,	165	static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu, pfn_t pfn,
165	unsigned long size,	166	unsigned long size,
166	bool ipa_uncached)	167	bool ipa_uncached)
167	{	168	{
168	if (!vcpu_has_cache_enabled(vcpu) \|\| ipa_uncached)
169	kvm_flush_dcache_to_poc((void *)hva, size);
170
171	/*	169	/*
172	* If we are going to insert an instruction page and the icache is	170	* If we are going to insert an instruction page and the icache is
173	* either VIPT or PIPT, there is a potential problem where the host	171	* either VIPT or PIPT, there is a potential problem where the host
174	* (or another VM) may have used the same page as this guest, and we	172	* (or another VM) may have used the same page as this guest, and we
175	* read incorrect data from the icache. If we're using a PIPT cache,	173	* read incorrect data from the icache. If we're using a PIPT cache,
176	* we can invalidate just that page, but if we are using a VIPT cache	174	* we can invalidate just that page, but if we are using a VIPT cache
177	* we need to invalidate the entire icache - damn shame - as written	175	* we need to invalidate the entire icache - damn shame - as written
178	* in the ARM ARM (DDI 0406C.b - Page B3-1393).	176	* in the ARM ARM (DDI 0406C.b - Page B3-1393).
179	*	177	*
180	* VIVT caches are tagged using both the ASID and the VMID and doesn't	178	* VIVT caches are tagged using both the ASID and the VMID and doesn't
181	* need any kind of flushing (DDI 0406C.b - Page B3-1392).	179	* need any kind of flushing (DDI 0406C.b - Page B3-1392).
		180	*
		181	* We need to do this through a kernel mapping (using the
		182	* user-space mapping has proved to be the wrong
		183	* solution). For that, we need to kmap one page at a time,
		184	* and iterate over the range.
182	*/	185	*/
183	if (icache_is_pipt()) {	186
184	__cpuc_coherent_user_range(hva, hva + size);	187	bool need_flush = !vcpu_has_cache_enabled(vcpu) \|\| ipa_uncached;
185	} else if (!icache_is_vivt_asid_tagged()) {	188
		189	VM_BUG_ON(size & PAGE_MASK);
		190
		191	if (!need_flush && !icache_is_pipt())
		192	goto vipt_cache;
		193
		194	while (size) {
		195	void *va = kmap_atomic_pfn(pfn);
		196
		197	if (need_flush)
		198	kvm_flush_dcache_to_poc(va, PAGE_SIZE);
		199
		200	if (icache_is_pipt())
		201	__cpuc_coherent_user_range((unsigned long)va,
		202	(unsigned long)va + PAGE_SIZE);
		203
		204	size -= PAGE_SIZE;
		205	pfn++;
		206
		207	kunmap_atomic(va);
		208	}
		209
		210	vipt_cache:
		211	if (!icache_is_pipt() && !icache_is_vivt_asid_tagged()) {
186	/* any kind of VIPT cache */	212	/* any kind of VIPT cache */
187	__flush_icache_all();	213	__flush_icache_all();
188	}	214	}
189	}	215	}
190		216
		217	static inline void __kvm_flush_dcache_pte(pte_t pte)
		218	{
		219	void *va = kmap_atomic(pte_page(pte));
		220
		221	kvm_flush_dcache_to_poc(va, PAGE_SIZE);
		222
		223	kunmap_atomic(va);
		224	}
		225
		226	static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
		227	{
		228	unsigned long size = PMD_SIZE;
		229	pfn_t pfn = pmd_pfn(pmd);
		230
		231	while (size) {
		232	void *va = kmap_atomic_pfn(pfn);
		233
		234	kvm_flush_dcache_to_poc(va, PAGE_SIZE);
		235
		236	pfn++;
		237	size -= PAGE_SIZE;
		238
		239	kunmap_atomic(va);
		240	}
		241	}
		242
		243	static inline void __kvm_flush_dcache_pud(pud_t pud)
		244	{
		245	}
		246
191	#define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))	247	#define kvm_virt_to_phys(x) virt_to_idmap((unsigned long)(x))
192		248
193	void stage2_flush_vm(struct kvm *kvm);	249	void kvm_set_way_flush(struct kvm_vcpu *vcpu);
		250	void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
194		251
195	#endif /* !__ASSEMBLY__ */	252	#endif /* !__ASSEMBLY__ */

arch/arm/kvm/arm.c

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #include <linux/cpu.h>
 #include <linux/cpu_pm.h>
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/kvm_host.h>
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/fs.h>
 #include <linux/mman.h>
 #include <linux/sched.h>
 #include <linux/kvm.h>
 #include <trace/events/kvm.h>
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 #include <asm/uaccess.h>
 #include <asm/ptrace.h>
 #include <asm/mman.h>
 #include <asm/tlbflush.h>
 #include <asm/cacheflush.h>
 #include <asm/virt.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_psci.h>
 #ifdef REQUIRES_VIRT
 __asm__(".arch_extension	virt");
 #endif
 static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
 static kvm_cpu_context_t __percpu *kvm_host_cpu_state;
 static unsigned long hyp_default_vectors;
 /* Per-CPU variable containing the currently running vcpu. */
 static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_arm_running_vcpu);
 /* The VMID used in the VTTBR */
 static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
 static u8 kvm_next_vmid;
 static DEFINE_SPINLOCK(kvm_vmid_lock);
 static bool vgic_present;
 static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
 {
 	BUG_ON(preemptible());
 	__this_cpu_write(kvm_arm_running_vcpu, vcpu);
 }
 /**
  * kvm_arm_get_running_vcpu - get the vcpu running on the current CPU.
  * Must be called from non-preemptible context
  */
 struct kvm_vcpu *kvm_arm_get_running_vcpu(void)
 {
 	BUG_ON(preemptible());
 	return __this_cpu_read(kvm_arm_running_vcpu);
 }
 /**
  * kvm_arm_get_running_vcpus - get the per-CPU array of currently running vcpus.
  */
 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
 {
 	return &kvm_arm_running_vcpu;
 }
 int kvm_arch_hardware_enable(void)
 {
 	return 0;
 }
 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
 }
 int kvm_arch_hardware_setup(void)
 {
 	return 0;
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 	*(int *)rtn = 0;
 }
 /**
  * kvm_arch_init_vm - initializes a VM data structure
  * @kvm:	pointer to the KVM struct
  */
 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 {
 	int ret = 0;
 	if (type)
 		return -EINVAL;
 	ret = kvm_alloc_stage2_pgd(kvm);
 	if (ret)
 		goto out_fail_alloc;
 	ret = create_hyp_mappings(kvm, kvm + 1);
 	if (ret)
 		goto out_free_stage2_pgd;
 	kvm_timer_init(kvm);
 	/* Mark the initial VMID generation invalid */
 	kvm->arch.vmid_gen = 0;
 	return ret;
 out_free_stage2_pgd:
 	kvm_free_stage2_pgd(kvm);
 out_fail_alloc:
 	return ret;
 }
 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
 {
 	return VM_FAULT_SIGBUS;
 }
 /**
  * kvm_arch_destroy_vm - destroy the VM data structure
  * @kvm:	pointer to the KVM struct
  */
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	int i;
 	kvm_free_stage2_pgd(kvm);
 	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 		if (kvm->vcpus[i]) {
 			kvm_arch_vcpu_free(kvm->vcpus[i]);
 			kvm->vcpus[i] = NULL;
 		}
 	}
 	kvm_vgic_destroy(kvm);
 }
 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_IRQCHIP:
 		r = vgic_present;
 		break;
 	case KVM_CAP_DEVICE_CTRL:
 	case KVM_CAP_USER_MEMORY:
 	case KVM_CAP_SYNC_MMU:
 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
 	case KVM_CAP_ONE_REG:
 	case KVM_CAP_ARM_PSCI:
 	case KVM_CAP_ARM_PSCI_0_2:
 	case KVM_CAP_READONLY_MEM:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 	case KVM_CAP_ARM_SET_DEVICE_ADDR:
 		r = 1;
 		break;
 	case KVM_CAP_NR_VCPUS:
 		r = num_online_cpus();
 		break;
 	case KVM_CAP_MAX_VCPUS:
 		r = KVM_MAX_VCPUS;
 		break;
 	default:
 		r = kvm_arch_dev_ioctl_check_extension(ext);
 		break;
 	}
 	return r;
 }
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
 	return -EINVAL;
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
 {
 	int err;
 	struct kvm_vcpu *vcpu;
 	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) {
 		err = -EBUSY;
 		goto out;
 	}
 	vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 	if (!vcpu) {
 		err = -ENOMEM;
 		goto out;
 	}
 	err = kvm_vcpu_init(vcpu, kvm, id);
 	if (err)
 		goto free_vcpu;
 	err = create_hyp_mappings(vcpu, vcpu + 1);
 	if (err)
 		goto vcpu_uninit;
 	return vcpu;
 vcpu_uninit:
 	kvm_vcpu_uninit(vcpu);
 free_vcpu:
 	kmem_cache_free(kvm_vcpu_cache, vcpu);
 out:
 	return ERR_PTR(err);
 }
 int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
 {
 	kvm_mmu_free_memory_caches(vcpu);
 	kvm_timer_vcpu_terminate(vcpu);
 	kvm_vgic_vcpu_destroy(vcpu);
 	kmem_cache_free(kvm_vcpu_cache, vcpu);
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	kvm_arch_vcpu_free(vcpu);
 }
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	/* Force users to call KVM_ARM_VCPU_INIT */
 	vcpu->arch.target = -1;
 	bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES);
 	/* Set up the timer */
 	kvm_timer_vcpu_init(vcpu);
 	return 0;
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	vcpu->cpu = cpu;
 	vcpu->arch.host_cpu_context = this_cpu_ptr(kvm_host_cpu_state);
-	/*
-	 * Check whether this vcpu requires the cache to be flushed on
-	 * this physical CPU. This is a consequence of doing dcache
-	 * operations by set/way on this vcpu. We do it here to be in
-	 * a non-preemptible section.
-	 */
-	if (cpumask_test_and_clear_cpu(cpu, &vcpu->arch.require_dcache_flush))
-		flush_cache_all(); /* We'd really want v7_flush_dcache_all() */
 	kvm_arm_set_running_vcpu(vcpu);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	/*
 	 * The arch-generic KVM code expects the cpu field of a vcpu to be -1
 	 * if the vcpu is no longer assigned to a cpu.  This is used for the
 	 * optimized make_all_cpus_request path.
 	 */
 	vcpu->cpu = -1;
 	kvm_arm_set_running_vcpu(NULL);
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	return -EINVAL;
 }
 /**
  * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
  * @v:		The VCPU pointer
  *
  * If the guest CPU is not waiting for interrupts or an interrupt line is
  * asserted, the CPU is by definition runnable.
  */
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 {
 	return !!v->arch.irq_lines || kvm_vgic_vcpu_pending_irq(v);
 }
 /* Just ensure a guest exit from a particular CPU */
 static void exit_vm_noop(void *info)
 {
 }
 void force_vm_exit(const cpumask_t *mask)
 {
 	smp_call_function_many(mask, exit_vm_noop, NULL, true);
 }
 /**
  * need_new_vmid_gen - check that the VMID is still valid
  * @kvm: The VM's VMID to checkt
  *
  * return true if there is a new generation of VMIDs being used
  *
  * The hardware supports only 256 values with the value zero reserved for the
  * host, so we check if an assigned value belongs to a previous generation,
  * which which requires us to assign a new value. If we're the first to use a
  * VMID for the new generation, we must flush necessary caches and TLBs on all
  * CPUs.
  */
 static bool need_new_vmid_gen(struct kvm *kvm)
 {
 	return unlikely(kvm->arch.vmid_gen != atomic64_read(&kvm_vmid_gen));
 }
 /**
  * update_vttbr - Update the VTTBR with a valid VMID before the guest runs
  * @kvm	The guest that we are about to run
  *
  * Called from kvm_arch_vcpu_ioctl_run before entering the guest to ensure the
  * VM has a valid VMID, otherwise assigns a new one and flushes corresponding
  * caches and TLBs.
  */
 static void update_vttbr(struct kvm *kvm)
 {
 	phys_addr_t pgd_phys;
 	u64 vmid;
 	if (!need_new_vmid_gen(kvm))
 		return;
 	spin_lock(&kvm_vmid_lock);
 	/*
 	 * We need to re-check the vmid_gen here to ensure that if another vcpu
 	 * already allocated a valid vmid for this vm, then this vcpu should
 	 * use the same vmid.
 	 */
 	if (!need_new_vmid_gen(kvm)) {
 		spin_unlock(&kvm_vmid_lock);
 		return;
 	}
 	/* First user of a new VMID generation? */
 	if (unlikely(kvm_next_vmid == 0)) {
 		atomic64_inc(&kvm_vmid_gen);
 		kvm_next_vmid = 1;
 		/*
 		 * On SMP we know no other CPUs can use this CPU's or each
 		 * other's VMID after force_vm_exit returns since the
 		 * kvm_vmid_lock blocks them from reentry to the guest.
 		 */
 		force_vm_exit(cpu_all_mask);
 		/*
 		 * Now broadcast TLB + ICACHE invalidation over the inner
 		 * shareable domain to make sure all data structures are
 		 * clean.
 		 */
 		kvm_call_hyp(__kvm_flush_vm_context);
 	}
 	kvm->arch.vmid_gen = atomic64_read(&kvm_vmid_gen);
 	kvm->arch.vmid = kvm_next_vmid;
 	kvm_next_vmid++;
 	/* update vttbr to be used with the new vmid */
 	pgd_phys = virt_to_phys(kvm_get_hwpgd(kvm));
 	BUG_ON(pgd_phys & ~VTTBR_BADDR_MASK);
 	vmid = ((u64)(kvm->arch.vmid) << VTTBR_VMID_SHIFT) & VTTBR_VMID_MASK;
 	kvm->arch.vttbr = pgd_phys | vmid;
 	spin_unlock(&kvm_vmid_lock);
 }
 static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
 	int ret;
 	if (likely(vcpu->arch.has_run_once))
 		return 0;
 	vcpu->arch.has_run_once = true;
 	/*
 	 * Map the VGIC hardware resources before running a vcpu the first
 	 * time on this VM.
 	 */
 	if (unlikely(!vgic_ready(kvm))) {
 		ret = kvm_vgic_map_resources(kvm);
 		if (ret)
 			return ret;
 	}
 	/*
 	 * Enable the arch timers only if we have an in-kernel VGIC
 	 * and it has been properly initialized, since we cannot handle
 	 * interrupts from the virtual timer with a userspace gic.
 	 */
 	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
 		kvm_timer_enable(kvm);
 	return 0;
 }
 static void vcpu_pause(struct kvm_vcpu *vcpu)
 {
 	wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
 	wait_event_interruptible(*wq, !vcpu->arch.pause);
 }
 static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.target >= 0;
 }
 /**
  * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
  * @vcpu:	The VCPU pointer
  * @run:	The kvm_run structure pointer used for userspace state exchange
  *
  * This function is called through the VCPU_RUN ioctl called from user space. It
  * will execute VM code in a loop until the time slice for the process is used
  * or some emulation is needed from user space in which case the function will
  * return with return value 0 and with the kvm_run structure filled in with the
  * required data for the requested emulation.
  */
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	int ret;
 	sigset_t sigsaved;
 	if (unlikely(!kvm_vcpu_initialized(vcpu)))
 		return -ENOEXEC;
 	ret = kvm_vcpu_first_run_init(vcpu);
 	if (ret)
 		return ret;
 	if (run->exit_reason == KVM_EXIT_MMIO) {
 		ret = kvm_handle_mmio_return(vcpu, vcpu->run);
 		if (ret)
 			return ret;
 	}
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	ret = 1;
 	run->exit_reason = KVM_EXIT_UNKNOWN;
 	while (ret > 0) {
 		/*
 		 * Check conditions before entering the guest
 		 */
 		cond_resched();
 		update_vttbr(vcpu->kvm);
 		if (vcpu->arch.pause)
 			vcpu_pause(vcpu);
 		kvm_vgic_flush_hwstate(vcpu);
 		kvm_timer_flush_hwstate(vcpu);
 		local_irq_disable();
 		/*
 		 * Re-check atomic conditions
 		 */
 		if (signal_pending(current)) {
 			ret = -EINTR;
 			run->exit_reason = KVM_EXIT_INTR;
 		}
 		if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) {
 			local_irq_enable();
 			kvm_timer_sync_hwstate(vcpu);
 			kvm_vgic_sync_hwstate(vcpu);
 			continue;
 		}
 		/**************************************************************
 		 * Enter the guest
 		 */
 		trace_kvm_entry(*vcpu_pc(vcpu));
 		kvm_guest_enter();
 		vcpu->mode = IN_GUEST_MODE;
 		ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 		vcpu->mode = OUTSIDE_GUEST_MODE;
-		vcpu->arch.last_pcpu = smp_processor_id();
 		kvm_guest_exit();
 		trace_kvm_exit(*vcpu_pc(vcpu));
 		/*
 		 * We may have taken a host interrupt in HYP mode (ie
 		 * while executing the guest). This interrupt is still
 		 * pending, as we haven't serviced it yet!
 		 *
 		 * We're now back in SVC mode, with interrupts
 		 * disabled.  Enabling the interrupts now will have
 		 * the effect of taking the interrupt again, in SVC
 		 * mode this time.
 		 */
 		local_irq_enable();
 		/*
 		 * Back from guest
 		 *************************************************************/
 		kvm_timer_sync_hwstate(vcpu);
 		kvm_vgic_sync_hwstate(vcpu);
 		ret = handle_exit(vcpu, run, ret);
 	}
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
 	return ret;
 }
 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
 {
 	int bit_index;
 	bool set;
 	unsigned long *ptr;
 	if (number == KVM_ARM_IRQ_CPU_IRQ)
 		bit_index = __ffs(HCR_VI);
 	else /* KVM_ARM_IRQ_CPU_FIQ */
 		bit_index = __ffs(HCR_VF);
 	ptr = (unsigned long *)&vcpu->arch.irq_lines;
 	if (level)
 		set = test_and_set_bit(bit_index, ptr);
 	else
 		set = test_and_clear_bit(bit_index, ptr);
 	/*
 	 * If we didn't change anything, no need to wake up or kick other CPUs
 	 */
 	if (set == level)
 		return 0;
 	/*
 	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
 	 * trigger a world-switch round on the running physical CPU to set the
 	 * virtual IRQ/FIQ fields in the HCR appropriately.
 	 */
 	kvm_vcpu_kick(vcpu);
 	return 0;
 }
 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 			  bool line_status)
 {
 	u32 irq = irq_level->irq;
 	unsigned int irq_type, vcpu_idx, irq_num;
 	int nrcpus = atomic_read(&kvm->online_vcpus);
 	struct kvm_vcpu *vcpu = NULL;
 	bool level = irq_level->level;
 	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
 	vcpu_idx = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
 	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
 	trace_kvm_irq_line(irq_type, vcpu_idx, irq_num, irq_level->level);
 	switch (irq_type) {
 	case KVM_ARM_IRQ_TYPE_CPU:
 		if (irqchip_in_kernel(kvm))
 			return -ENXIO;
 		if (vcpu_idx >= nrcpus)
 			return -EINVAL;
 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
 		if (!vcpu)
 			return -EINVAL;
 		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
 			return -EINVAL;
 		return vcpu_interrupt_line(vcpu, irq_num, level);
 	case KVM_ARM_IRQ_TYPE_PPI:
 		if (!irqchip_in_kernel(kvm))
 			return -ENXIO;
 		if (vcpu_idx >= nrcpus)
 			return -EINVAL;
 		vcpu = kvm_get_vcpu(kvm, vcpu_idx);
 		if (!vcpu)
 			return -EINVAL;
 		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
 			return -EINVAL;
 		return kvm_vgic_inject_irq(kvm, vcpu->vcpu_id, irq_num, level);
 	case KVM_ARM_IRQ_TYPE_SPI:
 		if (!irqchip_in_kernel(kvm))
 			return -ENXIO;
 		if (irq_num < VGIC_NR_PRIVATE_IRQS ||
 		    irq_num > KVM_ARM_IRQ_GIC_MAX)
 			return -EINVAL;
 		return kvm_vgic_inject_irq(kvm, 0, irq_num, level);
 	}
 	return -EINVAL;
 }
 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
 			       const struct kvm_vcpu_init *init)
 {
 	unsigned int i;
 	int phys_target = kvm_target_cpu();
 	if (init->target != phys_target)
 		return -EINVAL;
 	/*
 	 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
 	 * use the same target.
 	 */
 	if (vcpu->arch.target != -1 && vcpu->arch.target != init->target)
 		return -EINVAL;
 	/* -ENOENT for unknown features, -EINVAL for invalid combinations. */
 	for (i = 0; i < sizeof(init->features) * 8; i++) {
 		bool set = (init->features[i / 32] & (1 << (i % 32)));
 		if (set && i >= KVM_VCPU_MAX_FEATURES)
 			return -ENOENT;
 		/*
 		 * Secondary and subsequent calls to KVM_ARM_VCPU_INIT must
 		 * use the same feature set.
 		 */
 		if (vcpu->arch.target != -1 && i < KVM_VCPU_MAX_FEATURES &&
 		    test_bit(i, vcpu->arch.features) != set)
 			return -EINVAL;
 		if (set)
 			set_bit(i, vcpu->arch.features);
 	}
 	vcpu->arch.target = phys_target;
 	/* Now we know what it is, we can reset it. */
 	return kvm_reset_vcpu(vcpu);
 }
 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
 					 struct kvm_vcpu_init *init)
 {
 	int ret;
 	ret = kvm_vcpu_set_target(vcpu, init);
 	if (ret)
 		return ret;
 	/*
 	 * Ensure a rebooted VM will fault in RAM pages and detect if the
 	 * guest MMU is turned off and flush the caches as needed.
 	 */
 	if (vcpu->arch.has_run_once)
 		stage2_unmap_vm(vcpu->kvm);
 	vcpu_reset_hcr(vcpu);
 	/*
 	 * Handle the "start in power-off" case by marking the VCPU as paused.
 	 */
 	if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features))
 		vcpu->arch.pause = true;
 	else
 		vcpu->arch.pause = false;
 	return 0;
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	switch (ioctl) {
 	case KVM_ARM_VCPU_INIT: {
 		struct kvm_vcpu_init init;
 		if (copy_from_user(&init, argp, sizeof(init)))
 			return -EFAULT;
 		return kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
 	}
 	case KVM_SET_ONE_REG:
 	case KVM_GET_ONE_REG: {
 		struct kvm_one_reg reg;
 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
 			return -ENOEXEC;
 		if (copy_from_user(&reg, argp, sizeof(reg)))
 			return -EFAULT;
 		if (ioctl == KVM_SET_ONE_REG)
 			return kvm_arm_set_reg(vcpu, &reg);
 		else
 			return kvm_arm_get_reg(vcpu, &reg);
 	}
 	case KVM_GET_REG_LIST: {
 		struct kvm_reg_list __user *user_list = argp;
 		struct kvm_reg_list reg_list;
 		unsigned n;
 		if (unlikely(!kvm_vcpu_initialized(vcpu)))
 			return -ENOEXEC;
 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
 			return -EFAULT;
 		n = reg_list.n;
 		reg_list.n = kvm_arm_num_regs(vcpu);
 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
 			return -EFAULT;
 		if (n < reg_list.n)
 			return -E2BIG;
 		return kvm_arm_copy_reg_indices(vcpu, user_list->reg);
 	}
 	default:
 		return -EINVAL;
 	}
 }
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
 {
 	return -EINVAL;
 }
 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
 					struct kvm_arm_device_addr *dev_addr)
 {
 	unsigned long dev_id, type;
 	dev_id = (dev_addr->id & KVM_ARM_DEVICE_ID_MASK) >>
 		KVM_ARM_DEVICE_ID_SHIFT;
 	type = (dev_addr->id & KVM_ARM_DEVICE_TYPE_MASK) >>
 		KVM_ARM_DEVICE_TYPE_SHIFT;
 	switch (dev_id) {
 	case KVM_ARM_DEVICE_VGIC_V2:
 		if (!vgic_present)
 			return -ENXIO;
 		return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
 	default:
 		return -ENODEV;
 	}
 }
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	switch (ioctl) {
 	case KVM_CREATE_IRQCHIP: {
 		if (vgic_present)
 			return kvm_vgic_create(kvm);
 		else
 			return -ENXIO;
 	}
 	case KVM_ARM_SET_DEVICE_ADDR: {
 		struct kvm_arm_device_addr dev_addr;
 		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
 			return -EFAULT;
 		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
 	}
 	case KVM_ARM_PREFERRED_TARGET: {
 		int err;
 		struct kvm_vcpu_init init;
 		err = kvm_vcpu_preferred_target(&init);
 		if (err)
 			return err;
 		if (copy_to_user(argp, &init, sizeof(init)))
 			return -EFAULT;
 		return 0;
 	}
 	default:
 		return -EINVAL;
 	}
 }
 static void cpu_init_hyp_mode(void *dummy)
 {
 	phys_addr_t boot_pgd_ptr;
 	phys_addr_t pgd_ptr;
 	unsigned long hyp_stack_ptr;
 	unsigned long stack_page;
 	unsigned long vector_ptr;
 	/* Switch from the HYP stub to our own HYP init vector */
 	__hyp_set_vectors(kvm_get_idmap_vector());
 	boot_pgd_ptr = kvm_mmu_get_boot_httbr();
 	pgd_ptr = kvm_mmu_get_httbr();
 	stack_page = __this_cpu_read(kvm_arm_hyp_stack_page);
 	hyp_stack_ptr = stack_page + PAGE_SIZE;
 	vector_ptr = (unsigned long)__kvm_hyp_vector;
 	__cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr);
 }
 static int hyp_init_cpu_notify(struct notifier_block *self,
 			       unsigned long action, void *cpu)
 {
 	switch (action) {
 	case CPU_STARTING:
 	case CPU_STARTING_FROZEN:
 		if (__hyp_get_vectors() == hyp_default_vectors)
 			cpu_init_hyp_mode(NULL);
 		break;
 	}
 	return NOTIFY_OK;
 }
 static struct notifier_block hyp_init_cpu_nb = {
 	.notifier_call = hyp_init_cpu_notify,
 };
 #ifdef CONFIG_CPU_PM
 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
 				    unsigned long cmd,
 				    void *v)
 {
 	if (cmd == CPU_PM_EXIT &&
 	    __hyp_get_vectors() == hyp_default_vectors) {
 		cpu_init_hyp_mode(NULL);
 		return NOTIFY_OK;
 	}
 	return NOTIFY_DONE;
 }
 static struct notifier_block hyp_init_cpu_pm_nb = {
 	.notifier_call = hyp_init_cpu_pm_notifier,
 };
 static void __init hyp_cpu_pm_init(void)
 {
 	cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
 }
 #else
 static inline void hyp_cpu_pm_init(void)
 {
 }
 #endif
 /**
  * Inits Hyp-mode on all online CPUs
  */
 static int init_hyp_mode(void)
 {
 	int cpu;
 	int err = 0;
 	/*
 	 * Allocate Hyp PGD and setup Hyp identity mapping
 	 */
 	err = kvm_mmu_init();
 	if (err)
 		goto out_err;
 	/*
 	 * It is probably enough to obtain the default on one
 	 * CPU. It's unlikely to be different on the others.
 	 */
 	hyp_default_vectors = __hyp_get_vectors();
 	/*
 	 * Allocate stack pages for Hypervisor-mode
 	 */
 	for_each_possible_cpu(cpu) {
 		unsigned long stack_page;
 		stack_page = __get_free_page(GFP_KERNEL);
 		if (!stack_page) {
 			err = -ENOMEM;
 			goto out_free_stack_pages;
 		}
 		per_cpu(kvm_arm_hyp_stack_page, cpu) = stack_page;
 	}
 	/*
 	 * Map the Hyp-code called directly from the host
 	 */
 	err = create_hyp_mappings(__kvm_hyp_code_start, __kvm_hyp_code_end);
 	if (err) {
 		kvm_err("Cannot map world-switch code\n");
 		goto out_free_mappings;
 	}
 	/*
 	 * Map the Hyp stack pages
 	 */
 	for_each_possible_cpu(cpu) {
 		char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu);
 		err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE);
 		if (err) {
 			kvm_err("Cannot map hyp stack\n");
 			goto out_free_mappings;
 		}
 	}
 	/*
 	 * Map the host CPU structures
 	 */
 	kvm_host_cpu_state = alloc_percpu(kvm_cpu_context_t);
 	if (!kvm_host_cpu_state) {
 		err = -ENOMEM;
 		kvm_err("Cannot allocate host CPU state\n");
 		goto out_free_mappings;
 	}
 	for_each_possible_cpu(cpu) {
 		kvm_cpu_context_t *cpu_ctxt;
 		cpu_ctxt = per_cpu_ptr(kvm_host_cpu_state, cpu);
 		err = create_hyp_mappings(cpu_ctxt, cpu_ctxt + 1);
 		if (err) {
 			kvm_err("Cannot map host CPU state: %d\n", err);
 			goto out_free_context;
 		}
 	}
 	/*
 	 * Execute the init code on each CPU.
 	 */
 	on_each_cpu(cpu_init_hyp_mode, NULL, 1);
 	/*
 	 * Init HYP view of VGIC
 	 */
 	err = kvm_vgic_hyp_init();
 	if (err)
 		goto out_free_context;
 #ifdef CONFIG_KVM_ARM_VGIC
 		vgic_present = true;
 #endif
 	/*
 	 * Init HYP architected timer support
 	 */
 	err = kvm_timer_hyp_init();
 	if (err)
 		goto out_free_mappings;
 #ifndef CONFIG_HOTPLUG_CPU
 	free_boot_hyp_pgd();
 #endif
 	kvm_perf_init();
 	kvm_info("Hyp mode initialized successfully\n");
 	return 0;
 out_free_context:
 	free_percpu(kvm_host_cpu_state);
 out_free_mappings:
 	free_hyp_pgds();
 out_free_stack_pages:
 	for_each_possible_cpu(cpu)
 		free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
 out_err:
 	kvm_err("error initializing Hyp mode: %d\n", err);
 	return err;
 }
 static void check_kvm_target_cpu(void *ret)
 {
 	*(int *)ret = kvm_target_cpu();
 }
 /**
  * Initialize Hyp-mode and memory mappings on all CPUs.
  */
 int kvm_arch_init(void *opaque)
 {
 	int err;
 	int ret, cpu;
 	if (!is_hyp_mode_available()) {
 		kvm_err("HYP mode not available\n");
 		return -ENODEV;
 	}
 	for_each_online_cpu(cpu) {
 		smp_call_function_single(cpu, check_kvm_target_cpu, &ret, 1);
 		if (ret < 0) {
 			kvm_err("Error, CPU %d not supported!\n", cpu);
 			return -ENODEV;
 		}
 	}
 	cpu_notifier_register_begin();
 	err = init_hyp_mode();
 	if (err)
 		goto out_err;
 	err = __register_cpu_notifier(&hyp_init_cpu_nb);
 	if (err) {
 		kvm_err("Cannot register HYP init CPU notifier (%d)\n", err);
 		goto out_err;
 	}
 	cpu_notifier_register_done();
 	hyp_cpu_pm_init();
 	kvm_coproc_table_init();
 	return 0;
 out_err:
 	cpu_notifier_register_done();
 	return err;
 }
 /* NOP: Compiling as a module not supported */
 void kvm_arch_exit(void)
 {
 	kvm_perf_teardown();
 }
 static int arm_init(void)
 {
 	int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
 	return rc;
 }
 module_init(arm_init);

arch/arm/kvm/coproc.c

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Authors: Rusty Russell <rusty@rustcorp.com.au>
  *          Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #include <linux/mm.h>
 #include <linux/kvm_host.h>
 #include <linux/uaccess.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_mmu.h>
 #include <asm/cacheflush.h>
 #include <asm/cputype.h>
 #include <trace/events/kvm.h>
 #include <asm/vfp.h>
 #include "../vfp/vfpinstr.h"
 #include "trace.h"
 #include "coproc.h"
 /******************************************************************************
  * Co-processor emulation
  *****************************************************************************/
 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
 static u32 cache_levels;
 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
 #define CSSELR_MAX 12
 /*
  * kvm_vcpu_arch.cp15 holds cp15 registers as an array of u32, but some
  * of cp15 registers can be viewed either as couple of two u32 registers
  * or one u64 register. Current u64 register encoding is that least
  * significant u32 word is followed by most significant u32 word.
  */
 static inline void vcpu_cp15_reg64_set(struct kvm_vcpu *vcpu,
 				       const struct coproc_reg *r,
 				       u64 val)
 {
 	vcpu->arch.cp15[r->reg] = val & 0xffffffff;
 	vcpu->arch.cp15[r->reg + 1] = val >> 32;
 }
 static inline u64 vcpu_cp15_reg64_get(struct kvm_vcpu *vcpu,
 				      const struct coproc_reg *r)
 {
 	u64 val;
 	val = vcpu->arch.cp15[r->reg + 1];
 	val = val << 32;
 	val = val | vcpu->arch.cp15[r->reg];
 	return val;
 }
 int kvm_handle_cp10_id(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 int kvm_handle_cp_0_13_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	/*
 	 * We can get here, if the host has been built without VFPv3 support,
 	 * but the guest attempted a floating point operation.
 	 */
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 int kvm_handle_cp14_access(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
 {
 	/*
 	 * Compute guest MPIDR. We build a virtual cluster out of the
 	 * vcpu_id, but we read the 'U' bit from the underlying
 	 * hardware directly.
 	 */
 	vcpu->arch.cp15[c0_MPIDR] = ((read_cpuid_mpidr() & MPIDR_SMP_BITMASK) |
 				     ((vcpu->vcpu_id >> 2) << MPIDR_LEVEL_BITS) |
 				     (vcpu->vcpu_id & 3));
 }
 /* TRM entries A7:4.3.31 A15:4.3.28 - RO WI */
 static bool access_actlr(struct kvm_vcpu *vcpu,
 			 const struct coproc_params *p,
 			 const struct coproc_reg *r)
 {
 	if (p->is_write)
 		return ignore_write(vcpu, p);
 	*vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c1_ACTLR];
 	return true;
 }
 /* TRM entries A7:4.3.56, A15:4.3.60 - R/O. */
 static bool access_cbar(struct kvm_vcpu *vcpu,
 			const struct coproc_params *p,
 			const struct coproc_reg *r)
 {
 	if (p->is_write)
 		return write_to_read_only(vcpu, p);
 	return read_zero(vcpu, p);
 }
 /* TRM entries A7:4.3.49, A15:4.3.48 - R/O WI */
 static bool access_l2ctlr(struct kvm_vcpu *vcpu,
 			  const struct coproc_params *p,
 			  const struct coproc_reg *r)
 {
 	if (p->is_write)
 		return ignore_write(vcpu, p);
 	*vcpu_reg(vcpu, p->Rt1) = vcpu->arch.cp15[c9_L2CTLR];
 	return true;
 }
 static void reset_l2ctlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
 {
 	u32 l2ctlr, ncores;
 	asm volatile("mrc p15, 1, %0, c9, c0, 2\n" : "=r" (l2ctlr));
 	l2ctlr &= ~(3 << 24);
 	ncores = atomic_read(&vcpu->kvm->online_vcpus) - 1;
 	/* How many cores in the current cluster and the next ones */
 	ncores -= (vcpu->vcpu_id & ~3);
 	/* Cap it to the maximum number of cores in a single cluster */
 	ncores = min(ncores, 3U);
 	l2ctlr |= (ncores & 3) << 24;
 	vcpu->arch.cp15[c9_L2CTLR] = l2ctlr;
 }
 static void reset_actlr(struct kvm_vcpu *vcpu, const struct coproc_reg *r)
 {
 	u32 actlr;
 	/* ACTLR contains SMP bit: make sure you create all cpus first! */
 	asm volatile("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr));
 	/* Make the SMP bit consistent with the guest configuration */
 	if (atomic_read(&vcpu->kvm->online_vcpus) > 1)
 		actlr |= 1U << 6;
 	else
 		actlr &= ~(1U << 6);
 	vcpu->arch.cp15[c1_ACTLR] = actlr;
 }
 /*
  * TRM entries: A7:4.3.50, A15:4.3.49
  * R/O WI (even if NSACR.NS_L2ERR, a write of 1 is ignored).
  */
 static bool access_l2ectlr(struct kvm_vcpu *vcpu,
 			   const struct coproc_params *p,
 			   const struct coproc_reg *r)
 {
 	if (p->is_write)
 		return ignore_write(vcpu, p);
 	*vcpu_reg(vcpu, p->Rt1) = 0;
 	return true;
 }
-/* See note at ARM ARM B1.14.4 */
+/*
+ * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
+ */
 static bool access_dcsw(struct kvm_vcpu *vcpu,
 			const struct coproc_params *p,
 			const struct coproc_reg *r)
 {
-	unsigned long val;
-	int cpu;
 	if (!p->is_write)
 		return read_from_write_only(vcpu, p);
-	cpu = get_cpu();
+	kvm_set_way_flush(vcpu);
-	cpumask_setall(&vcpu->arch.require_dcache_flush);
-	cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
-	/* If we were already preempted, take the long way around */
-	if (cpu != vcpu->arch.last_pcpu) {
-		flush_cache_all();
-		goto done;
-	}
-	val = *vcpu_reg(vcpu, p->Rt1);
-	switch (p->CRm) {
-	case 6:			/* Upgrade DCISW to DCCISW, as per HCR.SWIO */
-	case 14:		/* DCCISW */
-		asm volatile("mcr p15, 0, %0, c7, c14, 2" : : "r" (val));
-		break;
-	case 10:		/* DCCSW */
-		asm volatile("mcr p15, 0, %0, c7, c10, 2" : : "r" (val));
-		break;
-	}
-done:
-	put_cpu();
 	return true;
 }
 /*
  * Generic accessor for VM registers. Only called as long as HCR_TVM
- * is set.
+ * is set.  If the guest enables the MMU, we stop trapping the VM
+ * sys_regs and leave it in complete control of the caches.
+ *
+ * Used by the cpu-specific code.
  */
-static bool access_vm_reg(struct kvm_vcpu *vcpu,
+bool access_vm_reg(struct kvm_vcpu *vcpu,
-			  const struct coproc_params *p,
+		   const struct coproc_params *p,
-			  const struct coproc_reg *r)
+		   const struct coproc_reg *r)
 {
+	bool was_enabled = vcpu_has_cache_enabled(vcpu);
 	BUG_ON(!p->is_write);
 	vcpu->arch.cp15[r->reg] = *vcpu_reg(vcpu, p->Rt1);
 	if (p->is_64bit)
 		vcpu->arch.cp15[r->reg + 1] = *vcpu_reg(vcpu, p->Rt2);
-	return true;
+	kvm_toggle_cache(vcpu, was_enabled);
-}
-/*
- * SCTLR accessor. Only called as long as HCR_TVM is set.  If the
- * guest enables the MMU, we stop trapping the VM sys_regs and leave
- * it in complete control of the caches.
- *
- * Used by the cpu-specific code.
- */
-bool access_sctlr(struct kvm_vcpu *vcpu,
-		  const struct coproc_params *p,
-		  const struct coproc_reg *r)
-{
-	access_vm_reg(vcpu, p, r);
-	if (vcpu_has_cache_enabled(vcpu)) {	/* MMU+Caches enabled? */
-		vcpu->arch.hcr &= ~HCR_TVM;
-		stage2_flush_vm(vcpu->kvm);
-	}
 	return true;
 }
 /*
  * We could trap ID_DFR0 and tell the guest we don't support performance
  * monitoring.  Unfortunately the patch to make the kernel check ID_DFR0 was
  * NAKed, so it will read the PMCR anyway.
  *
  * Therefore we tell the guest we have 0 counters.  Unfortunately, we
  * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
  * all PM registers, which doesn't crash the guest kernel at least.
  */
 static bool pm_fake(struct kvm_vcpu *vcpu,
 		    const struct coproc_params *p,
 		    const struct coproc_reg *r)
 {
 	if (p->is_write)
 		return ignore_write(vcpu, p);
 	else
 		return read_zero(vcpu, p);
 }
 #define access_pmcr pm_fake
 #define access_pmcntenset pm_fake
 #define access_pmcntenclr pm_fake
 #define access_pmovsr pm_fake
 #define access_pmselr pm_fake
 #define access_pmceid0 pm_fake
 #define access_pmceid1 pm_fake
 #define access_pmccntr pm_fake
 #define access_pmxevtyper pm_fake
 #define access_pmxevcntr pm_fake
 #define access_pmuserenr pm_fake
 #define access_pmintenset pm_fake
 #define access_pmintenclr pm_fake
 /* Architected CP15 registers.
  * CRn denotes the primary register number, but is copied to the CRm in the
  * user space API for 64-bit register access in line with the terminology used
  * in the ARM ARM.
  * Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit
  *            registers preceding 32-bit ones.
  */
 static const struct coproc_reg cp15_regs[] = {
 	/* MPIDR: we use VMPIDR for guest access. */
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 5), is32,
 			NULL, reset_mpidr, c0_MPIDR },
 	/* CSSELR: swapped by interrupt.S. */
 	{ CRn( 0), CRm( 0), Op1( 2), Op2( 0), is32,
 			NULL, reset_unknown, c0_CSSELR },
 	/* ACTLR: trapped by HCR.TAC bit. */
 	{ CRn( 1), CRm( 0), Op1( 0), Op2( 1), is32,
 			access_actlr, reset_actlr, c1_ACTLR },
 	/* CPACR: swapped by interrupt.S. */
 	{ CRn( 1), CRm( 0), Op1( 0), Op2( 2), is32,
 			NULL, reset_val, c1_CPACR, 0x00000000 },
 	/* TTBR0/TTBR1/TTBCR: swapped by interrupt.S. */
 	{ CRm64( 2), Op1( 0), is64, access_vm_reg, reset_unknown64, c2_TTBR0 },
 	{ CRn(2), CRm( 0), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c2_TTBR0 },
 	{ CRn(2), CRm( 0), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_unknown, c2_TTBR1 },
 	{ CRn( 2), CRm( 0), Op1( 0), Op2( 2), is32,
 			access_vm_reg, reset_val, c2_TTBCR, 0x00000000 },
 	{ CRm64( 2), Op1( 1), is64, access_vm_reg, reset_unknown64, c2_TTBR1 },
 	/* DACR: swapped by interrupt.S. */
 	{ CRn( 3), CRm( 0), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c3_DACR },
 	/* DFSR/IFSR/ADFSR/AIFSR: swapped by interrupt.S. */
 	{ CRn( 5), CRm( 0), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c5_DFSR },
 	{ CRn( 5), CRm( 0), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_unknown, c5_IFSR },
 	{ CRn( 5), CRm( 1), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c5_ADFSR },
 	{ CRn( 5), CRm( 1), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_unknown, c5_AIFSR },
 	/* DFAR/IFAR: swapped by interrupt.S. */
 	{ CRn( 6), CRm( 0), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c6_DFAR },
 	{ CRn( 6), CRm( 0), Op1( 0), Op2( 2), is32,
 			access_vm_reg, reset_unknown, c6_IFAR },
 	/* PAR swapped by interrupt.S */
 	{ CRm64( 7), Op1( 0), is64, NULL, reset_unknown64, c7_PAR },
 	/*
 	 * DC{C,I,CI}SW operations:
 	 */
 	{ CRn( 7), CRm( 6), Op1( 0), Op2( 2), is32, access_dcsw},
 	{ CRn( 7), CRm(10), Op1( 0), Op2( 2), is32, access_dcsw},
 	{ CRn( 7), CRm(14), Op1( 0), Op2( 2), is32, access_dcsw},
 	/*
 	 * L2CTLR access (guest wants to know #CPUs).
 	 */
 	{ CRn( 9), CRm( 0), Op1( 1), Op2( 2), is32,
 			access_l2ctlr, reset_l2ctlr, c9_L2CTLR },
 	{ CRn( 9), CRm( 0), Op1( 1), Op2( 3), is32, access_l2ectlr},
 	/*
 	 * Dummy performance monitor implementation.
 	 */
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 0), is32, access_pmcr},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 1), is32, access_pmcntenset},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 2), is32, access_pmcntenclr},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 3), is32, access_pmovsr},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 5), is32, access_pmselr},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 6), is32, access_pmceid0},
 	{ CRn( 9), CRm(12), Op1( 0), Op2( 7), is32, access_pmceid1},
 	{ CRn( 9), CRm(13), Op1( 0), Op2( 0), is32, access_pmccntr},
 	{ CRn( 9), CRm(13), Op1( 0), Op2( 1), is32, access_pmxevtyper},
 	{ CRn( 9), CRm(13), Op1( 0), Op2( 2), is32, access_pmxevcntr},
 	{ CRn( 9), CRm(14), Op1( 0), Op2( 0), is32, access_pmuserenr},
 	{ CRn( 9), CRm(14), Op1( 0), Op2( 1), is32, access_pmintenset},
 	{ CRn( 9), CRm(14), Op1( 0), Op2( 2), is32, access_pmintenclr},
 	/* PRRR/NMRR (aka MAIR0/MAIR1): swapped by interrupt.S. */
 	{ CRn(10), CRm( 2), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c10_PRRR},
 	{ CRn(10), CRm( 2), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_unknown, c10_NMRR},
 	/* AMAIR0/AMAIR1: swapped by interrupt.S. */
 	{ CRn(10), CRm( 3), Op1( 0), Op2( 0), is32,
 			access_vm_reg, reset_unknown, c10_AMAIR0},
 	{ CRn(10), CRm( 3), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_unknown, c10_AMAIR1},
 	/* VBAR: swapped by interrupt.S. */
 	{ CRn(12), CRm( 0), Op1( 0), Op2( 0), is32,
 			NULL, reset_val, c12_VBAR, 0x00000000 },
 	/* CONTEXTIDR/TPIDRURW/TPIDRURO/TPIDRPRW: swapped by interrupt.S. */
 	{ CRn(13), CRm( 0), Op1( 0), Op2( 1), is32,
 			access_vm_reg, reset_val, c13_CID, 0x00000000 },
 	{ CRn(13), CRm( 0), Op1( 0), Op2( 2), is32,
 			NULL, reset_unknown, c13_TID_URW },
 	{ CRn(13), CRm( 0), Op1( 0), Op2( 3), is32,
 			NULL, reset_unknown, c13_TID_URO },
 	{ CRn(13), CRm( 0), Op1( 0), Op2( 4), is32,
 			NULL, reset_unknown, c13_TID_PRIV },
 	/* CNTKCTL: swapped by interrupt.S. */
 	{ CRn(14), CRm( 1), Op1( 0), Op2( 0), is32,
 			NULL, reset_val, c14_CNTKCTL, 0x00000000 },
 	/* The Configuration Base Address Register. */
 	{ CRn(15), CRm( 0), Op1( 4), Op2( 0), is32, access_cbar},
 };
 /* Target specific emulation tables */
 static struct kvm_coproc_target_table *target_tables[KVM_ARM_NUM_TARGETS];
 void kvm_register_target_coproc_table(struct kvm_coproc_target_table *table)
 {
 	unsigned int i;
 	for (i = 1; i < table->num; i++)
 		BUG_ON(cmp_reg(&table->table[i-1],
 			       &table->table[i]) >= 0);
 	target_tables[table->target] = table;
 }
 /* Get specific register table for this target. */
 static const struct coproc_reg *get_target_table(unsigned target, size_t *num)
 {
 	struct kvm_coproc_target_table *table;
 	table = target_tables[target];
 	*num = table->num;
 	return table->table;
 }
 static const struct coproc_reg *find_reg(const struct coproc_params *params,
 					 const struct coproc_reg table[],
 					 unsigned int num)
 {
 	unsigned int i;
 	for (i = 0; i < num; i++) {
 		const struct coproc_reg *r = &table[i];
 		if (params->is_64bit != r->is_64)
 			continue;
 		if (params->CRn != r->CRn)
 			continue;
 		if (params->CRm != r->CRm)
 			continue;
 		if (params->Op1 != r->Op1)
 			continue;
 		if (params->Op2 != r->Op2)
 			continue;
 		return r;
 	}
 	return NULL;
 }
 static int emulate_cp15(struct kvm_vcpu *vcpu,
 			const struct coproc_params *params)
 {
 	size_t num;
 	const struct coproc_reg *table, *r;
 	trace_kvm_emulate_cp15_imp(params->Op1, params->Rt1, params->CRn,
 				   params->CRm, params->Op2, params->is_write);
 	table = get_target_table(vcpu->arch.target, &num);
 	/* Search target-specific then generic table. */
 	r = find_reg(params, table, num);
 	if (!r)
 		r = find_reg(params, cp15_regs, ARRAY_SIZE(cp15_regs));
 	if (likely(r)) {
 		/* If we don't have an accessor, we should never get here! */
 		BUG_ON(!r->access);
 		if (likely(r->access(vcpu, params, r))) {
 			/* Skip instruction, since it was emulated */
 			kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 			return 1;
 		}
 		/* If access function fails, it should complain. */
 	} else {
 		kvm_err("Unsupported guest CP15 access at: %08lx\n",
 			*vcpu_pc(vcpu));
 		print_cp_instr(params);
 	}
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 /**
  * kvm_handle_cp15_64 -- handles a mrrc/mcrr trap on a guest CP15 access
  * @vcpu: The VCPU pointer
  * @run:  The kvm_run struct
  */
 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	struct coproc_params params;
 	params.CRn = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf;
 	params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf;
 	params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0);
 	params.is_64bit = true;
 	params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 16) & 0xf;
 	params.Op2 = 0;
 	params.Rt2 = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf;
 	params.CRm = 0;
 	return emulate_cp15(vcpu, &params);
 }
 static void reset_coproc_regs(struct kvm_vcpu *vcpu,
 			      const struct coproc_reg *table, size_t num)
 {
 	unsigned long i;
 	for (i = 0; i < num; i++)
 		if (table[i].reset)
 			table[i].reset(vcpu, &table[i]);
 }
 /**
  * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access
  * @vcpu: The VCPU pointer
  * @run:  The kvm_run struct
  */
 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	struct coproc_params params;
 	params.CRm = (kvm_vcpu_get_hsr(vcpu) >> 1) & 0xf;
 	params.Rt1 = (kvm_vcpu_get_hsr(vcpu) >> 5) & 0xf;
 	params.is_write = ((kvm_vcpu_get_hsr(vcpu) & 1) == 0);
 	params.is_64bit = false;
 	params.CRn = (kvm_vcpu_get_hsr(vcpu) >> 10) & 0xf;
 	params.Op1 = (kvm_vcpu_get_hsr(vcpu) >> 14) & 0x7;
 	params.Op2 = (kvm_vcpu_get_hsr(vcpu) >> 17) & 0x7;
 	params.Rt2 = 0;
 	return emulate_cp15(vcpu, &params);
 }
 /******************************************************************************
  * Userspace API
  *****************************************************************************/
 static bool index_to_params(u64 id, struct coproc_params *params)
 {
 	switch (id & KVM_REG_SIZE_MASK) {
 	case KVM_REG_SIZE_U32:
 		/* Any unused index bits means it's not valid. */
 		if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
 			   | KVM_REG_ARM_COPROC_MASK
 			   | KVM_REG_ARM_32_CRN_MASK
 			   | KVM_REG_ARM_CRM_MASK
 			   | KVM_REG_ARM_OPC1_MASK
 			   | KVM_REG_ARM_32_OPC2_MASK))
 			return false;
 		params->is_64bit = false;
 		params->CRn = ((id & KVM_REG_ARM_32_CRN_MASK)
 			       >> KVM_REG_ARM_32_CRN_SHIFT);
 		params->CRm = ((id & KVM_REG_ARM_CRM_MASK)
 			       >> KVM_REG_ARM_CRM_SHIFT);
 		params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
 			       >> KVM_REG_ARM_OPC1_SHIFT);
 		params->Op2 = ((id & KVM_REG_ARM_32_OPC2_MASK)
 			       >> KVM_REG_ARM_32_OPC2_SHIFT);
 		return true;
 	case KVM_REG_SIZE_U64:
 		/* Any unused index bits means it's not valid. */
 		if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
 			      | KVM_REG_ARM_COPROC_MASK
 			      | KVM_REG_ARM_CRM_MASK
 			      | KVM_REG_ARM_OPC1_MASK))
 			return false;
 		params->is_64bit = true;
 		/* CRm to CRn: see cp15_to_index for details */
 		params->CRn = ((id & KVM_REG_ARM_CRM_MASK)
 			       >> KVM_REG_ARM_CRM_SHIFT);
 		params->Op1 = ((id & KVM_REG_ARM_OPC1_MASK)
 			       >> KVM_REG_ARM_OPC1_SHIFT);
 		params->Op2 = 0;
 		params->CRm = 0;
 		return true;
 	default:
 		return false;
 	}
 }
 /* Decode an index value, and find the cp15 coproc_reg entry. */
 static const struct coproc_reg *index_to_coproc_reg(struct kvm_vcpu *vcpu,
 						    u64 id)
 {
 	size_t num;
 	const struct coproc_reg *table, *r;
 	struct coproc_params params;
 	/* We only do cp15 for now. */
 	if ((id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT != 15)
 		return NULL;
 	if (!index_to_params(id, &params))
 		return NULL;
 	table = get_target_table(vcpu->arch.target, &num);
 	r = find_reg(&params, table, num);
 	if (!r)
 		r = find_reg(&params, cp15_regs, ARRAY_SIZE(cp15_regs));
 	/* Not saved in the cp15 array? */
 	if (r && !r->reg)
 		r = NULL;
 	return r;
 }
 /*
  * These are the invariant cp15 registers: we let the guest see the host
  * versions of these, so they're part of the guest state.
  *
  * A future CPU may provide a mechanism to present different values to
  * the guest, or a future kvm may trap them.
  */
 /* Unfortunately, there's no register-argument for mrc, so generate. */
 #define FUNCTION_FOR32(crn, crm, op1, op2, name)			\
 	static void get_##name(struct kvm_vcpu *v,			\
 			       const struct coproc_reg *r)		\
 	{								\
 		u32 val;						\
 									\
 		asm volatile("mrc p15, " __stringify(op1)		\
 			     ", %0, c" __stringify(crn)			\
 			     ", c" __stringify(crm)			\
 			     ", " __stringify(op2) "\n" : "=r" (val));	\
 		((struct coproc_reg *)r)->val = val;			\
 	}
 FUNCTION_FOR32(0, 0, 0, 0, MIDR)
 FUNCTION_FOR32(0, 0, 0, 1, CTR)
 FUNCTION_FOR32(0, 0, 0, 2, TCMTR)
 FUNCTION_FOR32(0, 0, 0, 3, TLBTR)
 FUNCTION_FOR32(0, 0, 0, 6, REVIDR)
 FUNCTION_FOR32(0, 1, 0, 0, ID_PFR0)
 FUNCTION_FOR32(0, 1, 0, 1, ID_PFR1)
 FUNCTION_FOR32(0, 1, 0, 2, ID_DFR0)
 FUNCTION_FOR32(0, 1, 0, 3, ID_AFR0)
 FUNCTION_FOR32(0, 1, 0, 4, ID_MMFR0)
 FUNCTION_FOR32(0, 1, 0, 5, ID_MMFR1)
 FUNCTION_FOR32(0, 1, 0, 6, ID_MMFR2)
 FUNCTION_FOR32(0, 1, 0, 7, ID_MMFR3)
 FUNCTION_FOR32(0, 2, 0, 0, ID_ISAR0)
 FUNCTION_FOR32(0, 2, 0, 1, ID_ISAR1)
 FUNCTION_FOR32(0, 2, 0, 2, ID_ISAR2)
 FUNCTION_FOR32(0, 2, 0, 3, ID_ISAR3)
 FUNCTION_FOR32(0, 2, 0, 4, ID_ISAR4)
 FUNCTION_FOR32(0, 2, 0, 5, ID_ISAR5)
 FUNCTION_FOR32(0, 0, 1, 1, CLIDR)
 FUNCTION_FOR32(0, 0, 1, 7, AIDR)
 /* ->val is filled in by kvm_invariant_coproc_table_init() */
 static struct coproc_reg invariant_cp15[] = {
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 0), is32, NULL, get_MIDR },
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 1), is32, NULL, get_CTR },
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 2), is32, NULL, get_TCMTR },
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 3), is32, NULL, get_TLBTR },
 	{ CRn( 0), CRm( 0), Op1( 0), Op2( 6), is32, NULL, get_REVIDR },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 0), is32, NULL, get_ID_PFR0 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 1), is32, NULL, get_ID_PFR1 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 2), is32, NULL, get_ID_DFR0 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 3), is32, NULL, get_ID_AFR0 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 4), is32, NULL, get_ID_MMFR0 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 5), is32, NULL, get_ID_MMFR1 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 6), is32, NULL, get_ID_MMFR2 },
 	{ CRn( 0), CRm( 1), Op1( 0), Op2( 7), is32, NULL, get_ID_MMFR3 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 0), is32, NULL, get_ID_ISAR0 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 1), is32, NULL, get_ID_ISAR1 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 2), is32, NULL, get_ID_ISAR2 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 3), is32, NULL, get_ID_ISAR3 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 4), is32, NULL, get_ID_ISAR4 },
 	{ CRn( 0), CRm( 2), Op1( 0), Op2( 5), is32, NULL, get_ID_ISAR5 },
 	{ CRn( 0), CRm( 0), Op1( 1), Op2( 1), is32, NULL, get_CLIDR },
 	{ CRn( 0), CRm( 0), Op1( 1), Op2( 7), is32, NULL, get_AIDR },
 };
 /*
  * Reads a register value from a userspace address to a kernel
  * variable. Make sure that register size matches sizeof(*__val).
  */
 static int reg_from_user(void *val, const void __user *uaddr, u64 id)
 {
 	if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
 		return -EFAULT;
 	return 0;
 }
 /*
  * Writes a register value to a userspace address from a kernel variable.
  * Make sure that register size matches sizeof(*__val).
  */
 static int reg_to_user(void __user *uaddr, const void *val, u64 id)
 {
 	if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
 		return -EFAULT;
 	return 0;
 }
 static int get_invariant_cp15(u64 id, void __user *uaddr)
 {
 	struct coproc_params params;
 	const struct coproc_reg *r;
 	int ret;
 	if (!index_to_params(id, &params))
 		return -ENOENT;
 	r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
 	if (!r)
 		return -ENOENT;
 	ret = -ENOENT;
 	if (KVM_REG_SIZE(id) == 4) {
 		u32 val = r->val;
 		ret = reg_to_user(uaddr, &val, id);
 	} else if (KVM_REG_SIZE(id) == 8) {
 		ret = reg_to_user(uaddr, &r->val, id);
 	}
 	return ret;
 }
 static int set_invariant_cp15(u64 id, void __user *uaddr)
 {
 	struct coproc_params params;
 	const struct coproc_reg *r;
 	int err;
 	u64 val;
 	if (!index_to_params(id, &params))
 		return -ENOENT;
 	r = find_reg(&params, invariant_cp15, ARRAY_SIZE(invariant_cp15));
 	if (!r)
 		return -ENOENT;
 	err = -ENOENT;
 	if (KVM_REG_SIZE(id) == 4) {
 		u32 val32;
 		err = reg_from_user(&val32, uaddr, id);
 		if (!err)
 			val = val32;
 	} else if (KVM_REG_SIZE(id) == 8) {
 		err = reg_from_user(&val, uaddr, id);
 	}
 	if (err)
 		return err;
 	/* This is what we mean by invariant: you can't change it. */
 	if (r->val != val)
 		return -EINVAL;
 	return 0;
 }
 static bool is_valid_cache(u32 val)
 {
 	u32 level, ctype;
 	if (val >= CSSELR_MAX)
 		return false;
 	/* Bottom bit is Instruction or Data bit.  Next 3 bits are level. */
         level = (val >> 1);
         ctype = (cache_levels >> (level * 3)) & 7;
 	switch (ctype) {
 	case 0: /* No cache */
 		return false;
 	case 1: /* Instruction cache only */
 		return (val & 1);
 	case 2: /* Data cache only */
 	case 4: /* Unified cache */
 		return !(val & 1);
 	case 3: /* Separate instruction and data caches */
 		return true;
 	default: /* Reserved: we can't know instruction or data. */
 		return false;
 	}
 }
 /* Which cache CCSIDR represents depends on CSSELR value. */
 static u32 get_ccsidr(u32 csselr)
 {
 	u32 ccsidr;
 	/* Make sure noone else changes CSSELR during this! */
 	local_irq_disable();
 	/* Put value into CSSELR */
 	asm volatile("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr));
 	isb();
 	/* Read result out of CCSIDR */
 	asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr));
 	local_irq_enable();
 	return ccsidr;
 }
 static int demux_c15_get(u64 id, void __user *uaddr)
 {
 	u32 val;
 	u32 __user *uval = uaddr;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
 		if (KVM_REG_SIZE(id) != 4)
 			return -ENOENT;
 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
 		if (!is_valid_cache(val))
 			return -ENOENT;
 		return put_user(get_ccsidr(val), uval);
 	default:
 		return -ENOENT;
 	}
 }
 static int demux_c15_set(u64 id, void __user *uaddr)
 {
 	u32 val, newval;
 	u32 __user *uval = uaddr;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
 		if (KVM_REG_SIZE(id) != 4)
 			return -ENOENT;
 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
 		if (!is_valid_cache(val))
 			return -ENOENT;
 		if (get_user(newval, uval))
 			return -EFAULT;
 		/* This is also invariant: you can't change it. */
 		if (newval != get_ccsidr(val))
 			return -EINVAL;
 		return 0;
 	default:
 		return -ENOENT;
 	}
 }
 #ifdef CONFIG_VFPv3
 static const int vfp_sysregs[] = { KVM_REG_ARM_VFP_FPEXC,
 				   KVM_REG_ARM_VFP_FPSCR,
 				   KVM_REG_ARM_VFP_FPINST,
 				   KVM_REG_ARM_VFP_FPINST2,
 				   KVM_REG_ARM_VFP_MVFR0,
 				   KVM_REG_ARM_VFP_MVFR1,
 				   KVM_REG_ARM_VFP_FPSID };
 static unsigned int num_fp_regs(void)
 {
 	if (((fmrx(MVFR0) & MVFR0_A_SIMD_MASK) >> MVFR0_A_SIMD_BIT) == 2)
 		return 32;
 	else
 		return 16;
 }
 static unsigned int num_vfp_regs(void)
 {
 	/* Normal FP regs + control regs. */
 	return num_fp_regs() + ARRAY_SIZE(vfp_sysregs);
 }
 static int copy_vfp_regids(u64 __user *uindices)
 {
 	unsigned int i;
 	const u64 u32reg = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_VFP;
 	const u64 u64reg = KVM_REG_ARM | KVM_REG_SIZE_U64 | KVM_REG_ARM_VFP;
 	for (i = 0; i < num_fp_regs(); i++) {
 		if (put_user((u64reg | KVM_REG_ARM_VFP_BASE_REG) + i,
 			     uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	for (i = 0; i < ARRAY_SIZE(vfp_sysregs); i++) {
 		if (put_user(u32reg | vfp_sysregs[i], uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	return num_vfp_regs();
 }
 static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
 {
 	u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
 	u32 val;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	if (vfpid < num_fp_regs()) {
 		if (KVM_REG_SIZE(id) != 8)
 			return -ENOENT;
 		return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpregs[vfpid],
 				   id);
 	}
 	/* FP control registers are all 32 bit. */
 	if (KVM_REG_SIZE(id) != 4)
 		return -ENOENT;
 	switch (vfpid) {
 	case KVM_REG_ARM_VFP_FPEXC:
 		return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpexc, id);
 	case KVM_REG_ARM_VFP_FPSCR:
 		return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpscr, id);
 	case KVM_REG_ARM_VFP_FPINST:
 		return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst, id);
 	case KVM_REG_ARM_VFP_FPINST2:
 		return reg_to_user(uaddr, &vcpu->arch.vfp_guest.fpinst2, id);
 	case KVM_REG_ARM_VFP_MVFR0:
 		val = fmrx(MVFR0);
 		return reg_to_user(uaddr, &val, id);
 	case KVM_REG_ARM_VFP_MVFR1:
 		val = fmrx(MVFR1);
 		return reg_to_user(uaddr, &val, id);
 	case KVM_REG_ARM_VFP_FPSID:
 		val = fmrx(FPSID);
 		return reg_to_user(uaddr, &val, id);
 	default:
 		return -ENOENT;
 	}
 }
 static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
 {
 	u32 vfpid = (id & KVM_REG_ARM_VFP_MASK);
 	u32 val;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	if (vfpid < num_fp_regs()) {
 		if (KVM_REG_SIZE(id) != 8)
 			return -ENOENT;
 		return reg_from_user(&vcpu->arch.vfp_guest.fpregs[vfpid],
 				     uaddr, id);
 	}
 	/* FP control registers are all 32 bit. */
 	if (KVM_REG_SIZE(id) != 4)
 		return -ENOENT;
 	switch (vfpid) {
 	case KVM_REG_ARM_VFP_FPEXC:
 		return reg_from_user(&vcpu->arch.vfp_guest.fpexc, uaddr, id);
 	case KVM_REG_ARM_VFP_FPSCR:
 		return reg_from_user(&vcpu->arch.vfp_guest.fpscr, uaddr, id);
 	case KVM_REG_ARM_VFP_FPINST:
 		return reg_from_user(&vcpu->arch.vfp_guest.fpinst, uaddr, id);
 	case KVM_REG_ARM_VFP_FPINST2:
 		return reg_from_user(&vcpu->arch.vfp_guest.fpinst2, uaddr, id);
 	/* These are invariant. */
 	case KVM_REG_ARM_VFP_MVFR0:
 		if (reg_from_user(&val, uaddr, id))
 			return -EFAULT;
 		if (val != fmrx(MVFR0))
 			return -EINVAL;
 		return 0;
 	case KVM_REG_ARM_VFP_MVFR1:
 		if (reg_from_user(&val, uaddr, id))
 			return -EFAULT;
 		if (val != fmrx(MVFR1))
 			return -EINVAL;
 		return 0;
 	case KVM_REG_ARM_VFP_FPSID:
 		if (reg_from_user(&val, uaddr, id))
 			return -EFAULT;
 		if (val != fmrx(FPSID))
 			return -EINVAL;
 		return 0;
 	default:
 		return -ENOENT;
 	}
 }
 #else /* !CONFIG_VFPv3 */
 static unsigned int num_vfp_regs(void)
 {
 	return 0;
 }
 static int copy_vfp_regids(u64 __user *uindices)
 {
 	return 0;
 }
 static int vfp_get_reg(const struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
 {
 	return -ENOENT;
 }
 static int vfp_set_reg(struct kvm_vcpu *vcpu, u64 id, const void __user *uaddr)
 {
 	return -ENOENT;
 }
 #endif /* !CONFIG_VFPv3 */
 int kvm_arm_coproc_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
 	const struct coproc_reg *r;
 	void __user *uaddr = (void __user *)(long)reg->addr;
 	int ret;
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
 		return demux_c15_get(reg->id, uaddr);
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
 		return vfp_get_reg(vcpu, reg->id, uaddr);
 	r = index_to_coproc_reg(vcpu, reg->id);
 	if (!r)
 		return get_invariant_cp15(reg->id, uaddr);
 	ret = -ENOENT;
 	if (KVM_REG_SIZE(reg->id) == 8) {
 		u64 val;
 		val = vcpu_cp15_reg64_get(vcpu, r);
 		ret = reg_to_user(uaddr, &val, reg->id);
 	} else if (KVM_REG_SIZE(reg->id) == 4) {
 		ret = reg_to_user(uaddr, &vcpu->arch.cp15[r->reg], reg->id);
 	}
 	return ret;
 }
 int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
 	const struct coproc_reg *r;
 	void __user *uaddr = (void __user *)(long)reg->addr;
 	int ret;
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
 		return demux_c15_set(reg->id, uaddr);
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_VFP)
 		return vfp_set_reg(vcpu, reg->id, uaddr);
 	r = index_to_coproc_reg(vcpu, reg->id);
 	if (!r)
 		return set_invariant_cp15(reg->id, uaddr);
 	ret = -ENOENT;
 	if (KVM_REG_SIZE(reg->id) == 8) {
 		u64 val;
 		ret = reg_from_user(&val, uaddr, reg->id);
 		if (!ret)
 			vcpu_cp15_reg64_set(vcpu, r, val);
 	} else if (KVM_REG_SIZE(reg->id) == 4) {
 		ret = reg_from_user(&vcpu->arch.cp15[r->reg], uaddr, reg->id);
 	}
 	return ret;
 }
 static unsigned int num_demux_regs(void)
 {
 	unsigned int i, count = 0;
 	for (i = 0; i < CSSELR_MAX; i++)
 		if (is_valid_cache(i))
 			count++;
 	return count;
 }
 static int write_demux_regids(u64 __user *uindices)
 {
 	u64 val = KVM_REG_ARM | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
 	unsigned int i;
 	val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
 	for (i = 0; i < CSSELR_MAX; i++) {
 		if (!is_valid_cache(i))
 			continue;
 		if (put_user(val | i, uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	return 0;
 }
 static u64 cp15_to_index(const struct coproc_reg *reg)
 {
 	u64 val = KVM_REG_ARM | (15 << KVM_REG_ARM_COPROC_SHIFT);
 	if (reg->is_64) {
 		val |= KVM_REG_SIZE_U64;
 		val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
 		/*
 		 * CRn always denotes the primary coproc. reg. nr. for the
 		 * in-kernel representation, but the user space API uses the
 		 * CRm for the encoding, because it is modelled after the
 		 * MRRC/MCRR instructions: see the ARM ARM rev. c page
 		 * B3-1445
 		 */
 		val |= (reg->CRn << KVM_REG_ARM_CRM_SHIFT);
 	} else {
 		val |= KVM_REG_SIZE_U32;
 		val |= (reg->Op1 << KVM_REG_ARM_OPC1_SHIFT);
 		val |= (reg->Op2 << KVM_REG_ARM_32_OPC2_SHIFT);
 		val |= (reg->CRm << KVM_REG_ARM_CRM_SHIFT);
 		val |= (reg->CRn << KVM_REG_ARM_32_CRN_SHIFT);
 	}
 	return val;
 }
 static bool copy_reg_to_user(const struct coproc_reg *reg, u64 __user **uind)
 {
 	if (!*uind)
 		return true;
 	if (put_user(cp15_to_index(reg), *uind))
 		return false;
 	(*uind)++;
 	return true;
 }
 /* Assumed ordered tables, see kvm_coproc_table_init. */
 static int walk_cp15(struct kvm_vcpu *vcpu, u64 __user *uind)
 {
 	const struct coproc_reg *i1, *i2, *end1, *end2;
 	unsigned int total = 0;
 	size_t num;
 	/* We check for duplicates here, to allow arch-specific overrides. */
 	i1 = get_target_table(vcpu->arch.target, &num);
 	end1 = i1 + num;
 	i2 = cp15_regs;
 	end2 = cp15_regs + ARRAY_SIZE(cp15_regs);
 	BUG_ON(i1 == end1 || i2 == end2);
 	/* Walk carefully, as both tables may refer to the same register. */
 	while (i1 || i2) {
 		int cmp = cmp_reg(i1, i2);
 		/* target-specific overrides generic entry. */
 		if (cmp <= 0) {
 			/* Ignore registers we trap but don't save. */
 			if (i1->reg) {
 				if (!copy_reg_to_user(i1, &uind))
 					return -EFAULT;
 				total++;
 			}
 		} else {
 			/* Ignore registers we trap but don't save. */
 			if (i2->reg) {
 				if (!copy_reg_to_user(i2, &uind))
 					return -EFAULT;
 				total++;
 			}
 		}
 		if (cmp <= 0 && ++i1 == end1)
 			i1 = NULL;
 		if (cmp >= 0 && ++i2 == end2)
 			i2 = NULL;
 	}
 	return total;
 }
 unsigned long kvm_arm_num_coproc_regs(struct kvm_vcpu *vcpu)
 {
 	return ARRAY_SIZE(invariant_cp15)
 		+ num_demux_regs()
 		+ num_vfp_regs()
 		+ walk_cp15(vcpu, (u64 __user *)NULL);
 }
 int kvm_arm_copy_coproc_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 {
 	unsigned int i;
 	int err;
 	/* Then give them all the invariant registers' indices. */
 	for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++) {
 		if (put_user(cp15_to_index(&invariant_cp15[i]), uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	err = walk_cp15(vcpu, uindices);
 	if (err < 0)
 		return err;
 	uindices += err;
 	err = copy_vfp_regids(uindices);
 	if (err < 0)
 		return err;
 	uindices += err;
 	return write_demux_regids(uindices);
 }
 void kvm_coproc_table_init(void)
 {
 	unsigned int i;
 	/* Make sure tables are unique and in order. */
 	for (i = 1; i < ARRAY_SIZE(cp15_regs); i++)
 		BUG_ON(cmp_reg(&cp15_regs[i-1], &cp15_regs[i]) >= 0);
 	/* We abuse the reset function to overwrite the table itself. */
 	for (i = 0; i < ARRAY_SIZE(invariant_cp15); i++)
 		invariant_cp15[i].reset(NULL, &invariant_cp15[i]);
 	/*
 	 * CLIDR format is awkward, so clean it up.  See ARM B4.1.20:
 	 *
 	 *   If software reads the Cache Type fields from Ctype1
 	 *   upwards, once it has seen a value of 0b000, no caches
 	 *   exist at further-out levels of the hierarchy. So, for
 	 *   example, if Ctype3 is the first Cache Type field with a
 	 *   value of 0b000, the values of Ctype4 to Ctype7 must be
 	 *   ignored.
 	 */
 	asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r" (cache_levels));
 	for (i = 0; i < 7; i++)
 		if (((cache_levels >> (i*3)) & 7) == 0)
 			break;
 	/* Clear all higher bits. */
 	cache_levels &= (1 << (i*3))-1;
 }
 /**
  * kvm_reset_coprocs - sets cp15 registers to reset value
  * @vcpu: The VCPU pointer
  *
  * This function finds the right table above and sets the registers on the
  * virtual CPU struct to their architecturally defined reset values.
  */
 void kvm_reset_coprocs(struct kvm_vcpu *vcpu)
 {
 	size_t num;
 	const struct coproc_reg *table;
 	/* Catch someone adding a register without putting in reset entry. */
 	memset(vcpu->arch.cp15, 0x42, sizeof(vcpu->arch.cp15));
 	/* Generic chip reset first (so target could override). */
 	reset_coproc_regs(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs));
 	table = get_target_table(vcpu->arch.target, &num);

arch/arm/kvm/coproc.h

Diff comments View file @ 02512b2

1	/*	1	/*
2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University	2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
3	* Authors: Christoffer Dall <c.dall@virtualopensystems.com>	3	* Authors: Christoffer Dall <c.dall@virtualopensystems.com>
4	*	4	*
5	* This program is free software; you can redistribute it and/or modify	5	* This program is free software; you can redistribute it and/or modify
6	* it under the terms of the GNU General Public License, version 2, as	6	* it under the terms of the GNU General Public License, version 2, as
7	* published by the Free Software Foundation.	7	* published by the Free Software Foundation.
8	*	8	*
9	* This program is distributed in the hope that it will be useful,	9	* This program is distributed in the hope that it will be useful,
10	* but WITHOUT ANY WARRANTY; without even the implied warranty of	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12	* GNU General Public License for more details.	12	* GNU General Public License for more details.
13	*	13	*
14	* You should have received a copy of the GNU General Public License	14	* You should have received a copy of the GNU General Public License
15	* along with this program; if not, write to the Free Software	15	* along with this program; if not, write to the Free Software
16	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.	16	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17	*/	17	*/
18		18
19	#ifndef __ARM_KVM_COPROC_LOCAL_H__	19	#ifndef __ARM_KVM_COPROC_LOCAL_H__
20	#define __ARM_KVM_COPROC_LOCAL_H__	20	#define __ARM_KVM_COPROC_LOCAL_H__
21		21
22	struct coproc_params {	22	struct coproc_params {
23	unsigned long CRn;	23	unsigned long CRn;
24	unsigned long CRm;	24	unsigned long CRm;
25	unsigned long Op1;	25	unsigned long Op1;
26	unsigned long Op2;	26	unsigned long Op2;
27	unsigned long Rt1;	27	unsigned long Rt1;
28	unsigned long Rt2;	28	unsigned long Rt2;
29	bool is_64bit;	29	bool is_64bit;
30	bool is_write;	30	bool is_write;
31	};	31	};
32		32
33	struct coproc_reg {	33	struct coproc_reg {
34	/* MRC/MCR/MRRC/MCRR instruction which accesses it. */	34	/* MRC/MCR/MRRC/MCRR instruction which accesses it. */
35	unsigned long CRn;	35	unsigned long CRn;
36	unsigned long CRm;	36	unsigned long CRm;
37	unsigned long Op1;	37	unsigned long Op1;
38	unsigned long Op2;	38	unsigned long Op2;
39		39
40	bool is_64;	40	bool is_64;
41		41
42	/* Trapped access from guest, if non-NULL. */	42	/* Trapped access from guest, if non-NULL. */
43	bool (access)(struct kvm_vcpu ,	43	bool (access)(struct kvm_vcpu ,
44	const struct coproc_params *,	44	const struct coproc_params *,
45	const struct coproc_reg *);	45	const struct coproc_reg *);
46		46
47	/* Initialization for vcpu. */	47	/* Initialization for vcpu. */
48	void (reset)(struct kvm_vcpu , const struct coproc_reg *);	48	void (reset)(struct kvm_vcpu , const struct coproc_reg *);
49		49
50	/* Index into vcpu->arch.cp15[], or 0 if we don't need to save it. */	50	/* Index into vcpu->arch.cp15[], or 0 if we don't need to save it. */
51	unsigned long reg;	51	unsigned long reg;
52		52
53	/* Value (usually reset value) */	53	/* Value (usually reset value) */
54	u64 val;	54	u64 val;
55	};	55	};
56		56
57	static inline void print_cp_instr(const struct coproc_params *p)	57	static inline void print_cp_instr(const struct coproc_params *p)
58	{	58	{
59	/* Look, we even formatted it for you to paste into the table! */	59	/* Look, we even formatted it for you to paste into the table! */
60	if (p->is_64bit) {	60	if (p->is_64bit) {
61	kvm_pr_unimpl(" { CRm64(%2lu), Op1(%2lu), is64, func_%s },\n",	61	kvm_pr_unimpl(" { CRm64(%2lu), Op1(%2lu), is64, func_%s },\n",
62	p->CRn, p->Op1, p->is_write ? "write" : "read");	62	p->CRn, p->Op1, p->is_write ? "write" : "read");
63	} else {	63	} else {
64	kvm_pr_unimpl(" { CRn(%2lu), CRm(%2lu), Op1(%2lu), Op2(%2lu), is32,"	64	kvm_pr_unimpl(" { CRn(%2lu), CRm(%2lu), Op1(%2lu), Op2(%2lu), is32,"
65	" func_%s },\n",	65	" func_%s },\n",
66	p->CRn, p->CRm, p->Op1, p->Op2,	66	p->CRn, p->CRm, p->Op1, p->Op2,
67	p->is_write ? "write" : "read");	67	p->is_write ? "write" : "read");
68	}	68	}
69	}	69	}
70		70
71	static inline bool ignore_write(struct kvm_vcpu *vcpu,	71	static inline bool ignore_write(struct kvm_vcpu *vcpu,
72	const struct coproc_params *p)	72	const struct coproc_params *p)
73	{	73	{
74	return true;	74	return true;
75	}	75	}
76		76
77	static inline bool read_zero(struct kvm_vcpu *vcpu,	77	static inline bool read_zero(struct kvm_vcpu *vcpu,
78	const struct coproc_params *p)	78	const struct coproc_params *p)
79	{	79	{
80	*vcpu_reg(vcpu, p->Rt1) = 0;	80	*vcpu_reg(vcpu, p->Rt1) = 0;
81	return true;	81	return true;
82	}	82	}
83		83
84	static inline bool write_to_read_only(struct kvm_vcpu *vcpu,	84	static inline bool write_to_read_only(struct kvm_vcpu *vcpu,
85	const struct coproc_params *params)	85	const struct coproc_params *params)
86	{	86	{
87	kvm_debug("CP15 write to read-only register at: %08lx\n",	87	kvm_debug("CP15 write to read-only register at: %08lx\n",
88	*vcpu_pc(vcpu));	88	*vcpu_pc(vcpu));
89	print_cp_instr(params);	89	print_cp_instr(params);
90	return false;	90	return false;
91	}	91	}
92		92
93	static inline bool read_from_write_only(struct kvm_vcpu *vcpu,	93	static inline bool read_from_write_only(struct kvm_vcpu *vcpu,
94	const struct coproc_params *params)	94	const struct coproc_params *params)
95	{	95	{
96	kvm_debug("CP15 read to write-only register at: %08lx\n",	96	kvm_debug("CP15 read to write-only register at: %08lx\n",
97	*vcpu_pc(vcpu));	97	*vcpu_pc(vcpu));
98	print_cp_instr(params);	98	print_cp_instr(params);
99	return false;	99	return false;
100	}	100	}
101		101
102	/* Reset functions */	102	/* Reset functions */
103	static inline void reset_unknown(struct kvm_vcpu *vcpu,	103	static inline void reset_unknown(struct kvm_vcpu *vcpu,
104	const struct coproc_reg *r)	104	const struct coproc_reg *r)
105	{	105	{
106	BUG_ON(!r->reg);	106	BUG_ON(!r->reg);
107	BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));	107	BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
108	vcpu->arch.cp15[r->reg] = 0xdecafbad;	108	vcpu->arch.cp15[r->reg] = 0xdecafbad;
109	}	109	}
110		110
111	static inline void reset_val(struct kvm_vcpu vcpu, const struct coproc_reg r)	111	static inline void reset_val(struct kvm_vcpu vcpu, const struct coproc_reg r)
112	{	112	{
113	BUG_ON(!r->reg);	113	BUG_ON(!r->reg);
114	BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));	114	BUG_ON(r->reg >= ARRAY_SIZE(vcpu->arch.cp15));
115	vcpu->arch.cp15[r->reg] = r->val;	115	vcpu->arch.cp15[r->reg] = r->val;
116	}	116	}
117		117
118	static inline void reset_unknown64(struct kvm_vcpu *vcpu,	118	static inline void reset_unknown64(struct kvm_vcpu *vcpu,
119	const struct coproc_reg *r)	119	const struct coproc_reg *r)
120	{	120	{
121	BUG_ON(!r->reg);	121	BUG_ON(!r->reg);
122	BUG_ON(r->reg + 1 >= ARRAY_SIZE(vcpu->arch.cp15));	122	BUG_ON(r->reg + 1 >= ARRAY_SIZE(vcpu->arch.cp15));
123		123
124	vcpu->arch.cp15[r->reg] = 0xdecafbad;	124	vcpu->arch.cp15[r->reg] = 0xdecafbad;
125	vcpu->arch.cp15[r->reg+1] = 0xd0c0ffee;	125	vcpu->arch.cp15[r->reg+1] = 0xd0c0ffee;
126	}	126	}
127		127
128	static inline int cmp_reg(const struct coproc_reg *i1,	128	static inline int cmp_reg(const struct coproc_reg *i1,
129	const struct coproc_reg *i2)	129	const struct coproc_reg *i2)
130	{	130	{
131	BUG_ON(i1 == i2);	131	BUG_ON(i1 == i2);
132	if (!i1)	132	if (!i1)
133	return 1;	133	return 1;
134	else if (!i2)	134	else if (!i2)
135	return -1;	135	return -1;
136	if (i1->CRn != i2->CRn)	136	if (i1->CRn != i2->CRn)
137	return i1->CRn - i2->CRn;	137	return i1->CRn - i2->CRn;
138	if (i1->CRm != i2->CRm)	138	if (i1->CRm != i2->CRm)
139	return i1->CRm - i2->CRm;	139	return i1->CRm - i2->CRm;
140	if (i1->Op1 != i2->Op1)	140	if (i1->Op1 != i2->Op1)
141	return i1->Op1 - i2->Op1;	141	return i1->Op1 - i2->Op1;
142	if (i1->Op2 != i2->Op2)	142	if (i1->Op2 != i2->Op2)
143	return i1->Op2 - i2->Op2;	143	return i1->Op2 - i2->Op2;
144	return i2->is_64 - i1->is_64;	144	return i2->is_64 - i1->is_64;
145	}	145	}
146		146
147		147
148	#define CRn(_x) .CRn = _x	148	#define CRn(_x) .CRn = _x
149	#define CRm(_x) .CRm = _x	149	#define CRm(_x) .CRm = _x
150	#define CRm64(_x) .CRn = _x, .CRm = 0	150	#define CRm64(_x) .CRn = _x, .CRm = 0
151	#define Op1(_x) .Op1 = _x	151	#define Op1(_x) .Op1 = _x
152	#define Op2(_x) .Op2 = _x	152	#define Op2(_x) .Op2 = _x
153	#define is64 .is_64 = true	153	#define is64 .is_64 = true
154	#define is32 .is_64 = false	154	#define is32 .is_64 = false
155		155
156	bool access_sctlr(struct kvm_vcpu *vcpu,	156	bool access_vm_reg(struct kvm_vcpu *vcpu,
157	const struct coproc_params *p,	157	const struct coproc_params *p,
158	const struct coproc_reg *r);	158	const struct coproc_reg *r);
159		159
160	#endif /* __ARM_KVM_COPROC_LOCAL_H__ */	160	#endif /* __ARM_KVM_COPROC_LOCAL_H__ */
161		161

arch/arm/kvm/coproc_a15.c

Diff comments View file @ 02512b2

arch/arm/kvm/coproc_a7.c

Diff comments View file @ 02512b2

arch/arm/kvm/mmu.c

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 #include <linux/mman.h>
 #include <linux/kvm_host.h>
 #include <linux/io.h>
 #include <linux/hugetlb.h>
 #include <trace/events/kvm.h>
 #include <asm/pgalloc.h>
 #include <asm/cacheflush.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_mmio.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
 #include "trace.h"
 extern char  __hyp_idmap_text_start[], __hyp_idmap_text_end[];
 static pgd_t *boot_hyp_pgd;
 static pgd_t *hyp_pgd;
 static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
 static void *init_bounce_page;
 static unsigned long hyp_idmap_start;
 static unsigned long hyp_idmap_end;
 static phys_addr_t hyp_idmap_vector;
 #define hyp_pgd_order get_order(PTRS_PER_PGD * sizeof(pgd_t))
 #define kvm_pmd_huge(_x)	(pmd_huge(_x) || pmd_trans_huge(_x))
 static void kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
 {
 	/*
 	 * This function also gets called when dealing with HYP page
 	 * tables. As HYP doesn't have an associated struct kvm (and
 	 * the HYP page tables are fairly static), we don't do
 	 * anything there.
 	 */
 	if (kvm)
 		kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, kvm, ipa);
 }
+/*
+ * D-Cache management functions. They take the page table entries by
+ * value, as they are flushing the cache using the kernel mapping (or
+ * kmap on 32bit).
+ */
+static void kvm_flush_dcache_pte(pte_t pte)
+{
+	__kvm_flush_dcache_pte(pte);
+}
+static void kvm_flush_dcache_pmd(pmd_t pmd)
+{
+	__kvm_flush_dcache_pmd(pmd);
+}
+static void kvm_flush_dcache_pud(pud_t pud)
+{
+	__kvm_flush_dcache_pud(pud);
+}
 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
 				  int min, int max)
 {
 	void *page;
 	BUG_ON(max > KVM_NR_MEM_OBJS);
 	if (cache->nobjs >= min)
 		return 0;
 	while (cache->nobjs < max) {
 		page = (void *)__get_free_page(PGALLOC_GFP);
 		if (!page)
 			return -ENOMEM;
 		cache->objects[cache->nobjs++] = page;
 	}
 	return 0;
 }
 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
 {
 	while (mc->nobjs)
 		free_page((unsigned long)mc->objects[--mc->nobjs]);
 }
 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
 {
 	void *p;
 	BUG_ON(!mc || !mc->nobjs);
 	p = mc->objects[--mc->nobjs];
 	return p;
 }
 static void clear_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
 {
 	pud_t *pud_table __maybe_unused = pud_offset(pgd, 0);
 	pgd_clear(pgd);
 	kvm_tlb_flush_vmid_ipa(kvm, addr);
 	pud_free(NULL, pud_table);
 	put_page(virt_to_page(pgd));
 }
 static void clear_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
 {
 	pmd_t *pmd_table = pmd_offset(pud, 0);
 	VM_BUG_ON(pud_huge(*pud));
 	pud_clear(pud);
 	kvm_tlb_flush_vmid_ipa(kvm, addr);
 	pmd_free(NULL, pmd_table);
 	put_page(virt_to_page(pud));
 }
 static void clear_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr)
 {
 	pte_t *pte_table = pte_offset_kernel(pmd, 0);
 	VM_BUG_ON(kvm_pmd_huge(*pmd));
 	pmd_clear(pmd);
 	kvm_tlb_flush_vmid_ipa(kvm, addr);
 	pte_free_kernel(NULL, pte_table);
 	put_page(virt_to_page(pmd));
 }
+/*
+ * Unmapping vs dcache management:
+ *
+ * If a guest maps certain memory pages as uncached, all writes will
+ * bypass the data cache and go directly to RAM.  However, the CPUs
+ * can still speculate reads (not writes) and fill cache lines with
+ * data.
+ *
+ * Those cache lines will be *clean* cache lines though, so a
+ * clean+invalidate operation is equivalent to an invalidate
+ * operation, because no cache lines are marked dirty.
+ *
+ * Those clean cache lines could be filled prior to an uncached write
+ * by the guest, and the cache coherent IO subsystem would therefore
+ * end up writing old data to disk.
+ *
+ * This is why right after unmapping a page/section and invalidating
+ * the corresponding TLBs, we call kvm_flush_dcache_p*() to make sure
+ * the IO subsystem will never hit in the cache.
+ */
 static void unmap_ptes(struct kvm *kvm, pmd_t *pmd,
 		       phys_addr_t addr, phys_addr_t end)
 {
 	phys_addr_t start_addr = addr;
 	pte_t *pte, *start_pte;
 	start_pte = pte = pte_offset_kernel(pmd, addr);
 	do {
 		if (!pte_none(*pte)) {
+			pte_t old_pte = *pte;
 			kvm_set_pte(pte, __pte(0));
-			put_page(virt_to_page(pte));
 			kvm_tlb_flush_vmid_ipa(kvm, addr);
+			/* No need to invalidate the cache for device mappings */
+			if ((pte_val(old_pte) & PAGE_S2_DEVICE) != PAGE_S2_DEVICE)
+				kvm_flush_dcache_pte(old_pte);
+			put_page(virt_to_page(pte));
 		}
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 	if (kvm_pte_table_empty(kvm, start_pte))
 		clear_pmd_entry(kvm, pmd, start_addr);
 }
 static void unmap_pmds(struct kvm *kvm, pud_t *pud,
 		       phys_addr_t addr, phys_addr_t end)
 {
 	phys_addr_t next, start_addr = addr;
 	pmd_t *pmd, *start_pmd;
 	start_pmd = pmd = pmd_offset(pud, addr);
 	do {
 		next = kvm_pmd_addr_end(addr, end);
 		if (!pmd_none(*pmd)) {
 			if (kvm_pmd_huge(*pmd)) {
+				pmd_t old_pmd = *pmd;
 				pmd_clear(pmd);
 				kvm_tlb_flush_vmid_ipa(kvm, addr);
+				kvm_flush_dcache_pmd(old_pmd);
 				put_page(virt_to_page(pmd));
 			} else {
 				unmap_ptes(kvm, pmd, addr, next);
 			}
 		}
 	} while (pmd++, addr = next, addr != end);
 	if (kvm_pmd_table_empty(kvm, start_pmd))
 		clear_pud_entry(kvm, pud, start_addr);
 }
 static void unmap_puds(struct kvm *kvm, pgd_t *pgd,
 		       phys_addr_t addr, phys_addr_t end)
 {
 	phys_addr_t next, start_addr = addr;
 	pud_t *pud, *start_pud;
 	start_pud = pud = pud_offset(pgd, addr);
 	do {
 		next = kvm_pud_addr_end(addr, end);
 		if (!pud_none(*pud)) {
 			if (pud_huge(*pud)) {
+				pud_t old_pud = *pud;
 				pud_clear(pud);
 				kvm_tlb_flush_vmid_ipa(kvm, addr);
+				kvm_flush_dcache_pud(old_pud);
 				put_page(virt_to_page(pud));
 			} else {
 				unmap_pmds(kvm, pud, addr, next);
 			}
 		}
 	} while (pud++, addr = next, addr != end);
 	if (kvm_pud_table_empty(kvm, start_pud))
 		clear_pgd_entry(kvm, pgd, start_addr);
 }
 static void unmap_range(struct kvm *kvm, pgd_t *pgdp,
 			phys_addr_t start, u64 size)
 {
 	pgd_t *pgd;
 	phys_addr_t addr = start, end = start + size;
 	phys_addr_t next;
 	pgd = pgdp + pgd_index(addr);
 	do {
 		next = kvm_pgd_addr_end(addr, end);
 		if (!pgd_none(*pgd))
 			unmap_puds(kvm, pgd, addr, next);
 	} while (pgd++, addr = next, addr != end);
 }
 static void stage2_flush_ptes(struct kvm *kvm, pmd_t *pmd,
 			      phys_addr_t addr, phys_addr_t end)
 {
 	pte_t *pte;
 	pte = pte_offset_kernel(pmd, addr);
 	do {
-		if (!pte_none(*pte)) {
+		if (!pte_none(*pte) &&
-			hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
+		    (pte_val(*pte) & PAGE_S2_DEVICE) != PAGE_S2_DEVICE)
-			kvm_flush_dcache_to_poc((void*)hva, PAGE_SIZE);
+			kvm_flush_dcache_pte(*pte);
-		}
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 }
 static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
 			      phys_addr_t addr, phys_addr_t end)
 {
 	pmd_t *pmd;
 	phys_addr_t next;
 	pmd = pmd_offset(pud, addr);
 	do {
 		next = kvm_pmd_addr_end(addr, end);
 		if (!pmd_none(*pmd)) {
-			if (kvm_pmd_huge(*pmd)) {
+			if (kvm_pmd_huge(*pmd))
-				hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
+				kvm_flush_dcache_pmd(*pmd);
-				kvm_flush_dcache_to_poc((void*)hva, PMD_SIZE);
+			else
-			} else {
 				stage2_flush_ptes(kvm, pmd, addr, next);
-			}
 		}
 	} while (pmd++, addr = next, addr != end);
 }
 static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
 			      phys_addr_t addr, phys_addr_t end)
 {
 	pud_t *pud;
 	phys_addr_t next;
 	pud = pud_offset(pgd, addr);
 	do {
 		next = kvm_pud_addr_end(addr, end);
 		if (!pud_none(*pud)) {
-			if (pud_huge(*pud)) {
+			if (pud_huge(*pud))
-				hva_t hva = gfn_to_hva(kvm, addr >> PAGE_SHIFT);
+				kvm_flush_dcache_pud(*pud);
-				kvm_flush_dcache_to_poc((void*)hva, PUD_SIZE);
+			else
-			} else {
 				stage2_flush_pmds(kvm, pud, addr, next);
-			}
 		}
 	} while (pud++, addr = next, addr != end);
 }
 static void stage2_flush_memslot(struct kvm *kvm,
 				 struct kvm_memory_slot *memslot)
 {
 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
 	phys_addr_t end = addr + PAGE_SIZE * memslot->npages;
 	phys_addr_t next;
 	pgd_t *pgd;
 	pgd = kvm->arch.pgd + pgd_index(addr);
 	do {
 		next = kvm_pgd_addr_end(addr, end);
 		stage2_flush_puds(kvm, pgd, addr, next);
 	} while (pgd++, addr = next, addr != end);
 }
 /**
  * stage2_flush_vm - Invalidate cache for pages mapped in stage 2
  * @kvm: The struct kvm pointer
  *
  * Go through the stage 2 page tables and invalidate any cache lines
  * backing memory already mapped to the VM.
  */
-void stage2_flush_vm(struct kvm *kvm)
+static void stage2_flush_vm(struct kvm *kvm)
 {
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 	int idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	slots = kvm_memslots(kvm);
 	kvm_for_each_memslot(memslot, slots)
 		stage2_flush_memslot(kvm, memslot);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 /**
  * free_boot_hyp_pgd - free HYP boot page tables
  *
  * Free the HYP boot page tables. The bounce page is also freed.
  */
 void free_boot_hyp_pgd(void)
 {
 	mutex_lock(&kvm_hyp_pgd_mutex);
 	if (boot_hyp_pgd) {
 		unmap_range(NULL, boot_hyp_pgd, hyp_idmap_start, PAGE_SIZE);
 		unmap_range(NULL, boot_hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
 		free_pages((unsigned long)boot_hyp_pgd, hyp_pgd_order);
 		boot_hyp_pgd = NULL;
 	}
 	if (hyp_pgd)
 		unmap_range(NULL, hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE);
 	free_page((unsigned long)init_bounce_page);
 	init_bounce_page = NULL;
 	mutex_unlock(&kvm_hyp_pgd_mutex);
 }
 /**
  * free_hyp_pgds - free Hyp-mode page tables
  *
  * Assumes hyp_pgd is a page table used strictly in Hyp-mode and
  * therefore contains either mappings in the kernel memory area (above
  * PAGE_OFFSET), or device mappings in the vmalloc range (from
  * VMALLOC_START to VMALLOC_END).
  *
  * boot_hyp_pgd should only map two pages for the init code.
  */
 void free_hyp_pgds(void)
 {
 	unsigned long addr;
 	free_boot_hyp_pgd();
 	mutex_lock(&kvm_hyp_pgd_mutex);
 	if (hyp_pgd) {
 		for (addr = PAGE_OFFSET; virt_addr_valid(addr); addr += PGDIR_SIZE)
 			unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
 		for (addr = VMALLOC_START; is_vmalloc_addr((void*)addr); addr += PGDIR_SIZE)
 			unmap_range(NULL, hyp_pgd, KERN_TO_HYP(addr), PGDIR_SIZE);
 		free_pages((unsigned long)hyp_pgd, hyp_pgd_order);
 		hyp_pgd = NULL;
 	}
 	mutex_unlock(&kvm_hyp_pgd_mutex);
 }
 static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long start,
 				    unsigned long end, unsigned long pfn,
 				    pgprot_t prot)
 {
 	pte_t *pte;
 	unsigned long addr;
 	addr = start;
 	do {
 		pte = pte_offset_kernel(pmd, addr);
 		kvm_set_pte(pte, pfn_pte(pfn, prot));
 		get_page(virt_to_page(pte));
 		kvm_flush_dcache_to_poc(pte, sizeof(*pte));
 		pfn++;
 	} while (addr += PAGE_SIZE, addr != end);
 }
 static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
 				   unsigned long end, unsigned long pfn,
 				   pgprot_t prot)
 {
 	pmd_t *pmd;
 	pte_t *pte;
 	unsigned long addr, next;
 	addr = start;
 	do {
 		pmd = pmd_offset(pud, addr);
 		BUG_ON(pmd_sect(*pmd));
 		if (pmd_none(*pmd)) {
 			pte = pte_alloc_one_kernel(NULL, addr);
 			if (!pte) {
 				kvm_err("Cannot allocate Hyp pte\n");
 				return -ENOMEM;
 			}
 			pmd_populate_kernel(NULL, pmd, pte);
 			get_page(virt_to_page(pmd));
 			kvm_flush_dcache_to_poc(pmd, sizeof(*pmd));
 		}
 		next = pmd_addr_end(addr, end);
 		create_hyp_pte_mappings(pmd, addr, next, pfn, prot);
 		pfn += (next - addr) >> PAGE_SHIFT;
 	} while (addr = next, addr != end);
 	return 0;
 }
 static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
 				   unsigned long end, unsigned long pfn,
 				   pgprot_t prot)
 {
 	pud_t *pud;
 	pmd_t *pmd;
 	unsigned long addr, next;
 	int ret;
 	addr = start;
 	do {
 		pud = pud_offset(pgd, addr);
 		if (pud_none_or_clear_bad(pud)) {
 			pmd = pmd_alloc_one(NULL, addr);
 			if (!pmd) {
 				kvm_err("Cannot allocate Hyp pmd\n");
 				return -ENOMEM;
 			}
 			pud_populate(NULL, pud, pmd);
 			get_page(virt_to_page(pud));
 			kvm_flush_dcache_to_poc(pud, sizeof(*pud));
 		}
 		next = pud_addr_end(addr, end);
 		ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot);
 		if (ret)
 			return ret;
 		pfn += (next - addr) >> PAGE_SHIFT;
 	} while (addr = next, addr != end);
 	return 0;
 }
 static int __create_hyp_mappings(pgd_t *pgdp,
 				 unsigned long start, unsigned long end,
 				 unsigned long pfn, pgprot_t prot)
 {
 	pgd_t *pgd;
 	pud_t *pud;
 	unsigned long addr, next;
 	int err = 0;
 	mutex_lock(&kvm_hyp_pgd_mutex);
 	addr = start & PAGE_MASK;
 	end = PAGE_ALIGN(end);
 	do {
 		pgd = pgdp + pgd_index(addr);
 		if (pgd_none(*pgd)) {
 			pud = pud_alloc_one(NULL, addr);
 			if (!pud) {
 				kvm_err("Cannot allocate Hyp pud\n");
 				err = -ENOMEM;
 				goto out;
 			}
 			pgd_populate(NULL, pgd, pud);
 			get_page(virt_to_page(pgd));
 			kvm_flush_dcache_to_poc(pgd, sizeof(*pgd));
 		}
 		next = pgd_addr_end(addr, end);
 		err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);
 		if (err)
 			goto out;
 		pfn += (next - addr) >> PAGE_SHIFT;
 	} while (addr = next, addr != end);
 out:
 	mutex_unlock(&kvm_hyp_pgd_mutex);
 	return err;
 }
 static phys_addr_t kvm_kaddr_to_phys(void *kaddr)
 {
 	if (!is_vmalloc_addr(kaddr)) {
 		BUG_ON(!virt_addr_valid(kaddr));
 		return __pa(kaddr);
 	} else {
 		return page_to_phys(vmalloc_to_page(kaddr)) +
 		       offset_in_page(kaddr);
 	}
 }
 /**
  * create_hyp_mappings - duplicate a kernel virtual address range in Hyp mode
  * @from:	The virtual kernel start address of the range
  * @to:		The virtual kernel end address of the range (exclusive)
  *
  * The same virtual address as the kernel virtual address is also used
  * in Hyp-mode mapping (modulo HYP_PAGE_OFFSET) to the same underlying
  * physical pages.
  */
 int create_hyp_mappings(void *from, void *to)
 {
 	phys_addr_t phys_addr;
 	unsigned long virt_addr;
 	unsigned long start = KERN_TO_HYP((unsigned long)from);
 	unsigned long end = KERN_TO_HYP((unsigned long)to);
 	start = start & PAGE_MASK;
 	end = PAGE_ALIGN(end);
 	for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
 		int err;
 		phys_addr = kvm_kaddr_to_phys(from + virt_addr - start);
 		err = __create_hyp_mappings(hyp_pgd, virt_addr,
 					    virt_addr + PAGE_SIZE,
 					    __phys_to_pfn(phys_addr),
 					    PAGE_HYP);
 		if (err)
 			return err;
 	}
 	return 0;
 }
 /**
  * create_hyp_io_mappings - duplicate a kernel IO mapping into Hyp mode
  * @from:	The kernel start VA of the range
  * @to:		The kernel end VA of the range (exclusive)
  * @phys_addr:	The physical start address which gets mapped
  *
  * The resulting HYP VA is the same as the kernel VA, modulo
  * HYP_PAGE_OFFSET.
  */
 int create_hyp_io_mappings(void *from, void *to, phys_addr_t phys_addr)
 {
 	unsigned long start = KERN_TO_HYP((unsigned long)from);
 	unsigned long end = KERN_TO_HYP((unsigned long)to);
 	/* Check for a valid kernel IO mapping */
 	if (!is_vmalloc_addr(from) || !is_vmalloc_addr(to - 1))
 		return -EINVAL;
 	return __create_hyp_mappings(hyp_pgd, start, end,
 				     __phys_to_pfn(phys_addr), PAGE_HYP_DEVICE);
 }
 /**
  * kvm_alloc_stage2_pgd - allocate level-1 table for stage-2 translation.
  * @kvm:	The KVM struct pointer for the VM.
  *
  * Allocates the 1st level table only of size defined by S2_PGD_ORDER (can
  * support either full 40-bit input addresses or limited to 32-bit input
  * addresses). Clears the allocated pages.
  *
  * Note we don't need locking here as this is only called when the VM is
  * created, which can only be done once.
  */
 int kvm_alloc_stage2_pgd(struct kvm *kvm)
 {
 	int ret;
 	pgd_t *pgd;
 	if (kvm->arch.pgd != NULL) {
 		kvm_err("kvm_arch already initialized?\n");
 		return -EINVAL;
 	}
 	if (KVM_PREALLOC_LEVEL > 0) {
 		/*
 		 * Allocate fake pgd for the page table manipulation macros to
 		 * work.  This is not used by the hardware and we have no
 		 * alignment requirement for this allocation.
 		 */
 		pgd = (pgd_t *)kmalloc(PTRS_PER_S2_PGD * sizeof(pgd_t),
 				       GFP_KERNEL | __GFP_ZERO);
 	} else {
 		/*
 		 * Allocate actual first-level Stage-2 page table used by the
 		 * hardware for Stage-2 page table walks.
 		 */
 		pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, S2_PGD_ORDER);
 	}
 	if (!pgd)
 		return -ENOMEM;
 	ret = kvm_prealloc_hwpgd(kvm, pgd);
 	if (ret)
 		goto out_err;
 	kvm_clean_pgd(pgd);
 	kvm->arch.pgd = pgd;
 	return 0;
 out_err:
 	if (KVM_PREALLOC_LEVEL > 0)
 		kfree(pgd);
 	else
 		free_pages((unsigned long)pgd, S2_PGD_ORDER);
 	return ret;
 }
 /**
  * unmap_stage2_range -- Clear stage2 page table entries to unmap a range
  * @kvm:   The VM pointer
  * @start: The intermediate physical base address of the range to unmap
  * @size:  The size of the area to unmap
  *
  * Clear a range of stage-2 mappings, lowering the various ref-counts.  Must
  * be called while holding mmu_lock (unless for freeing the stage2 pgd before
  * destroying the VM), otherwise another faulting VCPU may come in and mess
  * with things behind our backs.
  */
 static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
 {
 	unmap_range(kvm, kvm->arch.pgd, start, size);
 }
 static void stage2_unmap_memslot(struct kvm *kvm,
 				 struct kvm_memory_slot *memslot)
 {
 	hva_t hva = memslot->userspace_addr;
 	phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT;
 	phys_addr_t size = PAGE_SIZE * memslot->npages;
 	hva_t reg_end = hva + size;
 	/*
 	 * A memory region could potentially cover multiple VMAs, and any holes
 	 * between them, so iterate over all of them to find out if we should
 	 * unmap any of them.
 	 *
 	 *     +--------------------------------------------+
 	 * +---------------+----------------+   +----------------+
 	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
 	 * +---------------+----------------+   +----------------+
 	 *     |               memory region                |
 	 *     +--------------------------------------------+
 	 */
 	do {
 		struct vm_area_struct *vma = find_vma(current->mm, hva);
 		hva_t vm_start, vm_end;
 		if (!vma || vma->vm_start >= reg_end)
 			break;
 		/*
 		 * Take the intersection of this VMA with the memory region
 		 */
 		vm_start = max(hva, vma->vm_start);
 		vm_end = min(reg_end, vma->vm_end);
 		if (!(vma->vm_flags & VM_PFNMAP)) {
 			gpa_t gpa = addr + (vm_start - memslot->userspace_addr);
 			unmap_stage2_range(kvm, gpa, vm_end - vm_start);
 		}
 		hva = vm_end;
 	} while (hva < reg_end);
 }
 /**
  * stage2_unmap_vm - Unmap Stage-2 RAM mappings
  * @kvm: The struct kvm pointer
  *
  * Go through the memregions and unmap any reguler RAM
  * backing memory already mapped to the VM.
  */
 void stage2_unmap_vm(struct kvm *kvm)
 {
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 	int idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	slots = kvm_memslots(kvm);
 	kvm_for_each_memslot(memslot, slots)
 		stage2_unmap_memslot(kvm, memslot);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 /**
  * kvm_free_stage2_pgd - free all stage-2 tables
  * @kvm:	The KVM struct pointer for the VM.
  *
  * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all
  * underlying level-2 and level-3 tables before freeing the actual level-1 table
  * and setting the struct pointer to NULL.
  *
  * Note we don't need locking here as this is only called when the VM is
  * destroyed, which can only be done once.
  */
 void kvm_free_stage2_pgd(struct kvm *kvm)
 {
 	if (kvm->arch.pgd == NULL)
 		return;
 	unmap_stage2_range(kvm, 0, KVM_PHYS_SIZE);
 	kvm_free_hwpgd(kvm);
 	if (KVM_PREALLOC_LEVEL > 0)
 		kfree(kvm->arch.pgd);
 	else
 		free_pages((unsigned long)kvm->arch.pgd, S2_PGD_ORDER);
 	kvm->arch.pgd = NULL;
 }
 static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
 			     phys_addr_t addr)
 {
 	pgd_t *pgd;
 	pud_t *pud;
 	pgd = kvm->arch.pgd + pgd_index(addr);
 	if (WARN_ON(pgd_none(*pgd))) {
 		if (!cache)
 			return NULL;
 		pud = mmu_memory_cache_alloc(cache);
 		pgd_populate(NULL, pgd, pud);
 		get_page(virt_to_page(pgd));
 	}
 	return pud_offset(pgd, addr);
 }
 static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
 			     phys_addr_t addr)
 {
 	pud_t *pud;
 	pmd_t *pmd;
 	pud = stage2_get_pud(kvm, cache, addr);
 	if (pud_none(*pud)) {
 		if (!cache)
 			return NULL;
 		pmd = mmu_memory_cache_alloc(cache);
 		pud_populate(NULL, pud, pmd);
 		get_page(virt_to_page(pud));
 	}
 	return pmd_offset(pud, addr);
 }
 static int stage2_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache
 			       *cache, phys_addr_t addr, const pmd_t *new_pmd)
 {
 	pmd_t *pmd, old_pmd;
 	pmd = stage2_get_pmd(kvm, cache, addr);
 	VM_BUG_ON(!pmd);
 	/*
 	 * Mapping in huge pages should only happen through a fault.  If a
 	 * page is merged into a transparent huge page, the individual
 	 * subpages of that huge page should be unmapped through MMU
 	 * notifiers before we get here.
 	 *
 	 * Merging of CompoundPages is not supported; they should become
 	 * splitting first, unmapped, merged, and mapped back in on-demand.
 	 */
 	VM_BUG_ON(pmd_present(*pmd) && pmd_pfn(*pmd) != pmd_pfn(*new_pmd));
 	old_pmd = *pmd;
 	kvm_set_pmd(pmd, *new_pmd);
 	if (pmd_present(old_pmd))
 		kvm_tlb_flush_vmid_ipa(kvm, addr);
 	else
 		get_page(virt_to_page(pmd));
 	return 0;
 }
 static int stage2_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
 			  phys_addr_t addr, const pte_t *new_pte, bool iomap)
 {
 	pmd_t *pmd;
 	pte_t *pte, old_pte;
 	/* Create stage-2 page table mapping - Levels 0 and 1 */
 	pmd = stage2_get_pmd(kvm, cache, addr);
 	if (!pmd) {
 		/*
 		 * Ignore calls from kvm_set_spte_hva for unallocated
 		 * address ranges.
 		 */
 		return 0;
 	}
 	/* Create stage-2 page mappings - Level 2 */
 	if (pmd_none(*pmd)) {
 		if (!cache)
 			return 0; /* ignore calls from kvm_set_spte_hva */
 		pte = mmu_memory_cache_alloc(cache);
 		kvm_clean_pte(pte);
 		pmd_populate_kernel(NULL, pmd, pte);
 		get_page(virt_to_page(pmd));
 	}
 	pte = pte_offset_kernel(pmd, addr);
 	if (iomap && pte_present(*pte))
 		return -EFAULT;
 	/* Create 2nd stage page table mapping - Level 3 */
 	old_pte = *pte;
 	kvm_set_pte(pte, *new_pte);
 	if (pte_present(old_pte))
 		kvm_tlb_flush_vmid_ipa(kvm, addr);
 	else
 		get_page(virt_to_page(pte));
 	return 0;
 }
 /**
  * kvm_phys_addr_ioremap - map a device range to guest IPA
  *
  * @kvm:	The KVM pointer
  * @guest_ipa:	The IPA at which to insert the mapping
  * @pa:		The physical address of the device
  * @size:	The size of the mapping
  */
 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
 			  phys_addr_t pa, unsigned long size, bool writable)
 {
 	phys_addr_t addr, end;
 	int ret = 0;
 	unsigned long pfn;
 	struct kvm_mmu_memory_cache cache = { 0, };
 	end = (guest_ipa + size + PAGE_SIZE - 1) & PAGE_MASK;
 	pfn = __phys_to_pfn(pa);
 	for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) {
 		pte_t pte = pfn_pte(pfn, PAGE_S2_DEVICE);
 		if (writable)
 			kvm_set_s2pte_writable(&pte);
 		ret = mmu_topup_memory_cache(&cache, KVM_MMU_CACHE_MIN_PAGES,
 						KVM_NR_MEM_OBJS);
 		if (ret)
 			goto out;
 		spin_lock(&kvm->mmu_lock);
 		ret = stage2_set_pte(kvm, &cache, addr, &pte, true);
 		spin_unlock(&kvm->mmu_lock);
 		if (ret)
 			goto out;
 		pfn++;
 	}
 out:
 	mmu_free_memory_cache(&cache);
 	return ret;
 }
 static bool transparent_hugepage_adjust(pfn_t *pfnp, phys_addr_t *ipap)
 {
 	pfn_t pfn = *pfnp;
 	gfn_t gfn = *ipap >> PAGE_SHIFT;
 	if (PageTransCompound(pfn_to_page(pfn))) {
 		unsigned long mask;
 		/*
 		 * The address we faulted on is backed by a transparent huge
 		 * page.  However, because we map the compound huge page and
 		 * not the individual tail page, we need to transfer the
 		 * refcount to the head page.  We have to be careful that the
 		 * THP doesn't start to split while we are adjusting the
 		 * refcounts.
 		 *
 		 * We are sure this doesn't happen, because mmu_notifier_retry
 		 * was successful and we are holding the mmu_lock, so if this
 		 * THP is trying to split, it will be blocked in the mmu
 		 * notifier before touching any of the pages, specifically
 		 * before being able to call __split_huge_page_refcount().
 		 *
 		 * We can therefore safely transfer the refcount from PG_tail
 		 * to PG_head and switch the pfn from a tail page to the head
 		 * page accordingly.
 		 */
 		mask = PTRS_PER_PMD - 1;
 		VM_BUG_ON((gfn & mask) != (pfn & mask));
 		if (pfn & mask) {
 			*ipap &= PMD_MASK;
 			kvm_release_pfn_clean(pfn);
 			pfn &= ~mask;
 			kvm_get_pfn(pfn);
 			*pfnp = pfn;
 		}
 		return true;
 	}
 	return false;
 }
 static bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
 {
 	if (kvm_vcpu_trap_is_iabt(vcpu))
 		return false;
 	return kvm_vcpu_dabt_iswrite(vcpu);
 }
 static bool kvm_is_device_pfn(unsigned long pfn)
 {
 	return !pfn_valid(pfn);
 }
+static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, pfn_t pfn,
+				      unsigned long size, bool uncached)
+{
+	__coherent_cache_guest_page(vcpu, pfn, size, uncached);
+}
 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 			  struct kvm_memory_slot *memslot, unsigned long hva,
 			  unsigned long fault_status)
 {
 	int ret;
 	bool write_fault, writable, hugetlb = false, force_pte = false;
 	unsigned long mmu_seq;
 	gfn_t gfn = fault_ipa >> PAGE_SHIFT;
 	struct kvm *kvm = vcpu->kvm;
 	struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache;
 	struct vm_area_struct *vma;
 	pfn_t pfn;
 	pgprot_t mem_type = PAGE_S2;
 	bool fault_ipa_uncached;
 	write_fault = kvm_is_write_fault(vcpu);
 	if (fault_status == FSC_PERM && !write_fault) {
 		kvm_err("Unexpected L2 read permission error\n");
 		return -EFAULT;
 	}
 	/* Let's check if we will get back a huge page backed by hugetlbfs */
 	down_read(&current->mm->mmap_sem);
 	vma = find_vma_intersection(current->mm, hva, hva + 1);
 	if (unlikely(!vma)) {
 		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
 		up_read(&current->mm->mmap_sem);
 		return -EFAULT;
 	}
 	if (is_vm_hugetlb_page(vma)) {
 		hugetlb = true;
 		gfn = (fault_ipa & PMD_MASK) >> PAGE_SHIFT;
 	} else {
 		/*
 		 * Pages belonging to memslots that don't have the same
 		 * alignment for userspace and IPA cannot be mapped using
 		 * block descriptors even if the pages belong to a THP for
 		 * the process, because the stage-2 block descriptor will
 		 * cover more than a single THP and we loose atomicity for
 		 * unmapping, updates, and splits of the THP or other pages
 		 * in the stage-2 block range.
 		 */
 		if ((memslot->userspace_addr & ~PMD_MASK) !=
 		    ((memslot->base_gfn << PAGE_SHIFT) & ~PMD_MASK))
 			force_pte = true;
 	}
 	up_read(&current->mm->mmap_sem);
 	/* We need minimum second+third level pages */
 	ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES,
 				     KVM_NR_MEM_OBJS);
 	if (ret)
 		return ret;
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	/*
 	 * Ensure the read of mmu_notifier_seq happens before we call
 	 * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk
 	 * the page we just got a reference to gets unmapped before we have a
 	 * chance to grab the mmu_lock, which ensure that if the page gets
 	 * unmapped afterwards, the call to kvm_unmap_hva will take it away
 	 * from us again properly. This smp_rmb() interacts with the smp_wmb()
 	 * in kvm_mmu_notifier_invalidate_<page|range_end>.
 	 */
 	smp_rmb();
 	pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable);
 	if (is_error_pfn(pfn))
 		return -EFAULT;
 	if (kvm_is_device_pfn(pfn))
 		mem_type = PAGE_S2_DEVICE;
 	spin_lock(&kvm->mmu_lock);
 	if (mmu_notifier_retry(kvm, mmu_seq))
 		goto out_unlock;
 	if (!hugetlb && !force_pte)
 		hugetlb = transparent_hugepage_adjust(&pfn, &fault_ipa);
 	fault_ipa_uncached = memslot->flags & KVM_MEMSLOT_INCOHERENT;
 	if (hugetlb) {
 		pmd_t new_pmd = pfn_pmd(pfn, mem_type);
 		new_pmd = pmd_mkhuge(new_pmd);
 		if (writable) {
 			kvm_set_s2pmd_writable(&new_pmd);
 			kvm_set_pfn_dirty(pfn);
 		}
-		coherent_cache_guest_page(vcpu, hva & PMD_MASK, PMD_SIZE,
+		coherent_cache_guest_page(vcpu, pfn, PMD_SIZE, fault_ipa_uncached);
-					  fault_ipa_uncached);
 		ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
 	} else {
 		pte_t new_pte = pfn_pte(pfn, mem_type);
 		if (writable) {
 			kvm_set_s2pte_writable(&new_pte);
 			kvm_set_pfn_dirty(pfn);
 		}
-		coherent_cache_guest_page(vcpu, hva, PAGE_SIZE,
+		coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE, fault_ipa_uncached);
-					  fault_ipa_uncached);
 		ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte,
 			pgprot_val(mem_type) == pgprot_val(PAGE_S2_DEVICE));
 	}
 out_unlock:
 	spin_unlock(&kvm->mmu_lock);
 	kvm_release_pfn_clean(pfn);
 	return ret;
 }
 /**
  * kvm_handle_guest_abort - handles all 2nd stage aborts
  * @vcpu:	the VCPU pointer
  * @run:	the kvm_run structure
  *
  * Any abort that gets to the host is almost guaranteed to be caused by a
  * missing second stage translation table entry, which can mean that either the
  * guest simply needs more memory and we must allocate an appropriate page or it
  * can mean that the guest tried to access I/O memory, which is emulated by user
  * space. The distinction is based on the IPA causing the fault and whether this
  * memory region has been registered as standard RAM by user space.
  */
 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	unsigned long fault_status;
 	phys_addr_t fault_ipa;
 	struct kvm_memory_slot *memslot;
 	unsigned long hva;
 	bool is_iabt, write_fault, writable;
 	gfn_t gfn;
 	int ret, idx;
 	is_iabt = kvm_vcpu_trap_is_iabt(vcpu);
 	fault_ipa = kvm_vcpu_get_fault_ipa(vcpu);
 	trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_hsr(vcpu),
 			      kvm_vcpu_get_hfar(vcpu), fault_ipa);
 	/* Check the stage-2 fault is trans. fault or write fault */
 	fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
 	if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
 		kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
 			kvm_vcpu_trap_get_class(vcpu),
 			(unsigned long)kvm_vcpu_trap_get_fault(vcpu),
 			(unsigned long)kvm_vcpu_get_hsr(vcpu));
 		return -EFAULT;
 	}
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	gfn = fault_ipa >> PAGE_SHIFT;
 	memslot = gfn_to_memslot(vcpu->kvm, gfn);
 	hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable);
 	write_fault = kvm_is_write_fault(vcpu);
 	if (kvm_is_error_hva(hva) || (write_fault && !writable)) {
 		if (is_iabt) {
 			/* Prefetch Abort on I/O address */
 			kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
 			ret = 1;
 			goto out_unlock;
 		}
 		/*
 		 * The IPA is reported as [MAX:12], so we need to
 		 * complement it with the bottom 12 bits from the
 		 * faulting VA. This is always 12 bits, irrespective
 		 * of the page size.
 		 */
 		fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1);
 		ret = io_mem_abort(vcpu, run, fault_ipa);
 		goto out_unlock;
 	}
 	/* Userspace should not be able to register out-of-bounds IPAs */
 	VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
 	ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
 	if (ret == 0)
 		ret = 1;
 out_unlock:
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	return ret;
 }
 static void handle_hva_to_gpa(struct kvm *kvm,
 			      unsigned long start,
 			      unsigned long end,
 			      void (*handler)(struct kvm *kvm,
 					      gpa_t gpa, void *data),
 			      void *data)
 {
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 	slots = kvm_memslots(kvm);
 	/* we only care about the pages that the guest sees */
 	kvm_for_each_memslot(memslot, slots) {
 		unsigned long hva_start, hva_end;
 		gfn_t gfn, gfn_end;
 		hva_start = max(start, memslot->userspace_addr);
 		hva_end = min(end, memslot->userspace_addr +
 					(memslot->npages << PAGE_SHIFT));
 		if (hva_start >= hva_end)
 			continue;
 		/*
 		 * {gfn(page) | page intersects with [hva_start, hva_end)} =
 		 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
 		 */
 		gfn = hva_to_gfn_memslot(hva_start, memslot);
 		gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 		for (; gfn < gfn_end; ++gfn) {
 			gpa_t gpa = gfn << PAGE_SHIFT;
 			handler(kvm, gpa, data);
 		}
 	}
 }
 static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
 {
 	unmap_stage2_range(kvm, gpa, PAGE_SIZE);
 }
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
 {
 	unsigned long end = hva + PAGE_SIZE;
 	if (!kvm->arch.pgd)
 		return 0;
 	trace_kvm_unmap_hva(hva);
 	handle_hva_to_gpa(kvm, hva, end, &kvm_unmap_hva_handler, NULL);
 	return 0;
 }
 int kvm_unmap_hva_range(struct kvm *kvm,
 			unsigned long start, unsigned long end)
 {
 	if (!kvm->arch.pgd)
 		return 0;
 	trace_kvm_unmap_hva_range(start, end);
 	handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL);
 	return 0;
 }
 static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
 {
 	pte_t *pte = (pte_t *)data;
 	stage2_set_pte(kvm, NULL, gpa, pte, false);
 }
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
 {
 	unsigned long end = hva + PAGE_SIZE;
 	pte_t stage2_pte;
 	if (!kvm->arch.pgd)
 		return;
 	trace_kvm_set_spte_hva(hva);
 	stage2_pte = pfn_pte(pte_pfn(pte), PAGE_S2);
 	handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
 }
 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 {
 	mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
 }
 phys_addr_t kvm_mmu_get_httbr(void)
 {
 	return virt_to_phys(hyp_pgd);
 }
 phys_addr_t kvm_mmu_get_boot_httbr(void)
 {
 	return virt_to_phys(boot_hyp_pgd);
 }
 phys_addr_t kvm_get_idmap_vector(void)
 {
 	return hyp_idmap_vector;
 }
 int kvm_mmu_init(void)
 {
 	int err;
 	hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start);
 	hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end);
 	hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init);
 	if ((hyp_idmap_start ^ hyp_idmap_end) & PAGE_MASK) {
 		/*
 		 * Our init code is crossing a page boundary. Allocate
 		 * a bounce page, copy the code over and use that.
 		 */
 		size_t len = __hyp_idmap_text_end - __hyp_idmap_text_start;
 		phys_addr_t phys_base;
 		init_bounce_page = (void *)__get_free_page(GFP_KERNEL);
 		if (!init_bounce_page) {
 			kvm_err("Couldn't allocate HYP init bounce page\n");
 			err = -ENOMEM;
 			goto out;
 		}
 		memcpy(init_bounce_page, __hyp_idmap_text_start, len);
 		/*
 		 * Warning: the code we just copied to the bounce page
 		 * must be flushed to the point of coherency.
 		 * Otherwise, the data may be sitting in L2, and HYP
 		 * mode won't be able to observe it as it runs with
 		 * caches off at that point.
 		 */
 		kvm_flush_dcache_to_poc(init_bounce_page, len);
 		phys_base = kvm_virt_to_phys(init_bounce_page);
 		hyp_idmap_vector += phys_base - hyp_idmap_start;
 		hyp_idmap_start = phys_base;
 		hyp_idmap_end = phys_base + len;
 		kvm_info("Using HYP init bounce page @%lx\n",
 			 (unsigned long)phys_base);
 	}
 	hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order);
 	boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order);
 	if (!hyp_pgd || !boot_hyp_pgd) {
 		kvm_err("Hyp mode PGD not allocated\n");
 		err = -ENOMEM;
 		goto out;
 	}
 	/* Create the idmap in the boot page tables */
 	err = 	__create_hyp_mappings(boot_hyp_pgd,
 				      hyp_idmap_start, hyp_idmap_end,
 				      __phys_to_pfn(hyp_idmap_start),
 				      PAGE_HYP);
 	if (err) {
 		kvm_err("Failed to idmap %lx-%lx\n",
 			hyp_idmap_start, hyp_idmap_end);
 		goto out;
 	}
 	/* Map the very same page at the trampoline VA */
 	err = 	__create_hyp_mappings(boot_hyp_pgd,
 				      TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
 				      __phys_to_pfn(hyp_idmap_start),
 				      PAGE_HYP);
 	if (err) {
 		kvm_err("Failed to map trampoline @%lx into boot HYP pgd\n",
 			TRAMPOLINE_VA);
 		goto out;
 	}
 	/* Map the same page again into the runtime page tables */
 	err = 	__create_hyp_mappings(hyp_pgd,
 				      TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE,
 				      __phys_to_pfn(hyp_idmap_start),
 				      PAGE_HYP);
 	if (err) {
 		kvm_err("Failed to map trampoline @%lx into runtime HYP pgd\n",
 			TRAMPOLINE_VA);
 		goto out;
 	}
 	return 0;
 out:
 	free_hyp_pgds();
 	return err;
 }
 void kvm_arch_commit_memory_region(struct kvm *kvm,
 				   struct kvm_userspace_memory_region *mem,
 				   const struct kvm_memory_slot *old,
 				   enum kvm_mr_change change)
 {
 }
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 				   struct kvm_memory_slot *memslot,
 				   struct kvm_userspace_memory_region *mem,
 				   enum kvm_mr_change change)
 {
 	hva_t hva = mem->userspace_addr;
 	hva_t reg_end = hva + mem->memory_size;
 	bool writable = !(mem->flags & KVM_MEM_READONLY);
 	int ret = 0;
 	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE)
 		return 0;
 	/*
 	 * Prevent userspace from creating a memory region outside of the IPA
 	 * space addressable by the KVM guest IPA space.
 	 */
 	if (memslot->base_gfn + memslot->npages >=
 	    (KVM_PHYS_SIZE >> PAGE_SHIFT))
 		return -EFAULT;
 	/*
 	 * A memory region could potentially cover multiple VMAs, and any holes
 	 * between them, so iterate over all of them to find out if we can map
 	 * any of them right now.
 	 *
 	 *     +--------------------------------------------+
 	 * +---------------+----------------+   +----------------+
 	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
 	 * +---------------+----------------+   +----------------+
 	 *     |               memory region                |
 	 *     +--------------------------------------------+
 	 */
 	do {
 		struct vm_area_struct *vma = find_vma(current->mm, hva);
 		hva_t vm_start, vm_end;
 		if (!vma || vma->vm_start >= reg_end)
 			break;
 		/*
 		 * Mapping a read-only VMA is only allowed if the
 		 * memory region is configured as read-only.
 		 */
 		if (writable && !(vma->vm_flags & VM_WRITE)) {
 			ret = -EPERM;
 			break;
 		}
 		/*
 		 * Take the intersection of this VMA with the memory region
 		 */
 		vm_start = max(hva, vma->vm_start);
 		vm_end = min(reg_end, vma->vm_end);
 		if (vma->vm_flags & VM_PFNMAP) {
 			gpa_t gpa = mem->guest_phys_addr +
 				    (vm_start - mem->userspace_addr);
 			phys_addr_t pa = (vma->vm_pgoff << PAGE_SHIFT) +
 					 vm_start - vma->vm_start;
 			ret = kvm_phys_addr_ioremap(kvm, gpa, pa,
 						    vm_end - vm_start,
 						    writable);
 			if (ret)
 				break;
 		}
 		hva = vm_end;
 	} while (hva < reg_end);
 	spin_lock(&kvm->mmu_lock);
 	if (ret)
 		unmap_stage2_range(kvm, mem->guest_phys_addr, mem->memory_size);
 	else
 		stage2_flush_memslot(kvm, memslot);
 	spin_unlock(&kvm->mmu_lock);
 	return ret;
 }
 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 			   struct kvm_memory_slot *dont)
 {
 }
 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
 			    unsigned long npages)
 {
 	/*
 	 * Readonly memslots are not incoherent with the caches by definition,
 	 * but in practice, they are used mostly to emulate ROMs or NOR flashes
 	 * that the guest may consider devices and hence map as uncached.
 	 * To prevent incoherency issues in these cases, tag all readonly
 	 * regions as incoherent.
 	 */
 	if (slot->flags & KVM_MEM_READONLY)
 		slot->flags |= KVM_MEMSLOT_INCOHERENT;
 	return 0;
 }
 void kvm_arch_memslots_updated(struct kvm *kvm)
 {
 }
 void kvm_arch_flush_shadow_all(struct kvm *kvm)
 {
 }
 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 				   struct kvm_memory_slot *slot)
 {
 	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
 	phys_addr_t size = slot->npages << PAGE_SHIFT;
 	spin_lock(&kvm->mmu_lock);
 	unmap_stage2_range(kvm, gpa, size);
 	spin_unlock(&kvm->mmu_lock);
+}
+/*
+ * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
+ *
+ * Main problems:
+ * - S/W ops are local to a CPU (not broadcast)
+ * - We have line migration behind our back (speculation)
+ * - System caches don't support S/W at all (damn!)
+ *
+ * In the face of the above, the best we can do is to try and convert
+ * S/W ops to VA ops. Because the guest is not allowed to infer the
+ * S/W to PA mapping, it can only use S/W to nuke the whole cache,
+ * which is a rather good thing for us.
+ *
+ * Also, it is only used when turning caches on/off ("The expected
+ * usage of the cache maintenance instructions that operate by set/way
+ * is associated with the cache maintenance instructions associated
+ * with the powerdown and powerup of caches, if this is required by
+ * the implementation.").
+ *
+ * We use the following policy:
+ *
+ * - If we trap a S/W operation, we enable VM trapping to detect
+ *   caches being turned on/off, and do a full clean.
+ *
+ * - We flush the caches on both caches being turned on and off.
+ *
+ * - Once the caches are enabled, we stop trapping VM ops.
+ */
+void kvm_set_way_flush(struct kvm_vcpu *vcpu)
+{
+	unsigned long hcr = vcpu_get_hcr(vcpu);
+	/*
+	 * If this is the first time we do a S/W operation
+	 * (i.e. HCR_TVM not set) flush the whole memory, and set the
+	 * VM trapping.
+	 *
+	 * Otherwise, rely on the VM trapping to wait for the MMU +
+	 * Caches to be turned off. At that point, we'll be able to
+	 * clean the caches again.
+	 */
+	if (!(hcr & HCR_TVM)) {
+		trace_kvm_set_way_flush(*vcpu_pc(vcpu),
+					vcpu_has_cache_enabled(vcpu));
+		stage2_flush_vm(vcpu->kvm);
+		vcpu_set_hcr(vcpu, hcr | HCR_TVM);
+	}
+}
+void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled)
+{
+	bool now_enabled = vcpu_has_cache_enabled(vcpu);
+	/*
+	 * If switching the MMU+caches on, need to invalidate the caches.
+	 * If switching it off, need to clean the caches.
+	 * Clean + invalidate does the trick always.
+	 */
+	if (now_enabled != was_enabled)
+		stage2_flush_vm(vcpu->kvm);

arch/arm/kvm/trace.h

Diff comments View file @ 02512b2

 #if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
 #define _TRACE_KVM_H
 #include <linux/tracepoint.h>
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM kvm
 /*
  * Tracepoints for entry/exit to guest
  */
 TRACE_EVENT(kvm_entry,
 	TP_PROTO(unsigned long vcpu_pc),
 	TP_ARGS(vcpu_pc),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 	),
 	TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
 );
 TRACE_EVENT(kvm_exit,
 	TP_PROTO(unsigned long vcpu_pc),
 	TP_ARGS(vcpu_pc),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 	),
 	TP_printk("PC: 0x%08lx", __entry->vcpu_pc)
 );
 TRACE_EVENT(kvm_guest_fault,
 	TP_PROTO(unsigned long vcpu_pc, unsigned long hsr,
 		 unsigned long hxfar,
 		 unsigned long long ipa),
 	TP_ARGS(vcpu_pc, hsr, hxfar, ipa),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 		__field(	unsigned long,	hsr		)
 		__field(	unsigned long,	hxfar		)
 		__field(   unsigned long long,	ipa		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 		__entry->hsr			= hsr;
 		__entry->hxfar			= hxfar;
 		__entry->ipa			= ipa;
 	),
 	TP_printk("ipa %#llx, hsr %#08lx, hxfar %#08lx, pc %#08lx",
 		  __entry->ipa, __entry->hsr,
 		  __entry->hxfar, __entry->vcpu_pc)
 );
 TRACE_EVENT(kvm_irq_line,
 	TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
 	TP_ARGS(type, vcpu_idx, irq_num, level),
 	TP_STRUCT__entry(
 		__field(	unsigned int,	type		)
 		__field(	int,		vcpu_idx	)
 		__field(	int,		irq_num		)
 		__field(	int,		level		)
 	),
 	TP_fast_assign(
 		__entry->type		= type;
 		__entry->vcpu_idx	= vcpu_idx;
 		__entry->irq_num	= irq_num;
 		__entry->level		= level;
 	),
 	TP_printk("Inject %s interrupt (%d), vcpu->idx: %d, num: %d, level: %d",
 		  (__entry->type == KVM_ARM_IRQ_TYPE_CPU) ? "CPU" :
 		  (__entry->type == KVM_ARM_IRQ_TYPE_PPI) ? "VGIC PPI" :
 		  (__entry->type == KVM_ARM_IRQ_TYPE_SPI) ? "VGIC SPI" : "UNKNOWN",
 		  __entry->type, __entry->vcpu_idx, __entry->irq_num, __entry->level)
 );
 TRACE_EVENT(kvm_mmio_emulate,
 	TP_PROTO(unsigned long vcpu_pc, unsigned long instr,
 		 unsigned long cpsr),
 	TP_ARGS(vcpu_pc, instr, cpsr),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 		__field(	unsigned long,	instr		)
 		__field(	unsigned long,	cpsr		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 		__entry->instr			= instr;
 		__entry->cpsr			= cpsr;
 	),
 	TP_printk("Emulate MMIO at: 0x%08lx (instr: %08lx, cpsr: %08lx)",
 		  __entry->vcpu_pc, __entry->instr, __entry->cpsr)
 );
 /* Architecturally implementation defined CP15 register access */
 TRACE_EVENT(kvm_emulate_cp15_imp,
 	TP_PROTO(unsigned long Op1, unsigned long Rt1, unsigned long CRn,
 		 unsigned long CRm, unsigned long Op2, bool is_write),
 	TP_ARGS(Op1, Rt1, CRn, CRm, Op2, is_write),
 	TP_STRUCT__entry(
 		__field(	unsigned int,	Op1		)
 		__field(	unsigned int,	Rt1		)
 		__field(	unsigned int,	CRn		)
 		__field(	unsigned int,	CRm		)
 		__field(	unsigned int,	Op2		)
 		__field(	bool,		is_write	)
 	),
 	TP_fast_assign(
 		__entry->is_write		= is_write;
 		__entry->Op1			= Op1;
 		__entry->Rt1			= Rt1;
 		__entry->CRn			= CRn;
 		__entry->CRm			= CRm;
 		__entry->Op2			= Op2;
 	),
 	TP_printk("Implementation defined CP15: %s\tp15, %u, r%u, c%u, c%u, %u",
 			(__entry->is_write) ? "mcr" : "mrc",
 			__entry->Op1, __entry->Rt1, __entry->CRn,
 			__entry->CRm, __entry->Op2)
 );
 TRACE_EVENT(kvm_wfi,
 	TP_PROTO(unsigned long vcpu_pc),
 	TP_ARGS(vcpu_pc),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 	),
 	TP_printk("guest executed wfi at: 0x%08lx", __entry->vcpu_pc)
 );
 TRACE_EVENT(kvm_unmap_hva,
 	TP_PROTO(unsigned long hva),
 	TP_ARGS(hva),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	hva		)
 	),
 	TP_fast_assign(
 		__entry->hva		= hva;
 	),
 	TP_printk("mmu notifier unmap hva: %#08lx", __entry->hva)
 );
 TRACE_EVENT(kvm_unmap_hva_range,
 	TP_PROTO(unsigned long start, unsigned long end),
 	TP_ARGS(start, end),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	start		)
 		__field(	unsigned long,	end		)
 	),
 	TP_fast_assign(
 		__entry->start		= start;
 		__entry->end		= end;
 	),
 	TP_printk("mmu notifier unmap range: %#08lx -- %#08lx",
 		  __entry->start, __entry->end)
 );
 TRACE_EVENT(kvm_set_spte_hva,
 	TP_PROTO(unsigned long hva),
 	TP_ARGS(hva),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	hva		)
 	),
 	TP_fast_assign(
 		__entry->hva		= hva;
 	),
 	TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
 );
 TRACE_EVENT(kvm_hvc,
 	TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
 	TP_ARGS(vcpu_pc, r0, imm),
 	TP_STRUCT__entry(
 		__field(	unsigned long,	vcpu_pc		)
 		__field(	unsigned long,	r0		)
 		__field(	unsigned long,	imm		)
 	),
 	TP_fast_assign(
 		__entry->vcpu_pc		= vcpu_pc;
 		__entry->r0		= r0;
 		__entry->imm		= imm;
 	),
 	TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx",
 		  __entry->vcpu_pc, __entry->r0, __entry->imm)
 );
+TRACE_EVENT(kvm_set_way_flush,
+	    TP_PROTO(unsigned long vcpu_pc, bool cache),
+	    TP_ARGS(vcpu_pc, cache),
+	    TP_STRUCT__entry(
+		    __field(	unsigned long,	vcpu_pc		)
+		    __field(	bool,		cache		)
+	    ),
+	    TP_fast_assign(
+		    __entry->vcpu_pc		= vcpu_pc;
+		    __entry->cache		= cache;
+	    ),
+	    TP_printk("S/W flush at 0x%016lx (cache %s)",
+		      __entry->vcpu_pc, __entry->cache ? "on" : "off")
+);
+TRACE_EVENT(kvm_toggle_cache,
+	    TP_PROTO(unsigned long vcpu_pc, bool was, bool now),
+	    TP_ARGS(vcpu_pc, was, now),
+	    TP_STRUCT__entry(
+		    __field(	unsigned long,	vcpu_pc		)
+		    __field(	bool,		was		)
+		    __field(	bool,		now		)
+	    ),
+	    TP_fast_assign(
+		    __entry->vcpu_pc		= vcpu_pc;
+		    __entry->was		= was;
+		    __entry->now		= now;
+	    ),
+	    TP_printk("VM op at 0x%016lx (cache was %s, now %s)",
+		      __entry->vcpu_pc, __entry->was ? "on" : "off",
+		      __entry->now ? "on" : "off")
+);
 #endif /* _TRACE_KVM_H */
 #undef TRACE_INCLUDE_PATH
 #define TRACE_INCLUDE_PATH arch/arm/kvm
 #undef TRACE_INCLUDE_FILE
 #define TRACE_INCLUDE_FILE trace
 /* This part must be outside protection */
 #include <trace/define_trace.h>

arch/arm64/include/asm/kvm_emulate.h

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012,2013 - ARM Ltd
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * Derived from arch/arm/include/kvm_emulate.h
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ARM64_KVM_EMULATE_H__
 #define __ARM64_KVM_EMULATE_H__
 #include <linux/kvm_host.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_mmio.h>
 #include <asm/ptrace.h>
 unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num);
 unsigned long *vcpu_spsr32(const struct kvm_vcpu *vcpu);
 bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
 void kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr);
 void kvm_inject_undefined(struct kvm_vcpu *vcpu);
 void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
 void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
 	if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
 		vcpu->arch.hcr_el2 &= ~HCR_RW;
 }
+static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.hcr_el2;
+}
+static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
+{
+	vcpu->arch.hcr_el2 = hcr;
+}
 static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
 {
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
 }
 static inline unsigned long *vcpu_elr_el1(const struct kvm_vcpu *vcpu)
 {
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->elr_el1;
 }
 static inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
 {
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pstate;
 }
 static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
 {
 	return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
 }
 static inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		return kvm_condition_valid32(vcpu);
 	return true;
 }
 static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		kvm_skip_instr32(vcpu, is_wide_instr);
 	else
 		*vcpu_pc(vcpu) += 4;
 }
 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
 {
 	*vcpu_cpsr(vcpu) |= COMPAT_PSR_T_BIT;
 }
 static inline unsigned long *vcpu_reg(const struct kvm_vcpu *vcpu, u8 reg_num)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		return vcpu_reg32(vcpu, reg_num);
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.regs[reg_num];
 }
 /* Get vcpu SPSR for current mode */
 static inline unsigned long *vcpu_spsr(const struct kvm_vcpu *vcpu)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		return vcpu_spsr32(vcpu);
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1];
 }
 static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
 {
 	u32 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
 	if (vcpu_mode_is_32bit(vcpu))
 		return mode > COMPAT_PSR_MODE_USR;
 	return mode != PSR_MODE_EL0t;
 }
 static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.fault.esr_el2;
 }
 static inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.fault.far_el2;
 }
 static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
 {
 	return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
 }
 static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV);
 }
 static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR);
 }
 static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE);
 }
 static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
 {
 	return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT;
 }
 static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA);
 }
 static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW);
 }
 static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
 {
 	return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT);
 }
 /* This one is not specific to Data Abort */
 static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
 {
 	return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL);
 }
 static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT;
 }
 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT;
 }
 static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC;
 }
 static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE;
 }
 static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
 {
 	return vcpu_sys_reg(vcpu, MPIDR_EL1);
 }
 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		*vcpu_cpsr(vcpu) |= COMPAT_PSR_E_BIT;
 	else
 		vcpu_sys_reg(vcpu, SCTLR_EL1) |= (1 << 25);
 }
 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
 {
 	if (vcpu_mode_is_32bit(vcpu))
 		return !!(*vcpu_cpsr(vcpu) & COMPAT_PSR_E_BIT);
 	return !!(vcpu_sys_reg(vcpu, SCTLR_EL1) & (1 << 25));
 }
 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
 						    unsigned long data,
 						    unsigned int len)
 {
 	if (kvm_vcpu_is_be(vcpu)) {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return be16_to_cpu(data & 0xffff);
 		case 4:
 			return be32_to_cpu(data & 0xffffffff);
 		default:
 			return be64_to_cpu(data);
 		}
 	} else {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return le16_to_cpu(data & 0xffff);
 		case 4:
 			return le32_to_cpu(data & 0xffffffff);
 		default:
 			return le64_to_cpu(data);
 		}
 	}
 	return data;		/* Leave LE untouched */
 }
 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
 						    unsigned long data,
 						    unsigned int len)
 {
 	if (kvm_vcpu_is_be(vcpu)) {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return cpu_to_be16(data & 0xffff);
 		case 4:
 			return cpu_to_be32(data & 0xffffffff);
 		default:
 			return cpu_to_be64(data);
 		}
 	} else {
 		switch (len) {
 		case 1:
 			return data & 0xff;
 		case 2:
 			return cpu_to_le16(data & 0xffff);
 		case 4:
 			return cpu_to_le32(data & 0xffffffff);
 		default:
 			return cpu_to_le64(data);
 		}
 	}
 	return data;		/* Leave LE untouched */
 }
 #endif /* __ARM64_KVM_EMULATE_H__ */

arch/arm64/include/asm/kvm_host.h

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012,2013 - ARM Ltd
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * Derived from arch/arm/include/asm/kvm_host.h:
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ARM64_KVM_HOST_H__
 #define __ARM64_KVM_HOST_H__
 #include <linux/types.h>
 #include <linux/kvm_types.h>
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 #if defined(CONFIG_KVM_ARM_MAX_VCPUS)
 #define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
 #else
 #define KVM_MAX_VCPUS 0
 #endif
 #define KVM_USER_MEM_SLOTS 32
 #define KVM_PRIVATE_MEM_SLOTS 4
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
 #include <kvm/arm_vgic.h>
 #include <kvm/arm_arch_timer.h>
 #define KVM_VCPU_MAX_FEATURES 3
 int __attribute_const__ kvm_target_cpu(void);
 int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
 int kvm_arch_dev_ioctl_check_extension(long ext);
 struct kvm_arch {
 	/* The VMID generation used for the virt. memory system */
 	u64    vmid_gen;
 	u32    vmid;
 	/* 1-level 2nd stage table and lock */
 	spinlock_t pgd_lock;
 	pgd_t *pgd;
 	/* VTTBR value associated with above pgd and vmid */
 	u64    vttbr;
 	/* Interrupt controller */
 	struct vgic_dist	vgic;
 	/* Timer */
 	struct arch_timer_kvm	timer;
 };
 #define KVM_NR_MEM_OBJS     40
 /*
  * We don't want allocation failures within the mmu code, so we preallocate
  * enough memory for a single page fault in a cache.
  */
 struct kvm_mmu_memory_cache {
 	int nobjs;
 	void *objects[KVM_NR_MEM_OBJS];
 };
 struct kvm_vcpu_fault_info {
 	u32 esr_el2;		/* Hyp Syndrom Register */
 	u64 far_el2;		/* Hyp Fault Address Register */
 	u64 hpfar_el2;		/* Hyp IPA Fault Address Register */
 };
 struct kvm_cpu_context {
 	struct kvm_regs	gp_regs;
 	union {
 		u64 sys_regs[NR_SYS_REGS];
 		u32 copro[NR_COPRO_REGS];
 	};
 };
 typedef struct kvm_cpu_context kvm_cpu_context_t;
 struct kvm_vcpu_arch {
 	struct kvm_cpu_context ctxt;
 	/* HYP configuration */
 	u64 hcr_el2;
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
 	/* Debug state */
 	u64 debug_flags;
 	/* Pointer to host CPU context */
 	kvm_cpu_context_t *host_cpu_context;
 	/* VGIC state */
 	struct vgic_cpu vgic_cpu;
 	struct arch_timer_cpu timer_cpu;
 	/*
 	 * Anything that is not used directly from assembly code goes
 	 * here.
 	 */
-	/* dcache set/way operation pending */
-	int last_pcpu;
-	cpumask_t require_dcache_flush;
 	/* Don't run the guest */
 	bool pause;
 	/* IO related fields */
 	struct kvm_decode mmio_decode;
 	/* Interrupt related fields */
 	u64 irq_lines;		/* IRQ and FIQ levels */
 	/* Cache some mmu pages needed inside spinlock regions */
 	struct kvm_mmu_memory_cache mmu_page_cache;
 	/* Target CPU and feature flags */
 	int target;
 	DECLARE_BITMAP(features, KVM_VCPU_MAX_FEATURES);
 	/* Detect first run of a vcpu */
 	bool has_run_once;
 };
 #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
 #define vcpu_sys_reg(v,r)	((v)->arch.ctxt.sys_regs[(r)])
 /*
  * CP14 and CP15 live in the same array, as they are backed by the
  * same system registers.
  */
 #define vcpu_cp14(v,r)		((v)->arch.ctxt.copro[(r)])
 #define vcpu_cp15(v,r)		((v)->arch.ctxt.copro[(r)])
 #ifdef CONFIG_CPU_BIG_ENDIAN
 #define vcpu_cp15_64_high(v,r)	vcpu_cp15((v),(r))
 #define vcpu_cp15_64_low(v,r)	vcpu_cp15((v),(r) + 1)
 #else
 #define vcpu_cp15_64_high(v,r)	vcpu_cp15((v),(r) + 1)
 #define vcpu_cp15_64_low(v,r)	vcpu_cp15((v),(r))
 #endif
 struct kvm_vm_stat {
 	u32 remote_tlb_flush;
 };
 struct kvm_vcpu_stat {
 	u32 halt_wakeup;
 };
 int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init);
 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
 int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
 int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
 #define KVM_ARCH_WANT_MMU_NOTIFIER
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_unmap_hva_range(struct kvm *kvm,
 			unsigned long start, unsigned long end);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 /* We do not have shadow page tables, hence the empty hooks */
 static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
 			      unsigned long end)
 {
 	return 0;
 }
 static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
 {
 	return 0;
 }
 static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
 							 unsigned long address)
 {
 }
 struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
 struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void);
 u64 kvm_call_hyp(void *hypfn, ...);
 void force_vm_exit(const cpumask_t *mask);
 int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		int exception_index);
 int kvm_perf_init(void);
 int kvm_perf_teardown(void);
 static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
 				       phys_addr_t pgd_ptr,
 				       unsigned long hyp_stack_ptr,
 				       unsigned long vector_ptr)
 {
 	/*
 	 * Call initialization code, and switch to the full blown
 	 * HYP code.
 	 */
 	kvm_call_hyp((void *)boot_pgd_ptr, pgd_ptr,
 		     hyp_stack_ptr, vector_ptr);
 }
 struct vgic_sr_vectors {
 	void	*save_vgic;
 	void	*restore_vgic;
 };
 static inline void vgic_arch_setup(const struct vgic_params *vgic)
 {
 	extern struct vgic_sr_vectors __vgic_sr_vectors;
 	switch(vgic->type)
 	{
 	case VGIC_V2:
 		__vgic_sr_vectors.save_vgic	= __save_vgic_v2_state;
 		__vgic_sr_vectors.restore_vgic	= __restore_vgic_v2_state;
 		break;
 #ifdef CONFIG_ARM_GIC_V3
 	case VGIC_V3:
 		__vgic_sr_vectors.save_vgic	= __save_vgic_v3_state;
 		__vgic_sr_vectors.restore_vgic	= __restore_vgic_v3_state;
 		break;
 #endif
 	default:
 		BUG();
 	}
 }
 static inline void kvm_arch_hardware_disable(void) {}
 static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 #endif /* __ARM64_KVM_HOST_H__ */

arch/arm64/include/asm/kvm_mmu.h

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012,2013 - ARM Ltd
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ARM64_KVM_MMU_H__
 #define __ARM64_KVM_MMU_H__
 #include <asm/page.h>
 #include <asm/memory.h>
 /*
  * As we only have the TTBR0_EL2 register, we cannot express
  * "negative" addresses. This makes it impossible to directly share
  * mappings with the kernel.
  *
  * Instead, give the HYP mode its own VA region at a fixed offset from
  * the kernel by just masking the top bits (which are all ones for a
  * kernel address).
  */
 #define HYP_PAGE_OFFSET_SHIFT	VA_BITS
 #define HYP_PAGE_OFFSET_MASK	((UL(1) << HYP_PAGE_OFFSET_SHIFT) - 1)
 #define HYP_PAGE_OFFSET		(PAGE_OFFSET & HYP_PAGE_OFFSET_MASK)
 /*
  * Our virtual mapping for the idmap-ed MMU-enable code. Must be
  * shared across all the page-tables. Conveniently, we use the last
  * possible page, where no kernel mapping will ever exist.
  */
 #define TRAMPOLINE_VA		(HYP_PAGE_OFFSET_MASK & PAGE_MASK)
 /*
  * KVM_MMU_CACHE_MIN_PAGES is the number of stage2 page table translation
  * levels in addition to the PGD and potentially the PUD which are
  * pre-allocated (we pre-allocate the fake PGD and the PUD when the Stage-2
  * tables use one level of tables less than the kernel.
  */
 #ifdef CONFIG_ARM64_64K_PAGES
 #define KVM_MMU_CACHE_MIN_PAGES	1
 #else
 #define KVM_MMU_CACHE_MIN_PAGES	2
 #endif
 #ifdef __ASSEMBLY__
 /*
  * Convert a kernel VA into a HYP VA.
  * reg: VA to be converted.
  */
 .macro kern_hyp_va	reg
 	and	\reg, \reg, #HYP_PAGE_OFFSET_MASK
 .endm
 #else
 #include <asm/pgalloc.h>
 #include <asm/cachetype.h>
 #include <asm/cacheflush.h>
 #define KERN_TO_HYP(kva)	((unsigned long)kva - PAGE_OFFSET + HYP_PAGE_OFFSET)
 /*
  * We currently only support a 40bit IPA.
  */
 #define KVM_PHYS_SHIFT	(40)
 #define KVM_PHYS_SIZE	(1UL << KVM_PHYS_SHIFT)
 #define KVM_PHYS_MASK	(KVM_PHYS_SIZE - 1UL)
 int create_hyp_mappings(void *from, void *to);
 int create_hyp_io_mappings(void *from, void *to, phys_addr_t);
 void free_boot_hyp_pgd(void);
 void free_hyp_pgds(void);
 void stage2_unmap_vm(struct kvm *kvm);
 int kvm_alloc_stage2_pgd(struct kvm *kvm);
 void kvm_free_stage2_pgd(struct kvm *kvm);
 int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
 			  phys_addr_t pa, unsigned long size, bool writable);
 int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run);
 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
 phys_addr_t kvm_mmu_get_httbr(void);
 phys_addr_t kvm_mmu_get_boot_httbr(void);
 phys_addr_t kvm_get_idmap_vector(void);
 int kvm_mmu_init(void);
 void kvm_clear_hyp_idmap(void);
 #define	kvm_set_pte(ptep, pte)		set_pte(ptep, pte)
 #define	kvm_set_pmd(pmdp, pmd)		set_pmd(pmdp, pmd)
 static inline void kvm_clean_pgd(pgd_t *pgd) {}
 static inline void kvm_clean_pmd(pmd_t *pmd) {}
 static inline void kvm_clean_pmd_entry(pmd_t *pmd) {}
 static inline void kvm_clean_pte(pte_t *pte) {}
 static inline void kvm_clean_pte_entry(pte_t *pte) {}
 static inline void kvm_set_s2pte_writable(pte_t *pte)
 {
 	pte_val(*pte) |= PTE_S2_RDWR;
 }
 static inline void kvm_set_s2pmd_writable(pmd_t *pmd)
 {
 	pmd_val(*pmd) |= PMD_S2_RDWR;
 }
 #define kvm_pgd_addr_end(addr, end)	pgd_addr_end(addr, end)
 #define kvm_pud_addr_end(addr, end)	pud_addr_end(addr, end)
 #define kvm_pmd_addr_end(addr, end)	pmd_addr_end(addr, end)
 /*
  * In the case where PGDIR_SHIFT is larger than KVM_PHYS_SHIFT, we can address
  * the entire IPA input range with a single pgd entry, and we would only need
  * one pgd entry.  Note that in this case, the pgd is actually not used by
  * the MMU for Stage-2 translations, but is merely a fake pgd used as a data
  * structure for the kernel pgtable macros to work.
  */
 #if PGDIR_SHIFT > KVM_PHYS_SHIFT
 #define PTRS_PER_S2_PGD_SHIFT	0
 #else
 #define PTRS_PER_S2_PGD_SHIFT	(KVM_PHYS_SHIFT - PGDIR_SHIFT)
 #endif
 #define PTRS_PER_S2_PGD		(1 << PTRS_PER_S2_PGD_SHIFT)
 #define S2_PGD_ORDER		get_order(PTRS_PER_S2_PGD * sizeof(pgd_t))
 /*
  * If we are concatenating first level stage-2 page tables, we would have less
  * than or equal to 16 pointers in the fake PGD, because that's what the
  * architecture allows.  In this case, (4 - CONFIG_ARM64_PGTABLE_LEVELS)
  * represents the first level for the host, and we add 1 to go to the next
  * level (which uses contatenation) for the stage-2 tables.
  */
 #if PTRS_PER_S2_PGD <= 16
 #define KVM_PREALLOC_LEVEL	(4 - CONFIG_ARM64_PGTABLE_LEVELS + 1)
 #else
 #define KVM_PREALLOC_LEVEL	(0)
 #endif
 /**
  * kvm_prealloc_hwpgd - allocate inital table for VTTBR
  * @kvm:	The KVM struct pointer for the VM.
  * @pgd:	The kernel pseudo pgd
  *
  * When the kernel uses more levels of page tables than the guest, we allocate
  * a fake PGD and pre-populate it to point to the next-level page table, which
  * will be the real initial page table pointed to by the VTTBR.
  *
  * When KVM_PREALLOC_LEVEL==2, we allocate a single page for the PMD and
  * the kernel will use folded pud.  When KVM_PREALLOC_LEVEL==1, we
  * allocate 2 consecutive PUD pages.
  */
 static inline int kvm_prealloc_hwpgd(struct kvm *kvm, pgd_t *pgd)
 {
 	unsigned int i;
 	unsigned long hwpgd;
 	if (KVM_PREALLOC_LEVEL == 0)
 		return 0;
 	hwpgd = __get_free_pages(GFP_KERNEL | __GFP_ZERO, PTRS_PER_S2_PGD_SHIFT);
 	if (!hwpgd)
 		return -ENOMEM;
 	for (i = 0; i < PTRS_PER_S2_PGD; i++) {
 		if (KVM_PREALLOC_LEVEL == 1)
 			pgd_populate(NULL, pgd + i,
 				     (pud_t *)hwpgd + i * PTRS_PER_PUD);
 		else if (KVM_PREALLOC_LEVEL == 2)
 			pud_populate(NULL, pud_offset(pgd, 0) + i,
 				     (pmd_t *)hwpgd + i * PTRS_PER_PMD);
 	}
 	return 0;
 }
 static inline void *kvm_get_hwpgd(struct kvm *kvm)
 {
 	pgd_t *pgd = kvm->arch.pgd;
 	pud_t *pud;
 	if (KVM_PREALLOC_LEVEL == 0)
 		return pgd;
 	pud = pud_offset(pgd, 0);
 	if (KVM_PREALLOC_LEVEL == 1)
 		return pud;
 	BUG_ON(KVM_PREALLOC_LEVEL != 2);
 	return pmd_offset(pud, 0);
 }
 static inline void kvm_free_hwpgd(struct kvm *kvm)
 {
 	if (KVM_PREALLOC_LEVEL > 0) {
 		unsigned long hwpgd = (unsigned long)kvm_get_hwpgd(kvm);
 		free_pages(hwpgd, PTRS_PER_S2_PGD_SHIFT);
 	}
 }
 static inline bool kvm_page_empty(void *ptr)
 {
 	struct page *ptr_page = virt_to_page(ptr);
 	return page_count(ptr_page) == 1;
 }
 #define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)
 #ifdef __PAGETABLE_PMD_FOLDED
 #define kvm_pmd_table_empty(kvm, pmdp) (0)
 #else
 #define kvm_pmd_table_empty(kvm, pmdp) \
 	(kvm_page_empty(pmdp) && (!(kvm) || KVM_PREALLOC_LEVEL < 2))
 #endif
 #ifdef __PAGETABLE_PUD_FOLDED
 #define kvm_pud_table_empty(kvm, pudp) (0)
 #else
 #define kvm_pud_table_empty(kvm, pudp) \
 	(kvm_page_empty(pudp) && (!(kvm) || KVM_PREALLOC_LEVEL < 1))
 #endif
 struct kvm;
 #define kvm_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))
 static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
 {
 	return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
 }
-static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva,
+static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu, pfn_t pfn,
-					     unsigned long size,
+					       unsigned long size,
-					     bool ipa_uncached)
+					       bool ipa_uncached)
 {
+	void *va = page_address(pfn_to_page(pfn));
 	if (!vcpu_has_cache_enabled(vcpu) || ipa_uncached)
-		kvm_flush_dcache_to_poc((void *)hva, size);
+		kvm_flush_dcache_to_poc(va, size);
 	if (!icache_is_aliasing()) {		/* PIPT */
-		flush_icache_range(hva, hva + size);
+		flush_icache_range((unsigned long)va,
+				   (unsigned long)va + size);
 	} else if (!icache_is_aivivt()) {	/* non ASID-tagged VIVT */
 		/* any kind of VIPT cache */
 		__flush_icache_all();
 	}
 }
+static inline void __kvm_flush_dcache_pte(pte_t pte)
+{
+	struct page *page = pte_page(pte);
+	kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE);
+}
+static inline void __kvm_flush_dcache_pmd(pmd_t pmd)
+{
+	struct page *page = pmd_page(pmd);
+	kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE);
+}
+static inline void __kvm_flush_dcache_pud(pud_t pud)
+{
+	struct page *page = pud_page(pud);
+	kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE);
+}
 #define kvm_virt_to_phys(x)		__virt_to_phys((unsigned long)(x))
-void stage2_flush_vm(struct kvm *kvm);
+void kvm_set_way_flush(struct kvm_vcpu *vcpu);
+void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
 #endif /* __ASSEMBLY__ */
 #endif /* __ARM64_KVM_MMU_H__ */

arch/arm64/kvm/sys_regs.c

Diff comments View file @ 02512b2

 /*
  * Copyright (C) 2012,2013 - ARM Ltd
  * Author: Marc Zyngier <marc.zyngier@arm.com>
  *
  * Derived from arch/arm/kvm/coproc.c:
  * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  * Authors: Rusty Russell <rusty@rustcorp.com.au>
  *          Christoffer Dall <c.dall@virtualopensystems.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/mm.h>
 #include <linux/kvm_host.h>
 #include <linux/uaccess.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_mmu.h>
 #include <asm/cacheflush.h>
 #include <asm/cputype.h>
 #include <asm/debug-monitors.h>
 #include <trace/events/kvm.h>
 #include "sys_regs.h"
 /*
  * All of this file is extremly similar to the ARM coproc.c, but the
  * types are different. My gut feeling is that it should be pretty
  * easy to merge, but that would be an ABI breakage -- again. VFP
  * would also need to be abstracted.
  *
  * For AArch32, we only take care of what is being trapped. Anything
  * that has to do with init and userspace access has to go via the
  * 64bit interface.
  */
 /* 3 bits per cache level, as per CLIDR, but non-existent caches always 0 */
 static u32 cache_levels;
 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
 #define CSSELR_MAX 12
 /* Which cache CCSIDR represents depends on CSSELR value. */
 static u32 get_ccsidr(u32 csselr)
 {
 	u32 ccsidr;
 	/* Make sure noone else changes CSSELR during this! */
 	local_irq_disable();
 	/* Put value into CSSELR */
 	asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
 	isb();
 	/* Read result out of CCSIDR */
 	asm volatile("mrs %0, ccsidr_el1" : "=r" (ccsidr));
 	local_irq_enable();
 	return ccsidr;
 }
-static void do_dc_cisw(u32 val)
+/*
-{
+ * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
-	asm volatile("dc cisw, %x0" : : "r" (val));
+ */
-	dsb(ish);
-}
-static void do_dc_csw(u32 val)
-{
-	asm volatile("dc csw, %x0" : : "r" (val));
-	dsb(ish);
-}
-/* See note at ARM ARM B1.14.4 */
 static bool access_dcsw(struct kvm_vcpu *vcpu,
 			const struct sys_reg_params *p,
 			const struct sys_reg_desc *r)
 {
-	unsigned long val;
-	int cpu;
 	if (!p->is_write)
 		return read_from_write_only(vcpu, p);
-	cpu = get_cpu();
+	kvm_set_way_flush(vcpu);
-	cpumask_setall(&vcpu->arch.require_dcache_flush);
-	cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush);
-	/* If we were already preempted, take the long way around */
-	if (cpu != vcpu->arch.last_pcpu) {
-		flush_cache_all();
-		goto done;
-	}
-	val = *vcpu_reg(vcpu, p->Rt);
-	switch (p->CRm) {
-	case 6:			/* Upgrade DCISW to DCCISW, as per HCR.SWIO */
-	case 14:		/* DCCISW */
-		do_dc_cisw(val);
-		break;
-	case 10:		/* DCCSW */
-		do_dc_csw(val);
-		break;
-	}
-done:
-	put_cpu();
 	return true;
 }
 /*
  * Generic accessor for VM registers. Only called as long as HCR_TVM
- * is set.
+ * is set. If the guest enables the MMU, we stop trapping the VM
+ * sys_regs and leave it in complete control of the caches.
  */
 static bool access_vm_reg(struct kvm_vcpu *vcpu,
 			  const struct sys_reg_params *p,
 			  const struct sys_reg_desc *r)
 {
 	unsigned long val;
+	bool was_enabled = vcpu_has_cache_enabled(vcpu);
 	BUG_ON(!p->is_write);
 	val = *vcpu_reg(vcpu, p->Rt);
 	if (!p->is_aarch32) {
 		vcpu_sys_reg(vcpu, r->reg) = val;
 	} else {
 		if (!p->is_32bit)
 			vcpu_cp15_64_high(vcpu, r->reg) = val >> 32;
 		vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL;
 	}
+	kvm_toggle_cache(vcpu, was_enabled);
 	return true;
 }
-/*
- * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set.  If the
- * guest enables the MMU, we stop trapping the VM sys_regs and leave
- * it in complete control of the caches.
- */
-static bool access_sctlr(struct kvm_vcpu *vcpu,
-			 const struct sys_reg_params *p,
-			 const struct sys_reg_desc *r)
-{
-	access_vm_reg(vcpu, p, r);
-	if (vcpu_has_cache_enabled(vcpu)) {	/* MMU+Caches enabled? */
-		vcpu->arch.hcr_el2 &= ~HCR_TVM;
-		stage2_flush_vm(vcpu->kvm);
-	}
-	return true;
-}
 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
 			const struct sys_reg_params *p,
 			const struct sys_reg_desc *r)
 {
 	if (p->is_write)
 		return ignore_write(vcpu, p);
 	else
 		return read_zero(vcpu, p);
 }
 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
 			   const struct sys_reg_params *p,
 			   const struct sys_reg_desc *r)
 {
 	if (p->is_write) {
 		return ignore_write(vcpu, p);
 	} else {
 		*vcpu_reg(vcpu, p->Rt) = (1 << 3);
 		return true;
 	}
 }
 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
 				   const struct sys_reg_params *p,
 				   const struct sys_reg_desc *r)
 {
 	if (p->is_write) {
 		return ignore_write(vcpu, p);
 	} else {
 		u32 val;
 		asm volatile("mrs %0, dbgauthstatus_el1" : "=r" (val));
 		*vcpu_reg(vcpu, p->Rt) = val;
 		return true;
 	}
 }
 /*
  * We want to avoid world-switching all the DBG registers all the
  * time:
  *
  * - If we've touched any debug register, it is likely that we're
  *   going to touch more of them. It then makes sense to disable the
  *   traps and start doing the save/restore dance
  * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
  *   then mandatory to save/restore the registers, as the guest
  *   depends on them.
  *
  * For this, we use a DIRTY bit, indicating the guest has modified the
  * debug registers, used as follow:
  *
  * On guest entry:
  * - If the dirty bit is set (because we're coming back from trapping),
  *   disable the traps, save host registers, restore guest registers.
  * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
  *   set the dirty bit, disable the traps, save host registers,
  *   restore guest registers.
  * - Otherwise, enable the traps
  *
  * On guest exit:
  * - If the dirty bit is set, save guest registers, restore host
  *   registers and clear the dirty bit. This ensure that the host can
  *   now use the debug registers.
  */
 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 			    const struct sys_reg_params *p,
 			    const struct sys_reg_desc *r)
 {
 	if (p->is_write) {
 		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
 		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 	} else {
 		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
 	}
 	return true;
 }
 static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 {
 	u64 amair;
 	asm volatile("mrs %0, amair_el1\n" : "=r" (amair));
 	vcpu_sys_reg(vcpu, AMAIR_EL1) = amair;
 }
 static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
 {
 	/*
 	 * Simply map the vcpu_id into the Aff0 field of the MPIDR.
 	 */
 	vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff);
 }
 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
 #define DBG_BCR_BVR_WCR_WVR_EL1(n)					\
 	/* DBGBVRn_EL1 */						\
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100),	\
 	  trap_debug_regs, reset_val, (DBGBVR0_EL1 + (n)), 0 },		\
 	/* DBGBCRn_EL1 */						\
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101),	\
 	  trap_debug_regs, reset_val, (DBGBCR0_EL1 + (n)), 0 },		\
 	/* DBGWVRn_EL1 */						\
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110),	\
 	  trap_debug_regs, reset_val, (DBGWVR0_EL1 + (n)), 0 },		\
 	/* DBGWCRn_EL1 */						\
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111),	\
 	  trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 }
 /*
  * Architected system registers.
  * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
  *
  * We could trap ID_DFR0 and tell the guest we don't support performance
  * monitoring.  Unfortunately the patch to make the kernel check ID_DFR0 was
  * NAKed, so it will read the PMCR anyway.
  *
  * Therefore we tell the guest we have 0 counters.  Unfortunately, we
  * must always support PMCCNTR (the cycle counter): we just RAZ/WI for
  * all PM registers, which doesn't crash the guest kernel at least.
  *
  * Debug handling: We do trap most, if not all debug related system
  * registers. The implementation is good enough to ensure that a guest
  * can use these with minimal performance degradation. The drawback is
  * that we don't implement any of the external debug, none of the
  * OSlock protocol. This should be revisited if we ever encounter a
  * more demanding guest...
  */
 static const struct sys_reg_desc sys_reg_descs[] = {
 	/* DC ISW */
 	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b0110), Op2(0b010),
 	  access_dcsw },
 	/* DC CSW */
 	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1010), Op2(0b010),
 	  access_dcsw },
 	/* DC CISW */
 	{ Op0(0b01), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b010),
 	  access_dcsw },
 	DBG_BCR_BVR_WCR_WVR_EL1(0),
 	DBG_BCR_BVR_WCR_WVR_EL1(1),
 	/* MDCCINT_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
 	  trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
 	/* MDSCR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
 	  trap_debug_regs, reset_val, MDSCR_EL1, 0 },
 	DBG_BCR_BVR_WCR_WVR_EL1(2),
 	DBG_BCR_BVR_WCR_WVR_EL1(3),
 	DBG_BCR_BVR_WCR_WVR_EL1(4),
 	DBG_BCR_BVR_WCR_WVR_EL1(5),
 	DBG_BCR_BVR_WCR_WVR_EL1(6),
 	DBG_BCR_BVR_WCR_WVR_EL1(7),
 	DBG_BCR_BVR_WCR_WVR_EL1(8),
 	DBG_BCR_BVR_WCR_WVR_EL1(9),
 	DBG_BCR_BVR_WCR_WVR_EL1(10),
 	DBG_BCR_BVR_WCR_WVR_EL1(11),
 	DBG_BCR_BVR_WCR_WVR_EL1(12),
 	DBG_BCR_BVR_WCR_WVR_EL1(13),
 	DBG_BCR_BVR_WCR_WVR_EL1(14),
 	DBG_BCR_BVR_WCR_WVR_EL1(15),
 	/* MDRAR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
 	  trap_raz_wi },
 	/* OSLAR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b100),
 	  trap_raz_wi },
 	/* OSLSR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0001), Op2(0b100),
 	  trap_oslsr_el1 },
 	/* OSDLR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0011), Op2(0b100),
 	  trap_raz_wi },
 	/* DBGPRCR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0001), CRm(0b0100), Op2(0b100),
 	  trap_raz_wi },
 	/* DBGCLAIMSET_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1000), Op2(0b110),
 	  trap_raz_wi },
 	/* DBGCLAIMCLR_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1001), Op2(0b110),
 	  trap_raz_wi },
 	/* DBGAUTHSTATUS_EL1 */
 	{ Op0(0b10), Op1(0b000), CRn(0b0111), CRm(0b1110), Op2(0b110),
 	  trap_dbgauthstatus_el1 },
 	/* TEECR32_EL1 */
 	{ Op0(0b10), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
 	  NULL, reset_val, TEECR32_EL1, 0 },
 	/* TEEHBR32_EL1 */
 	{ Op0(0b10), Op1(0b010), CRn(0b0001), CRm(0b0000), Op2(0b000),
 	  NULL, reset_val, TEEHBR32_EL1, 0 },
 	/* MDCCSR_EL1 */
 	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0001), Op2(0b000),
 	  trap_raz_wi },
 	/* DBGDTR_EL0 */
 	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0100), Op2(0b000),
 	  trap_raz_wi },
 	/* DBGDTR[TR]X_EL0 */
 	{ Op0(0b10), Op1(0b011), CRn(0b0000), CRm(0b0101), Op2(0b000),
 	  trap_raz_wi },
 	/* DBGVCR32_EL2 */
 	{ Op0(0b10), Op1(0b100), CRn(0b0000), CRm(0b0111), Op2(0b000),
 	  NULL, reset_val, DBGVCR32_EL2, 0 },
 	/* MPIDR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b101),
 	  NULL, reset_mpidr, MPIDR_EL1 },
 	/* SCTLR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000),
-	  access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 },
+	  access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
 	/* CPACR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010),
 	  NULL, reset_val, CPACR_EL1, 0 },
 	/* TTBR0_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b000),
 	  access_vm_reg, reset_unknown, TTBR0_EL1 },
 	/* TTBR1_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b001),
 	  access_vm_reg, reset_unknown, TTBR1_EL1 },
 	/* TCR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0010), CRm(0b0000), Op2(0b010),
 	  access_vm_reg, reset_val, TCR_EL1, 0 },
 	/* AFSR0_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b000),
 	  access_vm_reg, reset_unknown, AFSR0_EL1 },
 	/* AFSR1_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0001), Op2(0b001),
 	  access_vm_reg, reset_unknown, AFSR1_EL1 },
 	/* ESR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0101), CRm(0b0010), Op2(0b000),
 	  access_vm_reg, reset_unknown, ESR_EL1 },
 	/* FAR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0110), CRm(0b0000), Op2(0b000),
 	  access_vm_reg, reset_unknown, FAR_EL1 },
 	/* PAR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b0111), CRm(0b0100), Op2(0b000),
 	  NULL, reset_unknown, PAR_EL1 },
 	/* PMINTENSET_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b001),
 	  trap_raz_wi },
 	/* PMINTENCLR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1001), CRm(0b1110), Op2(0b010),
 	  trap_raz_wi },
 	/* MAIR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0010), Op2(0b000),
 	  access_vm_reg, reset_unknown, MAIR_EL1 },
 	/* AMAIR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1010), CRm(0b0011), Op2(0b000),
 	  access_vm_reg, reset_amair_el1, AMAIR_EL1 },
 	/* VBAR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000),
 	  NULL, reset_val, VBAR_EL1, 0 },
 	/* ICC_SRE_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101),
 	  trap_raz_wi },
 	/* CONTEXTIDR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001),
 	  access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
 	/* TPIDR_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b100),
 	  NULL, reset_unknown, TPIDR_EL1 },
 	/* CNTKCTL_EL1 */
 	{ Op0(0b11), Op1(0b000), CRn(0b1110), CRm(0b0001), Op2(0b000),
 	  NULL, reset_val, CNTKCTL_EL1, 0},
 	/* CSSELR_EL1 */
 	{ Op0(0b11), Op1(0b010), CRn(0b0000), CRm(0b0000), Op2(0b000),
 	  NULL, reset_unknown, CSSELR_EL1 },
 	/* PMCR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b000),
 	  trap_raz_wi },
 	/* PMCNTENSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b001),
 	  trap_raz_wi },
 	/* PMCNTENCLR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b010),
 	  trap_raz_wi },
 	/* PMOVSCLR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b011),
 	  trap_raz_wi },
 	/* PMSWINC_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b100),
 	  trap_raz_wi },
 	/* PMSELR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b101),
 	  trap_raz_wi },
 	/* PMCEID0_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b110),
 	  trap_raz_wi },
 	/* PMCEID1_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1100), Op2(0b111),
 	  trap_raz_wi },
 	/* PMCCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b000),
 	  trap_raz_wi },
 	/* PMXEVTYPER_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b001),
 	  trap_raz_wi },
 	/* PMXEVCNTR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1101), Op2(0b010),
 	  trap_raz_wi },
 	/* PMUSERENR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b000),
 	  trap_raz_wi },
 	/* PMOVSSET_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1001), CRm(0b1110), Op2(0b011),
 	  trap_raz_wi },
 	/* TPIDR_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b010),
 	  NULL, reset_unknown, TPIDR_EL0 },
 	/* TPIDRRO_EL0 */
 	{ Op0(0b11), Op1(0b011), CRn(0b1101), CRm(0b0000), Op2(0b011),
 	  NULL, reset_unknown, TPIDRRO_EL0 },
 	/* DACR32_EL2 */
 	{ Op0(0b11), Op1(0b100), CRn(0b0011), CRm(0b0000), Op2(0b000),
 	  NULL, reset_unknown, DACR32_EL2 },
 	/* IFSR32_EL2 */
 	{ Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0000), Op2(0b001),
 	  NULL, reset_unknown, IFSR32_EL2 },
 	/* FPEXC32_EL2 */
 	{ Op0(0b11), Op1(0b100), CRn(0b0101), CRm(0b0011), Op2(0b000),
 	  NULL, reset_val, FPEXC32_EL2, 0x70 },
 };
 static bool trap_dbgidr(struct kvm_vcpu *vcpu,
 			const struct sys_reg_params *p,
 			const struct sys_reg_desc *r)
 {
 	if (p->is_write) {
 		return ignore_write(vcpu, p);
 	} else {
 		u64 dfr = read_cpuid(ID_AA64DFR0_EL1);
 		u64 pfr = read_cpuid(ID_AA64PFR0_EL1);
 		u32 el3 = !!((pfr >> 12) & 0xf);
 		*vcpu_reg(vcpu, p->Rt) = ((((dfr >> 20) & 0xf) << 28) |
 					  (((dfr >> 12) & 0xf) << 24) |
 					  (((dfr >> 28) & 0xf) << 20) |
 					  (6 << 16) | (el3 << 14) | (el3 << 12));
 		return true;
 	}
 }
 static bool trap_debug32(struct kvm_vcpu *vcpu,
 			 const struct sys_reg_params *p,
 			 const struct sys_reg_desc *r)
 {
 	if (p->is_write) {
 		vcpu_cp14(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
 		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 	} else {
 		*vcpu_reg(vcpu, p->Rt) = vcpu_cp14(vcpu, r->reg);
 	}
 	return true;
 }
 #define DBG_BCR_BVR_WCR_WVR(n)					\
 	/* DBGBVRn */						\
 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_debug32,	\
 	  NULL, (cp14_DBGBVR0 + (n) * 2) },			\
 	/* DBGBCRn */						\
 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_debug32,	\
 	  NULL, (cp14_DBGBCR0 + (n) * 2) },			\
 	/* DBGWVRn */						\
 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_debug32,	\
 	  NULL, (cp14_DBGWVR0 + (n) * 2) },			\
 	/* DBGWCRn */						\
 	{ Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_debug32,	\
 	  NULL, (cp14_DBGWCR0 + (n) * 2) }
 #define DBGBXVR(n)						\
 	{ Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_debug32,	\
 	  NULL, cp14_DBGBXVR0 + n * 2 }
 /*
  * Trapped cp14 registers. We generally ignore most of the external
  * debug, on the principle that they don't really make sense to a
  * guest. Revisit this one day, whould this principle change.
  */
 static const struct sys_reg_desc cp14_regs[] = {
 	/* DBGIDR */
 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgidr },
 	/* DBGDTRRXext */
 	{ Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
 	DBG_BCR_BVR_WCR_WVR(0),
 	/* DBGDSCRint */
 	{ Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
 	DBG_BCR_BVR_WCR_WVR(1),
 	/* DBGDCCINT */
 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug32 },
 	/* DBGDSCRext */
 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug32 },
 	DBG_BCR_BVR_WCR_WVR(2),
 	/* DBGDTR[RT]Xint */
 	{ Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
 	/* DBGDTR[RT]Xext */
 	{ Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
 	DBG_BCR_BVR_WCR_WVR(3),
 	DBG_BCR_BVR_WCR_WVR(4),
 	DBG_BCR_BVR_WCR_WVR(5),
 	/* DBGWFAR */
 	{ Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
 	/* DBGOSECCR */
 	{ Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
 	DBG_BCR_BVR_WCR_WVR(6),
 	/* DBGVCR */
 	{ Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug32 },
 	DBG_BCR_BVR_WCR_WVR(7),
 	DBG_BCR_BVR_WCR_WVR(8),
 	DBG_BCR_BVR_WCR_WVR(9),
 	DBG_BCR_BVR_WCR_WVR(10),
 	DBG_BCR_BVR_WCR_WVR(11),
 	DBG_BCR_BVR_WCR_WVR(12),
 	DBG_BCR_BVR_WCR_WVR(13),
 	DBG_BCR_BVR_WCR_WVR(14),
 	DBG_BCR_BVR_WCR_WVR(15),
 	/* DBGDRAR (32bit) */
 	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
 	DBGBXVR(0),
 	/* DBGOSLAR */
 	{ Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_raz_wi },
 	DBGBXVR(1),
 	/* DBGOSLSR */
 	{ Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1 },
 	DBGBXVR(2),
 	DBGBXVR(3),
 	/* DBGOSDLR */
 	{ Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
 	DBGBXVR(4),
 	/* DBGPRCR */
 	{ Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
 	DBGBXVR(5),
 	DBGBXVR(6),
 	DBGBXVR(7),
 	DBGBXVR(8),
 	DBGBXVR(9),
 	DBGBXVR(10),
 	DBGBXVR(11),
 	DBGBXVR(12),
 	DBGBXVR(13),
 	DBGBXVR(14),
 	DBGBXVR(15),
 	/* DBGDSAR (32bit) */
 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
 	/* DBGDEVID2 */
 	{ Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
 	/* DBGDEVID1 */
 	{ Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
 	/* DBGDEVID */
 	{ Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
 	/* DBGCLAIMSET */
 	{ Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
 	/* DBGCLAIMCLR */
 	{ Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
 	/* DBGAUTHSTATUS */
 	{ Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
 };
 /* Trapped cp14 64bit registers */
 static const struct sys_reg_desc cp14_64_regs[] = {
 	/* DBGDRAR (64bit) */
 	{ Op1( 0), CRm( 1), .access = trap_raz_wi },
 	/* DBGDSAR (64bit) */
 	{ Op1( 0), CRm( 2), .access = trap_raz_wi },
 };
 /*
  * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
  * depending on the way they are accessed (as a 32bit or a 64bit
  * register).
  */
 static const struct sys_reg_desc cp15_regs[] = {
-	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR },
+	{ Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR },
 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 },
 	{ Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR },
 	{ Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, c3_DACR },
 	{ Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, c5_DFSR },
 	{ Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, c5_IFSR },
 	{ Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, c5_ADFSR },
 	{ Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, c5_AIFSR },
 	{ Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, c6_DFAR },
 	{ Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, c6_IFAR },
 	/*
 	 * DC{C,I,CI}SW operations:
 	 */
 	{ Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
 	{ Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
 	{ Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
 	/* PMU */
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 0), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 1), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 2), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 3), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 5), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 6), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(12), Op2( 7), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(13), Op2( 0), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(13), Op2( 1), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(13), Op2( 2), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(14), Op2( 0), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(14), Op2( 1), trap_raz_wi },
 	{ Op1( 0), CRn( 9), CRm(14), Op2( 2), trap_raz_wi },
 	{ Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, c10_PRRR },
 	{ Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR },
 	{ Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 },
 	{ Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 },
 	/* ICC_SRE */
 	{ Op1( 0), CRn(12), CRm(12), Op2( 5), trap_raz_wi },
 	{ Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID },
 };
 static const struct sys_reg_desc cp15_64_regs[] = {
 	{ Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 },
 	{ Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 },
 };
 /* Target specific emulation tables */
 static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS];
 void kvm_register_target_sys_reg_table(unsigned int target,
 				       struct kvm_sys_reg_target_table *table)
 {
 	target_tables[target] = table;
 }
 /* Get specific register table for this target. */
 static const struct sys_reg_desc *get_target_table(unsigned target,
 						   bool mode_is_64,
 						   size_t *num)
 {
 	struct kvm_sys_reg_target_table *table;
 	table = target_tables[target];
 	if (mode_is_64) {
 		*num = table->table64.num;
 		return table->table64.table;
 	} else {
 		*num = table->table32.num;
 		return table->table32.table;
 	}
 }
 static const struct sys_reg_desc *find_reg(const struct sys_reg_params *params,
 					 const struct sys_reg_desc table[],
 					 unsigned int num)
 {
 	unsigned int i;
 	for (i = 0; i < num; i++) {
 		const struct sys_reg_desc *r = &table[i];
 		if (params->Op0 != r->Op0)
 			continue;
 		if (params->Op1 != r->Op1)
 			continue;
 		if (params->CRn != r->CRn)
 			continue;
 		if (params->CRm != r->CRm)
 			continue;
 		if (params->Op2 != r->Op2)
 			continue;
 		return r;
 	}
 	return NULL;
 }
 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 /*
  * emulate_cp --  tries to match a sys_reg access in a handling table, and
  *                call the corresponding trap handler.
  *
  * @params: pointer to the descriptor of the access
  * @table: array of trap descriptors
  * @num: size of the trap descriptor array
  *
  * Return 0 if the access has been handled, and -1 if not.
  */
 static int emulate_cp(struct kvm_vcpu *vcpu,
 		      const struct sys_reg_params *params,
 		      const struct sys_reg_desc *table,
 		      size_t num)
 {
 	const struct sys_reg_desc *r;
 	if (!table)
 		return -1;	/* Not handled */
 	r = find_reg(params, table, num);
 	if (r) {
 		/*
 		 * Not having an accessor means that we have
 		 * configured a trap that we don't know how to
 		 * handle. This certainly qualifies as a gross bug
 		 * that should be fixed right away.
 		 */
 		BUG_ON(!r->access);
 		if (likely(r->access(vcpu, params, r))) {
 			/* Skip instruction, since it was emulated */
 			kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 		}
 		/* Handled */
 		return 0;
 	}
 	/* Not handled */
 	return -1;
 }
 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
 				struct sys_reg_params *params)
 {
 	u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
 	int cp;
 	switch(hsr_ec) {
 	case ESR_EL2_EC_CP15_32:
 	case ESR_EL2_EC_CP15_64:
 		cp = 15;
 		break;
 	case ESR_EL2_EC_CP14_MR:
 	case ESR_EL2_EC_CP14_64:
 		cp = 14;
 		break;
 	default:
 		WARN_ON((cp = -1));
 	}
 	kvm_err("Unsupported guest CP%d access at: %08lx\n",
 		cp, *vcpu_pc(vcpu));
 	print_sys_reg_instr(params);
 	kvm_inject_undefined(vcpu);
 }
 /**
  * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP15 access
  * @vcpu: The VCPU pointer
  * @run:  The kvm_run struct
  */
 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
 			    const struct sys_reg_desc *global,
 			    size_t nr_global,
 			    const struct sys_reg_desc *target_specific,
 			    size_t nr_specific)
 {
 	struct sys_reg_params params;
 	u32 hsr = kvm_vcpu_get_hsr(vcpu);
 	int Rt2 = (hsr >> 10) & 0xf;
 	params.is_aarch32 = true;
 	params.is_32bit = false;
 	params.CRm = (hsr >> 1) & 0xf;
 	params.Rt = (hsr >> 5) & 0xf;
 	params.is_write = ((hsr & 1) == 0);
 	params.Op0 = 0;
 	params.Op1 = (hsr >> 16) & 0xf;
 	params.Op2 = 0;
 	params.CRn = 0;
 	/*
 	 * Massive hack here. Store Rt2 in the top 32bits so we only
 	 * have one register to deal with. As we use the same trap
 	 * backends between AArch32 and AArch64, we get away with it.
 	 */
 	if (params.is_write) {
 		u64 val = *vcpu_reg(vcpu, params.Rt);
 		val &= 0xffffffff;
 		val |= *vcpu_reg(vcpu, Rt2) << 32;
 		*vcpu_reg(vcpu, params.Rt) = val;
 	}
 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
 		goto out;
 	if (!emulate_cp(vcpu, &params, global, nr_global))
 		goto out;
 	unhandled_cp_access(vcpu, &params);
 out:
 	/* Do the opposite hack for the read side */
 	if (!params.is_write) {
 		u64 val = *vcpu_reg(vcpu, params.Rt);
 		val >>= 32;
 		*vcpu_reg(vcpu, Rt2) = val;
 	}
 	return 1;
 }
 /**
  * kvm_handle_cp15_32 -- handles a mrc/mcr trap on a guest CP15 access
  * @vcpu: The VCPU pointer
  * @run:  The kvm_run struct
  */
 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
 			    const struct sys_reg_desc *global,
 			    size_t nr_global,
 			    const struct sys_reg_desc *target_specific,
 			    size_t nr_specific)
 {
 	struct sys_reg_params params;
 	u32 hsr = kvm_vcpu_get_hsr(vcpu);
 	params.is_aarch32 = true;
 	params.is_32bit = true;
 	params.CRm = (hsr >> 1) & 0xf;
 	params.Rt  = (hsr >> 5) & 0xf;
 	params.is_write = ((hsr & 1) == 0);
 	params.CRn = (hsr >> 10) & 0xf;
 	params.Op0 = 0;
 	params.Op1 = (hsr >> 14) & 0x7;
 	params.Op2 = (hsr >> 17) & 0x7;
 	if (!emulate_cp(vcpu, &params, target_specific, nr_specific))
 		return 1;
 	if (!emulate_cp(vcpu, &params, global, nr_global))
 		return 1;
 	unhandled_cp_access(vcpu, &params);
 	return 1;
 }
 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	const struct sys_reg_desc *target_specific;
 	size_t num;
 	target_specific = get_target_table(vcpu->arch.target, false, &num);
 	return kvm_handle_cp_64(vcpu,
 				cp15_64_regs, ARRAY_SIZE(cp15_64_regs),
 				target_specific, num);
 }
 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	const struct sys_reg_desc *target_specific;
 	size_t num;
 	target_specific = get_target_table(vcpu->arch.target, false, &num);
 	return kvm_handle_cp_32(vcpu,
 				cp15_regs, ARRAY_SIZE(cp15_regs),
 				target_specific, num);
 }
 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	return kvm_handle_cp_64(vcpu,
 				cp14_64_regs, ARRAY_SIZE(cp14_64_regs),
 				NULL, 0);
 }
 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	return kvm_handle_cp_32(vcpu,
 				cp14_regs, ARRAY_SIZE(cp14_regs),
 				NULL, 0);
 }
 static int emulate_sys_reg(struct kvm_vcpu *vcpu,
 			   const struct sys_reg_params *params)
 {
 	size_t num;
 	const struct sys_reg_desc *table, *r;
 	table = get_target_table(vcpu->arch.target, true, &num);
 	/* Search target-specific then generic table. */
 	r = find_reg(params, table, num);
 	if (!r)
 		r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
 	if (likely(r)) {
 		/*
 		 * Not having an accessor means that we have
 		 * configured a trap that we don't know how to
 		 * handle. This certainly qualifies as a gross bug
 		 * that should be fixed right away.
 		 */
 		BUG_ON(!r->access);
 		if (likely(r->access(vcpu, params, r))) {
 			/* Skip instruction, since it was emulated */
 			kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 			return 1;
 		}
 		/* If access function fails, it should complain. */
 	} else {
 		kvm_err("Unsupported guest sys_reg access at: %lx\n",
 			*vcpu_pc(vcpu));
 		print_sys_reg_instr(params);
 	}
 	kvm_inject_undefined(vcpu);
 	return 1;
 }
 static void reset_sys_reg_descs(struct kvm_vcpu *vcpu,
 			      const struct sys_reg_desc *table, size_t num)
 {
 	unsigned long i;
 	for (i = 0; i < num; i++)
 		if (table[i].reset)
 			table[i].reset(vcpu, &table[i]);
 }
 /**
  * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
  * @vcpu: The VCPU pointer
  * @run:  The kvm_run struct
  */
 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	struct sys_reg_params params;
 	unsigned long esr = kvm_vcpu_get_hsr(vcpu);
 	params.is_aarch32 = false;
 	params.is_32bit = false;
 	params.Op0 = (esr >> 20) & 3;
 	params.Op1 = (esr >> 14) & 0x7;
 	params.CRn = (esr >> 10) & 0xf;
 	params.CRm = (esr >> 1) & 0xf;
 	params.Op2 = (esr >> 17) & 0x7;
 	params.Rt = (esr >> 5) & 0x1f;
 	params.is_write = !(esr & 1);
 	return emulate_sys_reg(vcpu, &params);
 }
 /******************************************************************************
  * Userspace API
  *****************************************************************************/
 static bool index_to_params(u64 id, struct sys_reg_params *params)
 {
 	switch (id & KVM_REG_SIZE_MASK) {
 	case KVM_REG_SIZE_U64:
 		/* Any unused index bits means it's not valid. */
 		if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
 			      | KVM_REG_ARM_COPROC_MASK
 			      | KVM_REG_ARM64_SYSREG_OP0_MASK
 			      | KVM_REG_ARM64_SYSREG_OP1_MASK
 			      | KVM_REG_ARM64_SYSREG_CRN_MASK
 			      | KVM_REG_ARM64_SYSREG_CRM_MASK
 			      | KVM_REG_ARM64_SYSREG_OP2_MASK))
 			return false;
 		params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
 			       >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
 		params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
 			       >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
 		params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
 			       >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
 		params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
 			       >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
 		params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
 			       >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
 		return true;
 	default:
 		return false;
 	}
 }
 /* Decode an index value, and find the sys_reg_desc entry. */
 static const struct sys_reg_desc *index_to_sys_reg_desc(struct kvm_vcpu *vcpu,
 						    u64 id)
 {
 	size_t num;
 	const struct sys_reg_desc *table, *r;
 	struct sys_reg_params params;
 	/* We only do sys_reg for now. */
 	if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
 		return NULL;
 	if (!index_to_params(id, &params))
 		return NULL;
 	table = get_target_table(vcpu->arch.target, true, &num);
 	r = find_reg(&params, table, num);
 	if (!r)
 		r = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
 	/* Not saved in the sys_reg array? */
 	if (r && !r->reg)
 		r = NULL;
 	return r;
 }
 /*
  * These are the invariant sys_reg registers: we let the guest see the
  * host versions of these, so they're part of the guest state.
  *
  * A future CPU may provide a mechanism to present different values to
  * the guest, or a future kvm may trap them.
  */
 #define FUNCTION_INVARIANT(reg)						\
 	static void get_##reg(struct kvm_vcpu *v,			\
 			      const struct sys_reg_desc *r)		\
 	{								\
 		u64 val;						\
 									\
 		asm volatile("mrs %0, " __stringify(reg) "\n"		\
 			     : "=r" (val));				\
 		((struct sys_reg_desc *)r)->val = val;			\
 	}
 FUNCTION_INVARIANT(midr_el1)
 FUNCTION_INVARIANT(ctr_el0)
 FUNCTION_INVARIANT(revidr_el1)
 FUNCTION_INVARIANT(id_pfr0_el1)
 FUNCTION_INVARIANT(id_pfr1_el1)
 FUNCTION_INVARIANT(id_dfr0_el1)
 FUNCTION_INVARIANT(id_afr0_el1)
 FUNCTION_INVARIANT(id_mmfr0_el1)
 FUNCTION_INVARIANT(id_mmfr1_el1)
 FUNCTION_INVARIANT(id_mmfr2_el1)
 FUNCTION_INVARIANT(id_mmfr3_el1)
 FUNCTION_INVARIANT(id_isar0_el1)
 FUNCTION_INVARIANT(id_isar1_el1)
 FUNCTION_INVARIANT(id_isar2_el1)
 FUNCTION_INVARIANT(id_isar3_el1)
 FUNCTION_INVARIANT(id_isar4_el1)
 FUNCTION_INVARIANT(id_isar5_el1)
 FUNCTION_INVARIANT(clidr_el1)
 FUNCTION_INVARIANT(aidr_el1)
 /* ->val is filled in by kvm_sys_reg_table_init() */
 static struct sys_reg_desc invariant_sys_regs[] = {
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b000),
 	  NULL, get_midr_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0000), Op2(0b110),
 	  NULL, get_revidr_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b000),
 	  NULL, get_id_pfr0_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b001),
 	  NULL, get_id_pfr1_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b010),
 	  NULL, get_id_dfr0_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b011),
 	  NULL, get_id_afr0_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b100),
 	  NULL, get_id_mmfr0_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b101),
 	  NULL, get_id_mmfr1_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b110),
 	  NULL, get_id_mmfr2_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0001), Op2(0b111),
 	  NULL, get_id_mmfr3_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b000),
 	  NULL, get_id_isar0_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b001),
 	  NULL, get_id_isar1_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b010),
 	  NULL, get_id_isar2_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b011),
 	  NULL, get_id_isar3_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b100),
 	  NULL, get_id_isar4_el1 },
 	{ Op0(0b11), Op1(0b000), CRn(0b0000), CRm(0b0010), Op2(0b101),
 	  NULL, get_id_isar5_el1 },
 	{ Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b001),
 	  NULL, get_clidr_el1 },
 	{ Op0(0b11), Op1(0b001), CRn(0b0000), CRm(0b0000), Op2(0b111),
 	  NULL, get_aidr_el1 },
 	{ Op0(0b11), Op1(0b011), CRn(0b0000), CRm(0b0000), Op2(0b001),
 	  NULL, get_ctr_el0 },
 };
 static int reg_from_user(u64 *val, const void __user *uaddr, u64 id)
 {
 	if (copy_from_user(val, uaddr, KVM_REG_SIZE(id)) != 0)
 		return -EFAULT;
 	return 0;
 }
 static int reg_to_user(void __user *uaddr, const u64 *val, u64 id)
 {
 	if (copy_to_user(uaddr, val, KVM_REG_SIZE(id)) != 0)
 		return -EFAULT;
 	return 0;
 }
 static int get_invariant_sys_reg(u64 id, void __user *uaddr)
 {
 	struct sys_reg_params params;
 	const struct sys_reg_desc *r;
 	if (!index_to_params(id, &params))
 		return -ENOENT;
 	r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
 	if (!r)
 		return -ENOENT;
 	return reg_to_user(uaddr, &r->val, id);
 }
 static int set_invariant_sys_reg(u64 id, void __user *uaddr)
 {
 	struct sys_reg_params params;
 	const struct sys_reg_desc *r;
 	int err;
 	u64 val = 0; /* Make sure high bits are 0 for 32-bit regs */
 	if (!index_to_params(id, &params))
 		return -ENOENT;
 	r = find_reg(&params, invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs));
 	if (!r)
 		return -ENOENT;
 	err = reg_from_user(&val, uaddr, id);
 	if (err)
 		return err;
 	/* This is what we mean by invariant: you can't change it. */
 	if (r->val != val)
 		return -EINVAL;
 	return 0;
 }
 static bool is_valid_cache(u32 val)
 {
 	u32 level, ctype;
 	if (val >= CSSELR_MAX)
 		return false;
 	/* Bottom bit is Instruction or Data bit.  Next 3 bits are level. */
 	level = (val >> 1);
 	ctype = (cache_levels >> (level * 3)) & 7;
 	switch (ctype) {
 	case 0: /* No cache */
 		return false;
 	case 1: /* Instruction cache only */
 		return (val & 1);
 	case 2: /* Data cache only */
 	case 4: /* Unified cache */
 		return !(val & 1);
 	case 3: /* Separate instruction and data caches */
 		return true;
 	default: /* Reserved: we can't know instruction or data. */
 		return false;
 	}
 }
 static int demux_c15_get(u64 id, void __user *uaddr)
 {
 	u32 val;
 	u32 __user *uval = uaddr;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
 		if (KVM_REG_SIZE(id) != 4)
 			return -ENOENT;
 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
 		if (!is_valid_cache(val))
 			return -ENOENT;
 		return put_user(get_ccsidr(val), uval);
 	default:
 		return -ENOENT;
 	}
 }
 static int demux_c15_set(u64 id, void __user *uaddr)
 {
 	u32 val, newval;
 	u32 __user *uval = uaddr;
 	/* Fail if we have unknown bits set. */
 	if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
 		   | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
 		return -ENOENT;
 	switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
 	case KVM_REG_ARM_DEMUX_ID_CCSIDR:
 		if (KVM_REG_SIZE(id) != 4)
 			return -ENOENT;
 		val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
 			>> KVM_REG_ARM_DEMUX_VAL_SHIFT;
 		if (!is_valid_cache(val))
 			return -ENOENT;
 		if (get_user(newval, uval))
 			return -EFAULT;
 		/* This is also invariant: you can't change it. */
 		if (newval != get_ccsidr(val))
 			return -EINVAL;
 		return 0;
 	default:
 		return -ENOENT;
 	}
 }
 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
 	const struct sys_reg_desc *r;
 	void __user *uaddr = (void __user *)(unsigned long)reg->addr;
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
 		return demux_c15_get(reg->id, uaddr);
 	if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
 		return -ENOENT;
 	r = index_to_sys_reg_desc(vcpu, reg->id);
 	if (!r)
 		return get_invariant_sys_reg(reg->id, uaddr);
 	return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id);
 }
 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
 	const struct sys_reg_desc *r;
 	void __user *uaddr = (void __user *)(unsigned long)reg->addr;
 	if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
 		return demux_c15_set(reg->id, uaddr);
 	if (KVM_REG_SIZE(reg->id) != sizeof(__u64))
 		return -ENOENT;
 	r = index_to_sys_reg_desc(vcpu, reg->id);
 	if (!r)
 		return set_invariant_sys_reg(reg->id, uaddr);
 	return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id);
 }
 static unsigned int num_demux_regs(void)
 {
 	unsigned int i, count = 0;
 	for (i = 0; i < CSSELR_MAX; i++)
 		if (is_valid_cache(i))
 			count++;
 	return count;
 }
 static int write_demux_regids(u64 __user *uindices)
 {
 	u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
 	unsigned int i;
 	val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
 	for (i = 0; i < CSSELR_MAX; i++) {
 		if (!is_valid_cache(i))
 			continue;
 		if (put_user(val | i, uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	return 0;
 }
 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
 {
 	return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
 		KVM_REG_ARM64_SYSREG |
 		(reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
 		(reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
 		(reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
 		(reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
 		(reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
 }
 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
 {
 	if (!*uind)
 		return true;
 	if (put_user(sys_reg_to_index(reg), *uind))
 		return false;
 	(*uind)++;
 	return true;
 }
 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
 {
 	const struct sys_reg_desc *i1, *i2, *end1, *end2;
 	unsigned int total = 0;
 	size_t num;
 	/* We check for duplicates here, to allow arch-specific overrides. */
 	i1 = get_target_table(vcpu->arch.target, true, &num);
 	end1 = i1 + num;
 	i2 = sys_reg_descs;
 	end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
 	BUG_ON(i1 == end1 || i2 == end2);
 	/* Walk carefully, as both tables may refer to the same register. */
 	while (i1 || i2) {
 		int cmp = cmp_sys_reg(i1, i2);
 		/* target-specific overrides generic entry. */
 		if (cmp <= 0) {
 			/* Ignore registers we trap but don't save. */
 			if (i1->reg) {
 				if (!copy_reg_to_user(i1, &uind))
 					return -EFAULT;
 				total++;
 			}
 		} else {
 			/* Ignore registers we trap but don't save. */
 			if (i2->reg) {
 				if (!copy_reg_to_user(i2, &uind))
 					return -EFAULT;
 				total++;
 			}
 		}
 		if (cmp <= 0 && ++i1 == end1)
 			i1 = NULL;
 		if (cmp >= 0 && ++i2 == end2)
 			i2 = NULL;
 	}
 	return total;
 }
 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
 {
 	return ARRAY_SIZE(invariant_sys_regs)
 		+ num_demux_regs()
 		+ walk_sys_regs(vcpu, (u64 __user *)NULL);
 }
 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
 {
 	unsigned int i;
 	int err;
 	/* Then give them all the invariant registers' indices. */
 	for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
 		if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
 			return -EFAULT;
 		uindices++;
 	}
 	err = walk_sys_regs(vcpu, uindices);
 	if (err < 0)
 		return err;
 	uindices += err;
 	return write_demux_regids(uindices);
 }
 static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n)
 {
 	unsigned int i;
 	for (i = 1; i < n; i++) {
 		if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
 			kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1);
 			return 1;
 		}
 	}
 	return 0;
 }
 void kvm_sys_reg_table_init(void)
 {
 	unsigned int i;
 	struct sys_reg_desc clidr;
 	/* Make sure tables are unique and in order. */
 	BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs)));
 	BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs)));
 	BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs)));
 	BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs)));
 	BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs)));
 	BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs)));
 	/* We abuse the reset function to overwrite the table itself. */
 	for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
 		invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
 	/*
 	 * CLIDR format is awkward, so clean it up.  See ARM B4.1.20:
 	 *
 	 *   If software reads the Cache Type fields from Ctype1
 	 *   upwards, once it has seen a value of 0b000, no caches
 	 *   exist at further-out levels of the hierarchy. So, for
 	 *   example, if Ctype3 is the first Cache Type field with a
 	 *   value of 0b000, the values of Ctype4 to Ctype7 must be
 	 *   ignored.
 	 */
 	get_clidr_el1(NULL, &clidr); /* Ugly... */
 	cache_levels = clidr.val;
 	for (i = 0; i < 7; i++)
 		if (((cache_levels >> (i*3)) & 7) == 0)
 			break;
 	/* Clear all higher bits. */
 	cache_levels &= (1 << (i*3))-1;
 }
 /**
  * kvm_reset_sys_regs - sets system registers to reset value
  * @vcpu: The VCPU pointer
  *
  * This function finds the right table above and sets the registers on the
  * virtual CPU struct to their architecturally defined reset values.
  */
 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
 {
 	size_t num;
 	const struct sys_reg_desc *table;
 	/* Catch someone adding a register without putting in reset entry. */
 	memset(&vcpu->arch.ctxt.sys_regs, 0x42, sizeof(vcpu->arch.ctxt.sys_regs));
 	/* Generic chip reset first (so target could override). */
 	reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
 	table = get_target_table(vcpu->arch.target, true, &num);
 	reset_sys_reg_descs(vcpu, table, num);
 	for (num = 1; num < NR_SYS_REGS; num++)
 		if (vcpu_sys_reg(vcpu, num) == 0x4242424242424242)

1	/*	1	/*
2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University	2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
3	* Authors: Rusty Russell <rusty@rustcorp.au>	3	* Authors: Rusty Russell <rusty@rustcorp.au>
4	* Christoffer Dall <c.dall@virtualopensystems.com>	4	* Christoffer Dall <c.dall@virtualopensystems.com>
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify	6	* This program is free software; you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License, version 2, as	7	* it under the terms of the GNU General Public License, version 2, as
8	* published by the Free Software Foundation.	8	* published by the Free Software Foundation.
9	*	9	*
10	* This program is distributed in the hope that it will be useful,	10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.	13	* GNU General Public License for more details.
14	*	14	*
15	* You should have received a copy of the GNU General Public License	15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software	16	* along with this program; if not, write to the Free Software
17	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.	17	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18	*/	18	*/
19	#include <linux/kvm_host.h>	19	#include <linux/kvm_host.h>
20	#include <asm/kvm_coproc.h>	20	#include <asm/kvm_coproc.h>
21	#include <asm/kvm_emulate.h>	21	#include <asm/kvm_emulate.h>
22	#include <linux/init.h>	22	#include <linux/init.h>
23		23
24	#include "coproc.h"	24	#include "coproc.h"
25		25
26	/*	26	/*
27	* A15-specific CP15 registers.	27	* A15-specific CP15 registers.
28	* CRn denotes the primary register number, but is copied to the CRm in the	28	* CRn denotes the primary register number, but is copied to the CRm in the
29	* user space API for 64-bit register access in line with the terminology used	29	* user space API for 64-bit register access in line with the terminology used
30	* in the ARM ARM.	30	* in the ARM ARM.
31	* Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit	31	* Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit
32	* registers preceding 32-bit ones.	32	* registers preceding 32-bit ones.
33	*/	33	*/
34	static const struct coproc_reg a15_regs[] = {	34	static const struct coproc_reg a15_regs[] = {
35	/* SCTLR: swapped by interrupt.S. */	35	/* SCTLR: swapped by interrupt.S. */
36	{ CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,	36	{ CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
37	access_sctlr, reset_val, c1_SCTLR, 0x00C50078 },	37	access_vm_reg, reset_val, c1_SCTLR, 0x00C50078 },
38	};	38	};
39		39
40	static struct kvm_coproc_target_table a15_target_table = {	40	static struct kvm_coproc_target_table a15_target_table = {
41	.target = KVM_ARM_TARGET_CORTEX_A15,	41	.target = KVM_ARM_TARGET_CORTEX_A15,
42	.table = a15_regs,	42	.table = a15_regs,
43	.num = ARRAY_SIZE(a15_regs),	43	.num = ARRAY_SIZE(a15_regs),
44	};	44	};
45		45
46	static int __init coproc_a15_init(void)	46	static int __init coproc_a15_init(void)
47	{	47	{
48	kvm_register_target_coproc_table(&a15_target_table);	48	kvm_register_target_coproc_table(&a15_target_table);
49	return 0;	49	return 0;
50	}	50	}
51	late_initcall(coproc_a15_init);	51	late_initcall(coproc_a15_init);
52		52

1	/*	1	/*
2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University	2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
3	* Copyright (C) 2013 - ARM Ltd	3	* Copyright (C) 2013 - ARM Ltd
4	*	4	*
5	* Authors: Rusty Russell <rusty@rustcorp.au>	5	* Authors: Rusty Russell <rusty@rustcorp.au>
6	* Christoffer Dall <c.dall@virtualopensystems.com>	6	* Christoffer Dall <c.dall@virtualopensystems.com>
7	* Jonathan Austin <jonathan.austin@arm.com>	7	* Jonathan Austin <jonathan.austin@arm.com>
8	*	8	*
9	* This program is free software; you can redistribute it and/or modify	9	* This program is free software; you can redistribute it and/or modify
10	* it under the terms of the GNU General Public License, version 2, as	10	* it under the terms of the GNU General Public License, version 2, as
11	* published by the Free Software Foundation.	11	* published by the Free Software Foundation.
12	*	12	*
13	* This program is distributed in the hope that it will be useful,	13	* This program is distributed in the hope that it will be useful,
14	* but WITHOUT ANY WARRANTY; without even the implied warranty of	14	* but WITHOUT ANY WARRANTY; without even the implied warranty of
15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	15	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16	* GNU General Public License for more details.	16	* GNU General Public License for more details.
17	*	17	*
18	* You should have received a copy of the GNU General Public License	18	* You should have received a copy of the GNU General Public License
19	* along with this program; if not, write to the Free Software	19	* along with this program; if not, write to the Free Software
20	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.	20	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21	*/	21	*/
22	#include <linux/kvm_host.h>	22	#include <linux/kvm_host.h>
23	#include <asm/kvm_coproc.h>	23	#include <asm/kvm_coproc.h>
24	#include <asm/kvm_emulate.h>	24	#include <asm/kvm_emulate.h>
25	#include <linux/init.h>	25	#include <linux/init.h>
26		26
27	#include "coproc.h"	27	#include "coproc.h"
28		28
29	/*	29	/*
30	* Cortex-A7 specific CP15 registers.	30	* Cortex-A7 specific CP15 registers.
31	* CRn denotes the primary register number, but is copied to the CRm in the	31	* CRn denotes the primary register number, but is copied to the CRm in the
32	* user space API for 64-bit register access in line with the terminology used	32	* user space API for 64-bit register access in line with the terminology used
33	* in the ARM ARM.	33	* in the ARM ARM.
34	* Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit	34	* Important: Must be sorted ascending by CRn, CRM, Op1, Op2 and with 64-bit
35	* registers preceding 32-bit ones.	35	* registers preceding 32-bit ones.
36	*/	36	*/
37	static const struct coproc_reg a7_regs[] = {	37	static const struct coproc_reg a7_regs[] = {
38	/* SCTLR: swapped by interrupt.S. */	38	/* SCTLR: swapped by interrupt.S. */
39	{ CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,	39	{ CRn( 1), CRm( 0), Op1( 0), Op2( 0), is32,
40	access_sctlr, reset_val, c1_SCTLR, 0x00C50878 },	40	access_vm_reg, reset_val, c1_SCTLR, 0x00C50878 },
41	};	41	};
42		42
43	static struct kvm_coproc_target_table a7_target_table = {	43	static struct kvm_coproc_target_table a7_target_table = {
44	.target = KVM_ARM_TARGET_CORTEX_A7,	44	.target = KVM_ARM_TARGET_CORTEX_A7,
45	.table = a7_regs,	45	.table = a7_regs,
46	.num = ARRAY_SIZE(a7_regs),	46	.num = ARRAY_SIZE(a7_regs),
47	};	47	};
48		48
49	static int __init coproc_a7_init(void)	49	static int __init coproc_a7_init(void)
50	{	50	{
51	kvm_register_target_coproc_table(&a7_target_table);	51	kvm_register_target_coproc_table(&a7_target_table);
52	return 0;	52	return 0;
53	}	53	}
54	late_initcall(coproc_a7_init);	54	late_initcall(coproc_a7_init);
55		55