Eric Lee / linux-smarc-t335x-v3.2

Commit 2122ff5eab8faec853e43f6de886e8dc8f31e317

Authored by Avi Kivity 2010-05-13 16:25:04 +0800

Exists in master and in 4 other branches

KVM: move vcpu locking to dispatcher for generic vcpu ioctls

All vcpu ioctls need to be locked, so instead of locking each one specifically
we lock at the generic dispatcher.

This patch only updates generic ioctls and leaves arch specific ioctls alone.

Signed-off-by: Avi Kivity <avi@redhat.com>

Showing 7 changed files with 17 additions and 95 deletions Inline Diff

arch/ia64/kvm/kvm-ia64.c
arch/powerpc/kvm/book3s.c
arch/powerpc/kvm/booke.c
arch/powerpc/kvm/powerpc.c
arch/s390/kvm/kvm-s390.c
arch/x86/kvm/x86.c
virt/kvm/kvm_main.c

arch/ia64/kvm/kvm-ia64.c

Diff comments View file @ 2122ff5

 /*
  * kvm_ia64.c: Basic KVM suppport On Itanium series processors
  *
  *
  * 	Copyright (C) 2007, Intel Corporation.
  *  	Xiantao Zhang  (xiantao.zhang@intel.com)
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  * Place - Suite 330, Boston, MA 02111-1307 USA.
  *
  */
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/percpu.h>
 #include <linux/fs.h>
 #include <linux/slab.h>
 #include <linux/smp.h>
 #include <linux/kvm_host.h>
 #include <linux/kvm.h>
 #include <linux/bitops.h>
 #include <linux/hrtimer.h>
 #include <linux/uaccess.h>
 #include <linux/iommu.h>
 #include <linux/intel-iommu.h>
 #include <asm/pgtable.h>
 #include <asm/gcc_intrin.h>
 #include <asm/pal.h>
 #include <asm/cacheflush.h>
 #include <asm/div64.h>
 #include <asm/tlb.h>
 #include <asm/elf.h>
 #include <asm/sn/addrs.h>
 #include <asm/sn/clksupport.h>
 #include <asm/sn/shub_mmr.h>
 #include "misc.h"
 #include "vti.h"
 #include "iodev.h"
 #include "ioapic.h"
 #include "lapic.h"
 #include "irq.h"
 static unsigned long kvm_vmm_base;
 static unsigned long kvm_vsa_base;
 static unsigned long kvm_vm_buffer;
 static unsigned long kvm_vm_buffer_size;
 unsigned long kvm_vmm_gp;
 static long vp_env_info;
 static struct kvm_vmm_info *kvm_vmm_info;
 static DEFINE_PER_CPU(struct kvm_vcpu *, last_vcpu);
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ NULL }
 };
 static unsigned long kvm_get_itc(struct kvm_vcpu *vcpu)
 {
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (vcpu->kvm->arch.is_sn2)
 		return rtc_time();
 	else
 #endif
 		return ia64_getreg(_IA64_REG_AR_ITC);
 }
 static void kvm_flush_icache(unsigned long start, unsigned long len)
 {
 	int l;
 	for (l = 0; l < (len + 32); l += 32)
 		ia64_fc((void *)(start + l));
 	ia64_sync_i();
 	ia64_srlz_i();
 }
 static void kvm_flush_tlb_all(void)
 {
 	unsigned long i, j, count0, count1, stride0, stride1, addr;
 	long flags;
 	addr    = local_cpu_data->ptce_base;
 	count0  = local_cpu_data->ptce_count[0];
 	count1  = local_cpu_data->ptce_count[1];
 	stride0 = local_cpu_data->ptce_stride[0];
 	stride1 = local_cpu_data->ptce_stride[1];
 	local_irq_save(flags);
 	for (i = 0; i < count0; ++i) {
 		for (j = 0; j < count1; ++j) {
 			ia64_ptce(addr);
 			addr += stride1;
 		}
 		addr += stride0;
 	}
 	local_irq_restore(flags);
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
 long ia64_pal_vp_create(u64 *vpd, u64 *host_iva, u64 *opt_handler)
 {
 	struct ia64_pal_retval iprv;
 	PAL_CALL_STK(iprv, PAL_VP_CREATE, (u64)vpd, (u64)host_iva,
 			(u64)opt_handler);
 	return iprv.status;
 }
 static  DEFINE_SPINLOCK(vp_lock);
 int kvm_arch_hardware_enable(void *garbage)
 {
 	long  status;
 	long  tmp_base;
 	unsigned long pte;
 	unsigned long saved_psr;
 	int slot;
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base), PAGE_KERNEL));
 	local_irq_save(saved_psr);
 	slot = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	local_irq_restore(saved_psr);
 	if (slot < 0)
 		return -EINVAL;
 	spin_lock(&vp_lock);
 	status = ia64_pal_vp_init_env(kvm_vsa_base ?
 				VP_INIT_ENV : VP_INIT_ENV_INITALIZE,
 			__pa(kvm_vm_buffer), KVM_VM_BUFFER_BASE, &tmp_base);
 	if (status != 0) {
 		spin_unlock(&vp_lock);
 		printk(KERN_WARNING"kvm: Failed to Enable VT Support!!!!\n");
 		return -EINVAL;
 	}
 	if (!kvm_vsa_base) {
 		kvm_vsa_base = tmp_base;
 		printk(KERN_INFO"kvm: kvm_vsa_base:0x%lx\n", kvm_vsa_base);
 	}
 	spin_unlock(&vp_lock);
 	ia64_ptr_entry(0x3, slot);
 	return 0;
 }
 void kvm_arch_hardware_disable(void *garbage)
 {
 	long status;
 	int slot;
 	unsigned long pte;
 	unsigned long saved_psr;
 	unsigned long host_iva = ia64_getreg(_IA64_REG_CR_IVA);
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base),
 				PAGE_KERNEL));
 	local_irq_save(saved_psr);
 	slot = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	local_irq_restore(saved_psr);
 	if (slot < 0)
 		return;
 	status = ia64_pal_vp_exit_env(host_iva);
 	if (status)
 		printk(KERN_DEBUG"kvm: Failed to disable VT support! :%ld\n",
 				status);
 	ia64_ptr_entry(0x3, slot);
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 	*(int *)rtn = 0;
 }
 int kvm_dev_ioctl_check_extension(long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_IRQCHIP:
 	case KVM_CAP_MP_STATE:
 	case KVM_CAP_IRQ_INJECT_STATUS:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 	case KVM_CAP_IOMMU:
 		r = iommu_found();
 		break;
 	default:
 		r = 0;
 	}
 	return r;
 }
 static int handle_vm_error(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 	kvm_run->hw.hardware_exit_reason = 1;
 	return 0;
 }
 static int handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct kvm_mmio_req *p;
 	struct kvm_io_device *mmio_dev;
 	int r;
 	p = kvm_get_vcpu_ioreq(vcpu);
 	if ((p->addr & PAGE_MASK) == IOAPIC_DEFAULT_BASE_ADDRESS)
 		goto mmio;
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_phys_addr = kvm_run->mmio.phys_addr = p->addr;
 	vcpu->mmio_size = kvm_run->mmio.len = p->size;
 	vcpu->mmio_is_write = kvm_run->mmio.is_write = !p->dir;
 	if (vcpu->mmio_is_write)
 		memcpy(vcpu->mmio_data, &p->data, p->size);
 	memcpy(kvm_run->mmio.data, &p->data, p->size);
 	kvm_run->exit_reason = KVM_EXIT_MMIO;
 	return 0;
 mmio:
 	if (p->dir)
 		r = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, p->addr,
 				    p->size, &p->data);
 	else
 		r = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, p->addr,
 				     p->size, &p->data);
 	if (r)
 		printk(KERN_ERR"kvm: No iodevice found! addr:%lx\n", p->addr);
 	p->state = STATE_IORESP_READY;
 	return 1;
 }
 static int handle_pal_call(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p;
 	p = kvm_get_exit_data(vcpu);
 	if (p->exit_reason == EXIT_REASON_PAL_CALL)
 		return kvm_pal_emul(vcpu, kvm_run);
 	else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = 2;
 		return 0;
 	}
 }
 static int handle_sal_call(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p;
 	p = kvm_get_exit_data(vcpu);
 	if (p->exit_reason == EXIT_REASON_SAL_CALL) {
 		kvm_sal_emul(vcpu);
 		return 1;
 	} else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = 3;
 		return 0;
 	}
 }
 static int __apic_accept_irq(struct kvm_vcpu *vcpu, uint64_t vector)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (!test_and_set_bit(vector, &vpd->irr[0])) {
 		vcpu->arch.irq_new_pending = 1;
 		kvm_vcpu_kick(vcpu);
 		return 1;
 	}
 	return 0;
 }
 /*
  *  offset: address offset to IPI space.
  *  value:  deliver value.
  */
 static void vcpu_deliver_ipi(struct kvm_vcpu *vcpu, uint64_t dm,
 				uint64_t vector)
 {
 	switch (dm) {
 	case SAPIC_FIXED:
 		break;
 	case SAPIC_NMI:
 		vector = 2;
 		break;
 	case SAPIC_EXTINT:
 		vector = 0;
 		break;
 	case SAPIC_INIT:
 	case SAPIC_PMI:
 	default:
 		printk(KERN_ERR"kvm: Unimplemented Deliver reserved IPI!\n");
 		return;
 	}
 	__apic_accept_irq(vcpu, vector);
 }
 static struct kvm_vcpu *lid_to_vcpu(struct kvm *kvm, unsigned long id,
 			unsigned long eid)
 {
 	union ia64_lid lid;
 	int i;
 	struct kvm_vcpu *vcpu;
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		lid.val = VCPU_LID(vcpu);
 		if (lid.id == id && lid.eid == eid)
 			return vcpu;
 	}
 	return NULL;
 }
 static int handle_ipi(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p = kvm_get_exit_data(vcpu);
 	struct kvm_vcpu *target_vcpu;
 	struct kvm_pt_regs *regs;
 	union ia64_ipi_a addr = p->u.ipi_data.addr;
 	union ia64_ipi_d data = p->u.ipi_data.data;
 	target_vcpu = lid_to_vcpu(vcpu->kvm, addr.id, addr.eid);
 	if (!target_vcpu)
 		return handle_vm_error(vcpu, kvm_run);
 	if (!target_vcpu->arch.launched) {
 		regs = vcpu_regs(target_vcpu);
 		regs->cr_iip = vcpu->kvm->arch.rdv_sal_data.boot_ip;
 		regs->r1 = vcpu->kvm->arch.rdv_sal_data.boot_gp;
 		target_vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		if (waitqueue_active(&target_vcpu->wq))
 			wake_up_interruptible(&target_vcpu->wq);
 	} else {
 		vcpu_deliver_ipi(target_vcpu, data.dm, data.vector);
 		if (target_vcpu != vcpu)
 			kvm_vcpu_kick(target_vcpu);
 	}
 	return 1;
 }
 struct call_data {
 	struct kvm_ptc_g ptc_g_data;
 	struct kvm_vcpu *vcpu;
 };
 static void vcpu_global_purge(void *info)
 {
 	struct call_data *p = (struct call_data *)info;
 	struct kvm_vcpu *vcpu = p->vcpu;
 	if (test_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
 		return;
 	set_bit(KVM_REQ_PTC_G, &vcpu->requests);
 	if (vcpu->arch.ptc_g_count < MAX_PTC_G_NUM) {
 		vcpu->arch.ptc_g_data[vcpu->arch.ptc_g_count++] =
 							p->ptc_g_data;
 	} else {
 		clear_bit(KVM_REQ_PTC_G, &vcpu->requests);
 		vcpu->arch.ptc_g_count = 0;
 		set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests);
 	}
 }
 static int handle_global_purge(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p = kvm_get_exit_data(vcpu);
 	struct kvm *kvm = vcpu->kvm;
 	struct call_data call_data;
 	int i;
 	struct kvm_vcpu *vcpui;
 	call_data.ptc_g_data = p->u.ptc_g_data;
 	kvm_for_each_vcpu(i, vcpui, kvm) {
 		if (vcpui->arch.mp_state == KVM_MP_STATE_UNINITIALIZED ||
 				vcpu == vcpui)
 			continue;
 		if (waitqueue_active(&vcpui->wq))
 			wake_up_interruptible(&vcpui->wq);
 		if (vcpui->cpu != -1) {
 			call_data.vcpu = vcpui;
 			smp_call_function_single(vcpui->cpu,
 					vcpu_global_purge, &call_data, 1);
 		} else
 			printk(KERN_WARNING"kvm: Uninit vcpu received ipi!\n");
 	}
 	return 1;
 }
 static int handle_switch_rr6(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	return 1;
 }
 static int kvm_sn2_setup_mappings(struct kvm_vcpu *vcpu)
 {
 	unsigned long pte, rtc_phys_addr, map_addr;
 	int slot;
 	map_addr = KVM_VMM_BASE + (1UL << KVM_VMM_SHIFT);
 	rtc_phys_addr = LOCAL_MMR_OFFSET | SH_RTC;
 	pte = pte_val(mk_pte_phys(rtc_phys_addr, PAGE_KERNEL_UC));
 	slot = ia64_itr_entry(0x3, map_addr, pte, PAGE_SHIFT);
 	vcpu->arch.sn_rtc_tr_slot = slot;
 	if (slot < 0) {
 		printk(KERN_ERR "Mayday mayday! RTC mapping failed!\n");
 		slot = 0;
 	}
 	return slot;
 }
 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 {
 	ktime_t kt;
 	long itc_diff;
 	unsigned long vcpu_now_itc;
 	unsigned long expires;
 	struct hrtimer *p_ht = &vcpu->arch.hlt_timer;
 	unsigned long cyc_per_usec = local_cpu_data->cyc_per_usec;
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (irqchip_in_kernel(vcpu->kvm)) {
 		vcpu_now_itc = kvm_get_itc(vcpu) + vcpu->arch.itc_offset;
 		if (time_after(vcpu_now_itc, vpd->itm)) {
 			vcpu->arch.timer_check = 1;
 			return 1;
 		}
 		itc_diff = vpd->itm - vcpu_now_itc;
 		if (itc_diff < 0)
 			itc_diff = -itc_diff;
 		expires = div64_u64(itc_diff, cyc_per_usec);
 		kt = ktime_set(0, 1000 * expires);
 		vcpu->arch.ht_active = 1;
 		hrtimer_start(p_ht, kt, HRTIMER_MODE_ABS);
 		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
 		kvm_vcpu_block(vcpu);
 		hrtimer_cancel(p_ht);
 		vcpu->arch.ht_active = 0;
 		if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests) ||
 				kvm_cpu_has_pending_timer(vcpu))
 			if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
 				vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE)
 			return -EINTR;
 		return 1;
 	} else {
 		printk(KERN_ERR"kvm: Unsupported userspace halt!");
 		return 0;
 	}
 }
 static int handle_vm_shutdown(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run)
 {
 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
 	return 0;
 }
 static int handle_external_interrupt(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run)
 {
 	return 1;
 }
 static int handle_vcpu_debug(struct kvm_vcpu *vcpu,
 				struct kvm_run *kvm_run)
 {
 	printk("VMM: %s", vcpu->arch.log_buf);
 	return 1;
 }
 static int (*kvm_vti_exit_handlers[])(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run) = {
 	[EXIT_REASON_VM_PANIC]              = handle_vm_error,
 	[EXIT_REASON_MMIO_INSTRUCTION]      = handle_mmio,
 	[EXIT_REASON_PAL_CALL]              = handle_pal_call,
 	[EXIT_REASON_SAL_CALL]              = handle_sal_call,
 	[EXIT_REASON_SWITCH_RR6]            = handle_switch_rr6,
 	[EXIT_REASON_VM_DESTROY]            = handle_vm_shutdown,
 	[EXIT_REASON_EXTERNAL_INTERRUPT]    = handle_external_interrupt,
 	[EXIT_REASON_IPI]		    = handle_ipi,
 	[EXIT_REASON_PTC_G]		    = handle_global_purge,
 	[EXIT_REASON_DEBUG]		    = handle_vcpu_debug,
 };
 static const int kvm_vti_max_exit_handlers =
 		sizeof(kvm_vti_exit_handlers)/sizeof(*kvm_vti_exit_handlers);
 static uint32_t kvm_get_exit_reason(struct kvm_vcpu *vcpu)
 {
 	struct exit_ctl_data *p_exit_data;
 	p_exit_data = kvm_get_exit_data(vcpu);
 	return p_exit_data->exit_reason;
 }
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
  */
 static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 {
 	u32 exit_reason = kvm_get_exit_reason(vcpu);
 	vcpu->arch.last_exit = exit_reason;
 	if (exit_reason < kvm_vti_max_exit_handlers
 			&& kvm_vti_exit_handlers[exit_reason])
 		return kvm_vti_exit_handlers[exit_reason](vcpu, kvm_run);
 	else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = exit_reason;
 	}
 	return 0;
 }
 static inline void vti_set_rr6(unsigned long rr6)
 {
 	ia64_set_rr(RR6, rr6);
 	ia64_srlz_i();
 }
 static int kvm_insert_vmm_mapping(struct kvm_vcpu *vcpu)
 {
 	unsigned long pte;
 	struct kvm *kvm = vcpu->kvm;
 	int r;
 	/*Insert a pair of tr to map vmm*/
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base), PAGE_KERNEL));
 	r = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	if (r < 0)
 		goto out;
 	vcpu->arch.vmm_tr_slot = r;
 	/*Insert a pairt of tr to map data of vm*/
 	pte = pte_val(mk_pte_phys(__pa(kvm->arch.vm_base), PAGE_KERNEL));
 	r = ia64_itr_entry(0x3, KVM_VM_DATA_BASE,
 					pte, KVM_VM_DATA_SHIFT);
 	if (r < 0)
 		goto out;
 	vcpu->arch.vm_tr_slot = r;
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (kvm->arch.is_sn2) {
 		r = kvm_sn2_setup_mappings(vcpu);
 		if (r < 0)
 			goto out;
 	}
 #endif
 	r = 0;
 out:
 	return r;
 }
 static void kvm_purge_vmm_mapping(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
 	ia64_ptr_entry(0x3, vcpu->arch.vmm_tr_slot);
 	ia64_ptr_entry(0x3, vcpu->arch.vm_tr_slot);
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (kvm->arch.is_sn2)
 		ia64_ptr_entry(0x3, vcpu->arch.sn_rtc_tr_slot);
 #endif
 }
 static int kvm_vcpu_pre_transition(struct kvm_vcpu *vcpu)
 {
 	unsigned long psr;
 	int r;
 	int cpu = smp_processor_id();
 	if (vcpu->arch.last_run_cpu != cpu ||
 			per_cpu(last_vcpu, cpu) != vcpu) {
 		per_cpu(last_vcpu, cpu) = vcpu;
 		vcpu->arch.last_run_cpu = cpu;
 		kvm_flush_tlb_all();
 	}
 	vcpu->arch.host_rr6 = ia64_get_rr(RR6);
 	vti_set_rr6(vcpu->arch.vmm_rr);
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	return r;
 }
 static void kvm_vcpu_post_transition(struct kvm_vcpu *vcpu)
 {
 	kvm_purge_vmm_mapping(vcpu);
 	vti_set_rr6(vcpu->arch.host_rr6);
 }
 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	union context *host_ctx, *guest_ctx;
 	int r, idx;
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 again:
 	if (signal_pending(current)) {
 		r = -EINTR;
 		kvm_run->exit_reason = KVM_EXIT_INTR;
 		goto out;
 	}
 	preempt_disable();
 	local_irq_disable();
 	/*Get host and guest context with guest address space.*/
 	host_ctx = kvm_get_host_context(vcpu);
 	guest_ctx = kvm_get_guest_context(vcpu);
 	clear_bit(KVM_REQ_KICK, &vcpu->requests);
 	r = kvm_vcpu_pre_transition(vcpu);
 	if (r < 0)
 		goto vcpu_run_fail;
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	kvm_guest_enter();
 	/*
 	 * Transition to the guest
 	 */
 	kvm_vmm_info->tramp_entry(host_ctx, guest_ctx);
 	kvm_vcpu_post_transition(vcpu);
 	vcpu->arch.launched = 1;
 	set_bit(KVM_REQ_KICK, &vcpu->requests);
 	local_irq_enable();
 	/*
 	 * We must have an instruction between local_irq_enable() and
 	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
 	 * the interrupt shadow.  The stat.exits increment will do nicely.
 	 * But we need to prevent reordering, hence this barrier():
 	 */
 	barrier();
 	kvm_guest_exit();
 	preempt_enable();
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	r = kvm_handle_exit(kvm_run, vcpu);
 	if (r > 0) {
 		if (!need_resched())
 			goto again;
 	}
 out:
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	if (r > 0) {
 		kvm_resched(vcpu);
 		idx = srcu_read_lock(&vcpu->kvm->srcu);
 		goto again;
 	}
 	return r;
 vcpu_run_fail:
 	local_irq_enable();
 	preempt_enable();
 	kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
 	goto out;
 }
 static void kvm_set_mmio_data(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmio_req *p = kvm_get_vcpu_ioreq(vcpu);
 	if (!vcpu->mmio_is_write)
 		memcpy(&p->data, vcpu->mmio_data, 8);
 	p->state = STATE_IORESP_READY;
 }
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
 	sigset_t sigsaved;
-	vcpu_load(vcpu);
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
 		kvm_vcpu_block(vcpu);
 		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 		r = -EAGAIN;
 		goto out;
 	}
 	if (vcpu->mmio_needed) {
 		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
 		kvm_set_mmio_data(vcpu);
 		vcpu->mmio_read_completed = 1;
 		vcpu->mmio_needed = 0;
 	}
 	r = __vcpu_run(vcpu, kvm_run);
 out:
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
-	vcpu_put(vcpu);
 	return r;
 }
 static struct kvm *kvm_alloc_kvm(void)
 {
 	struct kvm *kvm;
 	uint64_t  vm_base;
 	BUG_ON(sizeof(struct kvm) > KVM_VM_STRUCT_SIZE);
 	vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE));
 	if (!vm_base)
 		return ERR_PTR(-ENOMEM);
 	memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
 	kvm = (struct kvm *)(vm_base +
 			offsetof(struct kvm_vm_data, kvm_vm_struct));
 	kvm->arch.vm_base = vm_base;
 	printk(KERN_DEBUG"kvm: vm's data area:0x%lx\n", vm_base);
 	return kvm;
 }
 struct kvm_io_range {
 	unsigned long start;
 	unsigned long size;
 	unsigned long type;
 };
 static const struct kvm_io_range io_ranges[] = {
 	{VGA_IO_START, VGA_IO_SIZE, GPFN_FRAME_BUFFER},
 	{MMIO_START, MMIO_SIZE, GPFN_LOW_MMIO},
 	{LEGACY_IO_START, LEGACY_IO_SIZE, GPFN_LEGACY_IO},
 	{IO_SAPIC_START, IO_SAPIC_SIZE, GPFN_IOSAPIC},
 	{PIB_START, PIB_SIZE, GPFN_PIB},
 };
 static void kvm_build_io_pmt(struct kvm *kvm)
 {
 	unsigned long i, j;
 	/* Mark I/O ranges */
 	for (i = 0; i < (sizeof(io_ranges) / sizeof(struct kvm_io_range));
 							i++) {
 		for (j = io_ranges[i].start;
 				j < io_ranges[i].start + io_ranges[i].size;
 				j += PAGE_SIZE)
 			kvm_set_pmt_entry(kvm, j >> PAGE_SHIFT,
 					io_ranges[i].type, 0);
 	}
 }
 /*Use unused rids to virtualize guest rid.*/
 #define GUEST_PHYSICAL_RR0	0x1739
 #define GUEST_PHYSICAL_RR4	0x2739
 #define VMM_INIT_RR		0x1660
 static void kvm_init_vm(struct kvm *kvm)
 {
 	BUG_ON(!kvm);
 	kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0;
 	kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4;
 	kvm->arch.vmm_init_rr = VMM_INIT_RR;
 	/*
 	 *Fill P2M entries for MMIO/IO ranges
 	 */
 	kvm_build_io_pmt(kvm);
 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
 }
 struct  kvm *kvm_arch_create_vm(void)
 {
 	struct kvm *kvm = kvm_alloc_kvm();
 	if (IS_ERR(kvm))
 		return ERR_PTR(-ENOMEM);
 	kvm->arch.is_sn2 = ia64_platform_is("sn2");
 	kvm_init_vm(kvm);
 	return kvm;
 }
 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm,
 					struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_IOAPIC:
 		r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_IOAPIC:
 		r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 #define RESTORE_REGS(_x) vcpu->arch._x = regs->_x
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	int i;
-	vcpu_load(vcpu);
 	for (i = 0; i < 16; i++) {
 		vpd->vgr[i] = regs->vpd.vgr[i];
 		vpd->vbgr[i] = regs->vpd.vbgr[i];
 	}
 	for (i = 0; i < 128; i++)
 		vpd->vcr[i] = regs->vpd.vcr[i];
 	vpd->vhpi = regs->vpd.vhpi;
 	vpd->vnat = regs->vpd.vnat;
 	vpd->vbnat = regs->vpd.vbnat;
 	vpd->vpsr = regs->vpd.vpsr;
 	vpd->vpr = regs->vpd.vpr;
 	memcpy(&vcpu->arch.guest, &regs->saved_guest, sizeof(union context));
 	RESTORE_REGS(mp_state);
 	RESTORE_REGS(vmm_rr);
 	memcpy(vcpu->arch.itrs, regs->itrs, sizeof(struct thash_data) * NITRS);
 	memcpy(vcpu->arch.dtrs, regs->dtrs, sizeof(struct thash_data) * NDTRS);
 	RESTORE_REGS(itr_regions);
 	RESTORE_REGS(dtr_regions);
 	RESTORE_REGS(tc_regions);
 	RESTORE_REGS(irq_check);
 	RESTORE_REGS(itc_check);
 	RESTORE_REGS(timer_check);
 	RESTORE_REGS(timer_pending);
 	RESTORE_REGS(last_itc);
 	for (i = 0; i < 8; i++) {
 		vcpu->arch.vrr[i] = regs->vrr[i];
 		vcpu->arch.ibr[i] = regs->ibr[i];
 		vcpu->arch.dbr[i] = regs->dbr[i];
 	}
 	for (i = 0; i < 4; i++)
 		vcpu->arch.insvc[i] = regs->insvc[i];
 	RESTORE_REGS(xtp);
 	RESTORE_REGS(metaphysical_rr0);
 	RESTORE_REGS(metaphysical_rr4);
 	RESTORE_REGS(metaphysical_saved_rr0);
 	RESTORE_REGS(metaphysical_saved_rr4);
 	RESTORE_REGS(fp_psr);
 	RESTORE_REGS(saved_gp);
 	vcpu->arch.irq_new_pending = 1;
 	vcpu->arch.itc_offset = regs->saved_itc - kvm_get_itc(vcpu);
 	set_bit(KVM_REQ_RESUME, &vcpu->requests);
-	vcpu_put(vcpu);
 	return 0;
 }
 long kvm_arch_vm_ioctl(struct file *filp,
 		unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r = -ENOTTY;
 	switch (ioctl) {
 	case KVM_SET_MEMORY_REGION: {
 		struct kvm_memory_region kvm_mem;
 		struct kvm_userspace_memory_region kvm_userspace_mem;
 		r = -EFAULT;
 		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
 			goto out;
 		kvm_userspace_mem.slot = kvm_mem.slot;
 		kvm_userspace_mem.flags = kvm_mem.flags;
 		kvm_userspace_mem.guest_phys_addr =
 					kvm_mem.guest_phys_addr;
 		kvm_userspace_mem.memory_size = kvm_mem.memory_size;
 		r = kvm_vm_ioctl_set_memory_region(kvm,
 					&kvm_userspace_mem, 0);
 		if (r)
 			goto out;
 		break;
 		}
 	case KVM_CREATE_IRQCHIP:
 		r = -EFAULT;
 		r = kvm_ioapic_init(kvm);
 		if (r)
 			goto out;
 		r = kvm_setup_default_irq_routing(kvm);
 		if (r) {
 			kvm_ioapic_destroy(kvm);
 			goto out;
 		}
 		break;
 	case KVM_IRQ_LINE_STATUS:
 	case KVM_IRQ_LINE: {
 		struct kvm_irq_level irq_event;
 		r = -EFAULT;
 		if (copy_from_user(&irq_event, argp, sizeof irq_event))
 			goto out;
 		r = -ENXIO;
 		if (irqchip_in_kernel(kvm)) {
 			__s32 status;
 			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
 				    irq_event.irq, irq_event.level);
 			if (ioctl == KVM_IRQ_LINE_STATUS) {
 				r = -EFAULT;
 				irq_event.status = status;
 				if (copy_to_user(argp, &irq_event,
 							sizeof irq_event))
 					goto out;
 			}
 			r = 0;
 		}
 		break;
 		}
 	case KVM_GET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip chip;
 		r = -EFAULT;
 		if (copy_from_user(&chip, argp, sizeof chip))
 				goto out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto out;
 		r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &chip, sizeof chip))
 				goto out;
 		r = 0;
 		break;
 		}
 	case KVM_SET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip chip;
 		r = -EFAULT;
 		if (copy_from_user(&chip, argp, sizeof chip))
 				goto out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto out;
 		r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 		}
 	default:
 		;
 	}
 out:
 	return r;
 }
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 		struct kvm_sregs *sregs)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 		struct kvm_sregs *sregs)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 		struct kvm_translation *tr)
 {
 	return -EINVAL;
 }
 static int kvm_alloc_vmm_area(void)
 {
 	if (!kvm_vmm_base && (kvm_vm_buffer_size < KVM_VM_BUFFER_SIZE)) {
 		kvm_vmm_base = __get_free_pages(GFP_KERNEL,
 				get_order(KVM_VMM_SIZE));
 		if (!kvm_vmm_base)
 			return -ENOMEM;
 		memset((void *)kvm_vmm_base, 0, KVM_VMM_SIZE);
 		kvm_vm_buffer = kvm_vmm_base + VMM_SIZE;
 		printk(KERN_DEBUG"kvm:VMM's Base Addr:0x%lx, vm_buffer:0x%lx\n",
 				kvm_vmm_base, kvm_vm_buffer);
 	}
 	return 0;
 }
 static void kvm_free_vmm_area(void)
 {
 	if (kvm_vmm_base) {
 		/*Zero this area before free to avoid bits leak!!*/
 		memset((void *)kvm_vmm_base, 0, KVM_VMM_SIZE);
 		free_pages(kvm_vmm_base, get_order(KVM_VMM_SIZE));
 		kvm_vmm_base  = 0;
 		kvm_vm_buffer = 0;
 		kvm_vsa_base = 0;
 	}
 }
 static int vti_init_vpd(struct kvm_vcpu *vcpu)
 {
 	int i;
 	union cpuid3_t cpuid3;
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (IS_ERR(vpd))
 		return PTR_ERR(vpd);
 	/* CPUID init */
 	for (i = 0; i < 5; i++)
 		vpd->vcpuid[i] = ia64_get_cpuid(i);
 	/* Limit the CPUID number to 5 */
 	cpuid3.value = vpd->vcpuid[3];
 	cpuid3.number = 4;	/* 5 - 1 */
 	vpd->vcpuid[3] = cpuid3.value;
 	/*Set vac and vdc fields*/
 	vpd->vac.a_from_int_cr = 1;
 	vpd->vac.a_to_int_cr = 1;
 	vpd->vac.a_from_psr = 1;
 	vpd->vac.a_from_cpuid = 1;
 	vpd->vac.a_cover = 1;
 	vpd->vac.a_bsw = 1;
 	vpd->vac.a_int = 1;
 	vpd->vdc.d_vmsw = 1;
 	/*Set virtual buffer*/
 	vpd->virt_env_vaddr = KVM_VM_BUFFER_BASE;
 	return 0;
 }
 static int vti_create_vp(struct kvm_vcpu *vcpu)
 {
 	long ret;
 	struct vpd *vpd = vcpu->arch.vpd;
 	unsigned long  vmm_ivt;
 	vmm_ivt = kvm_vmm_info->vmm_ivt;
 	printk(KERN_DEBUG "kvm: vcpu:%p,ivt: 0x%lx\n", vcpu, vmm_ivt);
 	ret = ia64_pal_vp_create((u64 *)vpd, (u64 *)vmm_ivt, 0);
 	if (ret) {
 		printk(KERN_ERR"kvm: ia64_pal_vp_create failed!\n");
 		return -EINVAL;
 	}
 	return 0;
 }
 static void init_ptce_info(struct kvm_vcpu *vcpu)
 {
 	ia64_ptce_info_t ptce = {0};
 	ia64_get_ptce(&ptce);
 	vcpu->arch.ptce_base = ptce.base;
 	vcpu->arch.ptce_count[0] = ptce.count[0];
 	vcpu->arch.ptce_count[1] = ptce.count[1];
 	vcpu->arch.ptce_stride[0] = ptce.stride[0];
 	vcpu->arch.ptce_stride[1] = ptce.stride[1];
 }
 static void kvm_migrate_hlt_timer(struct kvm_vcpu *vcpu)
 {
 	struct hrtimer *p_ht = &vcpu->arch.hlt_timer;
 	if (hrtimer_cancel(p_ht))
 		hrtimer_start_expires(p_ht, HRTIMER_MODE_ABS);
 }
 static enum hrtimer_restart hlt_timer_fn(struct hrtimer *data)
 {
 	struct kvm_vcpu *vcpu;
 	wait_queue_head_t *q;
 	vcpu  = container_of(data, struct kvm_vcpu, arch.hlt_timer);
 	q = &vcpu->wq;
 	if (vcpu->arch.mp_state != KVM_MP_STATE_HALTED)
 		goto out;
 	if (waitqueue_active(q))
 		wake_up_interruptible(q);
 out:
 	vcpu->arch.timer_fired = 1;
 	vcpu->arch.timer_check = 1;
 	return HRTIMER_NORESTART;
 }
 #define PALE_RESET_ENTRY    0x80000000ffffffb0UL
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu *v;
 	int r;
 	int i;
 	long itc_offset;
 	struct kvm *kvm = vcpu->kvm;
 	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
 	union context *p_ctx = &vcpu->arch.guest;
 	struct kvm_vcpu *vmm_vcpu = to_guest(vcpu->kvm, vcpu);
 	/*Init vcpu context for first run.*/
 	if (IS_ERR(vmm_vcpu))
 		return PTR_ERR(vmm_vcpu);
 	if (kvm_vcpu_is_bsp(vcpu)) {
 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		/*Set entry address for first run.*/
 		regs->cr_iip = PALE_RESET_ENTRY;
 		/*Initialize itc offset for vcpus*/
 		itc_offset = 0UL - kvm_get_itc(vcpu);
 		for (i = 0; i < KVM_MAX_VCPUS; i++) {
 			v = (struct kvm_vcpu *)((char *)vcpu +
 					sizeof(struct kvm_vcpu_data) * i);
 			v->arch.itc_offset = itc_offset;
 			v->arch.last_itc = 0;
 		}
 	} else
 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
 	r = -ENOMEM;
 	vcpu->arch.apic = kzalloc(sizeof(struct kvm_lapic), GFP_KERNEL);
 	if (!vcpu->arch.apic)
 		goto out;
 	vcpu->arch.apic->vcpu = vcpu;
 	p_ctx->gr[1] = 0;
 	p_ctx->gr[12] = (unsigned long)((char *)vmm_vcpu + KVM_STK_OFFSET);
 	p_ctx->gr[13] = (unsigned long)vmm_vcpu;
 	p_ctx->psr = 0x1008522000UL;
 	p_ctx->ar[40] = FPSR_DEFAULT; /*fpsr*/
 	p_ctx->caller_unat = 0;
 	p_ctx->pr = 0x0;
 	p_ctx->ar[36] = 0x0; /*unat*/
 	p_ctx->ar[19] = 0x0; /*rnat*/
 	p_ctx->ar[18] = (unsigned long)vmm_vcpu +
 				((sizeof(struct kvm_vcpu)+15) & ~15);
 	p_ctx->ar[64] = 0x0; /*pfs*/
 	p_ctx->cr[0] = 0x7e04UL;
 	p_ctx->cr[2] = (unsigned long)kvm_vmm_info->vmm_ivt;
 	p_ctx->cr[8] = 0x3c;
 	/*Initilize region register*/
 	p_ctx->rr[0] = 0x30;
 	p_ctx->rr[1] = 0x30;
 	p_ctx->rr[2] = 0x30;
 	p_ctx->rr[3] = 0x30;
 	p_ctx->rr[4] = 0x30;
 	p_ctx->rr[5] = 0x30;
 	p_ctx->rr[7] = 0x30;
 	/*Initilize branch register 0*/
 	p_ctx->br[0] = *(unsigned long *)kvm_vmm_info->vmm_entry;
 	vcpu->arch.vmm_rr = kvm->arch.vmm_init_rr;
 	vcpu->arch.metaphysical_rr0 = kvm->arch.metaphysical_rr0;
 	vcpu->arch.metaphysical_rr4 = kvm->arch.metaphysical_rr4;
 	hrtimer_init(&vcpu->arch.hlt_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 	vcpu->arch.hlt_timer.function = hlt_timer_fn;
 	vcpu->arch.last_run_cpu = -1;
 	vcpu->arch.vpd = (struct vpd *)VPD_BASE(vcpu->vcpu_id);
 	vcpu->arch.vsa_base = kvm_vsa_base;
 	vcpu->arch.__gp = kvm_vmm_gp;
 	vcpu->arch.dirty_log_lock_pa = __pa(&kvm->arch.dirty_log_lock);
 	vcpu->arch.vhpt.hash = (struct thash_data *)VHPT_BASE(vcpu->vcpu_id);
 	vcpu->arch.vtlb.hash = (struct thash_data *)VTLB_BASE(vcpu->vcpu_id);
 	init_ptce_info(vcpu);
 	r = 0;
 out:
 	return r;
 }
 static int vti_vcpu_setup(struct kvm_vcpu *vcpu, int id)
 {
 	unsigned long psr;
 	int r;
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	if (r)
 		goto fail;
 	r = kvm_vcpu_init(vcpu, vcpu->kvm, id);
 	if (r)
 		goto fail;
 	r = vti_init_vpd(vcpu);
 	if (r) {
 		printk(KERN_DEBUG"kvm: vpd init error!!\n");
 		goto uninit;
 	}
 	r = vti_create_vp(vcpu);
 	if (r)
 		goto uninit;
 	kvm_purge_vmm_mapping(vcpu);
 	return 0;
 uninit:
 	kvm_vcpu_uninit(vcpu);
 fail:
 	return r;
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 		unsigned int id)
 {
 	struct kvm_vcpu *vcpu;
 	unsigned long vm_base = kvm->arch.vm_base;
 	int r;
 	int cpu;
 	BUG_ON(sizeof(struct kvm_vcpu) > VCPU_STRUCT_SIZE/2);
 	r = -EINVAL;
 	if (id >= KVM_MAX_VCPUS) {
 		printk(KERN_ERR"kvm: Can't configure vcpus > %ld",
 				KVM_MAX_VCPUS);
 		goto fail;
 	}
 	r = -ENOMEM;
 	if (!vm_base) {
 		printk(KERN_ERR"kvm: Create vcpu[%d] error!\n", id);
 		goto fail;
 	}
 	vcpu = (struct kvm_vcpu *)(vm_base + offsetof(struct kvm_vm_data,
 					vcpu_data[id].vcpu_struct));
 	vcpu->kvm = kvm;
 	cpu = get_cpu();
 	r = vti_vcpu_setup(vcpu, id);
 	put_cpu();
 	if (r) {
 		printk(KERN_DEBUG"kvm: vcpu_setup error!!\n");
 		goto fail;
 	}
 	return vcpu;
 fail:
 	return ERR_PTR(r);
 }
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
 	return -EINVAL;
 }
 static void free_kvm(struct kvm *kvm)
 {
 	unsigned long vm_base = kvm->arch.vm_base;
 	if (vm_base) {
 		memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
 		free_pages(vm_base, get_order(KVM_VM_DATA_SIZE));
 	}
 }
 static void kvm_release_vm_pages(struct kvm *kvm)
 {
 	struct kvm_memslots *slots;
 	struct kvm_memory_slot *memslot;
 	int i, j;
 	unsigned long base_gfn;
 	slots = kvm_memslots(kvm);
 	for (i = 0; i < slots->nmemslots; i++) {
 		memslot = &slots->memslots[i];
 		base_gfn = memslot->base_gfn;
 		for (j = 0; j < memslot->npages; j++) {
 			if (memslot->rmap[j])
 				put_page((struct page *)memslot->rmap[j]);
 		}
 	}
 }
 void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvm_iommu_unmap_guest(kvm);
 #ifdef  KVM_CAP_DEVICE_ASSIGNMENT
 	kvm_free_all_assigned_devices(kvm);
 #endif
 	kfree(kvm->arch.vioapic);
 	kvm_release_vm_pages(kvm);
 	kvm_free_physmem(kvm);
 	cleanup_srcu_struct(&kvm->srcu);
 	free_kvm(kvm);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	if (cpu != vcpu->cpu) {
 		vcpu->cpu = cpu;
 		if (vcpu->arch.ht_active)
 			kvm_migrate_hlt_timer(vcpu);
 	}
 }
 #define SAVE_REGS(_x) 	regs->_x = vcpu->arch._x
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	int i;
 	vcpu_load(vcpu);
 	for (i = 0; i < 16; i++) {
 		regs->vpd.vgr[i] = vpd->vgr[i];
 		regs->vpd.vbgr[i] = vpd->vbgr[i];
 	}
 	for (i = 0; i < 128; i++)
 		regs->vpd.vcr[i] = vpd->vcr[i];
 	regs->vpd.vhpi = vpd->vhpi;
 	regs->vpd.vnat = vpd->vnat;
 	regs->vpd.vbnat = vpd->vbnat;
 	regs->vpd.vpsr = vpd->vpsr;
 	regs->vpd.vpr = vpd->vpr;
 	memcpy(&regs->saved_guest, &vcpu->arch.guest, sizeof(union context));
 	SAVE_REGS(mp_state);
 	SAVE_REGS(vmm_rr);
 	memcpy(regs->itrs, vcpu->arch.itrs, sizeof(struct thash_data) * NITRS);
 	memcpy(regs->dtrs, vcpu->arch.dtrs, sizeof(struct thash_data) * NDTRS);
 	SAVE_REGS(itr_regions);
 	SAVE_REGS(dtr_regions);
 	SAVE_REGS(tc_regions);
 	SAVE_REGS(irq_check);
 	SAVE_REGS(itc_check);
 	SAVE_REGS(timer_check);
 	SAVE_REGS(timer_pending);
 	SAVE_REGS(last_itc);
 	for (i = 0; i < 8; i++) {
 		regs->vrr[i] = vcpu->arch.vrr[i];
 		regs->ibr[i] = vcpu->arch.ibr[i];
 		regs->dbr[i] = vcpu->arch.dbr[i];
 	}
 	for (i = 0; i < 4; i++)
 		regs->insvc[i] = vcpu->arch.insvc[i];
 	regs->saved_itc = vcpu->arch.itc_offset + kvm_get_itc(vcpu);
 	SAVE_REGS(xtp);
 	SAVE_REGS(metaphysical_rr0);
 	SAVE_REGS(metaphysical_rr4);
 	SAVE_REGS(metaphysical_saved_rr0);
 	SAVE_REGS(metaphysical_saved_rr4);
 	SAVE_REGS(fp_psr);
 	SAVE_REGS(saved_gp);
 	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_stack(struct kvm_vcpu *vcpu,
 				  struct kvm_ia64_vcpu_stack *stack)
 {
 	memcpy(stack, vcpu, sizeof(struct kvm_ia64_vcpu_stack));
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_stack(struct kvm_vcpu *vcpu,
 				  struct kvm_ia64_vcpu_stack *stack)
 {
 	memcpy(vcpu + 1, &stack->stack[0] + sizeof(struct kvm_vcpu),
 	       sizeof(struct kvm_ia64_vcpu_stack) - sizeof(struct kvm_vcpu));
 	vcpu->arch.exit_data = ((struct kvm_vcpu *)stack)->arch.exit_data;
 	return 0;
 }
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	hrtimer_cancel(&vcpu->arch.hlt_timer);
 	kfree(vcpu->arch.apic);
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	struct kvm_ia64_vcpu_stack *stack = NULL;
 	long r;
 	switch (ioctl) {
 	case KVM_IA64_VCPU_GET_STACK: {
 		struct kvm_ia64_vcpu_stack __user *user_stack;
 	        void __user *first_p = argp;
 		r = -EFAULT;
 		if (copy_from_user(&user_stack, first_p, sizeof(void *)))
 			goto out;
 		if (!access_ok(VERIFY_WRITE, user_stack,
 			       sizeof(struct kvm_ia64_vcpu_stack))) {
 			printk(KERN_INFO "KVM_IA64_VCPU_GET_STACK: "
 			       "Illegal user destination address for stack\n");
 			goto out;
 		}
 		stack = kzalloc(sizeof(struct kvm_ia64_vcpu_stack), GFP_KERNEL);
 		if (!stack) {
 			r = -ENOMEM;
 			goto out;
 		}
 		r = kvm_arch_vcpu_ioctl_get_stack(vcpu, stack);
 		if (r)
 			goto out;
 		if (copy_to_user(user_stack, stack,
 				 sizeof(struct kvm_ia64_vcpu_stack))) {
 			r = -EFAULT;
 			goto out;
 		}
 		break;
 	}
 	case KVM_IA64_VCPU_SET_STACK: {
 		struct kvm_ia64_vcpu_stack __user *user_stack;
 	        void __user *first_p = argp;
 		r = -EFAULT;
 		if (copy_from_user(&user_stack, first_p, sizeof(void *)))
 			goto out;
 		if (!access_ok(VERIFY_READ, user_stack,
 			    sizeof(struct kvm_ia64_vcpu_stack))) {
 			printk(KERN_INFO "KVM_IA64_VCPU_SET_STACK: "
 			       "Illegal user address for stack\n");
 			goto out;
 		}
 		stack = kmalloc(sizeof(struct kvm_ia64_vcpu_stack), GFP_KERNEL);
 		if (!stack) {
 			r = -ENOMEM;
 			goto out;
 		}
 		if (copy_from_user(stack, user_stack,
 				   sizeof(struct kvm_ia64_vcpu_stack)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_stack(vcpu, stack);
 		break;
 	}
 	default:
 		r = -EINVAL;
 	}
 out:
 	kfree(stack);
 	return r;
 }
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 		struct kvm_memory_slot *memslot,
 		struct kvm_memory_slot old,
 		struct kvm_userspace_memory_region *mem,
 		int user_alloc)
 {
 	unsigned long i;
 	unsigned long pfn;
 	int npages = memslot->npages;
 	unsigned long base_gfn = memslot->base_gfn;
 	if (base_gfn + npages > (KVM_MAX_MEM_SIZE >> PAGE_SHIFT))
 		return -ENOMEM;
 	for (i = 0; i < npages; i++) {
 		pfn = gfn_to_pfn(kvm, base_gfn + i);
 		if (!kvm_is_mmio_pfn(pfn)) {
 			kvm_set_pmt_entry(kvm, base_gfn + i,
 					pfn << PAGE_SHIFT,
 				_PAGE_AR_RWX | _PAGE_MA_WB);
 			memslot->rmap[i] = (unsigned long)pfn_to_page(pfn);
 		} else {
 			kvm_set_pmt_entry(kvm, base_gfn + i,
 					GPFN_PHYS_MMIO | (pfn << PAGE_SHIFT),
 					_PAGE_MA_UC);
 			memslot->rmap[i] = 0;
 			}
 	}
 	return 0;
 }
 void kvm_arch_commit_memory_region(struct kvm *kvm,
 		struct kvm_userspace_memory_region *mem,
 		struct kvm_memory_slot old,
 		int user_alloc)
 {
 	return;
 }
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
 	kvm_flush_remote_tlbs(kvm);
 }
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
 	return -EINVAL;
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	kvm_vcpu_uninit(vcpu);
 }
 static int vti_cpu_has_kvm_support(void)
 {
 	long  avail = 1, status = 1, control = 1;
 	long ret;
 	ret = ia64_pal_proc_get_features(&avail, &status, &control, 0);
 	if (ret)
 		goto out;
 	if (!(avail & PAL_PROC_VM_BIT))
 		goto out;
 	printk(KERN_DEBUG"kvm: Hardware Supports VT\n");
 	ret = ia64_pal_vp_env_info(&kvm_vm_buffer_size, &vp_env_info);
 	if (ret)
 		goto out;
 	printk(KERN_DEBUG"kvm: VM Buffer Size:0x%lx\n", kvm_vm_buffer_size);
 	if (!(vp_env_info & VP_OPCODE)) {
 		printk(KERN_WARNING"kvm: No opcode ability on hardware, "
 				"vm_env_info:0x%lx\n", vp_env_info);
 	}
 	return 1;
 out:
 	return 0;
 }
 /*
  * On SN2, the ITC isn't stable, so copy in fast path code to use the
  * SN2 RTC, replacing the ITC based default verion.
  */
 static void kvm_patch_vmm(struct kvm_vmm_info *vmm_info,
 			  struct module *module)
 {
 	unsigned long new_ar, new_ar_sn2;
 	unsigned long module_base;
 	if (!ia64_platform_is("sn2"))
 		return;
 	module_base = (unsigned long)module->module_core;
 	new_ar = kvm_vmm_base + vmm_info->patch_mov_ar - module_base;
 	new_ar_sn2 = kvm_vmm_base + vmm_info->patch_mov_ar_sn2 - module_base;
 	printk(KERN_INFO "kvm: Patching ITC emulation to use SGI SN2 RTC "
 	       "as source\n");
 	/*
 	 * Copy the SN2 version of mov_ar into place. They are both
 	 * the same size, so 6 bundles is sufficient (6 * 0x10).
 	 */
 	memcpy((void *)new_ar, (void *)new_ar_sn2, 0x60);
 }
 static int kvm_relocate_vmm(struct kvm_vmm_info *vmm_info,
 			    struct module *module)
 {
 	unsigned long module_base;
 	unsigned long vmm_size;
 	unsigned long vmm_offset, func_offset, fdesc_offset;
 	struct fdesc *p_fdesc;
 	BUG_ON(!module);
 	if (!kvm_vmm_base) {
 		printk("kvm: kvm area hasn't been initilized yet!!\n");
 		return -EFAULT;
 	}
 	/*Calculate new position of relocated vmm module.*/
 	module_base = (unsigned long)module->module_core;
 	vmm_size = module->core_size;
 	if (unlikely(vmm_size > KVM_VMM_SIZE))
 		return -EFAULT;
 	memcpy((void *)kvm_vmm_base, (void *)module_base, vmm_size);
 	kvm_patch_vmm(vmm_info, module);
 	kvm_flush_icache(kvm_vmm_base, vmm_size);
 	/*Recalculate kvm_vmm_info based on new VMM*/
 	vmm_offset = vmm_info->vmm_ivt - module_base;
 	kvm_vmm_info->vmm_ivt = KVM_VMM_BASE + vmm_offset;
 	printk(KERN_DEBUG"kvm: Relocated VMM's IVT Base Addr:%lx\n",
 			kvm_vmm_info->vmm_ivt);
 	fdesc_offset = (unsigned long)vmm_info->vmm_entry - module_base;
 	kvm_vmm_info->vmm_entry = (kvm_vmm_entry *)(KVM_VMM_BASE +
 							fdesc_offset);
 	func_offset = *(unsigned long *)vmm_info->vmm_entry - module_base;
 	p_fdesc = (struct fdesc *)(kvm_vmm_base + fdesc_offset);
 	p_fdesc->ip = KVM_VMM_BASE + func_offset;
 	p_fdesc->gp = KVM_VMM_BASE+(p_fdesc->gp - module_base);
 	printk(KERN_DEBUG"kvm: Relocated VMM's Init Entry Addr:%lx\n",
 			KVM_VMM_BASE+func_offset);
 	fdesc_offset = (unsigned long)vmm_info->tramp_entry - module_base;
 	kvm_vmm_info->tramp_entry = (kvm_tramp_entry *)(KVM_VMM_BASE +
 			fdesc_offset);
 	func_offset = *(unsigned long *)vmm_info->tramp_entry - module_base;
 	p_fdesc = (struct fdesc *)(kvm_vmm_base + fdesc_offset);
 	p_fdesc->ip = KVM_VMM_BASE + func_offset;
 	p_fdesc->gp = KVM_VMM_BASE + (p_fdesc->gp - module_base);
 	kvm_vmm_gp = p_fdesc->gp;
 	printk(KERN_DEBUG"kvm: Relocated VMM's Entry IP:%p\n",
 						kvm_vmm_info->vmm_entry);
 	printk(KERN_DEBUG"kvm: Relocated VMM's Trampoline Entry IP:0x%lx\n",
 						KVM_VMM_BASE + func_offset);
 	return 0;
 }
 int kvm_arch_init(void *opaque)
 {
 	int r;
 	struct kvm_vmm_info *vmm_info = (struct kvm_vmm_info *)opaque;
 	if (!vti_cpu_has_kvm_support()) {
 		printk(KERN_ERR "kvm: No Hardware Virtualization Support!\n");
 		r = -EOPNOTSUPP;
 		goto out;
 	}
 	if (kvm_vmm_info) {
 		printk(KERN_ERR "kvm: Already loaded VMM module!\n");
 		r = -EEXIST;
 		goto out;
 	}
 	r = -ENOMEM;
 	kvm_vmm_info = kzalloc(sizeof(struct kvm_vmm_info), GFP_KERNEL);
 	if (!kvm_vmm_info)
 		goto out;
 	if (kvm_alloc_vmm_area())
 		goto out_free0;
 	r = kvm_relocate_vmm(vmm_info, vmm_info->module);
 	if (r)
 		goto out_free1;
 	return 0;
 out_free1:
 	kvm_free_vmm_area();
 out_free0:
 	kfree(kvm_vmm_info);
 out:
 	return r;
 }
 void kvm_arch_exit(void)
 {
 	kvm_free_vmm_area();
 	kfree(kvm_vmm_info);
 	kvm_vmm_info = NULL;
 }
 static int kvm_ia64_sync_dirty_log(struct kvm *kvm,
 		struct kvm_dirty_log *log)
 {
 	struct kvm_memory_slot *memslot;
 	int r, i;
 	long base;
 	unsigned long n;
 	unsigned long *dirty_bitmap = (unsigned long *)(kvm->arch.vm_base +
 			offsetof(struct kvm_vm_data, kvm_mem_dirty_log));
 	r = -EINVAL;
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 	memslot = &kvm->memslots->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
 	n = kvm_dirty_bitmap_bytes(memslot);
 	base = memslot->base_gfn / BITS_PER_LONG;
 	for (i = 0; i < n/sizeof(long); ++i) {
 		memslot->dirty_bitmap[i] = dirty_bitmap[base + i];
 		dirty_bitmap[base + i] = 0;
 	}
 	r = 0;
 out:
 	return r;
 }
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 		struct kvm_dirty_log *log)
 {
 	int r;
 	unsigned long n;
 	struct kvm_memory_slot *memslot;
 	int is_dirty = 0;
 	mutex_lock(&kvm->slots_lock);
 	spin_lock(&kvm->arch.dirty_log_lock);
 	r = kvm_ia64_sync_dirty_log(kvm, log);
 	if (r)
 		goto out;
 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
 	if (r)
 		goto out;
 	/* If nothing is dirty, don't bother messing with page tables. */
 	if (is_dirty) {
 		kvm_flush_remote_tlbs(kvm);
 		memslot = &kvm->memslots->memslots[log->slot];
 		n = kvm_dirty_bitmap_bytes(memslot);
 		memset(memslot->dirty_bitmap, 0, n);
 	}
 	r = 0;
 out:
 	mutex_unlock(&kvm->slots_lock);
 	spin_unlock(&kvm->arch.dirty_log_lock);
 	return r;
 }
 int kvm_arch_hardware_setup(void)
 {
 	return 0;
 }
 void kvm_arch_hardware_unsetup(void)
 {
 }
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 {
 	int me;
 	int cpu = vcpu->cpu;
 	if (waitqueue_active(&vcpu->wq))
 		wake_up_interruptible(&vcpu->wq);
 	me = get_cpu();
 	if (cpu != me && (unsigned) cpu < nr_cpu_ids && cpu_online(cpu))
 		if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
 			smp_send_reschedule(cpu);
 	put_cpu();
 }
 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq)
 {
 	return __apic_accept_irq(vcpu, irq->vector);
 }
 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
 {
 	return apic->vcpu->vcpu_id == dest;
 }
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
 {
 	return 0;
 }
 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
 {
 	return vcpu1->arch.xtp - vcpu2->arch.xtp;
 }
 int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 		int short_hand, int dest, int dest_mode)
 {
 	struct kvm_lapic *target = vcpu->arch.apic;
 	return (dest_mode == 0) ?
 		kvm_apic_match_physical_addr(target, dest) :
 		kvm_apic_match_logical_addr(target, dest);
 }
 static int find_highest_bits(int *dat)
 {
 	u32  bits, bitnum;
 	int i;
 	/* loop for all 256 bits */
 	for (i = 7; i >= 0 ; i--) {
 		bits = dat[i];
 		if (bits) {
 			bitnum = fls(bits);
 			return i * 32 + bitnum - 1;
 		}
 	}
 	return -1;
 }
 int kvm_highest_pending_irq(struct kvm_vcpu *vcpu)
 {
     struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
     if (vpd->irr[0] & (1UL << NMI_VECTOR))
 		return NMI_VECTOR;
     if (vpd->irr[0] & (1UL << ExtINT_VECTOR))
 		return ExtINT_VECTOR;
     return find_highest_bits((int *)&vpd->irr[0]);
 }
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.timer_fired;
 }
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn;
 }
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE) ||
 		(kvm_highest_pending_irq(vcpu) != -1);
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
-	vcpu_load(vcpu);
 	mp_state->mp_state = vcpu->arch.mp_state;
-	vcpu_put(vcpu);
 	return 0;
 }
 static int vcpu_reset(struct kvm_vcpu *vcpu)
 {
 	int r;
 	long psr;
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	if (r)
 		goto fail;
 	vcpu->arch.launched = 0;
 	kvm_arch_vcpu_uninit(vcpu);
 	r = kvm_arch_vcpu_init(vcpu);
 	if (r)
 		goto fail;
 	kvm_purge_vmm_mapping(vcpu);
 	r = 0;
 fail:
 	return r;
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	int r = 0;
-	vcpu_load(vcpu);
 	vcpu->arch.mp_state = mp_state->mp_state;
 	if (vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)
 		r = vcpu_reset(vcpu);
-	vcpu_put(vcpu);
 	return r;
 }

arch/powerpc/kvm/book3s.c

Diff comments View file @ 2122ff5

 /*
  * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
  *
  * Authors:
  *    Alexander Graf <agraf@suse.de>
  *    Kevin Wolf <mail@kevin-wolf.de>
  *
  * Description:
  * This file is derived from arch/powerpc/kvm/44x.c,
  * by Hollis Blanchard <hollisb@us.ibm.com>.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  */
 #include <linux/kvm_host.h>
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <asm/reg.h>
 #include <asm/cputable.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 #include <asm/io.h>
 #include <asm/kvm_ppc.h>
 #include <asm/kvm_book3s.h>
 #include <asm/mmu_context.h>
 #include <linux/gfp.h>
 #include <linux/sched.h>
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 /* #define EXIT_DEBUG */
 /* #define EXIT_DEBUG_SIMPLE */
 /* #define DEBUG_EXT */
 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
 			     ulong msr);
 /* Some compatibility defines */
 #ifdef CONFIG_PPC_BOOK3S_32
 #define MSR_USER32 MSR_USER
 #define MSR_USER64 MSR_USER
 #define HW_PAGE_SIZE PAGE_SIZE
 #endif
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "exits",       VCPU_STAT(sum_exits) },
 	{ "mmio",        VCPU_STAT(mmio_exits) },
 	{ "sig",         VCPU_STAT(signal_exits) },
 	{ "sysc",        VCPU_STAT(syscall_exits) },
 	{ "inst_emu",    VCPU_STAT(emulated_inst_exits) },
 	{ "dec",         VCPU_STAT(dec_exits) },
 	{ "ext_intr",    VCPU_STAT(ext_intr_exits) },
 	{ "queue_intr",  VCPU_STAT(queue_intr) },
 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
 	{ "pf_storage",  VCPU_STAT(pf_storage) },
 	{ "sp_storage",  VCPU_STAT(sp_storage) },
 	{ "pf_instruc",  VCPU_STAT(pf_instruc) },
 	{ "sp_instruc",  VCPU_STAT(sp_instruc) },
 	{ "ld",          VCPU_STAT(ld) },
 	{ "ld_slow",     VCPU_STAT(ld_slow) },
 	{ "st",          VCPU_STAT(st) },
 	{ "st_slow",     VCPU_STAT(st_slow) },
 	{ NULL }
 };
 void kvmppc_core_load_host_debugstate(struct kvm_vcpu *vcpu)
 {
 }
 void kvmppc_core_load_guest_debugstate(struct kvm_vcpu *vcpu)
 {
 }
 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 #ifdef CONFIG_PPC_BOOK3S_64
 	memcpy(to_svcpu(vcpu)->slb, to_book3s(vcpu)->slb_shadow, sizeof(to_svcpu(vcpu)->slb));
 	memcpy(&get_paca()->shadow_vcpu, to_book3s(vcpu)->shadow_vcpu,
 	       sizeof(get_paca()->shadow_vcpu));
 	to_svcpu(vcpu)->slb_max = to_book3s(vcpu)->slb_shadow_max;
 #endif
 #ifdef CONFIG_PPC_BOOK3S_32
 	current->thread.kvm_shadow_vcpu = to_book3s(vcpu)->shadow_vcpu;
 #endif
 }
 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
 {
 #ifdef CONFIG_PPC_BOOK3S_64
 	memcpy(to_book3s(vcpu)->slb_shadow, to_svcpu(vcpu)->slb, sizeof(to_svcpu(vcpu)->slb));
 	memcpy(to_book3s(vcpu)->shadow_vcpu, &get_paca()->shadow_vcpu,
 	       sizeof(get_paca()->shadow_vcpu));
 	to_book3s(vcpu)->slb_shadow_max = to_svcpu(vcpu)->slb_max;
 #endif
 	kvmppc_giveup_ext(vcpu, MSR_FP);
 	kvmppc_giveup_ext(vcpu, MSR_VEC);
 	kvmppc_giveup_ext(vcpu, MSR_VSX);
 }
 #if defined(EXIT_DEBUG)
 static u32 kvmppc_get_dec(struct kvm_vcpu *vcpu)
 {
 	u64 jd = mftb() - vcpu->arch.dec_jiffies;
 	return vcpu->arch.dec - jd;
 }
 #endif
 static void kvmppc_recalc_shadow_msr(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.shadow_msr = vcpu->arch.msr;
 	/* Guest MSR values */
 	vcpu->arch.shadow_msr &= MSR_FE0 | MSR_FE1 | MSR_SF | MSR_SE |
 				 MSR_BE | MSR_DE;
 	/* Process MSR values */
 	vcpu->arch.shadow_msr |= MSR_ME | MSR_RI | MSR_IR | MSR_DR | MSR_PR |
 				 MSR_EE;
 	/* External providers the guest reserved */
 	vcpu->arch.shadow_msr |= (vcpu->arch.msr & vcpu->arch.guest_owned_ext);
 	/* 64-bit Process MSR values */
 #ifdef CONFIG_PPC_BOOK3S_64
 	vcpu->arch.shadow_msr |= MSR_ISF | MSR_HV;
 #endif
 }
 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
 {
 	ulong old_msr = vcpu->arch.msr;
 #ifdef EXIT_DEBUG
 	printk(KERN_INFO "KVM: Set MSR to 0x%llx\n", msr);
 #endif
 	msr &= to_book3s(vcpu)->msr_mask;
 	vcpu->arch.msr = msr;
 	kvmppc_recalc_shadow_msr(vcpu);
 	if (msr & (MSR_WE|MSR_POW)) {
 		if (!vcpu->arch.pending_exceptions) {
 			kvm_vcpu_block(vcpu);
 			vcpu->stat.halt_wakeup++;
 		}
 	}
 	if ((vcpu->arch.msr & (MSR_PR|MSR_IR|MSR_DR)) !=
 		   (old_msr & (MSR_PR|MSR_IR|MSR_DR))) {
 		kvmppc_mmu_flush_segments(vcpu);
 		kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
 	}
 	/* Preload FPU if it's enabled */
 	if (vcpu->arch.msr & MSR_FP)
 		kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
 }
 void kvmppc_inject_interrupt(struct kvm_vcpu *vcpu, int vec, u64 flags)
 {
 	vcpu->arch.srr0 = kvmppc_get_pc(vcpu);
 	vcpu->arch.srr1 = vcpu->arch.msr | flags;
 	kvmppc_set_pc(vcpu, to_book3s(vcpu)->hior + vec);
 	vcpu->arch.mmu.reset_msr(vcpu);
 }
 static int kvmppc_book3s_vec2irqprio(unsigned int vec)
 {
 	unsigned int prio;
 	switch (vec) {
 	case 0x100: prio = BOOK3S_IRQPRIO_SYSTEM_RESET;		break;
 	case 0x200: prio = BOOK3S_IRQPRIO_MACHINE_CHECK;	break;
 	case 0x300: prio = BOOK3S_IRQPRIO_DATA_STORAGE;		break;
 	case 0x380: prio = BOOK3S_IRQPRIO_DATA_SEGMENT;		break;
 	case 0x400: prio = BOOK3S_IRQPRIO_INST_STORAGE;		break;
 	case 0x480: prio = BOOK3S_IRQPRIO_INST_SEGMENT;		break;
 	case 0x500: prio = BOOK3S_IRQPRIO_EXTERNAL;		break;
 	case 0x600: prio = BOOK3S_IRQPRIO_ALIGNMENT;		break;
 	case 0x700: prio = BOOK3S_IRQPRIO_PROGRAM;		break;
 	case 0x800: prio = BOOK3S_IRQPRIO_FP_UNAVAIL;		break;
 	case 0x900: prio = BOOK3S_IRQPRIO_DECREMENTER;		break;
 	case 0xc00: prio = BOOK3S_IRQPRIO_SYSCALL;		break;
 	case 0xd00: prio = BOOK3S_IRQPRIO_DEBUG;		break;
 	case 0xf20: prio = BOOK3S_IRQPRIO_ALTIVEC;		break;
 	case 0xf40: prio = BOOK3S_IRQPRIO_VSX;			break;
 	default:    prio = BOOK3S_IRQPRIO_MAX;			break;
 	}
 	return prio;
 }
 static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu *vcpu,
 					  unsigned int vec)
 {
 	clear_bit(kvmppc_book3s_vec2irqprio(vec),
 		  &vcpu->arch.pending_exceptions);
 }
 void kvmppc_book3s_queue_irqprio(struct kvm_vcpu *vcpu, unsigned int vec)
 {
 	vcpu->stat.queue_intr++;
 	set_bit(kvmppc_book3s_vec2irqprio(vec),
 		&vcpu->arch.pending_exceptions);
 #ifdef EXIT_DEBUG
 	printk(KERN_INFO "Queueing interrupt %x\n", vec);
 #endif
 }
 void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags)
 {
 	to_book3s(vcpu)->prog_flags = flags;
 	kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_PROGRAM);
 }
 void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
 {
 	kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_DECREMENTER);
 }
 int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
 {
 	return test_bit(BOOK3S_INTERRUPT_DECREMENTER >> 7, &vcpu->arch.pending_exceptions);
 }
 void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
 {
 	kvmppc_book3s_dequeue_irqprio(vcpu, BOOK3S_INTERRUPT_DECREMENTER);
 }
 void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
                                 struct kvm_interrupt *irq)
 {
 	kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_EXTERNAL);
 }
 void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu,
                                   struct kvm_interrupt *irq)
 {
 	kvmppc_book3s_dequeue_irqprio(vcpu, BOOK3S_INTERRUPT_EXTERNAL);
 }
 int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority)
 {
 	int deliver = 1;
 	int vec = 0;
 	ulong flags = 0ULL;
 	switch (priority) {
 	case BOOK3S_IRQPRIO_DECREMENTER:
 		deliver = vcpu->arch.msr & MSR_EE;
 		vec = BOOK3S_INTERRUPT_DECREMENTER;
 		break;
 	case BOOK3S_IRQPRIO_EXTERNAL:
 		deliver = vcpu->arch.msr & MSR_EE;
 		vec = BOOK3S_INTERRUPT_EXTERNAL;
 		break;
 	case BOOK3S_IRQPRIO_SYSTEM_RESET:
 		vec = BOOK3S_INTERRUPT_SYSTEM_RESET;
 		break;
 	case BOOK3S_IRQPRIO_MACHINE_CHECK:
 		vec = BOOK3S_INTERRUPT_MACHINE_CHECK;
 		break;
 	case BOOK3S_IRQPRIO_DATA_STORAGE:
 		vec = BOOK3S_INTERRUPT_DATA_STORAGE;
 		break;
 	case BOOK3S_IRQPRIO_INST_STORAGE:
 		vec = BOOK3S_INTERRUPT_INST_STORAGE;
 		break;
 	case BOOK3S_IRQPRIO_DATA_SEGMENT:
 		vec = BOOK3S_INTERRUPT_DATA_SEGMENT;
 		break;
 	case BOOK3S_IRQPRIO_INST_SEGMENT:
 		vec = BOOK3S_INTERRUPT_INST_SEGMENT;
 		break;
 	case BOOK3S_IRQPRIO_ALIGNMENT:
 		vec = BOOK3S_INTERRUPT_ALIGNMENT;
 		break;
 	case BOOK3S_IRQPRIO_PROGRAM:
 		vec = BOOK3S_INTERRUPT_PROGRAM;
 		flags = to_book3s(vcpu)->prog_flags;
 		break;
 	case BOOK3S_IRQPRIO_VSX:
 		vec = BOOK3S_INTERRUPT_VSX;
 		break;
 	case BOOK3S_IRQPRIO_ALTIVEC:
 		vec = BOOK3S_INTERRUPT_ALTIVEC;
 		break;
 	case BOOK3S_IRQPRIO_FP_UNAVAIL:
 		vec = BOOK3S_INTERRUPT_FP_UNAVAIL;
 		break;
 	case BOOK3S_IRQPRIO_SYSCALL:
 		vec = BOOK3S_INTERRUPT_SYSCALL;
 		break;
 	case BOOK3S_IRQPRIO_DEBUG:
 		vec = BOOK3S_INTERRUPT_TRACE;
 		break;
 	case BOOK3S_IRQPRIO_PERFORMANCE_MONITOR:
 		vec = BOOK3S_INTERRUPT_PERFMON;
 		break;
 	default:
 		deliver = 0;
 		printk(KERN_ERR "KVM: Unknown interrupt: 0x%x\n", priority);
 		break;
 	}
 #if 0
 	printk(KERN_INFO "Deliver interrupt 0x%x? %x\n", vec, deliver);
 #endif
 	if (deliver)
 		kvmppc_inject_interrupt(vcpu, vec, flags);
 	return deliver;
 }
 void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu)
 {
 	unsigned long *pending = &vcpu->arch.pending_exceptions;
 	unsigned int priority;
 #ifdef EXIT_DEBUG
 	if (vcpu->arch.pending_exceptions)
 		printk(KERN_EMERG "KVM: Check pending: %lx\n", vcpu->arch.pending_exceptions);
 #endif
 	priority = __ffs(*pending);
 	while (priority < BOOK3S_IRQPRIO_MAX) {
 		if (kvmppc_book3s_irqprio_deliver(vcpu, priority) &&
 		    (priority != BOOK3S_IRQPRIO_DECREMENTER)) {
 			/* DEC interrupts get cleared by mtdec */
 			clear_bit(priority, &vcpu->arch.pending_exceptions);
 			break;
 		}
 		priority = find_next_bit(pending,
 					 BITS_PER_BYTE * sizeof(*pending),
 					 priority + 1);
 	}
 }
 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
 {
 	u32 host_pvr;
 	vcpu->arch.hflags &= ~BOOK3S_HFLAG_SLB;
 	vcpu->arch.pvr = pvr;
 #ifdef CONFIG_PPC_BOOK3S_64
 	if ((pvr >= 0x330000) && (pvr < 0x70330000)) {
 		kvmppc_mmu_book3s_64_init(vcpu);
 		to_book3s(vcpu)->hior = 0xfff00000;
 		to_book3s(vcpu)->msr_mask = 0xffffffffffffffffULL;
 	} else
 #endif
 	{
 		kvmppc_mmu_book3s_32_init(vcpu);
 		to_book3s(vcpu)->hior = 0;
 		to_book3s(vcpu)->msr_mask = 0xffffffffULL;
 	}
 	/* If we are in hypervisor level on 970, we can tell the CPU to
 	 * treat DCBZ as 32 bytes store */
 	vcpu->arch.hflags &= ~BOOK3S_HFLAG_DCBZ32;
 	if (vcpu->arch.mmu.is_dcbz32(vcpu) && (mfmsr() & MSR_HV) &&
 	    !strcmp(cur_cpu_spec->platform, "ppc970"))
 		vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
 	/* Cell performs badly if MSR_FEx are set. So let's hope nobody
 	   really needs them in a VM on Cell and force disable them. */
 	if (!strcmp(cur_cpu_spec->platform, "ppc-cell-be"))
 		to_book3s(vcpu)->msr_mask &= ~(MSR_FE0 | MSR_FE1);
 #ifdef CONFIG_PPC_BOOK3S_32
 	/* 32 bit Book3S always has 32 byte dcbz */
 	vcpu->arch.hflags |= BOOK3S_HFLAG_DCBZ32;
 #endif
 	/* On some CPUs we can execute paired single operations natively */
 	asm ( "mfpvr %0" : "=r"(host_pvr));
 	switch (host_pvr) {
 	case 0x00080200:	/* lonestar 2.0 */
 	case 0x00088202:	/* lonestar 2.2 */
 	case 0x70000100:	/* gekko 1.0 */
 	case 0x00080100:	/* gekko 2.0 */
 	case 0x00083203:	/* gekko 2.3a */
 	case 0x00083213:	/* gekko 2.3b */
 	case 0x00083204:	/* gekko 2.4 */
 	case 0x00083214:	/* gekko 2.4e (8SE) - retail HW2 */
 	case 0x00087200:	/* broadway */
 		vcpu->arch.hflags |= BOOK3S_HFLAG_NATIVE_PS;
 		/* Enable HID2.PSE - in case we need it later */
 		mtspr(SPRN_HID2_GEKKO, mfspr(SPRN_HID2_GEKKO) | (1 << 29));
 	}
 }
 /* Book3s_32 CPUs always have 32 bytes cache line size, which Linux assumes. To
  * make Book3s_32 Linux work on Book3s_64, we have to make sure we trap dcbz to
  * emulate 32 bytes dcbz length.
  *
  * The Book3s_64 inventors also realized this case and implemented a special bit
  * in the HID5 register, which is a hypervisor ressource. Thus we can't use it.
  *
  * My approach here is to patch the dcbz instruction on executing pages.
  */
 static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte)
 {
 	struct page *hpage;
 	u64 hpage_offset;
 	u32 *page;
 	int i;
 	hpage = gfn_to_page(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
 	if (is_error_page(hpage))
 		return;
 	hpage_offset = pte->raddr & ~PAGE_MASK;
 	hpage_offset &= ~0xFFFULL;
 	hpage_offset /= 4;
 	get_page(hpage);
 	page = kmap_atomic(hpage, KM_USER0);
 	/* patch dcbz into reserved instruction, so we trap */
 	for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++)
 		if ((page[i] & 0xff0007ff) == INS_DCBZ)
 			page[i] &= 0xfffffff7;
 	kunmap_atomic(page, KM_USER0);
 	put_page(hpage);
 }
 static int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, bool data,
 			 struct kvmppc_pte *pte)
 {
 	int relocated = (vcpu->arch.msr & (data ? MSR_DR : MSR_IR));
 	int r;
 	if (relocated) {
 		r = vcpu->arch.mmu.xlate(vcpu, eaddr, pte, data);
 	} else {
 		pte->eaddr = eaddr;
 		pte->raddr = eaddr & 0xffffffff;
 		pte->vpage = VSID_REAL | eaddr >> 12;
 		pte->may_read = true;
 		pte->may_write = true;
 		pte->may_execute = true;
 		r = 0;
 	}
 	return r;
 }
 static hva_t kvmppc_bad_hva(void)
 {
 	return PAGE_OFFSET;
 }
 static hva_t kvmppc_pte_to_hva(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte,
 			       bool read)
 {
 	hva_t hpage;
 	if (read && !pte->may_read)
 		goto err;
 	if (!read && !pte->may_write)
 		goto err;
 	hpage = gfn_to_hva(vcpu->kvm, pte->raddr >> PAGE_SHIFT);
 	if (kvm_is_error_hva(hpage))
 		goto err;
 	return hpage | (pte->raddr & ~PAGE_MASK);
 err:
 	return kvmppc_bad_hva();
 }
 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 	      bool data)
 {
 	struct kvmppc_pte pte;
 	vcpu->stat.st++;
 	if (kvmppc_xlate(vcpu, *eaddr, data, &pte))
 		return -ENOENT;
 	*eaddr = pte.raddr;
 	if (!pte.may_write)
 		return -EPERM;
 	if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
 		return EMULATE_DO_MMIO;
 	return EMULATE_DONE;
 }
 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 		      bool data)
 {
 	struct kvmppc_pte pte;
 	hva_t hva = *eaddr;
 	vcpu->stat.ld++;
 	if (kvmppc_xlate(vcpu, *eaddr, data, &pte))
 		goto nopte;
 	*eaddr = pte.raddr;
 	hva = kvmppc_pte_to_hva(vcpu, &pte, true);
 	if (kvm_is_error_hva(hva))
 		goto mmio;
 	if (copy_from_user(ptr, (void __user *)hva, size)) {
 		printk(KERN_INFO "kvmppc_ld at 0x%lx failed\n", hva);
 		goto mmio;
 	}
 	return EMULATE_DONE;
 nopte:
 	return -ENOENT;
 mmio:
 	return EMULATE_DO_MMIO;
 }
 static int kvmppc_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
 {
 	return kvm_is_visible_gfn(vcpu->kvm, gfn);
 }
 int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 			    ulong eaddr, int vec)
 {
 	bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE);
 	int r = RESUME_GUEST;
 	int relocated;
 	int page_found = 0;
 	struct kvmppc_pte pte;
 	bool is_mmio = false;
 	bool dr = (vcpu->arch.msr & MSR_DR) ? true : false;
 	bool ir = (vcpu->arch.msr & MSR_IR) ? true : false;
 	u64 vsid;
 	relocated = data ? dr : ir;
 	/* Resolve real address if translation turned on */
 	if (relocated) {
 		page_found = vcpu->arch.mmu.xlate(vcpu, eaddr, &pte, data);
 	} else {
 		pte.may_execute = true;
 		pte.may_read = true;
 		pte.may_write = true;
 		pte.raddr = eaddr & 0xffffffff;
 		pte.eaddr = eaddr;
 		pte.vpage = eaddr >> 12;
 	}
 	switch (vcpu->arch.msr & (MSR_DR|MSR_IR)) {
 	case 0:
 		pte.vpage |= ((u64)VSID_REAL << (SID_SHIFT - 12));
 		break;
 	case MSR_DR:
 	case MSR_IR:
 		vcpu->arch.mmu.esid_to_vsid(vcpu, eaddr >> SID_SHIFT, &vsid);
 		if ((vcpu->arch.msr & (MSR_DR|MSR_IR)) == MSR_DR)
 			pte.vpage |= ((u64)VSID_REAL_DR << (SID_SHIFT - 12));
 		else
 			pte.vpage |= ((u64)VSID_REAL_IR << (SID_SHIFT - 12));
 		pte.vpage |= vsid;
 		if (vsid == -1)
 			page_found = -EINVAL;
 		break;
 	}
 	if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 	   (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
 		/*
 		 * If we do the dcbz hack, we have to NX on every execution,
 		 * so we can patch the executing code. This renders our guest
 		 * NX-less.
 		 */
 		pte.may_execute = !data;
 	}
 	if (page_found == -ENOENT) {
 		/* Page not found in guest PTE entries */
 		vcpu->arch.dear = kvmppc_get_fault_dar(vcpu);
 		to_book3s(vcpu)->dsisr = to_svcpu(vcpu)->fault_dsisr;
 		vcpu->arch.msr |= (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
 		kvmppc_book3s_queue_irqprio(vcpu, vec);
 	} else if (page_found == -EPERM) {
 		/* Storage protection */
 		vcpu->arch.dear = kvmppc_get_fault_dar(vcpu);
 		to_book3s(vcpu)->dsisr = to_svcpu(vcpu)->fault_dsisr & ~DSISR_NOHPTE;
 		to_book3s(vcpu)->dsisr |= DSISR_PROTFAULT;
 		vcpu->arch.msr |= (to_svcpu(vcpu)->shadow_srr1 & 0x00000000f8000000ULL);
 		kvmppc_book3s_queue_irqprio(vcpu, vec);
 	} else if (page_found == -EINVAL) {
 		/* Page not found in guest SLB */
 		vcpu->arch.dear = kvmppc_get_fault_dar(vcpu);
 		kvmppc_book3s_queue_irqprio(vcpu, vec + 0x80);
 	} else if (!is_mmio &&
 		   kvmppc_visible_gfn(vcpu, pte.raddr >> PAGE_SHIFT)) {
 		/* The guest's PTE is not mapped yet. Map on the host */
 		kvmppc_mmu_map_page(vcpu, &pte);
 		if (data)
 			vcpu->stat.sp_storage++;
 		else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 			(!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32)))
 			kvmppc_patch_dcbz(vcpu, &pte);
 	} else {
 		/* MMIO */
 		vcpu->stat.mmio_exits++;
 		vcpu->arch.paddr_accessed = pte.raddr;
 		r = kvmppc_emulate_mmio(run, vcpu);
 		if ( r == RESUME_HOST_NV )
 			r = RESUME_HOST;
 	}
 	return r;
 }
 static inline int get_fpr_index(int i)
 {
 #ifdef CONFIG_VSX
 	i *= 2;
 #endif
 	return i;
 }
 /* Give up external provider (FPU, Altivec, VSX) */
 void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr)
 {
 	struct thread_struct *t = &current->thread;
 	u64 *vcpu_fpr = vcpu->arch.fpr;
 #ifdef CONFIG_VSX
 	u64 *vcpu_vsx = vcpu->arch.vsr;
 #endif
 	u64 *thread_fpr = (u64*)t->fpr;
 	int i;
 	if (!(vcpu->arch.guest_owned_ext & msr))
 		return;
 #ifdef DEBUG_EXT
 	printk(KERN_INFO "Giving up ext 0x%lx\n", msr);
 #endif
 	switch (msr) {
 	case MSR_FP:
 		giveup_fpu(current);
 		for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
 			vcpu_fpr[i] = thread_fpr[get_fpr_index(i)];
 		vcpu->arch.fpscr = t->fpscr.val;
 		break;
 	case MSR_VEC:
 #ifdef CONFIG_ALTIVEC
 		giveup_altivec(current);
 		memcpy(vcpu->arch.vr, t->vr, sizeof(vcpu->arch.vr));
 		vcpu->arch.vscr = t->vscr;
 #endif
 		break;
 	case MSR_VSX:
 #ifdef CONFIG_VSX
 		__giveup_vsx(current);
 		for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
 			vcpu_vsx[i] = thread_fpr[get_fpr_index(i) + 1];
 #endif
 		break;
 	default:
 		BUG();
 	}
 	vcpu->arch.guest_owned_ext &= ~msr;
 	current->thread.regs->msr &= ~msr;
 	kvmppc_recalc_shadow_msr(vcpu);
 }
 static int kvmppc_read_inst(struct kvm_vcpu *vcpu)
 {
 	ulong srr0 = kvmppc_get_pc(vcpu);
 	u32 last_inst = kvmppc_get_last_inst(vcpu);
 	int ret;
 	ret = kvmppc_ld(vcpu, &srr0, sizeof(u32), &last_inst, false);
 	if (ret == -ENOENT) {
 		vcpu->arch.msr = kvmppc_set_field(vcpu->arch.msr, 33, 33, 1);
 		vcpu->arch.msr = kvmppc_set_field(vcpu->arch.msr, 34, 36, 0);
 		vcpu->arch.msr = kvmppc_set_field(vcpu->arch.msr, 42, 47, 0);
 		kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE);
 		return EMULATE_AGAIN;
 	}
 	return EMULATE_DONE;
 }
 static int kvmppc_check_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr)
 {
 	/* Need to do paired single emulation? */
 	if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE))
 		return EMULATE_DONE;
 	/* Read out the instruction */
 	if (kvmppc_read_inst(vcpu) == EMULATE_DONE)
 		/* Need to emulate */
 		return EMULATE_FAIL;
 	return EMULATE_AGAIN;
 }
 /* Handle external providers (FPU, Altivec, VSX) */
 static int kvmppc_handle_ext(struct kvm_vcpu *vcpu, unsigned int exit_nr,
 			     ulong msr)
 {
 	struct thread_struct *t = &current->thread;
 	u64 *vcpu_fpr = vcpu->arch.fpr;
 #ifdef CONFIG_VSX
 	u64 *vcpu_vsx = vcpu->arch.vsr;
 #endif
 	u64 *thread_fpr = (u64*)t->fpr;
 	int i;
 	/* When we have paired singles, we emulate in software */
 	if (vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)
 		return RESUME_GUEST;
 	if (!(vcpu->arch.msr & msr)) {
 		kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 		return RESUME_GUEST;
 	}
 	/* We already own the ext */
 	if (vcpu->arch.guest_owned_ext & msr) {
 		return RESUME_GUEST;
 	}
 #ifdef DEBUG_EXT
 	printk(KERN_INFO "Loading up ext 0x%lx\n", msr);
 #endif
 	current->thread.regs->msr |= msr;
 	switch (msr) {
 	case MSR_FP:
 		for (i = 0; i < ARRAY_SIZE(vcpu->arch.fpr); i++)
 			thread_fpr[get_fpr_index(i)] = vcpu_fpr[i];
 		t->fpscr.val = vcpu->arch.fpscr;
 		t->fpexc_mode = 0;
 		kvmppc_load_up_fpu();
 		break;
 	case MSR_VEC:
 #ifdef CONFIG_ALTIVEC
 		memcpy(t->vr, vcpu->arch.vr, sizeof(vcpu->arch.vr));
 		t->vscr = vcpu->arch.vscr;
 		t->vrsave = -1;
 		kvmppc_load_up_altivec();
 #endif
 		break;
 	case MSR_VSX:
 #ifdef CONFIG_VSX
 		for (i = 0; i < ARRAY_SIZE(vcpu->arch.vsr); i++)
 			thread_fpr[get_fpr_index(i) + 1] = vcpu_vsx[i];
 		kvmppc_load_up_vsx();
 #endif
 		break;
 	default:
 		BUG();
 	}
 	vcpu->arch.guest_owned_ext |= msr;
 	kvmppc_recalc_shadow_msr(vcpu);
 	return RESUME_GUEST;
 }
 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
                        unsigned int exit_nr)
 {
 	int r = RESUME_HOST;
 	vcpu->stat.sum_exits++;
 	run->exit_reason = KVM_EXIT_UNKNOWN;
 	run->ready_for_interrupt_injection = 1;
 #ifdef EXIT_DEBUG
 	printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | dar=0x%lx | dec=0x%x | msr=0x%lx\n",
 		exit_nr, kvmppc_get_pc(vcpu), kvmppc_get_fault_dar(vcpu),
 		kvmppc_get_dec(vcpu), to_svcpu(vcpu)->shadow_srr1);
 #elif defined (EXIT_DEBUG_SIMPLE)
 	if ((exit_nr != 0x900) && (exit_nr != 0x500))
 		printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | dar=0x%lx | msr=0x%lx\n",
 			exit_nr, kvmppc_get_pc(vcpu), kvmppc_get_fault_dar(vcpu),
 			vcpu->arch.msr);
 #endif
 	kvm_resched(vcpu);
 	switch (exit_nr) {
 	case BOOK3S_INTERRUPT_INST_STORAGE:
 		vcpu->stat.pf_instruc++;
 #ifdef CONFIG_PPC_BOOK3S_32
 		/* We set segments as unused segments when invalidating them. So
 		 * treat the respective fault as segment fault. */
 		if (to_svcpu(vcpu)->sr[kvmppc_get_pc(vcpu) >> SID_SHIFT]
 		    == SR_INVALID) {
 			kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu));
 			r = RESUME_GUEST;
 			break;
 		}
 #endif
 		/* only care about PTEG not found errors, but leave NX alone */
 		if (to_svcpu(vcpu)->shadow_srr1 & 0x40000000) {
 			r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr);
 			vcpu->stat.sp_instruc++;
 		} else if (vcpu->arch.mmu.is_dcbz32(vcpu) &&
 			  (!(vcpu->arch.hflags & BOOK3S_HFLAG_DCBZ32))) {
 			/*
 			 * XXX If we do the dcbz hack we use the NX bit to flush&patch the page,
 			 *     so we can't use the NX bit inside the guest. Let's cross our fingers,
 			 *     that no guest that needs the dcbz hack does NX.
 			 */
 			kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
 			r = RESUME_GUEST;
 		} else {
 			vcpu->arch.msr |= to_svcpu(vcpu)->shadow_srr1 & 0x58000000;
 			kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 			kvmppc_mmu_pte_flush(vcpu, kvmppc_get_pc(vcpu), ~0xFFFUL);
 			r = RESUME_GUEST;
 		}
 		break;
 	case BOOK3S_INTERRUPT_DATA_STORAGE:
 	{
 		ulong dar = kvmppc_get_fault_dar(vcpu);
 		vcpu->stat.pf_storage++;
 #ifdef CONFIG_PPC_BOOK3S_32
 		/* We set segments as unused segments when invalidating them. So
 		 * treat the respective fault as segment fault. */
 		if ((to_svcpu(vcpu)->sr[dar >> SID_SHIFT]) == SR_INVALID) {
 			kvmppc_mmu_map_segment(vcpu, dar);
 			r = RESUME_GUEST;
 			break;
 		}
 #endif
 		/* The only case we need to handle is missing shadow PTEs */
 		if (to_svcpu(vcpu)->fault_dsisr & DSISR_NOHPTE) {
 			r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr);
 		} else {
 			vcpu->arch.dear = dar;
 			to_book3s(vcpu)->dsisr = to_svcpu(vcpu)->fault_dsisr;
 			kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 			kvmppc_mmu_pte_flush(vcpu, vcpu->arch.dear, ~0xFFFUL);
 			r = RESUME_GUEST;
 		}
 		break;
 	}
 	case BOOK3S_INTERRUPT_DATA_SEGMENT:
 		if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_fault_dar(vcpu)) < 0) {
 			vcpu->arch.dear = kvmppc_get_fault_dar(vcpu);
 			kvmppc_book3s_queue_irqprio(vcpu,
 				BOOK3S_INTERRUPT_DATA_SEGMENT);
 		}
 		r = RESUME_GUEST;
 		break;
 	case BOOK3S_INTERRUPT_INST_SEGMENT:
 		if (kvmppc_mmu_map_segment(vcpu, kvmppc_get_pc(vcpu)) < 0) {
 			kvmppc_book3s_queue_irqprio(vcpu,
 				BOOK3S_INTERRUPT_INST_SEGMENT);
 		}
 		r = RESUME_GUEST;
 		break;
 	/* We're good on these - the host merely wanted to get our attention */
 	case BOOK3S_INTERRUPT_DECREMENTER:
 		vcpu->stat.dec_exits++;
 		r = RESUME_GUEST;
 		break;
 	case BOOK3S_INTERRUPT_EXTERNAL:
 		vcpu->stat.ext_intr_exits++;
 		r = RESUME_GUEST;
 		break;
 	case BOOK3S_INTERRUPT_PERFMON:
 		r = RESUME_GUEST;
 		break;
 	case BOOK3S_INTERRUPT_PROGRAM:
 	{
 		enum emulation_result er;
 		ulong flags;
 program_interrupt:
 		flags = to_svcpu(vcpu)->shadow_srr1 & 0x1f0000ull;
 		if (vcpu->arch.msr & MSR_PR) {
 #ifdef EXIT_DEBUG
 			printk(KERN_INFO "Userspace triggered 0x700 exception at 0x%lx (0x%x)\n", kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
 #endif
 			if ((kvmppc_get_last_inst(vcpu) & 0xff0007ff) !=
 			    (INS_DCBZ & 0xfffffff7)) {
 				kvmppc_core_queue_program(vcpu, flags);
 				r = RESUME_GUEST;
 				break;
 			}
 		}
 		vcpu->stat.emulated_inst_exits++;
 		er = kvmppc_emulate_instruction(run, vcpu);
 		switch (er) {
 		case EMULATE_DONE:
 			r = RESUME_GUEST_NV;
 			break;
 		case EMULATE_AGAIN:
 			r = RESUME_GUEST;
 			break;
 		case EMULATE_FAIL:
 			printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
 			       __func__, kvmppc_get_pc(vcpu), kvmppc_get_last_inst(vcpu));
 			kvmppc_core_queue_program(vcpu, flags);
 			r = RESUME_GUEST;
 			break;
 		case EMULATE_DO_MMIO:
 			run->exit_reason = KVM_EXIT_MMIO;
 			r = RESUME_HOST_NV;
 			break;
 		default:
 			BUG();
 		}
 		break;
 	}
 	case BOOK3S_INTERRUPT_SYSCALL:
 		// XXX make user settable
 		if (vcpu->arch.osi_enabled &&
 		    (((u32)kvmppc_get_gpr(vcpu, 3)) == OSI_SC_MAGIC_R3) &&
 		    (((u32)kvmppc_get_gpr(vcpu, 4)) == OSI_SC_MAGIC_R4)) {
 			u64 *gprs = run->osi.gprs;
 			int i;
 			run->exit_reason = KVM_EXIT_OSI;
 			for (i = 0; i < 32; i++)
 				gprs[i] = kvmppc_get_gpr(vcpu, i);
 			vcpu->arch.osi_needed = 1;
 			r = RESUME_HOST_NV;
 		} else {
 			vcpu->stat.syscall_exits++;
 			kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 			r = RESUME_GUEST;
 		}
 		break;
 	case BOOK3S_INTERRUPT_FP_UNAVAIL:
 	case BOOK3S_INTERRUPT_ALTIVEC:
 	case BOOK3S_INTERRUPT_VSX:
 	{
 		int ext_msr = 0;
 		switch (exit_nr) {
 		case BOOK3S_INTERRUPT_FP_UNAVAIL: ext_msr = MSR_FP;  break;
 		case BOOK3S_INTERRUPT_ALTIVEC:    ext_msr = MSR_VEC; break;
 		case BOOK3S_INTERRUPT_VSX:        ext_msr = MSR_VSX; break;
 		}
 		switch (kvmppc_check_ext(vcpu, exit_nr)) {
 		case EMULATE_DONE:
 			/* everything ok - let's enable the ext */
 			r = kvmppc_handle_ext(vcpu, exit_nr, ext_msr);
 			break;
 		case EMULATE_FAIL:
 			/* we need to emulate this instruction */
 			goto program_interrupt;
 			break;
 		default:
 			/* nothing to worry about - go again */
 			break;
 		}
 		break;
 	}
 	case BOOK3S_INTERRUPT_ALIGNMENT:
 		if (kvmppc_read_inst(vcpu) == EMULATE_DONE) {
 			to_book3s(vcpu)->dsisr = kvmppc_alignment_dsisr(vcpu,
 				kvmppc_get_last_inst(vcpu));
 			vcpu->arch.dear = kvmppc_alignment_dar(vcpu,
 				kvmppc_get_last_inst(vcpu));
 			kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 		}
 		r = RESUME_GUEST;
 		break;
 	case BOOK3S_INTERRUPT_MACHINE_CHECK:
 	case BOOK3S_INTERRUPT_TRACE:
 		kvmppc_book3s_queue_irqprio(vcpu, exit_nr);
 		r = RESUME_GUEST;
 		break;
 	default:
 		/* Ugh - bork here! What did we get? */
 		printk(KERN_EMERG "exit_nr=0x%x | pc=0x%lx | msr=0x%lx\n",
 			exit_nr, kvmppc_get_pc(vcpu), to_svcpu(vcpu)->shadow_srr1);
 		r = RESUME_HOST;
 		BUG();
 		break;
 	}
 	if (!(r & RESUME_HOST)) {
 		/* To avoid clobbering exit_reason, only check for signals if
 		 * we aren't already exiting to userspace for some other
 		 * reason. */
 		if (signal_pending(current)) {
 #ifdef EXIT_DEBUG
 			printk(KERN_EMERG "KVM: Going back to host\n");
 #endif
 			vcpu->stat.signal_exits++;
 			run->exit_reason = KVM_EXIT_INTR;
 			r = -EINTR;
 		} else {
 			/* In case an interrupt came in that was triggered
 			 * from userspace (like DEC), we need to check what
 			 * to inject now! */
 			kvmppc_core_deliver_interrupts(vcpu);
 		}
 	}
 #ifdef EXIT_DEBUG
 	printk(KERN_EMERG "KVM exit: vcpu=0x%p pc=0x%lx r=0x%x\n", vcpu, kvmppc_get_pc(vcpu), r);
 #endif
 	return r;
 }
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
-	vcpu_load(vcpu);
 	regs->pc = kvmppc_get_pc(vcpu);
 	regs->cr = kvmppc_get_cr(vcpu);
 	regs->ctr = kvmppc_get_ctr(vcpu);
 	regs->lr = kvmppc_get_lr(vcpu);
 	regs->xer = kvmppc_get_xer(vcpu);
 	regs->msr = vcpu->arch.msr;
 	regs->srr0 = vcpu->arch.srr0;
 	regs->srr1 = vcpu->arch.srr1;
 	regs->pid = vcpu->arch.pid;
 	regs->sprg0 = vcpu->arch.sprg0;
 	regs->sprg1 = vcpu->arch.sprg1;
 	regs->sprg2 = vcpu->arch.sprg2;
 	regs->sprg3 = vcpu->arch.sprg3;
 	regs->sprg5 = vcpu->arch.sprg4;
 	regs->sprg6 = vcpu->arch.sprg5;
 	regs->sprg7 = vcpu->arch.sprg6;
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
-	vcpu_load(vcpu);
 	kvmppc_set_pc(vcpu, regs->pc);
 	kvmppc_set_cr(vcpu, regs->cr);
 	kvmppc_set_ctr(vcpu, regs->ctr);
 	kvmppc_set_lr(vcpu, regs->lr);
 	kvmppc_set_xer(vcpu, regs->xer);
 	kvmppc_set_msr(vcpu, regs->msr);
 	vcpu->arch.srr0 = regs->srr0;
 	vcpu->arch.srr1 = regs->srr1;
 	vcpu->arch.sprg0 = regs->sprg0;
 	vcpu->arch.sprg1 = regs->sprg1;
 	vcpu->arch.sprg2 = regs->sprg2;
 	vcpu->arch.sprg3 = regs->sprg3;
 	vcpu->arch.sprg5 = regs->sprg4;
 	vcpu->arch.sprg6 = regs->sprg5;
 	vcpu->arch.sprg7 = regs->sprg6;
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
 	struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
 	int i;
-	vcpu_load(vcpu);
 	sregs->pvr = vcpu->arch.pvr;
 	sregs->u.s.sdr1 = to_book3s(vcpu)->sdr1;
 	if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
 		for (i = 0; i < 64; i++) {
 			sregs->u.s.ppc64.slb[i].slbe = vcpu3s->slb[i].orige | i;
 			sregs->u.s.ppc64.slb[i].slbv = vcpu3s->slb[i].origv;
 		}
 	} else {
 		for (i = 0; i < 16; i++) {
 			sregs->u.s.ppc32.sr[i] = vcpu3s->sr[i].raw;
 			sregs->u.s.ppc32.sr[i] = vcpu3s->sr[i].raw;
 		}
 		for (i = 0; i < 8; i++) {
 			sregs->u.s.ppc32.ibat[i] = vcpu3s->ibat[i].raw;
 			sregs->u.s.ppc32.dbat[i] = vcpu3s->dbat[i].raw;
 		}
 	}
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
 	struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu);
 	int i;
-	vcpu_load(vcpu);
 	kvmppc_set_pvr(vcpu, sregs->pvr);
 	vcpu3s->sdr1 = sregs->u.s.sdr1;
 	if (vcpu->arch.hflags & BOOK3S_HFLAG_SLB) {
 		for (i = 0; i < 64; i++) {
 			vcpu->arch.mmu.slbmte(vcpu, sregs->u.s.ppc64.slb[i].slbv,
 						    sregs->u.s.ppc64.slb[i].slbe);
 		}
 	} else {
 		for (i = 0; i < 16; i++) {
 			vcpu->arch.mmu.mtsrin(vcpu, i, sregs->u.s.ppc32.sr[i]);
 		}
 		for (i = 0; i < 8; i++) {
 			kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), false,
 				       (u32)sregs->u.s.ppc32.ibat[i]);
 			kvmppc_set_bat(vcpu, &(vcpu3s->ibat[i]), true,
 				       (u32)(sregs->u.s.ppc32.ibat[i] >> 32));
 			kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), false,
 				       (u32)sregs->u.s.ppc32.dbat[i]);
 			kvmppc_set_bat(vcpu, &(vcpu3s->dbat[i]), true,
 				       (u32)(sregs->u.s.ppc32.dbat[i] >> 32));
 		}
 	}
 	/* Flush the MMU after messing with the segments */
 	kvmppc_mmu_pte_flush(vcpu, 0, 0);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
                                   struct kvm_translation *tr)
 {
 	return 0;
 }
 /*
  * Get (and clear) the dirty memory log for a memory slot.
  */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 				      struct kvm_dirty_log *log)
 {
 	struct kvm_memory_slot *memslot;
 	struct kvm_vcpu *vcpu;
 	ulong ga, ga_end;
 	int is_dirty = 0;
 	int r;
 	unsigned long n;
 	mutex_lock(&kvm->slots_lock);
 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
 	if (r)
 		goto out;
 	/* If nothing is dirty, don't bother messing with page tables. */
 	if (is_dirty) {
 		memslot = &kvm->memslots->memslots[log->slot];
 		ga = memslot->base_gfn << PAGE_SHIFT;
 		ga_end = ga + (memslot->npages << PAGE_SHIFT);
 		kvm_for_each_vcpu(n, vcpu, kvm)
 			kvmppc_mmu_pte_pflush(vcpu, ga, ga_end);
 		n = kvm_dirty_bitmap_bytes(memslot);
 		memset(memslot->dirty_bitmap, 0, n);
 	}
 	r = 0;
 out:
 	mutex_unlock(&kvm->slots_lock);
 	return r;
 }
 int kvmppc_core_check_processor_compat(void)
 {
 	return 0;
 }
 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
 {
 	struct kvmppc_vcpu_book3s *vcpu_book3s;
 	struct kvm_vcpu *vcpu;
 	int err = -ENOMEM;
 	vcpu_book3s = vmalloc(sizeof(struct kvmppc_vcpu_book3s));
 	if (!vcpu_book3s)
 		goto out;
 	memset(vcpu_book3s, 0, sizeof(struct kvmppc_vcpu_book3s));
 	vcpu_book3s->shadow_vcpu = (struct kvmppc_book3s_shadow_vcpu *)
 		kzalloc(sizeof(*vcpu_book3s->shadow_vcpu), GFP_KERNEL);
 	if (!vcpu_book3s->shadow_vcpu)
 		goto free_vcpu;
 	vcpu = &vcpu_book3s->vcpu;
 	err = kvm_vcpu_init(vcpu, kvm, id);
 	if (err)
 		goto free_shadow_vcpu;
 	vcpu->arch.host_retip = kvm_return_point;
 	vcpu->arch.host_msr = mfmsr();
 #ifdef CONFIG_PPC_BOOK3S_64
 	/* default to book3s_64 (970fx) */
 	vcpu->arch.pvr = 0x3C0301;
 #else
 	/* default to book3s_32 (750) */
 	vcpu->arch.pvr = 0x84202;
 #endif
 	kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
 	vcpu_book3s->slb_nr = 64;
 	/* remember where some real-mode handlers are */
 	vcpu->arch.trampoline_lowmem = kvmppc_trampoline_lowmem;
 	vcpu->arch.trampoline_enter = kvmppc_trampoline_enter;
 	vcpu->arch.highmem_handler = (ulong)kvmppc_handler_highmem;
 #ifdef CONFIG_PPC_BOOK3S_64
 	vcpu->arch.rmcall = *(ulong*)kvmppc_rmcall;
 #else
 	vcpu->arch.rmcall = (ulong)kvmppc_rmcall;
 #endif
 	vcpu->arch.shadow_msr = MSR_USER64;
 	err = kvmppc_mmu_init(vcpu);
 	if (err < 0)
 		goto free_shadow_vcpu;
 	return vcpu;
 free_shadow_vcpu:
 	kfree(vcpu_book3s->shadow_vcpu);
 free_vcpu:
 	vfree(vcpu_book3s);
 out:
 	return ERR_PTR(err);
 }
 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
 {
 	struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
 	kvm_vcpu_uninit(vcpu);
 	kfree(vcpu_book3s->shadow_vcpu);
 	vfree(vcpu_book3s);
 }
 extern int __kvmppc_vcpu_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
 int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 {
 	int ret;
 	struct thread_struct ext_bkp;
 #ifdef CONFIG_ALTIVEC
 	bool save_vec = current->thread.used_vr;
 #endif
 #ifdef CONFIG_VSX
 	bool save_vsx = current->thread.used_vsr;
 #endif
 	ulong ext_msr;
 	/* No need to go into the guest when all we do is going out */
 	if (signal_pending(current)) {
 		kvm_run->exit_reason = KVM_EXIT_INTR;
 		return -EINTR;
 	}
 	/* Save FPU state in stack */
 	if (current->thread.regs->msr & MSR_FP)
 		giveup_fpu(current);
 	memcpy(ext_bkp.fpr, current->thread.fpr, sizeof(current->thread.fpr));
 	ext_bkp.fpscr = current->thread.fpscr;
 	ext_bkp.fpexc_mode = current->thread.fpexc_mode;
 #ifdef CONFIG_ALTIVEC
 	/* Save Altivec state in stack */
 	if (save_vec) {
 		if (current->thread.regs->msr & MSR_VEC)
 			giveup_altivec(current);
 		memcpy(ext_bkp.vr, current->thread.vr, sizeof(ext_bkp.vr));
 		ext_bkp.vscr = current->thread.vscr;
 		ext_bkp.vrsave = current->thread.vrsave;
 	}
 	ext_bkp.used_vr = current->thread.used_vr;
 #endif
 #ifdef CONFIG_VSX
 	/* Save VSX state in stack */
 	if (save_vsx && (current->thread.regs->msr & MSR_VSX))
 			__giveup_vsx(current);
 	ext_bkp.used_vsr = current->thread.used_vsr;
 #endif
 	/* Remember the MSR with disabled extensions */
 	ext_msr = current->thread.regs->msr;
 	/* XXX we get called with irq disabled - change that! */
 	local_irq_enable();
 	/* Preload FPU if it's enabled */
 	if (vcpu->arch.msr & MSR_FP)
 		kvmppc_handle_ext(vcpu, BOOK3S_INTERRUPT_FP_UNAVAIL, MSR_FP);
 	ret = __kvmppc_vcpu_entry(kvm_run, vcpu);
 	local_irq_disable();
 	current->thread.regs->msr = ext_msr;
 	/* Make sure we save the guest FPU/Altivec/VSX state */
 	kvmppc_giveup_ext(vcpu, MSR_FP);
 	kvmppc_giveup_ext(vcpu, MSR_VEC);
 	kvmppc_giveup_ext(vcpu, MSR_VSX);
 	/* Restore FPU state from stack */
 	memcpy(current->thread.fpr, ext_bkp.fpr, sizeof(ext_bkp.fpr));
 	current->thread.fpscr = ext_bkp.fpscr;
 	current->thread.fpexc_mode = ext_bkp.fpexc_mode;
 #ifdef CONFIG_ALTIVEC
 	/* Restore Altivec state from stack */
 	if (save_vec && current->thread.used_vr) {
 		memcpy(current->thread.vr, ext_bkp.vr, sizeof(ext_bkp.vr));
 		current->thread.vscr = ext_bkp.vscr;
 		current->thread.vrsave= ext_bkp.vrsave;
 	}
 	current->thread.used_vr = ext_bkp.used_vr;
 #endif
 #ifdef CONFIG_VSX
 	current->thread.used_vsr = ext_bkp.used_vsr;
 #endif
 	return ret;
 }
 static int kvmppc_book3s_init(void)
 {
 	return kvm_init(NULL, sizeof(struct kvmppc_vcpu_book3s), 0,
 			THIS_MODULE);
 }
 static void kvmppc_book3s_exit(void)
 {
 	kvm_exit();
 }
 module_init(kvmppc_book3s_init);
 module_exit(kvmppc_book3s_exit);

arch/powerpc/kvm/booke.c

Diff comments View file @ 2122ff5

 /*
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  *
  * Copyright IBM Corp. 2007
  *
  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  */
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/kvm_host.h>
 #include <linux/gfp.h>
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/fs.h>
 #include <asm/cputable.h>
 #include <asm/uaccess.h>
 #include <asm/kvm_ppc.h>
 #include "timing.h"
 #include <asm/cacheflush.h>
 #include "booke.h"
 unsigned long kvmppc_booke_handlers;
 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "mmio",       VCPU_STAT(mmio_exits) },
 	{ "dcr",        VCPU_STAT(dcr_exits) },
 	{ "sig",        VCPU_STAT(signal_exits) },
 	{ "itlb_r",     VCPU_STAT(itlb_real_miss_exits) },
 	{ "itlb_v",     VCPU_STAT(itlb_virt_miss_exits) },
 	{ "dtlb_r",     VCPU_STAT(dtlb_real_miss_exits) },
 	{ "dtlb_v",     VCPU_STAT(dtlb_virt_miss_exits) },
 	{ "sysc",       VCPU_STAT(syscall_exits) },
 	{ "isi",        VCPU_STAT(isi_exits) },
 	{ "dsi",        VCPU_STAT(dsi_exits) },
 	{ "inst_emu",   VCPU_STAT(emulated_inst_exits) },
 	{ "dec",        VCPU_STAT(dec_exits) },
 	{ "ext_intr",   VCPU_STAT(ext_intr_exits) },
 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
 	{ NULL }
 };
 /* TODO: use vcpu_printf() */
 void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
 {
 	int i;
 	printk("pc:   %08lx msr:  %08lx\n", vcpu->arch.pc, vcpu->arch.msr);
 	printk("lr:   %08lx ctr:  %08lx\n", vcpu->arch.lr, vcpu->arch.ctr);
 	printk("srr0: %08lx srr1: %08lx\n", vcpu->arch.srr0, vcpu->arch.srr1);
 	printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
 	for (i = 0; i < 32; i += 4) {
 		printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
 		       kvmppc_get_gpr(vcpu, i),
 		       kvmppc_get_gpr(vcpu, i+1),
 		       kvmppc_get_gpr(vcpu, i+2),
 		       kvmppc_get_gpr(vcpu, i+3));
 	}
 }
 static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
                                        unsigned int priority)
 {
 	set_bit(priority, &vcpu->arch.pending_exceptions);
 }
 static void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
                                         ulong dear_flags, ulong esr_flags)
 {
 	vcpu->arch.queued_dear = dear_flags;
 	vcpu->arch.queued_esr = esr_flags;
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
 }
 static void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu,
                                            ulong dear_flags, ulong esr_flags)
 {
 	vcpu->arch.queued_dear = dear_flags;
 	vcpu->arch.queued_esr = esr_flags;
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
 }
 static void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu,
                                            ulong esr_flags)
 {
 	vcpu->arch.queued_esr = esr_flags;
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
 }
 void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
 {
 	vcpu->arch.queued_esr = esr_flags;
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
 }
 void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
 {
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
 }
 int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
 {
 	return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 }
 void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
 {
 	clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
 }
 void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
                                 struct kvm_interrupt *irq)
 {
 	kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_EXTERNAL);
 }
 void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu,
                                   struct kvm_interrupt *irq)
 {
 	clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
 }
 /* Deliver the interrupt of the corresponding priority, if possible. */
 static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
                                         unsigned int priority)
 {
 	int allowed = 0;
 	ulong msr_mask;
 	bool update_esr = false, update_dear = false;
 	switch (priority) {
 	case BOOKE_IRQPRIO_DTLB_MISS:
 	case BOOKE_IRQPRIO_DATA_STORAGE:
 		update_dear = true;
 		/* fall through */
 	case BOOKE_IRQPRIO_INST_STORAGE:
 	case BOOKE_IRQPRIO_PROGRAM:
 		update_esr = true;
 		/* fall through */
 	case BOOKE_IRQPRIO_ITLB_MISS:
 	case BOOKE_IRQPRIO_SYSCALL:
 	case BOOKE_IRQPRIO_FP_UNAVAIL:
 	case BOOKE_IRQPRIO_SPE_UNAVAIL:
 	case BOOKE_IRQPRIO_SPE_FP_DATA:
 	case BOOKE_IRQPRIO_SPE_FP_ROUND:
 	case BOOKE_IRQPRIO_AP_UNAVAIL:
 	case BOOKE_IRQPRIO_ALIGNMENT:
 		allowed = 1;
 		msr_mask = MSR_CE|MSR_ME|MSR_DE;
 		break;
 	case BOOKE_IRQPRIO_CRITICAL:
 	case BOOKE_IRQPRIO_WATCHDOG:
 		allowed = vcpu->arch.msr & MSR_CE;
 		msr_mask = MSR_ME;
 		break;
 	case BOOKE_IRQPRIO_MACHINE_CHECK:
 		allowed = vcpu->arch.msr & MSR_ME;
 		msr_mask = 0;
 		break;
 	case BOOKE_IRQPRIO_EXTERNAL:
 	case BOOKE_IRQPRIO_DECREMENTER:
 	case BOOKE_IRQPRIO_FIT:
 		allowed = vcpu->arch.msr & MSR_EE;
 		msr_mask = MSR_CE|MSR_ME|MSR_DE;
 		break;
 	case BOOKE_IRQPRIO_DEBUG:
 		allowed = vcpu->arch.msr & MSR_DE;
 		msr_mask = MSR_ME;
 		break;
 	}
 	if (allowed) {
 		vcpu->arch.srr0 = vcpu->arch.pc;
 		vcpu->arch.srr1 = vcpu->arch.msr;
 		vcpu->arch.pc = vcpu->arch.ivpr | vcpu->arch.ivor[priority];
 		if (update_esr == true)
 			vcpu->arch.esr = vcpu->arch.queued_esr;
 		if (update_dear == true)
 			vcpu->arch.dear = vcpu->arch.queued_dear;
 		kvmppc_set_msr(vcpu, vcpu->arch.msr & msr_mask);
 		clear_bit(priority, &vcpu->arch.pending_exceptions);
 	}
 	return allowed;
 }
 /* Check pending exceptions and deliver one, if possible. */
 void kvmppc_core_deliver_interrupts(struct kvm_vcpu *vcpu)
 {
 	unsigned long *pending = &vcpu->arch.pending_exceptions;
 	unsigned int priority;
 	priority = __ffs(*pending);
 	while (priority <= BOOKE_IRQPRIO_MAX) {
 		if (kvmppc_booke_irqprio_deliver(vcpu, priority))
 			break;
 		priority = find_next_bit(pending,
 		                         BITS_PER_BYTE * sizeof(*pending),
 		                         priority + 1);
 	}
 }
 /**
  * kvmppc_handle_exit
  *
  * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
  */
 int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
                        unsigned int exit_nr)
 {
 	enum emulation_result er;
 	int r = RESUME_HOST;
 	/* update before a new last_exit_type is rewritten */
 	kvmppc_update_timing_stats(vcpu);
 	local_irq_enable();
 	run->exit_reason = KVM_EXIT_UNKNOWN;
 	run->ready_for_interrupt_injection = 1;
 	switch (exit_nr) {
 	case BOOKE_INTERRUPT_MACHINE_CHECK:
 		printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
 		kvmppc_dump_vcpu(vcpu);
 		r = RESUME_HOST;
 		break;
 	case BOOKE_INTERRUPT_EXTERNAL:
 		kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
 		if (need_resched())
 			cond_resched();
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_DECREMENTER:
 		/* Since we switched IVPR back to the host's value, the host
 		 * handled this interrupt the moment we enabled interrupts.
 		 * Now we just offer it a chance to reschedule the guest. */
 		kvmppc_account_exit(vcpu, DEC_EXITS);
 		if (need_resched())
 			cond_resched();
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_PROGRAM:
 		if (vcpu->arch.msr & MSR_PR) {
 			/* Program traps generated by user-level software must be handled
 			 * by the guest kernel. */
 			kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
 			r = RESUME_GUEST;
 			kvmppc_account_exit(vcpu, USR_PR_INST);
 			break;
 		}
 		er = kvmppc_emulate_instruction(run, vcpu);
 		switch (er) {
 		case EMULATE_DONE:
 			/* don't overwrite subtypes, just account kvm_stats */
 			kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
 			/* Future optimization: only reload non-volatiles if
 			 * they were actually modified by emulation. */
 			r = RESUME_GUEST_NV;
 			break;
 		case EMULATE_DO_DCR:
 			run->exit_reason = KVM_EXIT_DCR;
 			r = RESUME_HOST;
 			break;
 		case EMULATE_FAIL:
 			/* XXX Deliver Program interrupt to guest. */
 			printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
 			       __func__, vcpu->arch.pc, vcpu->arch.last_inst);
 			/* For debugging, encode the failing instruction and
 			 * report it to userspace. */
 			run->hw.hardware_exit_reason = ~0ULL << 32;
 			run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
 			r = RESUME_HOST;
 			break;
 		default:
 			BUG();
 		}
 		break;
 	case BOOKE_INTERRUPT_FP_UNAVAIL:
 		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
 		kvmppc_account_exit(vcpu, FP_UNAVAIL);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_SPE_UNAVAIL:
 		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_UNAVAIL);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_SPE_FP_DATA:
 		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_SPE_FP_ROUND:
 		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_DATA_STORAGE:
 		kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
 		                               vcpu->arch.fault_esr);
 		kvmppc_account_exit(vcpu, DSI_EXITS);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_INST_STORAGE:
 		kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
 		kvmppc_account_exit(vcpu, ISI_EXITS);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_SYSCALL:
 		kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
 		kvmppc_account_exit(vcpu, SYSCALL_EXITS);
 		r = RESUME_GUEST;
 		break;
 	case BOOKE_INTERRUPT_DTLB_MISS: {
 		unsigned long eaddr = vcpu->arch.fault_dear;
 		int gtlb_index;
 		gpa_t gpaddr;
 		gfn_t gfn;
 		/* Check the guest TLB. */
 		gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
 		if (gtlb_index < 0) {
 			/* The guest didn't have a mapping for it. */
 			kvmppc_core_queue_dtlb_miss(vcpu,
 			                            vcpu->arch.fault_dear,
 			                            vcpu->arch.fault_esr);
 			kvmppc_mmu_dtlb_miss(vcpu);
 			kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
 			r = RESUME_GUEST;
 			break;
 		}
 		gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
 		gfn = gpaddr >> PAGE_SHIFT;
 		if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
 			/* The guest TLB had a mapping, but the shadow TLB
 			 * didn't, and it is RAM. This could be because:
 			 * a) the entry is mapping the host kernel, or
 			 * b) the guest used a large mapping which we're faking
 			 * Either way, we need to satisfy the fault without
 			 * invoking the guest. */
 			kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
 			kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
 			r = RESUME_GUEST;
 		} else {
 			/* Guest has mapped and accessed a page which is not
 			 * actually RAM. */
 			vcpu->arch.paddr_accessed = gpaddr;
 			r = kvmppc_emulate_mmio(run, vcpu);
 			kvmppc_account_exit(vcpu, MMIO_EXITS);
 		}
 		break;
 	}
 	case BOOKE_INTERRUPT_ITLB_MISS: {
 		unsigned long eaddr = vcpu->arch.pc;
 		gpa_t gpaddr;
 		gfn_t gfn;
 		int gtlb_index;
 		r = RESUME_GUEST;
 		/* Check the guest TLB. */
 		gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
 		if (gtlb_index < 0) {
 			/* The guest didn't have a mapping for it. */
 			kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
 			kvmppc_mmu_itlb_miss(vcpu);
 			kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
 			break;
 		}
 		kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
 		gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
 		gfn = gpaddr >> PAGE_SHIFT;
 		if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
 			/* The guest TLB had a mapping, but the shadow TLB
 			 * didn't. This could be because:
 			 * a) the entry is mapping the host kernel, or
 			 * b) the guest used a large mapping which we're faking
 			 * Either way, we need to satisfy the fault without
 			 * invoking the guest. */
 			kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
 		} else {
 			/* Guest mapped and leaped at non-RAM! */
 			kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
 		}
 		break;
 	}
 	case BOOKE_INTERRUPT_DEBUG: {
 		u32 dbsr;
 		vcpu->arch.pc = mfspr(SPRN_CSRR0);
 		/* clear IAC events in DBSR register */
 		dbsr = mfspr(SPRN_DBSR);
 		dbsr &= DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4;
 		mtspr(SPRN_DBSR, dbsr);
 		run->exit_reason = KVM_EXIT_DEBUG;
 		kvmppc_account_exit(vcpu, DEBUG_EXITS);
 		r = RESUME_HOST;
 		break;
 	}
 	default:
 		printk(KERN_EMERG "exit_nr %d\n", exit_nr);
 		BUG();
 	}
 	local_irq_disable();
 	kvmppc_core_deliver_interrupts(vcpu);
 	if (!(r & RESUME_HOST)) {
 		/* To avoid clobbering exit_reason, only check for signals if
 		 * we aren't already exiting to userspace for some other
 		 * reason. */
 		if (signal_pending(current)) {
 			run->exit_reason = KVM_EXIT_INTR;
 			r = (-EINTR << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
 			kvmppc_account_exit(vcpu, SIGNAL_EXITS);
 		}
 	}
 	return r;
 }
 /* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.pc = 0;
 	vcpu->arch.msr = 0;
 	kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
 	vcpu->arch.shadow_pid = 1;
 	/* Eye-catching number so we know if the guest takes an interrupt
 	 * before it's programmed its own IVPR. */
 	vcpu->arch.ivpr = 0x55550000;
 	kvmppc_init_timing_stats(vcpu);
 	return kvmppc_core_vcpu_setup(vcpu);
 }
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
-	vcpu_load(vcpu);
 	regs->pc = vcpu->arch.pc;
 	regs->cr = kvmppc_get_cr(vcpu);
 	regs->ctr = vcpu->arch.ctr;
 	regs->lr = vcpu->arch.lr;
 	regs->xer = kvmppc_get_xer(vcpu);
 	regs->msr = vcpu->arch.msr;
 	regs->srr0 = vcpu->arch.srr0;
 	regs->srr1 = vcpu->arch.srr1;
 	regs->pid = vcpu->arch.pid;
 	regs->sprg0 = vcpu->arch.sprg0;
 	regs->sprg1 = vcpu->arch.sprg1;
 	regs->sprg2 = vcpu->arch.sprg2;
 	regs->sprg3 = vcpu->arch.sprg3;
 	regs->sprg5 = vcpu->arch.sprg4;
 	regs->sprg6 = vcpu->arch.sprg5;
 	regs->sprg7 = vcpu->arch.sprg6;
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
-	vcpu_load(vcpu);
 	vcpu->arch.pc = regs->pc;
 	kvmppc_set_cr(vcpu, regs->cr);
 	vcpu->arch.ctr = regs->ctr;
 	vcpu->arch.lr = regs->lr;
 	kvmppc_set_xer(vcpu, regs->xer);
 	kvmppc_set_msr(vcpu, regs->msr);
 	vcpu->arch.srr0 = regs->srr0;
 	vcpu->arch.srr1 = regs->srr1;
 	vcpu->arch.sprg0 = regs->sprg0;
 	vcpu->arch.sprg1 = regs->sprg1;
 	vcpu->arch.sprg2 = regs->sprg2;
 	vcpu->arch.sprg3 = regs->sprg3;
 	vcpu->arch.sprg5 = regs->sprg4;
 	vcpu->arch.sprg6 = regs->sprg5;
 	vcpu->arch.sprg7 = regs->sprg6;
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -ENOTSUPP;
 }
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
                                   struct kvm_translation *tr)
 {
 	int r;
-	vcpu_load(vcpu);
 	r = kvmppc_core_vcpu_translate(vcpu, tr);
-	vcpu_put(vcpu);
 	return r;
 }
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
 {
 	return -ENOTSUPP;
 }
 int __init kvmppc_booke_init(void)
 {
 	unsigned long ivor[16];
 	unsigned long max_ivor = 0;
 	int i;
 	/* We install our own exception handlers by hijacking IVPR. IVPR must
 	 * be 16-bit aligned, so we need a 64KB allocation. */
 	kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
 	                                         VCPU_SIZE_ORDER);
 	if (!kvmppc_booke_handlers)
 		return -ENOMEM;
 	/* XXX make sure our handlers are smaller than Linux's */
 	/* Copy our interrupt handlers to match host IVORs. That way we don't
 	 * have to swap the IVORs on every guest/host transition. */
 	ivor[0] = mfspr(SPRN_IVOR0);
 	ivor[1] = mfspr(SPRN_IVOR1);
 	ivor[2] = mfspr(SPRN_IVOR2);
 	ivor[3] = mfspr(SPRN_IVOR3);
 	ivor[4] = mfspr(SPRN_IVOR4);
 	ivor[5] = mfspr(SPRN_IVOR5);
 	ivor[6] = mfspr(SPRN_IVOR6);
 	ivor[7] = mfspr(SPRN_IVOR7);
 	ivor[8] = mfspr(SPRN_IVOR8);
 	ivor[9] = mfspr(SPRN_IVOR9);
 	ivor[10] = mfspr(SPRN_IVOR10);
 	ivor[11] = mfspr(SPRN_IVOR11);
 	ivor[12] = mfspr(SPRN_IVOR12);
 	ivor[13] = mfspr(SPRN_IVOR13);
 	ivor[14] = mfspr(SPRN_IVOR14);
 	ivor[15] = mfspr(SPRN_IVOR15);
 	for (i = 0; i < 16; i++) {
 		if (ivor[i] > max_ivor)
 			max_ivor = ivor[i];
 		memcpy((void *)kvmppc_booke_handlers + ivor[i],
 		       kvmppc_handlers_start + i * kvmppc_handler_len,
 		       kvmppc_handler_len);
 	}
 	flush_icache_range(kvmppc_booke_handlers,
 	                   kvmppc_booke_handlers + max_ivor + kvmppc_handler_len);
 	return 0;
 }
 void __exit kvmppc_booke_exit(void)
 {
 	free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
 	kvm_exit();
 }

arch/powerpc/kvm/powerpc.c

Diff comments View file @ 2122ff5

 /*
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 2, as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  *
  * Copyright IBM Corp. 2007
  *
  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  */
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/kvm_host.h>
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/hrtimer.h>
 #include <linux/fs.h>
 #include <linux/slab.h>
 #include <asm/cputable.h>
 #include <asm/uaccess.h>
 #include <asm/kvm_ppc.h>
 #include <asm/tlbflush.h>
 #include "timing.h"
 #include "../mm/mmu_decl.h"
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn;
 }
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 {
 	return !(v->arch.msr & MSR_WE) || !!(v->arch.pending_exceptions);
 }
 int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu)
 {
 	enum emulation_result er;
 	int r;
 	er = kvmppc_emulate_instruction(run, vcpu);
 	switch (er) {
 	case EMULATE_DONE:
 		/* Future optimization: only reload non-volatiles if they were
 		 * actually modified. */
 		r = RESUME_GUEST_NV;
 		break;
 	case EMULATE_DO_MMIO:
 		run->exit_reason = KVM_EXIT_MMIO;
 		/* We must reload nonvolatiles because "update" load/store
 		 * instructions modify register state. */
 		/* Future optimization: only reload non-volatiles if they were
 		 * actually modified. */
 		r = RESUME_HOST_NV;
 		break;
 	case EMULATE_FAIL:
 		/* XXX Deliver Program interrupt to guest. */
 		printk(KERN_EMERG "%s: emulation failed (%08x)\n", __func__,
 		       kvmppc_get_last_inst(vcpu));
 		r = RESUME_HOST;
 		break;
 	default:
 		BUG();
 	}
 	return r;
 }
 int kvm_arch_hardware_enable(void *garbage)
 {
 	return 0;
 }
 void kvm_arch_hardware_disable(void *garbage)
 {
 }
 int kvm_arch_hardware_setup(void)
 {
 	return 0;
 }
 void kvm_arch_hardware_unsetup(void)
 {
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 	*(int *)rtn = kvmppc_core_check_processor_compat();
 }
 struct kvm *kvm_arch_create_vm(void)
 {
 	struct kvm *kvm;
 	kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 	if (!kvm)
 		return ERR_PTR(-ENOMEM);
 	return kvm;
 }
 static void kvmppc_free_vcpus(struct kvm *kvm)
 {
 	unsigned int i;
 	struct kvm_vcpu *vcpu;
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		kvm_arch_vcpu_free(vcpu);
 	mutex_lock(&kvm->lock);
 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 		kvm->vcpus[i] = NULL;
 	atomic_set(&kvm->online_vcpus, 0);
 	mutex_unlock(&kvm->lock);
 }
 void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvmppc_free_vcpus(kvm);
 	kvm_free_physmem(kvm);
 	cleanup_srcu_struct(&kvm->srcu);
 	kfree(kvm);
 }
 int kvm_dev_ioctl_check_extension(long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_PPC_SEGSTATE:
 	case KVM_CAP_PPC_PAIRED_SINGLES:
 	case KVM_CAP_PPC_UNSET_IRQ:
 	case KVM_CAP_ENABLE_CAP:
 	case KVM_CAP_PPC_OSI:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 	default:
 		r = 0;
 		break;
 	}
 	return r;
 }
 long kvm_arch_dev_ioctl(struct file *filp,
                         unsigned int ioctl, unsigned long arg)
 {
 	return -EINVAL;
 }
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
                                    struct kvm_memory_slot *memslot,
                                    struct kvm_memory_slot old,
                                    struct kvm_userspace_memory_region *mem,
                                    int user_alloc)
 {
 	return 0;
 }
 void kvm_arch_commit_memory_region(struct kvm *kvm,
                struct kvm_userspace_memory_region *mem,
                struct kvm_memory_slot old,
                int user_alloc)
 {
        return;
 }
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
 {
 	struct kvm_vcpu *vcpu;
 	vcpu = kvmppc_core_vcpu_create(kvm, id);
 	if (!IS_ERR(vcpu))
 		kvmppc_create_vcpu_debugfs(vcpu, id);
 	return vcpu;
 }
 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
 {
 	/* Make sure we're not using the vcpu anymore */
 	hrtimer_cancel(&vcpu->arch.dec_timer);
 	tasklet_kill(&vcpu->arch.tasklet);
 	kvmppc_remove_vcpu_debugfs(vcpu);
 	kvmppc_core_vcpu_free(vcpu);
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	kvm_arch_vcpu_free(vcpu);
 }
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
 	return kvmppc_core_pending_dec(vcpu);
 }
 static void kvmppc_decrementer_func(unsigned long data)
 {
 	struct kvm_vcpu *vcpu = (struct kvm_vcpu *)data;
 	kvmppc_core_queue_dec(vcpu);
 	if (waitqueue_active(&vcpu->wq)) {
 		wake_up_interruptible(&vcpu->wq);
 		vcpu->stat.halt_wakeup++;
 	}
 }
 /*
  * low level hrtimer wake routine. Because this runs in hardirq context
  * we schedule a tasklet to do the real work.
  */
 enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer)
 {
 	struct kvm_vcpu *vcpu;
 	vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer);
 	tasklet_schedule(&vcpu->arch.tasklet);
 	return HRTIMER_NORESTART;
 }
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 	tasklet_init(&vcpu->arch.tasklet, kvmppc_decrementer_func, (ulong)vcpu);
 	vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
 	return 0;
 }
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	kvmppc_mmu_destroy(vcpu);
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	kvmppc_core_vcpu_load(vcpu, cpu);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	kvmppc_core_vcpu_put(vcpu);
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
                                         struct kvm_guest_debug *dbg)
 {
 	return -EINVAL;
 }
 static void kvmppc_complete_dcr_load(struct kvm_vcpu *vcpu,
                                      struct kvm_run *run)
 {
 	kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, run->dcr.data);
 }
 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
                                       struct kvm_run *run)
 {
 	u64 gpr;
 	if (run->mmio.len > sizeof(gpr)) {
 		printk(KERN_ERR "bad MMIO length: %d\n", run->mmio.len);
 		return;
 	}
 	if (vcpu->arch.mmio_is_bigendian) {
 		switch (run->mmio.len) {
 		case 8: gpr = *(u64 *)run->mmio.data; break;
 		case 4: gpr = *(u32 *)run->mmio.data; break;
 		case 2: gpr = *(u16 *)run->mmio.data; break;
 		case 1: gpr = *(u8 *)run->mmio.data; break;
 		}
 	} else {
 		/* Convert BE data from userland back to LE. */
 		switch (run->mmio.len) {
 		case 4: gpr = ld_le32((u32 *)run->mmio.data); break;
 		case 2: gpr = ld_le16((u16 *)run->mmio.data); break;
 		case 1: gpr = *(u8 *)run->mmio.data; break;
 		}
 	}
 	if (vcpu->arch.mmio_sign_extend) {
 		switch (run->mmio.len) {
 #ifdef CONFIG_PPC64
 		case 4:
 			gpr = (s64)(s32)gpr;
 			break;
 #endif
 		case 2:
 			gpr = (s64)(s16)gpr;
 			break;
 		case 1:
 			gpr = (s64)(s8)gpr;
 			break;
 		}
 	}
 	kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
 	switch (vcpu->arch.io_gpr & KVM_REG_EXT_MASK) {
 	case KVM_REG_GPR:
 		kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
 		break;
 	case KVM_REG_FPR:
 		vcpu->arch.fpr[vcpu->arch.io_gpr & KVM_REG_MASK] = gpr;
 		break;
 #ifdef CONFIG_PPC_BOOK3S
 	case KVM_REG_QPR:
 		vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_REG_MASK] = gpr;
 		break;
 	case KVM_REG_FQPR:
 		vcpu->arch.fpr[vcpu->arch.io_gpr & KVM_REG_MASK] = gpr;
 		vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_REG_MASK] = gpr;
 		break;
 #endif
 	default:
 		BUG();
 	}
 }
 int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
                        unsigned int rt, unsigned int bytes, int is_bigendian)
 {
 	if (bytes > sizeof(run->mmio.data)) {
 		printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
 		       run->mmio.len);
 	}
 	run->mmio.phys_addr = vcpu->arch.paddr_accessed;
 	run->mmio.len = bytes;
 	run->mmio.is_write = 0;
 	vcpu->arch.io_gpr = rt;
 	vcpu->arch.mmio_is_bigendian = is_bigendian;
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_is_write = 0;
 	vcpu->arch.mmio_sign_extend = 0;
 	return EMULATE_DO_MMIO;
 }
 /* Same as above, but sign extends */
 int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu,
                         unsigned int rt, unsigned int bytes, int is_bigendian)
 {
 	int r;
 	r = kvmppc_handle_load(run, vcpu, rt, bytes, is_bigendian);
 	vcpu->arch.mmio_sign_extend = 1;
 	return r;
 }
 int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
                         u64 val, unsigned int bytes, int is_bigendian)
 {
 	void *data = run->mmio.data;
 	if (bytes > sizeof(run->mmio.data)) {
 		printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
 		       run->mmio.len);
 	}
 	run->mmio.phys_addr = vcpu->arch.paddr_accessed;
 	run->mmio.len = bytes;
 	run->mmio.is_write = 1;
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_is_write = 1;
 	/* Store the value at the lowest bytes in 'data'. */
 	if (is_bigendian) {
 		switch (bytes) {
 		case 8: *(u64 *)data = val; break;
 		case 4: *(u32 *)data = val; break;
 		case 2: *(u16 *)data = val; break;
 		case 1: *(u8  *)data = val; break;
 		}
 	} else {
 		/* Store LE value into 'data'. */
 		switch (bytes) {
 		case 4: st_le32(data, val); break;
 		case 2: st_le16(data, val); break;
 		case 1: *(u8 *)data = val; break;
 		}
 	}
 	return EMULATE_DO_MMIO;
 }
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	int r;
 	sigset_t sigsaved;
-	vcpu_load(vcpu);
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	if (vcpu->mmio_needed) {
 		if (!vcpu->mmio_is_write)
 			kvmppc_complete_mmio_load(vcpu, run);
 		vcpu->mmio_needed = 0;
 	} else if (vcpu->arch.dcr_needed) {
 		if (!vcpu->arch.dcr_is_write)
 			kvmppc_complete_dcr_load(vcpu, run);
 		vcpu->arch.dcr_needed = 0;
 	} else if (vcpu->arch.osi_needed) {
 		u64 *gprs = run->osi.gprs;
 		int i;
 		for (i = 0; i < 32; i++)
 			kvmppc_set_gpr(vcpu, i, gprs[i]);
 		vcpu->arch.osi_needed = 0;
 	}
 	kvmppc_core_deliver_interrupts(vcpu);
 	local_irq_disable();
 	kvm_guest_enter();
 	r = __kvmppc_vcpu_run(run, vcpu);
 	kvm_guest_exit();
 	local_irq_enable();
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
-	vcpu_put(vcpu);
 	return r;
 }
 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
 {
 	if (irq->irq == KVM_INTERRUPT_UNSET)
 		kvmppc_core_dequeue_external(vcpu, irq);
 	else
 		kvmppc_core_queue_external(vcpu, irq);
 	if (waitqueue_active(&vcpu->wq)) {
 		wake_up_interruptible(&vcpu->wq);
 		vcpu->stat.halt_wakeup++;
 	}
 	return 0;
 }
 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
 				     struct kvm_enable_cap *cap)
 {
 	int r;
 	if (cap->flags)
 		return -EINVAL;
 	switch (cap->cap) {
 	case KVM_CAP_PPC_OSI:
 		r = 0;
 		vcpu->arch.osi_enabled = true;
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
                                     struct kvm_mp_state *mp_state)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
                                     struct kvm_mp_state *mp_state)
 {
 	return -EINVAL;
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
                          unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	long r;
 	switch (ioctl) {
 	case KVM_INTERRUPT: {
 		struct kvm_interrupt irq;
 		r = -EFAULT;
 		if (copy_from_user(&irq, argp, sizeof(irq)))
 			goto out;
 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
 		break;
 	}
 	case KVM_ENABLE_CAP:
 	{
 		struct kvm_enable_cap cap;
 		r = -EFAULT;
 		if (copy_from_user(&cap, argp, sizeof(cap)))
 			goto out;
 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
 		break;
 	}
 	default:
 		r = -EINVAL;
 	}
 out:
 	return r;
 }
 long kvm_arch_vm_ioctl(struct file *filp,
                        unsigned int ioctl, unsigned long arg)
 {
 	long r;
 	switch (ioctl) {
 	default:
 		r = -ENOTTY;
 	}
 	return r;
 }
 int kvm_arch_init(void *opaque)
 {
 	return 0;
 }
 void kvm_arch_exit(void)
 {
 }

arch/s390/kvm/kvm-s390.c

Diff comments View file @ 2122ff5

 /*
  * s390host.c --  hosting zSeries kernel virtual machines
  *
  * Copyright IBM Corp. 2008,2009
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License (version 2 only)
  * as published by the Free Software Foundation.
  *
  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  *               Christian Borntraeger <borntraeger@de.ibm.com>
  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  */
 #include <linux/compiler.h>
 #include <linux/err.h>
 #include <linux/fs.h>
 #include <linux/hrtimer.h>
 #include <linux/init.h>
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/timer.h>
 #include <asm/asm-offsets.h>
 #include <asm/lowcore.h>
 #include <asm/pgtable.h>
 #include <asm/nmi.h>
 #include <asm/system.h>
 #include "kvm-s390.h"
 #include "gaccess.h"
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "userspace_handled", VCPU_STAT(exit_userspace) },
 	{ "exit_null", VCPU_STAT(exit_null) },
 	{ "exit_validity", VCPU_STAT(exit_validity) },
 	{ "exit_stop_request", VCPU_STAT(exit_stop_request) },
 	{ "exit_external_request", VCPU_STAT(exit_external_request) },
 	{ "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
 	{ "exit_instruction", VCPU_STAT(exit_instruction) },
 	{ "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
 	{ "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
 	{ "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
 	{ "instruction_lctl", VCPU_STAT(instruction_lctl) },
 	{ "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
 	{ "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
 	{ "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
 	{ "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
 	{ "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
 	{ "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
 	{ "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
 	{ "exit_wait_state", VCPU_STAT(exit_wait_state) },
 	{ "instruction_stidp", VCPU_STAT(instruction_stidp) },
 	{ "instruction_spx", VCPU_STAT(instruction_spx) },
 	{ "instruction_stpx", VCPU_STAT(instruction_stpx) },
 	{ "instruction_stap", VCPU_STAT(instruction_stap) },
 	{ "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
 	{ "instruction_stsch", VCPU_STAT(instruction_stsch) },
 	{ "instruction_chsc", VCPU_STAT(instruction_chsc) },
 	{ "instruction_stsi", VCPU_STAT(instruction_stsi) },
 	{ "instruction_stfl", VCPU_STAT(instruction_stfl) },
 	{ "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 	{ "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 	{ "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 	{ "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 	{ "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 	{ "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 	{ "diagnose_44", VCPU_STAT(diagnose_44) },
 	{ NULL }
 };
 static unsigned long long *facilities;
 /* Section: not file related */
 int kvm_arch_hardware_enable(void *garbage)
 {
 	/* every s390 is virtualization enabled ;-) */
 	return 0;
 }
 void kvm_arch_hardware_disable(void *garbage)
 {
 }
 int kvm_arch_hardware_setup(void)
 {
 	return 0;
 }
 void kvm_arch_hardware_unsetup(void)
 {
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 }
 int kvm_arch_init(void *opaque)
 {
 	return 0;
 }
 void kvm_arch_exit(void)
 {
 }
 /* Section: device related */
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
 	if (ioctl == KVM_S390_ENABLE_SIE)
 		return s390_enable_sie();
 	return -EINVAL;
 }
 int kvm_dev_ioctl_check_extension(long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_S390_PSW:
 		r = 1;
 		break;
 	default:
 		r = 0;
 	}
 	return r;
 }
 /* Section: vm related */
 /*
  * Get (and clear) the dirty memory log for a memory slot.
  */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 			       struct kvm_dirty_log *log)
 {
 	return 0;
 }
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	switch (ioctl) {
 	case KVM_S390_INTERRUPT: {
 		struct kvm_s390_interrupt s390int;
 		r = -EFAULT;
 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
 			break;
 		r = kvm_s390_inject_vm(kvm, &s390int);
 		break;
 	}
 	default:
 		r = -ENOTTY;
 	}
 	return r;
 }
 struct kvm *kvm_arch_create_vm(void)
 {
 	struct kvm *kvm;
 	int rc;
 	char debug_name[16];
 	rc = s390_enable_sie();
 	if (rc)
 		goto out_nokvm;
 	rc = -ENOMEM;
 	kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 	if (!kvm)
 		goto out_nokvm;
 	kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 	if (!kvm->arch.sca)
 		goto out_nosca;
 	sprintf(debug_name, "kvm-%u", current->pid);
 	kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 	if (!kvm->arch.dbf)
 		goto out_nodbf;
 	spin_lock_init(&kvm->arch.float_int.lock);
 	INIT_LIST_HEAD(&kvm->arch.float_int.list);
 	debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 	VM_EVENT(kvm, 3, "%s", "vm created");
 	return kvm;
 out_nodbf:
 	free_page((unsigned long)(kvm->arch.sca));
 out_nosca:
 	kfree(kvm);
 out_nokvm:
 	return ERR_PTR(rc);
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 	if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 		(__u64) vcpu->arch.sie_block)
 		vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 	smp_mb();
 	free_page((unsigned long)(vcpu->arch.sie_block));
 	kvm_vcpu_uninit(vcpu);
 	kfree(vcpu);
 }
 static void kvm_free_vcpus(struct kvm *kvm)
 {
 	unsigned int i;
 	struct kvm_vcpu *vcpu;
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		kvm_arch_vcpu_destroy(vcpu);
 	mutex_lock(&kvm->lock);
 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 		kvm->vcpus[i] = NULL;
 	atomic_set(&kvm->online_vcpus, 0);
 	mutex_unlock(&kvm->lock);
 }
 void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvm_free_vcpus(kvm);
 	kvm_free_physmem(kvm);
 	free_page((unsigned long)(kvm->arch.sca));
 	debug_unregister(kvm->arch.dbf);
 	cleanup_srcu_struct(&kvm->srcu);
 	kfree(kvm);
 }
 /* Section: vcpu related */
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	/* Nothing todo */
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	save_fp_regs(&vcpu->arch.host_fpregs);
 	save_access_regs(vcpu->arch.host_acrs);
 	vcpu->arch.guest_fpregs.fpc &= FPC_VALID_MASK;
 	restore_fp_regs(&vcpu->arch.guest_fpregs);
 	restore_access_regs(vcpu->arch.guest_acrs);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	save_fp_regs(&vcpu->arch.guest_fpregs);
 	save_access_regs(vcpu->arch.guest_acrs);
 	restore_fp_regs(&vcpu->arch.host_fpregs);
 	restore_access_regs(vcpu->arch.host_acrs);
 }
 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 {
 	/* this equals initial cpu reset in pop, but we don't switch to ESA */
 	vcpu->arch.sie_block->gpsw.mask = 0UL;
 	vcpu->arch.sie_block->gpsw.addr = 0UL;
 	vcpu->arch.sie_block->prefix    = 0UL;
 	vcpu->arch.sie_block->ihcpu     = 0xffff;
 	vcpu->arch.sie_block->cputm     = 0UL;
 	vcpu->arch.sie_block->ckc       = 0UL;
 	vcpu->arch.sie_block->todpr     = 0;
 	memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 	vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 	vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 	vcpu->arch.guest_fpregs.fpc = 0;
 	asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 	vcpu->arch.sie_block->gbea = 1;
 }
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH);
 	set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests);
 	vcpu->arch.sie_block->ecb   = 2;
 	vcpu->arch.sie_block->eca   = 0xC1002001U;
 	vcpu->arch.sie_block->fac   = (int) (long) facilities;
 	hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 	tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
 		     (unsigned long) vcpu);
 	vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 	get_cpu_id(&vcpu->arch.cpu_id);
 	vcpu->arch.cpu_id.version = 0xff;
 	return 0;
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 				      unsigned int id)
 {
 	struct kvm_vcpu *vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL);
 	int rc = -ENOMEM;
 	if (!vcpu)
 		goto out_nomem;
 	vcpu->arch.sie_block = (struct kvm_s390_sie_block *)
 					get_zeroed_page(GFP_KERNEL);
 	if (!vcpu->arch.sie_block)
 		goto out_free_cpu;
 	vcpu->arch.sie_block->icpua = id;
 	BUG_ON(!kvm->arch.sca);
 	if (!kvm->arch.sca->cpu[id].sda)
 		kvm->arch.sca->cpu[id].sda = (__u64) vcpu->arch.sie_block;
 	vcpu->arch.sie_block->scaoh = (__u32)(((__u64)kvm->arch.sca) >> 32);
 	vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 	spin_lock_init(&vcpu->arch.local_int.lock);
 	INIT_LIST_HEAD(&vcpu->arch.local_int.list);
 	vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 	spin_lock(&kvm->arch.float_int.lock);
 	kvm->arch.float_int.local_int[id] = &vcpu->arch.local_int;
 	init_waitqueue_head(&vcpu->arch.local_int.wq);
 	vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 	spin_unlock(&kvm->arch.float_int.lock);
 	rc = kvm_vcpu_init(vcpu, kvm, id);
 	if (rc)
 		goto out_free_sie_block;
 	VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 		 vcpu->arch.sie_block);
 	return vcpu;
 out_free_sie_block:
 	free_page((unsigned long)(vcpu->arch.sie_block));
 out_free_cpu:
 	kfree(vcpu);
 out_nomem:
 	return ERR_PTR(rc);
 }
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	/* kvm common code refers to this, but never calls it */
 	BUG();
 	return 0;
 }
 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 {
 	vcpu_load(vcpu);
 	kvm_s390_vcpu_initial_reset(vcpu);
 	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
-	vcpu_load(vcpu);
 	memcpy(&vcpu->arch.guest_gprs, &regs->gprs, sizeof(regs->gprs));
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
-	vcpu_load(vcpu);
 	memcpy(&regs->gprs, &vcpu->arch.guest_gprs, sizeof(regs->gprs));
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
-	vcpu_load(vcpu);
 	memcpy(&vcpu->arch.guest_acrs, &sregs->acrs, sizeof(sregs->acrs));
 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
-	vcpu_load(vcpu);
 	memcpy(&sregs->acrs, &vcpu->arch.guest_acrs, sizeof(sregs->acrs));
 	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	vcpu_load(vcpu);
 	memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 	vcpu->arch.guest_fpregs.fpc = fpu->fpc;
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	vcpu_load(vcpu);
 	memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 	fpu->fpc = vcpu->arch.guest_fpregs.fpc;
-	vcpu_put(vcpu);
 	return 0;
 }
 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 {
 	int rc = 0;
 	vcpu_load(vcpu);
 	if (atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_RUNNING)
 		rc = -EBUSY;
 	else {
 		vcpu->run->psw_mask = psw.mask;
 		vcpu->run->psw_addr = psw.addr;
 	}
 	vcpu_put(vcpu);
 	return rc;
 }
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 				  struct kvm_translation *tr)
 {
 	return -EINVAL; /* not implemented yet */
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
 	return -EINVAL; /* not implemented yet */
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	return -EINVAL; /* not implemented yet */
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	return -EINVAL; /* not implemented yet */
 }
 static void __vcpu_run(struct kvm_vcpu *vcpu)
 {
 	memcpy(&vcpu->arch.sie_block->gg14, &vcpu->arch.guest_gprs[14], 16);
 	if (need_resched())
 		schedule();
 	if (test_thread_flag(TIF_MCCK_PENDING))
 		s390_handle_mcck();
 	kvm_s390_deliver_pending_interrupts(vcpu);
 	vcpu->arch.sie_block->icptcode = 0;
 	local_irq_disable();
 	kvm_guest_enter();
 	local_irq_enable();
 	VCPU_EVENT(vcpu, 6, "entering sie flags %x",
 		   atomic_read(&vcpu->arch.sie_block->cpuflags));
 	if (sie64a(vcpu->arch.sie_block, vcpu->arch.guest_gprs)) {
 		VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
 		kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
 	}
 	VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
 		   vcpu->arch.sie_block->icptcode);
 	local_irq_disable();
 	kvm_guest_exit();
 	local_irq_enable();
 	memcpy(&vcpu->arch.guest_gprs[14], &vcpu->arch.sie_block->gg14, 16);
 }
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int rc;
 	sigset_t sigsaved;
-	vcpu_load(vcpu);
 rerun_vcpu:
 	if (vcpu->requests)
 		if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
 			kvm_s390_vcpu_set_mem(vcpu);
 	/* verify, that memory has been registered */
 	if (!vcpu->arch.sie_block->gmslm) {
 		vcpu_put(vcpu);
 		VCPU_EVENT(vcpu, 3, "%s", "no memory registered to run vcpu");
 		return -EINVAL;
 	}
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 	BUG_ON(vcpu->kvm->arch.float_int.local_int[vcpu->vcpu_id] == NULL);
 	switch (kvm_run->exit_reason) {
 	case KVM_EXIT_S390_SIEIC:
 	case KVM_EXIT_UNKNOWN:
 	case KVM_EXIT_INTR:
 	case KVM_EXIT_S390_RESET:
 		break;
 	default:
 		BUG();
 	}
 	vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
 	vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
 	might_fault();
 	do {
 		__vcpu_run(vcpu);
 		rc = kvm_handle_sie_intercept(vcpu);
 	} while (!signal_pending(current) && !rc);
 	if (rc == SIE_INTERCEPT_RERUNVCPU)
 		goto rerun_vcpu;
 	if (signal_pending(current) && !rc) {
 		kvm_run->exit_reason = KVM_EXIT_INTR;
 		rc = -EINTR;
 	}
 	if (rc == -EOPNOTSUPP) {
 		/* intercept cannot be handled in-kernel, prepare kvm-run */
 		kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
 		kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
 		kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
 		kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
 		rc = 0;
 	}
 	if (rc == -EREMOTE) {
 		/* intercept was handled, but userspace support is needed
 		 * kvm_run has been prepared by the handler */
 		rc = 0;
 	}
 	kvm_run->psw_mask     = vcpu->arch.sie_block->gpsw.mask;
 	kvm_run->psw_addr     = vcpu->arch.sie_block->gpsw.addr;
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
-	vcpu_put(vcpu);
 	vcpu->stat.exit_userspace++;
 	return rc;
 }
 static int __guestcopy(struct kvm_vcpu *vcpu, u64 guestdest, const void *from,
 		       unsigned long n, int prefix)
 {
 	if (prefix)
 		return copy_to_guest(vcpu, guestdest, from, n);
 	else
 		return copy_to_guest_absolute(vcpu, guestdest, from, n);
 }
 /*
  * store status at address
  * we use have two special cases:
  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
  */
 int __kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
 {
 	const unsigned char archmode = 1;
 	int prefix;
 	if (addr == KVM_S390_STORE_STATUS_NOADDR) {
 		if (copy_to_guest_absolute(vcpu, 163ul, &archmode, 1))
 			return -EFAULT;
 		addr = SAVE_AREA_BASE;
 		prefix = 0;
 	} else if (addr == KVM_S390_STORE_STATUS_PREFIXED) {
 		if (copy_to_guest(vcpu, 163ul, &archmode, 1))
 			return -EFAULT;
 		addr = SAVE_AREA_BASE;
 		prefix = 1;
 	} else
 		prefix = 0;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, fp_regs),
 			vcpu->arch.guest_fpregs.fprs, 128, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, gp_regs),
 			vcpu->arch.guest_gprs, 128, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, psw),
 			&vcpu->arch.sie_block->gpsw, 16, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, pref_reg),
 			&vcpu->arch.sie_block->prefix, 4, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu,
 			addr + offsetof(struct save_area, fp_ctrl_reg),
 			&vcpu->arch.guest_fpregs.fpc, 4, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, tod_reg),
 			&vcpu->arch.sie_block->todpr, 4, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, timer),
 			&vcpu->arch.sie_block->cputm, 8, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, clk_cmp),
 			&vcpu->arch.sie_block->ckc, 8, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu, addr + offsetof(struct save_area, acc_regs),
 			&vcpu->arch.guest_acrs, 64, prefix))
 		return -EFAULT;
 	if (__guestcopy(vcpu,
 			addr + offsetof(struct save_area, ctrl_regs),
 			&vcpu->arch.sie_block->gcr, 128, prefix))
 		return -EFAULT;
 	return 0;
 }
 static int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
 {
 	int rc;
 	vcpu_load(vcpu);
 	rc = __kvm_s390_vcpu_store_status(vcpu, addr);
 	vcpu_put(vcpu);
 	return rc;
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	switch (ioctl) {
 	case KVM_S390_INTERRUPT: {
 		struct kvm_s390_interrupt s390int;
 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
 			return -EFAULT;
 		return kvm_s390_inject_vcpu(vcpu, &s390int);
 	}
 	case KVM_S390_STORE_STATUS:
 		return kvm_s390_vcpu_store_status(vcpu, arg);
 	case KVM_S390_SET_INITIAL_PSW: {
 		psw_t psw;
 		if (copy_from_user(&psw, argp, sizeof(psw)))
 			return -EFAULT;
 		return kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
 	}
 	case KVM_S390_INITIAL_RESET:
 		return kvm_arch_vcpu_ioctl_initial_reset(vcpu);
 	default:
 		;
 	}
 	return -EINVAL;
 }
 /* Section: memory related */
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 				   struct kvm_memory_slot *memslot,
 				   struct kvm_memory_slot old,
 				   struct kvm_userspace_memory_region *mem,
 				   int user_alloc)
 {
 	/* A few sanity checks. We can have exactly one memory slot which has
 	   to start at guest virtual zero and which has to be located at a
 	   page boundary in userland and which has to end at a page boundary.
 	   The memory in userland is ok to be fragmented into various different
 	   vmas. It is okay to mmap() and munmap() stuff in this slot after
 	   doing this call at any time */
 	if (mem->slot)
 		return -EINVAL;
 	if (mem->guest_phys_addr)
 		return -EINVAL;
 	if (mem->userspace_addr & (PAGE_SIZE - 1))
 		return -EINVAL;
 	if (mem->memory_size & (PAGE_SIZE - 1))
 		return -EINVAL;
 	if (!user_alloc)
 		return -EINVAL;
 	return 0;
 }
 void kvm_arch_commit_memory_region(struct kvm *kvm,
 				struct kvm_userspace_memory_region *mem,
 				struct kvm_memory_slot old,
 				int user_alloc)
 {
 	int i;
 	struct kvm_vcpu *vcpu;
 	/* request update of sie control block for all available vcpus */
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
 			continue;
 		kvm_s390_inject_sigp_stop(vcpu, ACTION_RELOADVCPU_ON_STOP);
 	}
 }
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
 }
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn;
 }
 static int __init kvm_s390_init(void)
 {
 	int ret;
 	ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
 	if (ret)
 		return ret;
 	/*
 	 * guests can ask for up to 255+1 double words, we need a full page
 	 * to hold the maximum amount of facilites. On the other hand, we
 	 * only set facilities that are known to work in KVM.
 	 */
 	facilities = (unsigned long long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
 	if (!facilities) {
 		kvm_exit();
 		return -ENOMEM;
 	}
 	stfle(facilities, 1);
 	facilities[0] &= 0xff00fff3f0700000ULL;
 	return 0;
 }
 static void __exit kvm_s390_exit(void)
 {
 	free_page((unsigned long) facilities);
 	kvm_exit();
 }
 module_init(kvm_s390_init);
 module_exit(kvm_s390_exit);

arch/x86/kvm/x86.c

Diff comments View file @ 2122ff5

 /*
  * Kernel-based Virtual Machine driver for Linux
  *
  * derived from drivers/kvm/kvm_main.c
  *
  * Copyright (C) 2006 Qumranet, Inc.
  * Copyright (C) 2008 Qumranet, Inc.
  * Copyright IBM Corporation, 2008
  *
  * Authors:
  *   Avi Kivity   <avi@qumranet.com>
  *   Yaniv Kamay  <yaniv@qumranet.com>
  *   Amit Shah    <amit.shah@qumranet.com>
  *   Ben-Ami Yassour <benami@il.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  *
  */
 #include <linux/kvm_host.h>
 #include "irq.h"
 #include "mmu.h"
 #include "i8254.h"
 #include "tss.h"
 #include "kvm_cache_regs.h"
 #include "x86.h"
 #include <linux/clocksource.h>
 #include <linux/interrupt.h>
 #include <linux/kvm.h>
 #include <linux/fs.h>
 #include <linux/vmalloc.h>
 #include <linux/module.h>
 #include <linux/mman.h>
 #include <linux/highmem.h>
 #include <linux/iommu.h>
 #include <linux/intel-iommu.h>
 #include <linux/cpufreq.h>
 #include <linux/user-return-notifier.h>
 #include <linux/srcu.h>
 #include <linux/slab.h>
 #include <linux/perf_event.h>
 #include <trace/events/kvm.h>
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 #include <asm/debugreg.h>
 #include <asm/uaccess.h>
 #include <asm/msr.h>
 #include <asm/desc.h>
 #include <asm/mtrr.h>
 #include <asm/mce.h>
 #define MAX_IO_MSRS 256
 #define CR0_RESERVED_BITS						\
 	(~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \
 			  | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \
 			  | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG))
 #define CR4_RESERVED_BITS						\
 	(~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\
 			  | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE	\
 			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR	\
 			  | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE))
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
 #define KVM_MAX_MCE_BANKS 32
 #define KVM_MCE_CAP_SUPPORTED MCG_CTL_P
 /* EFER defaults:
  * - enable syscall per default because its emulated by KVM
  * - enable LME and LMA per default on 64 bit KVM
  */
 #ifdef CONFIG_X86_64
 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffafeULL;
 #else
 static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL;
 #endif
 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
 				    struct kvm_cpuid_entry2 __user *entries);
 struct kvm_x86_ops *kvm_x86_ops;
 EXPORT_SYMBOL_GPL(kvm_x86_ops);
 int ignore_msrs = 0;
 module_param_named(ignore_msrs, ignore_msrs, bool, S_IRUGO | S_IWUSR);
 #define KVM_NR_SHARED_MSRS 16
 struct kvm_shared_msrs_global {
 	int nr;
 	u32 msrs[KVM_NR_SHARED_MSRS];
 };
 struct kvm_shared_msrs {
 	struct user_return_notifier urn;
 	bool registered;
 	struct kvm_shared_msr_values {
 		u64 host;
 		u64 curr;
 	} values[KVM_NR_SHARED_MSRS];
 };
 static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
 static DEFINE_PER_CPU(struct kvm_shared_msrs, shared_msrs);
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "pf_fixed", VCPU_STAT(pf_fixed) },
 	{ "pf_guest", VCPU_STAT(pf_guest) },
 	{ "tlb_flush", VCPU_STAT(tlb_flush) },
 	{ "invlpg", VCPU_STAT(invlpg) },
 	{ "exits", VCPU_STAT(exits) },
 	{ "io_exits", VCPU_STAT(io_exits) },
 	{ "mmio_exits", VCPU_STAT(mmio_exits) },
 	{ "signal_exits", VCPU_STAT(signal_exits) },
 	{ "irq_window", VCPU_STAT(irq_window_exits) },
 	{ "nmi_window", VCPU_STAT(nmi_window_exits) },
 	{ "halt_exits", VCPU_STAT(halt_exits) },
 	{ "halt_wakeup", VCPU_STAT(halt_wakeup) },
 	{ "hypercalls", VCPU_STAT(hypercalls) },
 	{ "request_irq", VCPU_STAT(request_irq_exits) },
 	{ "irq_exits", VCPU_STAT(irq_exits) },
 	{ "host_state_reload", VCPU_STAT(host_state_reload) },
 	{ "efer_reload", VCPU_STAT(efer_reload) },
 	{ "fpu_reload", VCPU_STAT(fpu_reload) },
 	{ "insn_emulation", VCPU_STAT(insn_emulation) },
 	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
 	{ "irq_injections", VCPU_STAT(irq_injections) },
 	{ "nmi_injections", VCPU_STAT(nmi_injections) },
 	{ "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
 	{ "mmu_pte_write", VM_STAT(mmu_pte_write) },
 	{ "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
 	{ "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
 	{ "mmu_flooded", VM_STAT(mmu_flooded) },
 	{ "mmu_recycled", VM_STAT(mmu_recycled) },
 	{ "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
 	{ "mmu_unsync", VM_STAT(mmu_unsync) },
 	{ "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
 	{ "largepages", VM_STAT(lpages) },
 	{ NULL }
 };
 static void kvm_on_user_return(struct user_return_notifier *urn)
 {
 	unsigned slot;
 	struct kvm_shared_msrs *locals
 		= container_of(urn, struct kvm_shared_msrs, urn);
 	struct kvm_shared_msr_values *values;
 	for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
 		values = &locals->values[slot];
 		if (values->host != values->curr) {
 			wrmsrl(shared_msrs_global.msrs[slot], values->host);
 			values->curr = values->host;
 		}
 	}
 	locals->registered = false;
 	user_return_notifier_unregister(urn);
 }
 static void shared_msr_update(unsigned slot, u32 msr)
 {
 	struct kvm_shared_msrs *smsr;
 	u64 value;
 	smsr = &__get_cpu_var(shared_msrs);
 	/* only read, and nobody should modify it at this time,
 	 * so don't need lock */
 	if (slot >= shared_msrs_global.nr) {
 		printk(KERN_ERR "kvm: invalid MSR slot!");
 		return;
 	}
 	rdmsrl_safe(msr, &value);
 	smsr->values[slot].host = value;
 	smsr->values[slot].curr = value;
 }
 void kvm_define_shared_msr(unsigned slot, u32 msr)
 {
 	if (slot >= shared_msrs_global.nr)
 		shared_msrs_global.nr = slot + 1;
 	shared_msrs_global.msrs[slot] = msr;
 	/* we need ensured the shared_msr_global have been updated */
 	smp_wmb();
 }
 EXPORT_SYMBOL_GPL(kvm_define_shared_msr);
 static void kvm_shared_msr_cpu_online(void)
 {
 	unsigned i;
 	for (i = 0; i < shared_msrs_global.nr; ++i)
 		shared_msr_update(i, shared_msrs_global.msrs[i]);
 }
 void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
 {
 	struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
 	if (((value ^ smsr->values[slot].curr) & mask) == 0)
 		return;
 	smsr->values[slot].curr = value;
 	wrmsrl(shared_msrs_global.msrs[slot], value);
 	if (!smsr->registered) {
 		smsr->urn.on_user_return = kvm_on_user_return;
 		user_return_notifier_register(&smsr->urn);
 		smsr->registered = true;
 	}
 }
 EXPORT_SYMBOL_GPL(kvm_set_shared_msr);
 static void drop_user_return_notifiers(void *ignore)
 {
 	struct kvm_shared_msrs *smsr = &__get_cpu_var(shared_msrs);
 	if (smsr->registered)
 		kvm_on_user_return(&smsr->urn);
 }
 u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
 {
 	if (irqchip_in_kernel(vcpu->kvm))
 		return vcpu->arch.apic_base;
 	else
 		return vcpu->arch.apic_base;
 }
 EXPORT_SYMBOL_GPL(kvm_get_apic_base);
 void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
 {
 	/* TODO: reserve bits check */
 	if (irqchip_in_kernel(vcpu->kvm))
 		kvm_lapic_set_base(vcpu, data);
 	else
 		vcpu->arch.apic_base = data;
 }
 EXPORT_SYMBOL_GPL(kvm_set_apic_base);
 #define EXCPT_BENIGN		0
 #define EXCPT_CONTRIBUTORY	1
 #define EXCPT_PF		2
 static int exception_class(int vector)
 {
 	switch (vector) {
 	case PF_VECTOR:
 		return EXCPT_PF;
 	case DE_VECTOR:
 	case TS_VECTOR:
 	case NP_VECTOR:
 	case SS_VECTOR:
 	case GP_VECTOR:
 		return EXCPT_CONTRIBUTORY;
 	default:
 		break;
 	}
 	return EXCPT_BENIGN;
 }
 static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
 		unsigned nr, bool has_error, u32 error_code,
 		bool reinject)
 {
 	u32 prev_nr;
 	int class1, class2;
 	if (!vcpu->arch.exception.pending) {
 	queue:
 		vcpu->arch.exception.pending = true;
 		vcpu->arch.exception.has_error_code = has_error;
 		vcpu->arch.exception.nr = nr;
 		vcpu->arch.exception.error_code = error_code;
 		vcpu->arch.exception.reinject = reinject;
 		return;
 	}
 	/* to check exception */
 	prev_nr = vcpu->arch.exception.nr;
 	if (prev_nr == DF_VECTOR) {
 		/* triple fault -> shutdown */
 		set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
 		return;
 	}
 	class1 = exception_class(prev_nr);
 	class2 = exception_class(nr);
 	if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
 		|| (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
 		/* generate double fault per SDM Table 5-5 */
 		vcpu->arch.exception.pending = true;
 		vcpu->arch.exception.has_error_code = true;
 		vcpu->arch.exception.nr = DF_VECTOR;
 		vcpu->arch.exception.error_code = 0;
 	} else
 		/* replace previous exception with a new one in a hope
 		   that instruction re-execution will regenerate lost
 		   exception */
 		goto queue;
 }
 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 {
 	kvm_multiple_exception(vcpu, nr, false, 0, false);
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 {
 	kvm_multiple_exception(vcpu, nr, false, 0, true);
 }
 EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr,
 			   u32 error_code)
 {
 	++vcpu->stat.pf_guest;
 	vcpu->arch.cr2 = addr;
 	kvm_queue_exception_e(vcpu, PF_VECTOR, error_code);
 }
 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.nmi_pending = 1;
 }
 EXPORT_SYMBOL_GPL(kvm_inject_nmi);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 {
 	kvm_multiple_exception(vcpu, nr, true, error_code, false);
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 {
 	kvm_multiple_exception(vcpu, nr, true, error_code, true);
 }
 EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
 /*
  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
  * a #GP and return false.
  */
 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
 {
 	if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
 		return true;
 	kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
 	return false;
 }
 EXPORT_SYMBOL_GPL(kvm_require_cpl);
 /*
  * Load the pae pdptrs.  Return true is they are all valid.
  */
 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
 	gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
 	unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
 	int i;
 	int ret;
 	u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 	ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte,
 				  offset * sizeof(u64), sizeof(pdpte));
 	if (ret < 0) {
 		ret = 0;
 		goto out;
 	}
 	for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
 		if (is_present_gpte(pdpte[i]) &&
 		    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
 			ret = 0;
 			goto out;
 		}
 	}
 	ret = 1;
 	memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs));
 	__set_bit(VCPU_EXREG_PDPTR,
 		  (unsigned long *)&vcpu->arch.regs_avail);
 	__set_bit(VCPU_EXREG_PDPTR,
 		  (unsigned long *)&vcpu->arch.regs_dirty);
 out:
 	return ret;
 }
 EXPORT_SYMBOL_GPL(load_pdptrs);
 static bool pdptrs_changed(struct kvm_vcpu *vcpu)
 {
 	u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)];
 	bool changed = true;
 	int r;
 	if (is_long_mode(vcpu) || !is_pae(vcpu))
 		return false;
 	if (!test_bit(VCPU_EXREG_PDPTR,
 		      (unsigned long *)&vcpu->arch.regs_avail))
 		return true;
 	r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte));
 	if (r < 0)
 		goto out;
 	changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0;
 out:
 	return changed;
 }
 static int __kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
 	unsigned long old_cr0 = kvm_read_cr0(vcpu);
 	unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
 				    X86_CR0_CD | X86_CR0_NW;
 	cr0 |= X86_CR0_ET;
 #ifdef CONFIG_X86_64
 	if (cr0 & 0xffffffff00000000UL)
 		return 1;
 #endif
 	cr0 &= ~CR0_RESERVED_BITS;
 	if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
 		return 1;
 	if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
 		return 1;
 	if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
 #ifdef CONFIG_X86_64
 		if ((vcpu->arch.efer & EFER_LME)) {
 			int cs_db, cs_l;
 			if (!is_pae(vcpu))
 				return 1;
 			kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 			if (cs_l)
 				return 1;
 		} else
 #endif
 		if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3))
 			return 1;
 	}
 	kvm_x86_ops->set_cr0(vcpu, cr0);
 	if ((cr0 ^ old_cr0) & update_bits)
 		kvm_mmu_reset_context(vcpu);
 	return 0;
 }
 void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 {
 	if (__kvm_set_cr0(vcpu, cr0))
 		kvm_inject_gp(vcpu, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr0);
 void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
 {
 	kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
 }
 EXPORT_SYMBOL_GPL(kvm_lmsw);
 int __kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 {
 	unsigned long old_cr4 = kvm_read_cr4(vcpu);
 	unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE;
 	if (cr4 & CR4_RESERVED_BITS)
 		return 1;
 	if (is_long_mode(vcpu)) {
 		if (!(cr4 & X86_CR4_PAE))
 			return 1;
 	} else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
 		   && ((cr4 ^ old_cr4) & pdptr_bits)
 		   && !load_pdptrs(vcpu, vcpu->arch.cr3))
 		return 1;
 	if (cr4 & X86_CR4_VMXE)
 		return 1;
 	kvm_x86_ops->set_cr4(vcpu, cr4);
 	if ((cr4 ^ old_cr4) & pdptr_bits)
 		kvm_mmu_reset_context(vcpu);
 	return 0;
 }
 void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 {
 	if (__kvm_set_cr4(vcpu, cr4))
 		kvm_inject_gp(vcpu, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr4);
 static int __kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
 	if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) {
 		kvm_mmu_sync_roots(vcpu);
 		kvm_mmu_flush_tlb(vcpu);
 		return 0;
 	}
 	if (is_long_mode(vcpu)) {
 		if (cr3 & CR3_L_MODE_RESERVED_BITS)
 			return 1;
 	} else {
 		if (is_pae(vcpu)) {
 			if (cr3 & CR3_PAE_RESERVED_BITS)
 				return 1;
 			if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3))
 				return 1;
 		}
 		/*
 		 * We don't check reserved bits in nonpae mode, because
 		 * this isn't enforced, and VMware depends on this.
 		 */
 	}
 	/*
 	 * Does the new cr3 value map to physical memory? (Note, we
 	 * catch an invalid cr3 even in real-mode, because it would
 	 * cause trouble later on when we turn on paging anyway.)
 	 *
 	 * A real CPU would silently accept an invalid cr3 and would
 	 * attempt to use it - with largely undefined (and often hard
 	 * to debug) behavior on the guest side.
 	 */
 	if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
 		return 1;
 	vcpu->arch.cr3 = cr3;
 	vcpu->arch.mmu.new_cr3(vcpu);
 	return 0;
 }
 void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 {
 	if (__kvm_set_cr3(vcpu, cr3))
 		kvm_inject_gp(vcpu, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr3);
 int __kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
 	if (cr8 & CR8_RESERVED_BITS)
 		return 1;
 	if (irqchip_in_kernel(vcpu->kvm))
 		kvm_lapic_set_tpr(vcpu, cr8);
 	else
 		vcpu->arch.cr8 = cr8;
 	return 0;
 }
 void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
 {
 	if (__kvm_set_cr8(vcpu, cr8))
 		kvm_inject_gp(vcpu, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_set_cr8);
 unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
 {
 	if (irqchip_in_kernel(vcpu->kvm))
 		return kvm_lapic_get_cr8(vcpu);
 	else
 		return vcpu->arch.cr8;
 }
 EXPORT_SYMBOL_GPL(kvm_get_cr8);
 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 {
 	switch (dr) {
 	case 0 ... 3:
 		vcpu->arch.db[dr] = val;
 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
 			vcpu->arch.eff_db[dr] = val;
 		break;
 	case 4:
 		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 			return 1; /* #UD */
 		/* fall through */
 	case 6:
 		if (val & 0xffffffff00000000ULL)
 			return -1; /* #GP */
 		vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
 		break;
 	case 5:
 		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 			return 1; /* #UD */
 		/* fall through */
 	default: /* 7 */
 		if (val & 0xffffffff00000000ULL)
 			return -1; /* #GP */
 		vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
 		if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) {
 			kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
 			vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
 		}
 		break;
 	}
 	return 0;
 }
 int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
 {
 	int res;
 	res = __kvm_set_dr(vcpu, dr, val);
 	if (res > 0)
 		kvm_queue_exception(vcpu, UD_VECTOR);
 	else if (res < 0)
 		kvm_inject_gp(vcpu, 0);
 	return res;
 }
 EXPORT_SYMBOL_GPL(kvm_set_dr);
 static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
 {
 	switch (dr) {
 	case 0 ... 3:
 		*val = vcpu->arch.db[dr];
 		break;
 	case 4:
 		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 			return 1;
 		/* fall through */
 	case 6:
 		*val = vcpu->arch.dr6;
 		break;
 	case 5:
 		if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
 			return 1;
 		/* fall through */
 	default: /* 7 */
 		*val = vcpu->arch.dr7;
 		break;
 	}
 	return 0;
 }
 int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
 {
 	if (_kvm_get_dr(vcpu, dr, val)) {
 		kvm_queue_exception(vcpu, UD_VECTOR);
 		return 1;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_get_dr);
 static inline u32 bit(int bitno)
 {
 	return 1 << (bitno & 31);
 }
 /*
  * List of msr numbers which we expose to userspace through KVM_GET_MSRS
  * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
  *
  * This list is modified at module load time to reflect the
  * capabilities of the host cpu. This capabilities test skips MSRs that are
  * kvm-specific. Those are put in the beginning of the list.
  */
 #define KVM_SAVE_MSRS_BEGIN	7
 static u32 msrs_to_save[] = {
 	MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
 	MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
 	HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
 	HV_X64_MSR_APIC_ASSIST_PAGE,
 	MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 	MSR_K6_STAR,
 #ifdef CONFIG_X86_64
 	MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
 #endif
 	MSR_IA32_TSC, MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA
 };
 static unsigned num_msrs_to_save;
 static u32 emulated_msrs[] = {
 	MSR_IA32_MISC_ENABLE,
 };
 static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
 {
 	u64 old_efer = vcpu->arch.efer;
 	if (efer & efer_reserved_bits)
 		return 1;
 	if (is_paging(vcpu)
 	    && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
 		return 1;
 	if (efer & EFER_FFXSR) {
 		struct kvm_cpuid_entry2 *feat;
 		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 		if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
 			return 1;
 	}
 	if (efer & EFER_SVME) {
 		struct kvm_cpuid_entry2 *feat;
 		feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
 		if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
 			return 1;
 	}
 	efer &= ~EFER_LMA;
 	efer |= vcpu->arch.efer & EFER_LMA;
 	kvm_x86_ops->set_efer(vcpu, efer);
 	vcpu->arch.mmu.base_role.nxe = (efer & EFER_NX) && !tdp_enabled;
 	kvm_mmu_reset_context(vcpu);
 	/* Update reserved bits */
 	if ((efer ^ old_efer) & EFER_NX)
 		kvm_mmu_reset_context(vcpu);
 	return 0;
 }
 void kvm_enable_efer_bits(u64 mask)
 {
        efer_reserved_bits &= ~mask;
 }
 EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);
 /*
  * Writes msr value into into the appropriate "register".
  * Returns 0 on success, non-0 otherwise.
  * Assumes vcpu_load() was already called.
  */
 int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
 {
 	return kvm_x86_ops->set_msr(vcpu, msr_index, data);
 }
 /*
  * Adapt set_msr() to msr_io()'s calling convention
  */
 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
 {
 	return kvm_set_msr(vcpu, index, *data);
 }
 static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
 {
 	int version;
 	int r;
 	struct pvclock_wall_clock wc;
 	struct timespec boot;
 	if (!wall_clock)
 		return;
 	r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
 	if (r)
 		return;
 	if (version & 1)
 		++version;  /* first time write, random junk */
 	++version;
 	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 	/*
 	 * The guest calculates current wall clock time by adding
 	 * system time (updated by kvm_write_guest_time below) to the
 	 * wall clock specified here.  guest system time equals host
 	 * system time for us, thus we must fill in host boot time here.
 	 */
 	getboottime(&boot);
 	wc.sec = boot.tv_sec;
 	wc.nsec = boot.tv_nsec;
 	wc.version = version;
 	kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));
 	version++;
 	kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
 }
 static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
 {
 	uint32_t quotient, remainder;
 	/* Don't try to replace with do_div(), this one calculates
 	 * "(dividend << 32) / divisor" */
 	__asm__ ( "divl %4"
 		  : "=a" (quotient), "=d" (remainder)
 		  : "0" (0), "1" (dividend), "r" (divisor) );
 	return quotient;
 }
 static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *hv_clock)
 {
 	uint64_t nsecs = 1000000000LL;
 	int32_t  shift = 0;
 	uint64_t tps64;
 	uint32_t tps32;
 	tps64 = tsc_khz * 1000LL;
 	while (tps64 > nsecs*2) {
 		tps64 >>= 1;
 		shift--;
 	}
 	tps32 = (uint32_t)tps64;
 	while (tps32 <= (uint32_t)nsecs) {
 		tps32 <<= 1;
 		shift++;
 	}
 	hv_clock->tsc_shift = shift;
 	hv_clock->tsc_to_system_mul = div_frac(nsecs, tps32);
 	pr_debug("%s: tsc_khz %u, tsc_shift %d, tsc_mul %u\n",
 		 __func__, tsc_khz, hv_clock->tsc_shift,
 		 hv_clock->tsc_to_system_mul);
 }
 static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
 static void kvm_write_guest_time(struct kvm_vcpu *v)
 {
 	struct timespec ts;
 	unsigned long flags;
 	struct kvm_vcpu_arch *vcpu = &v->arch;
 	void *shared_kaddr;
 	unsigned long this_tsc_khz;
 	if ((!vcpu->time_page))
 		return;
 	this_tsc_khz = get_cpu_var(cpu_tsc_khz);
 	if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
 		kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
 		vcpu->hv_clock_tsc_khz = this_tsc_khz;
 	}
 	put_cpu_var(cpu_tsc_khz);
 	/* Keep irq disabled to prevent changes to the clock */
 	local_irq_save(flags);
 	kvm_get_msr(v, MSR_IA32_TSC, &vcpu->hv_clock.tsc_timestamp);
 	ktime_get_ts(&ts);
 	monotonic_to_bootbased(&ts);
 	local_irq_restore(flags);
 	/* With all the info we got, fill in the values */
 	vcpu->hv_clock.system_time = ts.tv_nsec +
 				     (NSEC_PER_SEC * (u64)ts.tv_sec) + v->kvm->arch.kvmclock_offset;
 	vcpu->hv_clock.flags = 0;
 	/*
 	 * The interface expects us to write an even number signaling that the
 	 * update is finished. Since the guest won't see the intermediate
 	 * state, we just increase by 2 at the end.
 	 */
 	vcpu->hv_clock.version += 2;
 	shared_kaddr = kmap_atomic(vcpu->time_page, KM_USER0);
 	memcpy(shared_kaddr + vcpu->time_offset, &vcpu->hv_clock,
 	       sizeof(vcpu->hv_clock));
 	kunmap_atomic(shared_kaddr, KM_USER0);
 	mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT);
 }
 static int kvm_request_guest_time_update(struct kvm_vcpu *v)
 {
 	struct kvm_vcpu_arch *vcpu = &v->arch;
 	if (!vcpu->time_page)
 		return 0;
 	set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests);
 	return 1;
 }
 static bool msr_mtrr_valid(unsigned msr)
 {
 	switch (msr) {
 	case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
 	case MSR_MTRRfix64K_00000:
 	case MSR_MTRRfix16K_80000:
 	case MSR_MTRRfix16K_A0000:
 	case MSR_MTRRfix4K_C0000:
 	case MSR_MTRRfix4K_C8000:
 	case MSR_MTRRfix4K_D0000:
 	case MSR_MTRRfix4K_D8000:
 	case MSR_MTRRfix4K_E0000:
 	case MSR_MTRRfix4K_E8000:
 	case MSR_MTRRfix4K_F0000:
 	case MSR_MTRRfix4K_F8000:
 	case MSR_MTRRdefType:
 	case MSR_IA32_CR_PAT:
 		return true;
 	case 0x2f8:
 		return true;
 	}
 	return false;
 }
 static bool valid_pat_type(unsigned t)
 {
 	return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
 }
 static bool valid_mtrr_type(unsigned t)
 {
 	return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
 }
 static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	int i;
 	if (!msr_mtrr_valid(msr))
 		return false;
 	if (msr == MSR_IA32_CR_PAT) {
 		for (i = 0; i < 8; i++)
 			if (!valid_pat_type((data >> (i * 8)) & 0xff))
 				return false;
 		return true;
 	} else if (msr == MSR_MTRRdefType) {
 		if (data & ~0xcff)
 			return false;
 		return valid_mtrr_type(data & 0xff);
 	} else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
 		for (i = 0; i < 8 ; i++)
 			if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
 				return false;
 		return true;
 	}
 	/* variable MTRRs */
 	return valid_mtrr_type(data & 0xff);
 }
 static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
 	if (!mtrr_valid(vcpu, msr, data))
 		return 1;
 	if (msr == MSR_MTRRdefType) {
 		vcpu->arch.mtrr_state.def_type = data;
 		vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
 	} else if (msr == MSR_MTRRfix64K_00000)
 		p[0] = data;
 	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
 		p[1 + msr - MSR_MTRRfix16K_80000] = data;
 	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
 		p[3 + msr - MSR_MTRRfix4K_C0000] = data;
 	else if (msr == MSR_IA32_CR_PAT)
 		vcpu->arch.pat = data;
 	else {	/* Variable MTRRs */
 		int idx, is_mtrr_mask;
 		u64 *pt;
 		idx = (msr - 0x200) / 2;
 		is_mtrr_mask = msr - 0x200 - 2 * idx;
 		if (!is_mtrr_mask)
 			pt =
 			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
 		else
 			pt =
 			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
 		*pt = data;
 	}
 	kvm_mmu_reset_context(vcpu);
 	return 0;
 }
 static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	u64 mcg_cap = vcpu->arch.mcg_cap;
 	unsigned bank_num = mcg_cap & 0xff;
 	switch (msr) {
 	case MSR_IA32_MCG_STATUS:
 		vcpu->arch.mcg_status = data;
 		break;
 	case MSR_IA32_MCG_CTL:
 		if (!(mcg_cap & MCG_CTL_P))
 			return 1;
 		if (data != 0 && data != ~(u64)0)
 			return -1;
 		vcpu->arch.mcg_ctl = data;
 		break;
 	default:
 		if (msr >= MSR_IA32_MC0_CTL &&
 		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
 			u32 offset = msr - MSR_IA32_MC0_CTL;
 			/* only 0 or all 1s can be written to IA32_MCi_CTL
 			 * some Linux kernels though clear bit 10 in bank 4 to
 			 * workaround a BIOS/GART TBL issue on AMD K8s, ignore
 			 * this to avoid an uncatched #GP in the guest
 			 */
 			if ((offset & 0x3) == 0 &&
 			    data != 0 && (data | (1 << 10)) != ~(u64)0)
 				return -1;
 			vcpu->arch.mce_banks[offset] = data;
 			break;
 		}
 		return 1;
 	}
 	return 0;
 }
 static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
 {
 	struct kvm *kvm = vcpu->kvm;
 	int lm = is_long_mode(vcpu);
 	u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
 		: (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
 	u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
 		: kvm->arch.xen_hvm_config.blob_size_32;
 	u32 page_num = data & ~PAGE_MASK;
 	u64 page_addr = data & PAGE_MASK;
 	u8 *page;
 	int r;
 	r = -E2BIG;
 	if (page_num >= blob_size)
 		goto out;
 	r = -ENOMEM;
 	page = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (!page)
 		goto out;
 	r = -EFAULT;
 	if (copy_from_user(page, blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE))
 		goto out_free;
 	if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
 		goto out_free;
 	r = 0;
 out_free:
 	kfree(page);
 out:
 	return r;
 }
 static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
 {
 	return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
 }
 static bool kvm_hv_msr_partition_wide(u32 msr)
 {
 	bool r = false;
 	switch (msr) {
 	case HV_X64_MSR_GUEST_OS_ID:
 	case HV_X64_MSR_HYPERCALL:
 		r = true;
 		break;
 	}
 	return r;
 }
 static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	struct kvm *kvm = vcpu->kvm;
 	switch (msr) {
 	case HV_X64_MSR_GUEST_OS_ID:
 		kvm->arch.hv_guest_os_id = data;
 		/* setting guest os id to zero disables hypercall page */
 		if (!kvm->arch.hv_guest_os_id)
 			kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
 		break;
 	case HV_X64_MSR_HYPERCALL: {
 		u64 gfn;
 		unsigned long addr;
 		u8 instructions[4];
 		/* if guest os id is not set hypercall should remain disabled */
 		if (!kvm->arch.hv_guest_os_id)
 			break;
 		if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
 			kvm->arch.hv_hypercall = data;
 			break;
 		}
 		gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
 		addr = gfn_to_hva(kvm, gfn);
 		if (kvm_is_error_hva(addr))
 			return 1;
 		kvm_x86_ops->patch_hypercall(vcpu, instructions);
 		((unsigned char *)instructions)[3] = 0xc3; /* ret */
 		if (copy_to_user((void __user *)addr, instructions, 4))
 			return 1;
 		kvm->arch.hv_hypercall = data;
 		break;
 	}
 	default:
 		pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
 			  "data 0x%llx\n", msr, data);
 		return 1;
 	}
 	return 0;
 }
 static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	switch (msr) {
 	case HV_X64_MSR_APIC_ASSIST_PAGE: {
 		unsigned long addr;
 		if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
 			vcpu->arch.hv_vapic = data;
 			break;
 		}
 		addr = gfn_to_hva(vcpu->kvm, data >>
 				  HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT);
 		if (kvm_is_error_hva(addr))
 			return 1;
 		if (clear_user((void __user *)addr, PAGE_SIZE))
 			return 1;
 		vcpu->arch.hv_vapic = data;
 		break;
 	}
 	case HV_X64_MSR_EOI:
 		return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
 	case HV_X64_MSR_ICR:
 		return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
 	case HV_X64_MSR_TPR:
 		return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
 	default:
 		pr_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
 			  "data 0x%llx\n", msr, data);
 		return 1;
 	}
 	return 0;
 }
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 {
 	switch (msr) {
 	case MSR_EFER:
 		return set_efer(vcpu, data);
 	case MSR_K7_HWCR:
 		data &= ~(u64)0x40;	/* ignore flush filter disable */
 		data &= ~(u64)0x100;	/* ignore ignne emulation enable */
 		if (data != 0) {
 			pr_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
 				data);
 			return 1;
 		}
 		break;
 	case MSR_FAM10H_MMIO_CONF_BASE:
 		if (data != 0) {
 			pr_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
 				"0x%llx\n", data);
 			return 1;
 		}
 		break;
 	case MSR_AMD64_NB_CFG:
 		break;
 	case MSR_IA32_DEBUGCTLMSR:
 		if (!data) {
 			/* We support the non-activated case already */
 			break;
 		} else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
 			/* Values other than LBR and BTF are vendor-specific,
 			   thus reserved and should throw a #GP */
 			return 1;
 		}
 		pr_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
 			__func__, data);
 		break;
 	case MSR_IA32_UCODE_REV:
 	case MSR_IA32_UCODE_WRITE:
 	case MSR_VM_HSAVE_PA:
 	case MSR_AMD64_PATCH_LOADER:
 		break;
 	case 0x200 ... 0x2ff:
 		return set_msr_mtrr(vcpu, msr, data);
 	case MSR_IA32_APICBASE:
 		kvm_set_apic_base(vcpu, data);
 		break;
 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
 		return kvm_x2apic_msr_write(vcpu, msr, data);
 	case MSR_IA32_MISC_ENABLE:
 		vcpu->arch.ia32_misc_enable_msr = data;
 		break;
 	case MSR_KVM_WALL_CLOCK_NEW:
 	case MSR_KVM_WALL_CLOCK:
 		vcpu->kvm->arch.wall_clock = data;
 		kvm_write_wall_clock(vcpu->kvm, data);
 		break;
 	case MSR_KVM_SYSTEM_TIME_NEW:
 	case MSR_KVM_SYSTEM_TIME: {
 		if (vcpu->arch.time_page) {
 			kvm_release_page_dirty(vcpu->arch.time_page);
 			vcpu->arch.time_page = NULL;
 		}
 		vcpu->arch.time = data;
 		/* we verify if the enable bit is set... */
 		if (!(data & 1))
 			break;
 		/* ...but clean it before doing the actual write */
 		vcpu->arch.time_offset = data & ~(PAGE_MASK | 1);
 		vcpu->arch.time_page =
 				gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
 		if (is_error_page(vcpu->arch.time_page)) {
 			kvm_release_page_clean(vcpu->arch.time_page);
 			vcpu->arch.time_page = NULL;
 		}
 		kvm_request_guest_time_update(vcpu);
 		break;
 	}
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
 		return set_msr_mce(vcpu, msr, data);
 	/* Performance counters are not protected by a CPUID bit,
 	 * so we should check all of them in the generic path for the sake of
 	 * cross vendor migration.
 	 * Writing a zero into the event select MSRs disables them,
 	 * which we perfectly emulate ;-). Any other value should be at least
 	 * reported, some guests depend on them.
 	 */
 	case MSR_P6_EVNTSEL0:
 	case MSR_P6_EVNTSEL1:
 	case MSR_K7_EVNTSEL0:
 	case MSR_K7_EVNTSEL1:
 	case MSR_K7_EVNTSEL2:
 	case MSR_K7_EVNTSEL3:
 		if (data != 0)
 			pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
 				"0x%x data 0x%llx\n", msr, data);
 		break;
 	/* at least RHEL 4 unconditionally writes to the perfctr registers,
 	 * so we ignore writes to make it happy.
 	 */
 	case MSR_P6_PERFCTR0:
 	case MSR_P6_PERFCTR1:
 	case MSR_K7_PERFCTR0:
 	case MSR_K7_PERFCTR1:
 	case MSR_K7_PERFCTR2:
 	case MSR_K7_PERFCTR3:
 		pr_unimpl(vcpu, "unimplemented perfctr wrmsr: "
 			"0x%x data 0x%llx\n", msr, data);
 		break;
 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
 		if (kvm_hv_msr_partition_wide(msr)) {
 			int r;
 			mutex_lock(&vcpu->kvm->lock);
 			r = set_msr_hyperv_pw(vcpu, msr, data);
 			mutex_unlock(&vcpu->kvm->lock);
 			return r;
 		} else
 			return set_msr_hyperv(vcpu, msr, data);
 		break;
 	default:
 		if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
 			return xen_hvm_config(vcpu, data);
 		if (!ignore_msrs) {
 			pr_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
 				msr, data);
 			return 1;
 		} else {
 			pr_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
 				msr, data);
 			break;
 		}
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
 /*
  * Reads an msr value (of 'msr_index') into 'pdata'.
  * Returns 0 on success, non-0 otherwise.
  * Assumes vcpu_load() was already called.
  */
 int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 {
 	return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
 }
 static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
 	u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;
 	if (!msr_mtrr_valid(msr))
 		return 1;
 	if (msr == MSR_MTRRdefType)
 		*pdata = vcpu->arch.mtrr_state.def_type +
 			 (vcpu->arch.mtrr_state.enabled << 10);
 	else if (msr == MSR_MTRRfix64K_00000)
 		*pdata = p[0];
 	else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
 		*pdata = p[1 + msr - MSR_MTRRfix16K_80000];
 	else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
 		*pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
 	else if (msr == MSR_IA32_CR_PAT)
 		*pdata = vcpu->arch.pat;
 	else {	/* Variable MTRRs */
 		int idx, is_mtrr_mask;
 		u64 *pt;
 		idx = (msr - 0x200) / 2;
 		is_mtrr_mask = msr - 0x200 - 2 * idx;
 		if (!is_mtrr_mask)
 			pt =
 			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
 		else
 			pt =
 			  (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
 		*pdata = *pt;
 	}
 	return 0;
 }
 static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
 	u64 data;
 	u64 mcg_cap = vcpu->arch.mcg_cap;
 	unsigned bank_num = mcg_cap & 0xff;
 	switch (msr) {
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 		data = 0;
 		break;
 	case MSR_IA32_MCG_CAP:
 		data = vcpu->arch.mcg_cap;
 		break;
 	case MSR_IA32_MCG_CTL:
 		if (!(mcg_cap & MCG_CTL_P))
 			return 1;
 		data = vcpu->arch.mcg_ctl;
 		break;
 	case MSR_IA32_MCG_STATUS:
 		data = vcpu->arch.mcg_status;
 		break;
 	default:
 		if (msr >= MSR_IA32_MC0_CTL &&
 		    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
 			u32 offset = msr - MSR_IA32_MC0_CTL;
 			data = vcpu->arch.mce_banks[offset];
 			break;
 		}
 		return 1;
 	}
 	*pdata = data;
 	return 0;
 }
 static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
 	u64 data = 0;
 	struct kvm *kvm = vcpu->kvm;
 	switch (msr) {
 	case HV_X64_MSR_GUEST_OS_ID:
 		data = kvm->arch.hv_guest_os_id;
 		break;
 	case HV_X64_MSR_HYPERCALL:
 		data = kvm->arch.hv_hypercall;
 		break;
 	default:
 		pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
 		return 1;
 	}
 	*pdata = data;
 	return 0;
 }
 static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
 	u64 data = 0;
 	switch (msr) {
 	case HV_X64_MSR_VP_INDEX: {
 		int r;
 		struct kvm_vcpu *v;
 		kvm_for_each_vcpu(r, v, vcpu->kvm)
 			if (v == vcpu)
 				data = r;
 		break;
 	}
 	case HV_X64_MSR_EOI:
 		return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
 	case HV_X64_MSR_ICR:
 		return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
 	case HV_X64_MSR_TPR:
 		return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
 	default:
 		pr_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
 		return 1;
 	}
 	*pdata = data;
 	return 0;
 }
 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
 {
 	u64 data;
 	switch (msr) {
 	case MSR_IA32_PLATFORM_ID:
 	case MSR_IA32_UCODE_REV:
 	case MSR_IA32_EBL_CR_POWERON:
 	case MSR_IA32_DEBUGCTLMSR:
 	case MSR_IA32_LASTBRANCHFROMIP:
 	case MSR_IA32_LASTBRANCHTOIP:
 	case MSR_IA32_LASTINTFROMIP:
 	case MSR_IA32_LASTINTTOIP:
 	case MSR_K8_SYSCFG:
 	case MSR_K7_HWCR:
 	case MSR_VM_HSAVE_PA:
 	case MSR_P6_PERFCTR0:
 	case MSR_P6_PERFCTR1:
 	case MSR_P6_EVNTSEL0:
 	case MSR_P6_EVNTSEL1:
 	case MSR_K7_EVNTSEL0:
 	case MSR_K7_PERFCTR0:
 	case MSR_K8_INT_PENDING_MSG:
 	case MSR_AMD64_NB_CFG:
 	case MSR_FAM10H_MMIO_CONF_BASE:
 		data = 0;
 		break;
 	case MSR_MTRRcap:
 		data = 0x500 | KVM_NR_VAR_MTRR;
 		break;
 	case 0x200 ... 0x2ff:
 		return get_msr_mtrr(vcpu, msr, pdata);
 	case 0xcd: /* fsb frequency */
 		data = 3;
 		break;
 	case MSR_IA32_APICBASE:
 		data = kvm_get_apic_base(vcpu);
 		break;
 	case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
 		return kvm_x2apic_msr_read(vcpu, msr, pdata);
 		break;
 	case MSR_IA32_MISC_ENABLE:
 		data = vcpu->arch.ia32_misc_enable_msr;
 		break;
 	case MSR_IA32_PERF_STATUS:
 		/* TSC increment by tick */
 		data = 1000ULL;
 		/* CPU multiplier */
 		data |= (((uint64_t)4ULL) << 40);
 		break;
 	case MSR_EFER:
 		data = vcpu->arch.efer;
 		break;
 	case MSR_KVM_WALL_CLOCK:
 	case MSR_KVM_WALL_CLOCK_NEW:
 		data = vcpu->kvm->arch.wall_clock;
 		break;
 	case MSR_KVM_SYSTEM_TIME:
 	case MSR_KVM_SYSTEM_TIME_NEW:
 		data = vcpu->arch.time;
 		break;
 	case MSR_IA32_P5_MC_ADDR:
 	case MSR_IA32_P5_MC_TYPE:
 	case MSR_IA32_MCG_CAP:
 	case MSR_IA32_MCG_CTL:
 	case MSR_IA32_MCG_STATUS:
 	case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
 		return get_msr_mce(vcpu, msr, pdata);
 	case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
 		if (kvm_hv_msr_partition_wide(msr)) {
 			int r;
 			mutex_lock(&vcpu->kvm->lock);
 			r = get_msr_hyperv_pw(vcpu, msr, pdata);
 			mutex_unlock(&vcpu->kvm->lock);
 			return r;
 		} else
 			return get_msr_hyperv(vcpu, msr, pdata);
 		break;
 	default:
 		if (!ignore_msrs) {
 			pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
 			return 1;
 		} else {
 			pr_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
 			data = 0;
 		}
 		break;
 	}
 	*pdata = data;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
 /*
  * Read or write a bunch of msrs. All parameters are kernel addresses.
  *
  * @return number of msrs set successfully.
  */
 static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
 		    struct kvm_msr_entry *entries,
 		    int (*do_msr)(struct kvm_vcpu *vcpu,
 				  unsigned index, u64 *data))
 {
 	int i, idx;
 	vcpu_load(vcpu);
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	for (i = 0; i < msrs->nmsrs; ++i)
 		if (do_msr(vcpu, entries[i].index, &entries[i].data))
 			break;
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	vcpu_put(vcpu);
 	return i;
 }
 /*
  * Read or write a bunch of msrs. Parameters are user addresses.
  *
  * @return number of msrs set successfully.
  */
 static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
 		  int (*do_msr)(struct kvm_vcpu *vcpu,
 				unsigned index, u64 *data),
 		  int writeback)
 {
 	struct kvm_msrs msrs;
 	struct kvm_msr_entry *entries;
 	int r, n;
 	unsigned size;
 	r = -EFAULT;
 	if (copy_from_user(&msrs, user_msrs, sizeof msrs))
 		goto out;
 	r = -E2BIG;
 	if (msrs.nmsrs >= MAX_IO_MSRS)
 		goto out;
 	r = -ENOMEM;
 	size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
 	entries = kmalloc(size, GFP_KERNEL);
 	if (!entries)
 		goto out;
 	r = -EFAULT;
 	if (copy_from_user(entries, user_msrs->entries, size))
 		goto out_free;
 	r = n = __msr_io(vcpu, &msrs, entries, do_msr);
 	if (r < 0)
 		goto out_free;
 	r = -EFAULT;
 	if (writeback && copy_to_user(user_msrs->entries, entries, size))
 		goto out_free;
 	r = n;
 out_free:
 	kfree(entries);
 out:
 	return r;
 }
 int kvm_dev_ioctl_check_extension(long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_IRQCHIP:
 	case KVM_CAP_HLT:
 	case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
 	case KVM_CAP_SET_TSS_ADDR:
 	case KVM_CAP_EXT_CPUID:
 	case KVM_CAP_CLOCKSOURCE:
 	case KVM_CAP_PIT:
 	case KVM_CAP_NOP_IO_DELAY:
 	case KVM_CAP_MP_STATE:
 	case KVM_CAP_SYNC_MMU:
 	case KVM_CAP_REINJECT_CONTROL:
 	case KVM_CAP_IRQ_INJECT_STATUS:
 	case KVM_CAP_ASSIGN_DEV_IRQ:
 	case KVM_CAP_IRQFD:
 	case KVM_CAP_IOEVENTFD:
 	case KVM_CAP_PIT2:
 	case KVM_CAP_PIT_STATE2:
 	case KVM_CAP_SET_IDENTITY_MAP_ADDR:
 	case KVM_CAP_XEN_HVM:
 	case KVM_CAP_ADJUST_CLOCK:
 	case KVM_CAP_VCPU_EVENTS:
 	case KVM_CAP_HYPERV:
 	case KVM_CAP_HYPERV_VAPIC:
 	case KVM_CAP_HYPERV_SPIN:
 	case KVM_CAP_PCI_SEGMENT:
 	case KVM_CAP_DEBUGREGS:
 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 	case KVM_CAP_VAPIC:
 		r = !kvm_x86_ops->cpu_has_accelerated_tpr();
 		break;
 	case KVM_CAP_NR_VCPUS:
 		r = KVM_MAX_VCPUS;
 		break;
 	case KVM_CAP_NR_MEMSLOTS:
 		r = KVM_MEMORY_SLOTS;
 		break;
 	case KVM_CAP_PV_MMU:	/* obsolete */
 		r = 0;
 		break;
 	case KVM_CAP_IOMMU:
 		r = iommu_found();
 		break;
 	case KVM_CAP_MCE:
 		r = KVM_MAX_MCE_BANKS;
 		break;
 	default:
 		r = 0;
 		break;
 	}
 	return r;
 }
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
 	long r;
 	switch (ioctl) {
 	case KVM_GET_MSR_INDEX_LIST: {
 		struct kvm_msr_list __user *user_msr_list = argp;
 		struct kvm_msr_list msr_list;
 		unsigned n;
 		r = -EFAULT;
 		if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
 			goto out;
 		n = msr_list.nmsrs;
 		msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
 		if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
 			goto out;
 		r = -E2BIG;
 		if (n < msr_list.nmsrs)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(user_msr_list->indices, &msrs_to_save,
 				 num_msrs_to_save * sizeof(u32)))
 			goto out;
 		if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
 				 &emulated_msrs,
 				 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_GET_SUPPORTED_CPUID: {
 		struct kvm_cpuid2 __user *cpuid_arg = argp;
 		struct kvm_cpuid2 cpuid;
 		r = -EFAULT;
 		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 			goto out;
 		r = kvm_dev_ioctl_get_supported_cpuid(&cpuid,
 						      cpuid_arg->entries);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_X86_GET_MCE_CAP_SUPPORTED: {
 		u64 mce_cap;
 		mce_cap = KVM_MCE_CAP_SUPPORTED;
 		r = -EFAULT;
 		if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
 			goto out;
 		r = 0;
 		break;
 	}
 	default:
 		r = -EINVAL;
 	}
 out:
 	return r;
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	kvm_x86_ops->vcpu_load(vcpu, cpu);
 	if (unlikely(per_cpu(cpu_tsc_khz, cpu) == 0)) {
 		unsigned long khz = cpufreq_quick_get(cpu);
 		if (!khz)
 			khz = tsc_khz;
 		per_cpu(cpu_tsc_khz, cpu) = khz;
 	}
 	kvm_request_guest_time_update(vcpu);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	kvm_x86_ops->vcpu_put(vcpu);
 	kvm_put_guest_fpu(vcpu);
 }
 static int is_efer_nx(void)
 {
 	unsigned long long efer = 0;
 	rdmsrl_safe(MSR_EFER, &efer);
 	return efer & EFER_NX;
 }
 static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
 {
 	int i;
 	struct kvm_cpuid_entry2 *e, *entry;
 	entry = NULL;
 	for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
 		e = &vcpu->arch.cpuid_entries[i];
 		if (e->function == 0x80000001) {
 			entry = e;
 			break;
 		}
 	}
 	if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) {
 		entry->edx &= ~(1 << 20);
 		printk(KERN_INFO "kvm: guest NX capability removed\n");
 	}
 }
 /* when an old userspace process fills a new kernel module */
 static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
 				    struct kvm_cpuid *cpuid,
 				    struct kvm_cpuid_entry __user *entries)
 {
 	int r, i;
 	struct kvm_cpuid_entry *cpuid_entries;
 	r = -E2BIG;
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 		goto out;
 	r = -ENOMEM;
 	cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent);
 	if (!cpuid_entries)
 		goto out;
 	r = -EFAULT;
 	if (copy_from_user(cpuid_entries, entries,
 			   cpuid->nent * sizeof(struct kvm_cpuid_entry)))
 		goto out_free;
 	vcpu_load(vcpu);
 	for (i = 0; i < cpuid->nent; i++) {
 		vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
 		vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
 		vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
 		vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
 		vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
 		vcpu->arch.cpuid_entries[i].index = 0;
 		vcpu->arch.cpuid_entries[i].flags = 0;
 		vcpu->arch.cpuid_entries[i].padding[0] = 0;
 		vcpu->arch.cpuid_entries[i].padding[1] = 0;
 		vcpu->arch.cpuid_entries[i].padding[2] = 0;
 	}
 	vcpu->arch.cpuid_nent = cpuid->nent;
 	cpuid_fix_nx_cap(vcpu);
 	r = 0;
 	kvm_apic_set_version(vcpu);
 	kvm_x86_ops->cpuid_update(vcpu);
 	vcpu_put(vcpu);
 out_free:
 	vfree(cpuid_entries);
 out:
 	return r;
 }
 static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
 				     struct kvm_cpuid2 *cpuid,
 				     struct kvm_cpuid_entry2 __user *entries)
 {
 	int r;
 	r = -E2BIG;
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 		goto out;
 	r = -EFAULT;
 	if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
 			   cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
 		goto out;
 	vcpu_load(vcpu);
 	vcpu->arch.cpuid_nent = cpuid->nent;
 	kvm_apic_set_version(vcpu);
 	kvm_x86_ops->cpuid_update(vcpu);
 	vcpu_put(vcpu);
 	return 0;
 out:
 	return r;
 }
 static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
 				     struct kvm_cpuid2 *cpuid,
 				     struct kvm_cpuid_entry2 __user *entries)
 {
 	int r;
 	vcpu_load(vcpu);
 	r = -E2BIG;
 	if (cpuid->nent < vcpu->arch.cpuid_nent)
 		goto out;
 	r = -EFAULT;
 	if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
 			 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
 		goto out;
 	return 0;
 out:
 	cpuid->nent = vcpu->arch.cpuid_nent;
 	vcpu_put(vcpu);
 	return r;
 }
 static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 			   u32 index)
 {
 	entry->function = function;
 	entry->index = index;
 	cpuid_count(entry->function, entry->index,
 		    &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
 	entry->flags = 0;
 }
 #define F(x) bit(X86_FEATURE_##x)
 static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 			 u32 index, int *nent, int maxnent)
 {
 	unsigned f_nx = is_efer_nx() ? F(NX) : 0;
 #ifdef CONFIG_X86_64
 	unsigned f_gbpages = (kvm_x86_ops->get_lpage_level() == PT_PDPE_LEVEL)
 				? F(GBPAGES) : 0;
 	unsigned f_lm = F(LM);
 #else
 	unsigned f_gbpages = 0;
 	unsigned f_lm = 0;
 #endif
 	unsigned f_rdtscp = kvm_x86_ops->rdtscp_supported() ? F(RDTSCP) : 0;
 	/* cpuid 1.edx */
 	const u32 kvm_supported_word0_x86_features =
 		F(FPU) | F(VME) | F(DE) | F(PSE) |
 		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
 		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SEP) |
 		F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
 		F(PAT) | F(PSE36) | 0 /* PSN */ | F(CLFLSH) |
 		0 /* Reserved, DS, ACPI */ | F(MMX) |
 		F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
 		0 /* HTT, TM, Reserved, PBE */;
 	/* cpuid 0x80000001.edx */
 	const u32 kvm_supported_word1_x86_features =
 		F(FPU) | F(VME) | F(DE) | F(PSE) |
 		F(TSC) | F(MSR) | F(PAE) | F(MCE) |
 		F(CX8) | F(APIC) | 0 /* Reserved */ | F(SYSCALL) |
 		F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
 		F(PAT) | F(PSE36) | 0 /* Reserved */ |
 		f_nx | 0 /* Reserved */ | F(MMXEXT) | F(MMX) |
 		F(FXSR) | F(FXSR_OPT) | f_gbpages | f_rdtscp |
 		0 /* Reserved */ | f_lm | F(3DNOWEXT) | F(3DNOW);
 	/* cpuid 1.ecx */
 	const u32 kvm_supported_word4_x86_features =
 		F(XMM3) | 0 /* Reserved, DTES64, MONITOR */ |
 		0 /* DS-CPL, VMX, SMX, EST */ |
 		0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ |
 		0 /* Reserved */ | F(CX16) | 0 /* xTPR Update, PDCM */ |
 		0 /* Reserved, DCA */ | F(XMM4_1) |
 		F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
 		0 /* Reserved, XSAVE, OSXSAVE */;
 	/* cpuid 0x80000001.ecx */
 	const u32 kvm_supported_word6_x86_features =
 		F(LAHF_LM) | F(CMP_LEGACY) | F(SVM) | 0 /* ExtApicSpace */ |
 		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
 		F(3DNOWPREFETCH) | 0 /* OSVW */ | 0 /* IBS */ | F(SSE5) |
 		0 /* SKINIT */ | 0 /* WDT */;
 	/* all calls to cpuid_count() should be made on the same cpu */
 	get_cpu();
 	do_cpuid_1_ent(entry, function, index);
 	++*nent;
 	switch (function) {
 	case 0:
 		entry->eax = min(entry->eax, (u32)0xb);
 		break;
 	case 1:
 		entry->edx &= kvm_supported_word0_x86_features;
 		entry->ecx &= kvm_supported_word4_x86_features;
 		/* we support x2apic emulation even if host does not support
 		 * it since we emulate x2apic in software */
 		entry->ecx |= F(X2APIC);
 		break;
 	/* function 2 entries are STATEFUL. That is, repeated cpuid commands
 	 * may return different values. This forces us to get_cpu() before
 	 * issuing the first command, and also to emulate this annoying behavior
 	 * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */
 	case 2: {
 		int t, times = entry->eax & 0xff;
 		entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
 		entry->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
 		for (t = 1; t < times && *nent < maxnent; ++t) {
 			do_cpuid_1_ent(&entry[t], function, 0);
 			entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC;
 			++*nent;
 		}
 		break;
 	}
 	/* function 4 and 0xb have additional index. */
 	case 4: {
 		int i, cache_type;
 		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 		/* read more entries until cache_type is zero */
 		for (i = 1; *nent < maxnent; ++i) {
 			cache_type = entry[i - 1].eax & 0x1f;
 			if (!cache_type)
 				break;
 			do_cpuid_1_ent(&entry[i], function, i);
 			entry[i].flags |=
 			       KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 			++*nent;
 		}
 		break;
 	}
 	case 0xb: {
 		int i, level_type;
 		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 		/* read more entries until level_type is zero */
 		for (i = 1; *nent < maxnent; ++i) {
 			level_type = entry[i - 1].ecx & 0xff00;
 			if (!level_type)
 				break;
 			do_cpuid_1_ent(&entry[i], function, i);
 			entry[i].flags |=
 			       KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 			++*nent;
 		}
 		break;
 	}
 	case KVM_CPUID_SIGNATURE: {
 		char signature[12] = "KVMKVMKVM\0\0";
 		u32 *sigptr = (u32 *)signature;
 		entry->eax = 0;
 		entry->ebx = sigptr[0];
 		entry->ecx = sigptr[1];
 		entry->edx = sigptr[2];
 		break;
 	}
 	case KVM_CPUID_FEATURES:
 		entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
 			     (1 << KVM_FEATURE_NOP_IO_DELAY) |
 			     (1 << KVM_FEATURE_CLOCKSOURCE2) |
 			     (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
 		entry->ebx = 0;
 		entry->ecx = 0;
 		entry->edx = 0;
 		break;
 	case 0x80000000:
 		entry->eax = min(entry->eax, 0x8000001a);
 		break;
 	case 0x80000001:
 		entry->edx &= kvm_supported_word1_x86_features;
 		entry->ecx &= kvm_supported_word6_x86_features;
 		break;
 	}
 	kvm_x86_ops->set_supported_cpuid(function, entry);
 	put_cpu();
 }
 #undef F
 static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
 				     struct kvm_cpuid_entry2 __user *entries)
 {
 	struct kvm_cpuid_entry2 *cpuid_entries;
 	int limit, nent = 0, r = -E2BIG;
 	u32 func;
 	if (cpuid->nent < 1)
 		goto out;
 	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 		cpuid->nent = KVM_MAX_CPUID_ENTRIES;
 	r = -ENOMEM;
 	cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent);
 	if (!cpuid_entries)
 		goto out;
 	do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent);
 	limit = cpuid_entries[0].eax;
 	for (func = 1; func <= limit && nent < cpuid->nent; ++func)
 		do_cpuid_ent(&cpuid_entries[nent], func, 0,
 			     &nent, cpuid->nent);
 	r = -E2BIG;
 	if (nent >= cpuid->nent)
 		goto out_free;
 	do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent);
 	limit = cpuid_entries[nent - 1].eax;
 	for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func)
 		do_cpuid_ent(&cpuid_entries[nent], func, 0,
 			     &nent, cpuid->nent);
 	r = -E2BIG;
 	if (nent >= cpuid->nent)
 		goto out_free;
 	do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_SIGNATURE, 0, &nent,
 		     cpuid->nent);
 	r = -E2BIG;
 	if (nent >= cpuid->nent)
 		goto out_free;
 	do_cpuid_ent(&cpuid_entries[nent], KVM_CPUID_FEATURES, 0, &nent,
 		     cpuid->nent);
 	r = -E2BIG;
 	if (nent >= cpuid->nent)
 		goto out_free;
 	r = -EFAULT;
 	if (copy_to_user(entries, cpuid_entries,
 			 nent * sizeof(struct kvm_cpuid_entry2)))
 		goto out_free;
 	cpuid->nent = nent;
 	r = 0;
 out_free:
 	vfree(cpuid_entries);
 out:
 	return r;
 }
 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
 	vcpu_load(vcpu);
 	memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);
 	vcpu_put(vcpu);
 	return 0;
 }
 static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
 	vcpu_load(vcpu);
 	memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
 	kvm_apic_post_state_restore(vcpu);
 	update_cr8_intercept(vcpu);
 	vcpu_put(vcpu);
 	return 0;
 }
 static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
 				    struct kvm_interrupt *irq)
 {
 	if (irq->irq < 0 || irq->irq >= 256)
 		return -EINVAL;
 	if (irqchip_in_kernel(vcpu->kvm))
 		return -ENXIO;
 	vcpu_load(vcpu);
 	kvm_queue_interrupt(vcpu, irq->irq, false);
 	vcpu_put(vcpu);
 	return 0;
 }
 static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
 {
 	vcpu_load(vcpu);
 	kvm_inject_nmi(vcpu);
 	vcpu_put(vcpu);
 	return 0;
 }
 static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
 					   struct kvm_tpr_access_ctl *tac)
 {
 	if (tac->flags)
 		return -EINVAL;
 	vcpu->arch.tpr_access_reporting = !!tac->enabled;
 	return 0;
 }
 static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
 					u64 mcg_cap)
 {
 	int r;
 	unsigned bank_num = mcg_cap & 0xff, bank;
 	vcpu_load(vcpu);
 	r = -EINVAL;
 	if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
 		goto out;
 	if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
 		goto out;
 	r = 0;
 	vcpu->arch.mcg_cap = mcg_cap;
 	/* Init IA32_MCG_CTL to all 1s */
 	if (mcg_cap & MCG_CTL_P)
 		vcpu->arch.mcg_ctl = ~(u64)0;
 	/* Init IA32_MCi_CTL to all 1s */
 	for (bank = 0; bank < bank_num; bank++)
 		vcpu->arch.mce_banks[bank*4] = ~(u64)0;
 out:
 	vcpu_put(vcpu);
 	return r;
 }
 static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
 				      struct kvm_x86_mce *mce)
 {
 	u64 mcg_cap = vcpu->arch.mcg_cap;
 	unsigned bank_num = mcg_cap & 0xff;
 	u64 *banks = vcpu->arch.mce_banks;
 	if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
 		return -EINVAL;
 	/*
 	 * if IA32_MCG_CTL is not all 1s, the uncorrected error
 	 * reporting is disabled
 	 */
 	if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
 	    vcpu->arch.mcg_ctl != ~(u64)0)
 		return 0;
 	banks += 4 * mce->bank;
 	/*
 	 * if IA32_MCi_CTL is not all 1s, the uncorrected error
 	 * reporting is disabled for the bank
 	 */
 	if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
 		return 0;
 	if (mce->status & MCI_STATUS_UC) {
 		if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
 		    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
 			printk(KERN_DEBUG "kvm: set_mce: "
 			       "injects mce exception while "
 			       "previous one is in progress!\n");
 			set_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests);
 			return 0;
 		}
 		if (banks[1] & MCI_STATUS_VAL)
 			mce->status |= MCI_STATUS_OVER;
 		banks[2] = mce->addr;
 		banks[3] = mce->misc;
 		vcpu->arch.mcg_status = mce->mcg_status;
 		banks[1] = mce->status;
 		kvm_queue_exception(vcpu, MC_VECTOR);
 	} else if (!(banks[1] & MCI_STATUS_VAL)
 		   || !(banks[1] & MCI_STATUS_UC)) {
 		if (banks[1] & MCI_STATUS_VAL)
 			mce->status |= MCI_STATUS_OVER;
 		banks[2] = mce->addr;
 		banks[3] = mce->misc;
 		banks[1] = mce->status;
 	} else
 		banks[1] |= MCI_STATUS_OVER;
 	return 0;
 }
 static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
 					       struct kvm_vcpu_events *events)
 {
 	vcpu_load(vcpu);
 	events->exception.injected =
 		vcpu->arch.exception.pending &&
 		!kvm_exception_is_soft(vcpu->arch.exception.nr);
 	events->exception.nr = vcpu->arch.exception.nr;
 	events->exception.has_error_code = vcpu->arch.exception.has_error_code;
 	events->exception.error_code = vcpu->arch.exception.error_code;
 	events->interrupt.injected =
 		vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
 	events->interrupt.nr = vcpu->arch.interrupt.nr;
 	events->interrupt.soft = 0;
 	events->interrupt.shadow =
 		kvm_x86_ops->get_interrupt_shadow(vcpu,
 			KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);
 	events->nmi.injected = vcpu->arch.nmi_injected;
 	events->nmi.pending = vcpu->arch.nmi_pending;
 	events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
 	events->sipi_vector = vcpu->arch.sipi_vector;
 	events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
 			 | KVM_VCPUEVENT_VALID_SIPI_VECTOR
 			 | KVM_VCPUEVENT_VALID_SHADOW);
 	vcpu_put(vcpu);
 }
 static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu,
 					      struct kvm_vcpu_events *events)
 {
 	if (events->flags & ~(KVM_VCPUEVENT_VALID_NMI_PENDING
 			      | KVM_VCPUEVENT_VALID_SIPI_VECTOR
 			      | KVM_VCPUEVENT_VALID_SHADOW))
 		return -EINVAL;
 	vcpu_load(vcpu);
 	vcpu->arch.exception.pending = events->exception.injected;
 	vcpu->arch.exception.nr = events->exception.nr;
 	vcpu->arch.exception.has_error_code = events->exception.has_error_code;
 	vcpu->arch.exception.error_code = events->exception.error_code;
 	vcpu->arch.interrupt.pending = events->interrupt.injected;
 	vcpu->arch.interrupt.nr = events->interrupt.nr;
 	vcpu->arch.interrupt.soft = events->interrupt.soft;
 	if (vcpu->arch.interrupt.pending && irqchip_in_kernel(vcpu->kvm))
 		kvm_pic_clear_isr_ack(vcpu->kvm);
 	if (events->flags & KVM_VCPUEVENT_VALID_SHADOW)
 		kvm_x86_ops->set_interrupt_shadow(vcpu,
 						  events->interrupt.shadow);
 	vcpu->arch.nmi_injected = events->nmi.injected;
 	if (events->flags & KVM_VCPUEVENT_VALID_NMI_PENDING)
 		vcpu->arch.nmi_pending = events->nmi.pending;
 	kvm_x86_ops->set_nmi_mask(vcpu, events->nmi.masked);
 	if (events->flags & KVM_VCPUEVENT_VALID_SIPI_VECTOR)
 		vcpu->arch.sipi_vector = events->sipi_vector;
 	vcpu_put(vcpu);
 	return 0;
 }
 static void kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu,
 					     struct kvm_debugregs *dbgregs)
 {
 	vcpu_load(vcpu);
 	memcpy(dbgregs->db, vcpu->arch.db, sizeof(vcpu->arch.db));
 	dbgregs->dr6 = vcpu->arch.dr6;
 	dbgregs->dr7 = vcpu->arch.dr7;
 	dbgregs->flags = 0;
 	vcpu_put(vcpu);
 }
 static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu,
 					    struct kvm_debugregs *dbgregs)
 {
 	if (dbgregs->flags)
 		return -EINVAL;
 	vcpu_load(vcpu);
 	memcpy(vcpu->arch.db, dbgregs->db, sizeof(vcpu->arch.db));
 	vcpu->arch.dr6 = dbgregs->dr6;
 	vcpu->arch.dr7 = dbgregs->dr7;
 	vcpu_put(vcpu);
 	return 0;
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	struct kvm_lapic_state *lapic = NULL;
 	switch (ioctl) {
 	case KVM_GET_LAPIC: {
 		r = -EINVAL;
 		if (!vcpu->arch.apic)
 			goto out;
 		lapic = kzalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!lapic)
 			goto out;
 		r = kvm_vcpu_ioctl_get_lapic(vcpu, lapic);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, lapic, sizeof(struct kvm_lapic_state)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_LAPIC: {
 		r = -EINVAL;
 		if (!vcpu->arch.apic)
 			goto out;
 		lapic = kmalloc(sizeof(struct kvm_lapic_state), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!lapic)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(lapic, argp, sizeof(struct kvm_lapic_state)))
 			goto out;
 		r = kvm_vcpu_ioctl_set_lapic(vcpu, lapic);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_INTERRUPT: {
 		struct kvm_interrupt irq;
 		r = -EFAULT;
 		if (copy_from_user(&irq, argp, sizeof irq))
 			goto out;
 		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_NMI: {
 		r = kvm_vcpu_ioctl_nmi(vcpu);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_CPUID: {
 		struct kvm_cpuid __user *cpuid_arg = argp;
 		struct kvm_cpuid cpuid;
 		r = -EFAULT;
 		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 			goto out;
 		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_SET_CPUID2: {
 		struct kvm_cpuid2 __user *cpuid_arg = argp;
 		struct kvm_cpuid2 cpuid;
 		r = -EFAULT;
 		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 			goto out;
 		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
 					      cpuid_arg->entries);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_GET_CPUID2: {
 		struct kvm_cpuid2 __user *cpuid_arg = argp;
 		struct kvm_cpuid2 cpuid;
 		r = -EFAULT;
 		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
 			goto out;
 		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
 					      cpuid_arg->entries);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_GET_MSRS:
 		r = msr_io(vcpu, argp, kvm_get_msr, 1);
 		break;
 	case KVM_SET_MSRS:
 		r = msr_io(vcpu, argp, do_set_msr, 0);
 		break;
 	case KVM_TPR_ACCESS_REPORTING: {
 		struct kvm_tpr_access_ctl tac;
 		r = -EFAULT;
 		if (copy_from_user(&tac, argp, sizeof tac))
 			goto out;
 		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &tac, sizeof tac))
 			goto out;
 		r = 0;
 		break;
 	};
 	case KVM_SET_VAPIC_ADDR: {
 		struct kvm_vapic_addr va;
 		r = -EINVAL;
 		if (!irqchip_in_kernel(vcpu->kvm))
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(&va, argp, sizeof va))
 			goto out;
 		r = 0;
 		kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
 		break;
 	}
 	case KVM_X86_SETUP_MCE: {
 		u64 mcg_cap;
 		r = -EFAULT;
 		if (copy_from_user(&mcg_cap, argp, sizeof mcg_cap))
 			goto out;
 		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
 		break;
 	}
 	case KVM_X86_SET_MCE: {
 		struct kvm_x86_mce mce;
 		r = -EFAULT;
 		if (copy_from_user(&mce, argp, sizeof mce))
 			goto out;
 		vcpu_load(vcpu);
 		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
 		vcpu_put(vcpu);
 		break;
 	}
 	case KVM_GET_VCPU_EVENTS: {
 		struct kvm_vcpu_events events;
 		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);
 		r = -EFAULT;
 		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
 			break;
 		r = 0;
 		break;
 	}
 	case KVM_SET_VCPU_EVENTS: {
 		struct kvm_vcpu_events events;
 		r = -EFAULT;
 		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
 			break;
 		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
 		break;
 	}
 	case KVM_GET_DEBUGREGS: {
 		struct kvm_debugregs dbgregs;
 		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);
 		r = -EFAULT;
 		if (copy_to_user(argp, &dbgregs,
 				 sizeof(struct kvm_debugregs)))
 			break;
 		r = 0;
 		break;
 	}
 	case KVM_SET_DEBUGREGS: {
 		struct kvm_debugregs dbgregs;
 		r = -EFAULT;
 		if (copy_from_user(&dbgregs, argp,
 				   sizeof(struct kvm_debugregs)))
 			break;
 		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
 		break;
 	}
 	default:
 		r = -EINVAL;
 	}
 out:
 	kfree(lapic);
 	return r;
 }
 static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
 {
 	int ret;
 	if (addr > (unsigned int)(-3 * PAGE_SIZE))
 		return -1;
 	ret = kvm_x86_ops->set_tss_addr(kvm, addr);
 	return ret;
 }
 static int kvm_vm_ioctl_set_identity_map_addr(struct kvm *kvm,
 					      u64 ident_addr)
 {
 	kvm->arch.ept_identity_map_addr = ident_addr;
 	return 0;
 }
 static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm,
 					  u32 kvm_nr_mmu_pages)
 {
 	if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
 		return -EINVAL;
 	mutex_lock(&kvm->slots_lock);
 	spin_lock(&kvm->mmu_lock);
 	kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
 	kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages;
 	spin_unlock(&kvm->mmu_lock);
 	mutex_unlock(&kvm->slots_lock);
 	return 0;
 }
 static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm)
 {
 	return kvm->arch.n_alloc_mmu_pages;
 }
 gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_mem_alias *alias;
 	struct kvm_mem_aliases *aliases;
 	aliases = kvm_aliases(kvm);
 	for (i = 0; i < aliases->naliases; ++i) {
 		alias = &aliases->aliases[i];
 		if (alias->flags & KVM_ALIAS_INVALID)
 			continue;
 		if (gfn >= alias->base_gfn
 		    && gfn < alias->base_gfn + alias->npages)
 			return alias->target_gfn + gfn - alias->base_gfn;
 	}
 	return gfn;
 }
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_mem_alias *alias;
 	struct kvm_mem_aliases *aliases;
 	aliases = kvm_aliases(kvm);
 	for (i = 0; i < aliases->naliases; ++i) {
 		alias = &aliases->aliases[i];
 		if (gfn >= alias->base_gfn
 		    && gfn < alias->base_gfn + alias->npages)
 			return alias->target_gfn + gfn - alias->base_gfn;
 	}
 	return gfn;
 }
 /*
  * Set a new alias region.  Aliases map a portion of physical memory into
  * another portion.  This is useful for memory windows, for example the PC
  * VGA region.
  */
 static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm,
 					 struct kvm_memory_alias *alias)
 {
 	int r, n;
 	struct kvm_mem_alias *p;
 	struct kvm_mem_aliases *aliases, *old_aliases;
 	r = -EINVAL;
 	/* General sanity checks */
 	if (alias->memory_size & (PAGE_SIZE - 1))
 		goto out;
 	if (alias->guest_phys_addr & (PAGE_SIZE - 1))
 		goto out;
 	if (alias->slot >= KVM_ALIAS_SLOTS)
 		goto out;
 	if (alias->guest_phys_addr + alias->memory_size
 	    < alias->guest_phys_addr)
 		goto out;
 	if (alias->target_phys_addr + alias->memory_size
 	    < alias->target_phys_addr)
 		goto out;
 	r = -ENOMEM;
 	aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
 	if (!aliases)
 		goto out;
 	mutex_lock(&kvm->slots_lock);
 	/* invalidate any gfn reference in case of deletion/shrinking */
 	memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
 	aliases->aliases[alias->slot].flags |= KVM_ALIAS_INVALID;
 	old_aliases = kvm->arch.aliases;
 	rcu_assign_pointer(kvm->arch.aliases, aliases);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kvm_mmu_zap_all(kvm);
 	kfree(old_aliases);
 	r = -ENOMEM;
 	aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
 	if (!aliases)
 		goto out_unlock;
 	memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
 	p = &aliases->aliases[alias->slot];
 	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
 	p->npages = alias->memory_size >> PAGE_SHIFT;
 	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
 	p->flags &= ~(KVM_ALIAS_INVALID);
 	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
 		if (aliases->aliases[n - 1].npages)
 			break;
 	aliases->naliases = n;
 	old_aliases = kvm->arch.aliases;
 	rcu_assign_pointer(kvm->arch.aliases, aliases);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kfree(old_aliases);
 	r = 0;
 out_unlock:
 	mutex_unlock(&kvm->slots_lock);
 out:
 	return r;
 }
 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_PIC_MASTER:
 		memcpy(&chip->chip.pic,
 			&pic_irqchip(kvm)->pics[0],
 			sizeof(struct kvm_pic_state));
 		break;
 	case KVM_IRQCHIP_PIC_SLAVE:
 		memcpy(&chip->chip.pic,
 			&pic_irqchip(kvm)->pics[1],
 			sizeof(struct kvm_pic_state));
 		break;
 	case KVM_IRQCHIP_IOAPIC:
 		r = kvm_get_ioapic(kvm, &chip->chip.ioapic);
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_PIC_MASTER:
 		raw_spin_lock(&pic_irqchip(kvm)->lock);
 		memcpy(&pic_irqchip(kvm)->pics[0],
 			&chip->chip.pic,
 			sizeof(struct kvm_pic_state));
 		raw_spin_unlock(&pic_irqchip(kvm)->lock);
 		break;
 	case KVM_IRQCHIP_PIC_SLAVE:
 		raw_spin_lock(&pic_irqchip(kvm)->lock);
 		memcpy(&pic_irqchip(kvm)->pics[1],
 			&chip->chip.pic,
 			sizeof(struct kvm_pic_state));
 		raw_spin_unlock(&pic_irqchip(kvm)->lock);
 		break;
 	case KVM_IRQCHIP_IOAPIC:
 		r = kvm_set_ioapic(kvm, &chip->chip.ioapic);
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	kvm_pic_update_irq(pic_irqchip(kvm));
 	return r;
 }
 static int kvm_vm_ioctl_get_pit(struct kvm *kvm, struct kvm_pit_state *ps)
 {
 	int r = 0;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
 	memcpy(ps, &kvm->arch.vpit->pit_state, sizeof(struct kvm_pit_state));
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return r;
 }
 static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps)
 {
 	int r = 0;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
 	memcpy(&kvm->arch.vpit->pit_state, ps, sizeof(struct kvm_pit_state));
 	kvm_pit_load_count(kvm, 0, ps->channels[0].count, 0);
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return r;
 }
 static int kvm_vm_ioctl_get_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
 {
 	int r = 0;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
 	memcpy(ps->channels, &kvm->arch.vpit->pit_state.channels,
 		sizeof(ps->channels));
 	ps->flags = kvm->arch.vpit->pit_state.flags;
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return r;
 }
 static int kvm_vm_ioctl_set_pit2(struct kvm *kvm, struct kvm_pit_state2 *ps)
 {
 	int r = 0, start = 0;
 	u32 prev_legacy, cur_legacy;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
 	prev_legacy = kvm->arch.vpit->pit_state.flags & KVM_PIT_FLAGS_HPET_LEGACY;
 	cur_legacy = ps->flags & KVM_PIT_FLAGS_HPET_LEGACY;
 	if (!prev_legacy && cur_legacy)
 		start = 1;
 	memcpy(&kvm->arch.vpit->pit_state.channels, &ps->channels,
 	       sizeof(kvm->arch.vpit->pit_state.channels));
 	kvm->arch.vpit->pit_state.flags = ps->flags;
 	kvm_pit_load_count(kvm, 0, kvm->arch.vpit->pit_state.channels[0].count, start);
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return r;
 }
 static int kvm_vm_ioctl_reinject(struct kvm *kvm,
 				 struct kvm_reinject_control *control)
 {
 	if (!kvm->arch.vpit)
 		return -ENXIO;
 	mutex_lock(&kvm->arch.vpit->pit_state.lock);
 	kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject;
 	mutex_unlock(&kvm->arch.vpit->pit_state.lock);
 	return 0;
 }
 /*
  * Get (and clear) the dirty memory log for a memory slot.
  */
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 				      struct kvm_dirty_log *log)
 {
 	int r, i;
 	struct kvm_memory_slot *memslot;
 	unsigned long n;
 	unsigned long is_dirty = 0;
 	mutex_lock(&kvm->slots_lock);
 	r = -EINVAL;
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 	memslot = &kvm->memslots->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
 	n = kvm_dirty_bitmap_bytes(memslot);
 	for (i = 0; !is_dirty && i < n/sizeof(long); i++)
 		is_dirty = memslot->dirty_bitmap[i];
 	/* If nothing is dirty, don't bother messing with page tables. */
 	if (is_dirty) {
 		struct kvm_memslots *slots, *old_slots;
 		unsigned long *dirty_bitmap;
 		spin_lock(&kvm->mmu_lock);
 		kvm_mmu_slot_remove_write_access(kvm, log->slot);
 		spin_unlock(&kvm->mmu_lock);
 		r = -ENOMEM;
 		dirty_bitmap = vmalloc(n);
 		if (!dirty_bitmap)
 			goto out;
 		memset(dirty_bitmap, 0, n);
 		r = -ENOMEM;
 		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 		if (!slots) {
 			vfree(dirty_bitmap);
 			goto out;
 		}
 		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
 		slots->memslots[log->slot].dirty_bitmap = dirty_bitmap;
 		old_slots = kvm->memslots;
 		rcu_assign_pointer(kvm->memslots, slots);
 		synchronize_srcu_expedited(&kvm->srcu);
 		dirty_bitmap = old_slots->memslots[log->slot].dirty_bitmap;
 		kfree(old_slots);
 		r = -EFAULT;
 		if (copy_to_user(log->dirty_bitmap, dirty_bitmap, n)) {
 			vfree(dirty_bitmap);
 			goto out;
 		}
 		vfree(dirty_bitmap);
 	} else {
 		r = -EFAULT;
 		if (clear_user(log->dirty_bitmap, n))
 			goto out;
 	}
 	r = 0;
 out:
 	mutex_unlock(&kvm->slots_lock);
 	return r;
 }
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r = -ENOTTY;
 	/*
 	 * This union makes it completely explicit to gcc-3.x
 	 * that these two variables' stack usage should be
 	 * combined, not added together.
 	 */
 	union {
 		struct kvm_pit_state ps;
 		struct kvm_pit_state2 ps2;
 		struct kvm_memory_alias alias;
 		struct kvm_pit_config pit_config;
 	} u;
 	switch (ioctl) {
 	case KVM_SET_TSS_ADDR:
 		r = kvm_vm_ioctl_set_tss_addr(kvm, arg);
 		if (r < 0)
 			goto out;
 		break;
 	case KVM_SET_IDENTITY_MAP_ADDR: {
 		u64 ident_addr;
 		r = -EFAULT;
 		if (copy_from_user(&ident_addr, argp, sizeof ident_addr))
 			goto out;
 		r = kvm_vm_ioctl_set_identity_map_addr(kvm, ident_addr);
 		if (r < 0)
 			goto out;
 		break;
 	}
 	case KVM_SET_MEMORY_REGION: {
 		struct kvm_memory_region kvm_mem;
 		struct kvm_userspace_memory_region kvm_userspace_mem;
 		r = -EFAULT;
 		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
 			goto out;
 		kvm_userspace_mem.slot = kvm_mem.slot;
 		kvm_userspace_mem.flags = kvm_mem.flags;
 		kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr;
 		kvm_userspace_mem.memory_size = kvm_mem.memory_size;
 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_SET_NR_MMU_PAGES:
 		r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg);
 		if (r)
 			goto out;
 		break;
 	case KVM_GET_NR_MMU_PAGES:
 		r = kvm_vm_ioctl_get_nr_mmu_pages(kvm);
 		break;
 	case KVM_SET_MEMORY_ALIAS:
 		r = -EFAULT;
 		if (copy_from_user(&u.alias, argp, sizeof(struct kvm_memory_alias)))
 			goto out;
 		r = kvm_vm_ioctl_set_memory_alias(kvm, &u.alias);
 		if (r)
 			goto out;
 		break;
 	case KVM_CREATE_IRQCHIP: {
 		struct kvm_pic *vpic;
 		mutex_lock(&kvm->lock);
 		r = -EEXIST;
 		if (kvm->arch.vpic)
 			goto create_irqchip_unlock;
 		r = -ENOMEM;
 		vpic = kvm_create_pic(kvm);
 		if (vpic) {
 			r = kvm_ioapic_init(kvm);
 			if (r) {
 				kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS,
 							  &vpic->dev);
 				kfree(vpic);
 				goto create_irqchip_unlock;
 			}
 		} else
 			goto create_irqchip_unlock;
 		smp_wmb();
 		kvm->arch.vpic = vpic;
 		smp_wmb();
 		r = kvm_setup_default_irq_routing(kvm);
 		if (r) {
 			mutex_lock(&kvm->irq_lock);
 			kvm_ioapic_destroy(kvm);
 			kvm_destroy_pic(kvm);
 			mutex_unlock(&kvm->irq_lock);
 		}
 	create_irqchip_unlock:
 		mutex_unlock(&kvm->lock);
 		break;
 	}
 	case KVM_CREATE_PIT:
 		u.pit_config.flags = KVM_PIT_SPEAKER_DUMMY;
 		goto create_pit;
 	case KVM_CREATE_PIT2:
 		r = -EFAULT;
 		if (copy_from_user(&u.pit_config, argp,
 				   sizeof(struct kvm_pit_config)))
 			goto out;
 	create_pit:
 		mutex_lock(&kvm->slots_lock);
 		r = -EEXIST;
 		if (kvm->arch.vpit)
 			goto create_pit_unlock;
 		r = -ENOMEM;
 		kvm->arch.vpit = kvm_create_pit(kvm, u.pit_config.flags);
 		if (kvm->arch.vpit)
 			r = 0;
 	create_pit_unlock:
 		mutex_unlock(&kvm->slots_lock);
 		break;
 	case KVM_IRQ_LINE_STATUS:
 	case KVM_IRQ_LINE: {
 		struct kvm_irq_level irq_event;
 		r = -EFAULT;
 		if (copy_from_user(&irq_event, argp, sizeof irq_event))
 			goto out;
 		r = -ENXIO;
 		if (irqchip_in_kernel(kvm)) {
 			__s32 status;
 			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
 					irq_event.irq, irq_event.level);
 			if (ioctl == KVM_IRQ_LINE_STATUS) {
 				r = -EFAULT;
 				irq_event.status = status;
 				if (copy_to_user(argp, &irq_event,
 							sizeof irq_event))
 					goto out;
 			}
 			r = 0;
 		}
 		break;
 	}
 	case KVM_GET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!chip)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(chip, argp, sizeof *chip))
 			goto get_irqchip_out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto get_irqchip_out;
 		r = kvm_vm_ioctl_get_irqchip(kvm, chip);
 		if (r)
 			goto get_irqchip_out;
 		r = -EFAULT;
 		if (copy_to_user(argp, chip, sizeof *chip))
 			goto get_irqchip_out;
 		r = 0;
 	get_irqchip_out:
 		kfree(chip);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_SET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip *chip = kmalloc(sizeof(*chip), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!chip)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(chip, argp, sizeof *chip))
 			goto set_irqchip_out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto set_irqchip_out;
 		r = kvm_vm_ioctl_set_irqchip(kvm, chip);
 		if (r)
 			goto set_irqchip_out;
 		r = 0;
 	set_irqchip_out:
 		kfree(chip);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_GET_PIT: {
 		r = -EFAULT;
 		if (copy_from_user(&u.ps, argp, sizeof(struct kvm_pit_state)))
 			goto out;
 		r = -ENXIO;
 		if (!kvm->arch.vpit)
 			goto out;
 		r = kvm_vm_ioctl_get_pit(kvm, &u.ps);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &u.ps, sizeof(struct kvm_pit_state)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_PIT: {
 		r = -EFAULT;
 		if (copy_from_user(&u.ps, argp, sizeof u.ps))
 			goto out;
 		r = -ENXIO;
 		if (!kvm->arch.vpit)
 			goto out;
 		r = kvm_vm_ioctl_set_pit(kvm, &u.ps);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_GET_PIT2: {
 		r = -ENXIO;
 		if (!kvm->arch.vpit)
 			goto out;
 		r = kvm_vm_ioctl_get_pit2(kvm, &u.ps2);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &u.ps2, sizeof(u.ps2)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_PIT2: {
 		r = -EFAULT;
 		if (copy_from_user(&u.ps2, argp, sizeof(u.ps2)))
 			goto out;
 		r = -ENXIO;
 		if (!kvm->arch.vpit)
 			goto out;
 		r = kvm_vm_ioctl_set_pit2(kvm, &u.ps2);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_REINJECT_CONTROL: {
 		struct kvm_reinject_control control;
 		r =  -EFAULT;
 		if (copy_from_user(&control, argp, sizeof(control)))
 			goto out;
 		r = kvm_vm_ioctl_reinject(kvm, &control);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_XEN_HVM_CONFIG: {
 		r = -EFAULT;
 		if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
 				   sizeof(struct kvm_xen_hvm_config)))
 			goto out;
 		r = -EINVAL;
 		if (kvm->arch.xen_hvm_config.flags)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_CLOCK: {
 		struct timespec now;
 		struct kvm_clock_data user_ns;
 		u64 now_ns;
 		s64 delta;
 		r = -EFAULT;
 		if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
 			goto out;
 		r = -EINVAL;
 		if (user_ns.flags)
 			goto out;
 		r = 0;
 		ktime_get_ts(&now);
 		now_ns = timespec_to_ns(&now);
 		delta = user_ns.clock - now_ns;
 		kvm->arch.kvmclock_offset = delta;
 		break;
 	}
 	case KVM_GET_CLOCK: {
 		struct timespec now;
 		struct kvm_clock_data user_ns;
 		u64 now_ns;
 		ktime_get_ts(&now);
 		now_ns = timespec_to_ns(&now);
 		user_ns.clock = kvm->arch.kvmclock_offset + now_ns;
 		user_ns.flags = 0;
 		r = -EFAULT;
 		if (copy_to_user(argp, &user_ns, sizeof(user_ns)))
 			goto out;
 		r = 0;
 		break;
 	}
 	default:
 		;
 	}
 out:
 	return r;
 }
 static void kvm_init_msr_list(void)
 {
 	u32 dummy[2];
 	unsigned i, j;
 	/* skip the first msrs in the list. KVM-specific */
 	for (i = j = KVM_SAVE_MSRS_BEGIN; i < ARRAY_SIZE(msrs_to_save); i++) {
 		if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
 			continue;
 		if (j < i)
 			msrs_to_save[j] = msrs_to_save[i];
 		j++;
 	}
 	num_msrs_to_save = j;
 }
 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
 			   const void *v)
 {
 	if (vcpu->arch.apic &&
 	    !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
 		return 0;
 	return kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
 }
 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
 {
 	if (vcpu->arch.apic &&
 	    !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
 		return 0;
 	return kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
 }
 static void kvm_set_segment(struct kvm_vcpu *vcpu,
 			struct kvm_segment *var, int seg)
 {
 	kvm_x86_ops->set_segment(vcpu, var, seg);
 }
 void kvm_get_segment(struct kvm_vcpu *vcpu,
 		     struct kvm_segment *var, int seg)
 {
 	kvm_x86_ops->get_segment(vcpu, var, seg);
 }
 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
 {
 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
 }
  gpa_t kvm_mmu_gva_to_gpa_fetch(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
 {
 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	access |= PFERR_FETCH_MASK;
 	return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
 }
 gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
 {
 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	access |= PFERR_WRITE_MASK;
 	return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, access, error);
 }
 /* uses this to access any guest's mapped memory without checking CPL */
 gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, u32 *error)
 {
 	return vcpu->arch.mmu.gva_to_gpa(vcpu, gva, 0, error);
 }
 static int kvm_read_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
 				      struct kvm_vcpu *vcpu, u32 access,
 				      u32 *error)
 {
 	void *data = val;
 	int r = X86EMUL_CONTINUE;
 	while (bytes) {
 		gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr, access, error);
 		unsigned offset = addr & (PAGE_SIZE-1);
 		unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset);
 		int ret;
 		if (gpa == UNMAPPED_GVA) {
 			r = X86EMUL_PROPAGATE_FAULT;
 			goto out;
 		}
 		ret = kvm_read_guest(vcpu->kvm, gpa, data, toread);
 		if (ret < 0) {
 			r = X86EMUL_IO_NEEDED;
 			goto out;
 		}
 		bytes -= toread;
 		data += toread;
 		addr += toread;
 	}
 out:
 	return r;
 }
 /* used for instruction fetching */
 static int kvm_fetch_guest_virt(gva_t addr, void *val, unsigned int bytes,
 				struct kvm_vcpu *vcpu, u32 *error)
 {
 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu,
 					  access | PFERR_FETCH_MASK, error);
 }
 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
 			       struct kvm_vcpu *vcpu, u32 *error)
 {
 	u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0;
 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, access,
 					  error);
 }
 static int kvm_read_guest_virt_system(gva_t addr, void *val, unsigned int bytes,
 			       struct kvm_vcpu *vcpu, u32 *error)
 {
 	return kvm_read_guest_virt_helper(addr, val, bytes, vcpu, 0, error);
 }
 static int kvm_write_guest_virt_system(gva_t addr, void *val,
 				       unsigned int bytes,
 				       struct kvm_vcpu *vcpu,
 				       u32 *error)
 {
 	void *data = val;
 	int r = X86EMUL_CONTINUE;
 	while (bytes) {
 		gpa_t gpa =  vcpu->arch.mmu.gva_to_gpa(vcpu, addr,
 						       PFERR_WRITE_MASK, error);
 		unsigned offset = addr & (PAGE_SIZE-1);
 		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
 		int ret;
 		if (gpa == UNMAPPED_GVA) {
 			r = X86EMUL_PROPAGATE_FAULT;
 			goto out;
 		}
 		ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite);
 		if (ret < 0) {
 			r = X86EMUL_IO_NEEDED;
 			goto out;
 		}
 		bytes -= towrite;
 		data += towrite;
 		addr += towrite;
 	}
 out:
 	return r;
 }
 static int emulator_read_emulated(unsigned long addr,
 				  void *val,
 				  unsigned int bytes,
 				  unsigned int *error_code,
 				  struct kvm_vcpu *vcpu)
 {
 	gpa_t                 gpa;
 	if (vcpu->mmio_read_completed) {
 		memcpy(val, vcpu->mmio_data, bytes);
 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes,
 			       vcpu->mmio_phys_addr, *(u64 *)val);
 		vcpu->mmio_read_completed = 0;
 		return X86EMUL_CONTINUE;
 	}
 	gpa = kvm_mmu_gva_to_gpa_read(vcpu, addr, error_code);
 	if (gpa == UNMAPPED_GVA)
 		return X86EMUL_PROPAGATE_FAULT;
 	/* For APIC access vmexit */
 	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
 		goto mmio;
 	if (kvm_read_guest_virt(addr, val, bytes, vcpu, NULL)
 				== X86EMUL_CONTINUE)
 		return X86EMUL_CONTINUE;
 mmio:
 	/*
 	 * Is this MMIO handled locally?
 	 */
 	if (!vcpu_mmio_read(vcpu, gpa, bytes, val)) {
 		trace_kvm_mmio(KVM_TRACE_MMIO_READ, bytes, gpa, *(u64 *)val);
 		return X86EMUL_CONTINUE;
 	}
 	trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, bytes, gpa, 0);
 	vcpu->mmio_needed = 1;
 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
 	vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
 	vcpu->run->mmio.len = vcpu->mmio_size = bytes;
 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = 0;
 	return X86EMUL_IO_NEEDED;
 }
 int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
 			  const void *val, int bytes)
 {
 	int ret;
 	ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes);
 	if (ret < 0)
 		return 0;
 	kvm_mmu_pte_write(vcpu, gpa, val, bytes, 1);
 	return 1;
 }
 static int emulator_write_emulated_onepage(unsigned long addr,
 					   const void *val,
 					   unsigned int bytes,
 					   unsigned int *error_code,
 					   struct kvm_vcpu *vcpu)
 {
 	gpa_t                 gpa;
 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, error_code);
 	if (gpa == UNMAPPED_GVA)
 		return X86EMUL_PROPAGATE_FAULT;
 	/* For APIC access vmexit */
 	if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
 		goto mmio;
 	if (emulator_write_phys(vcpu, gpa, val, bytes))
 		return X86EMUL_CONTINUE;
 mmio:
 	trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, bytes, gpa, *(u64 *)val);
 	/*
 	 * Is this MMIO handled locally?
 	 */
 	if (!vcpu_mmio_write(vcpu, gpa, bytes, val))
 		return X86EMUL_CONTINUE;
 	vcpu->mmio_needed = 1;
 	vcpu->run->exit_reason = KVM_EXIT_MMIO;
 	vcpu->run->mmio.phys_addr = vcpu->mmio_phys_addr = gpa;
 	vcpu->run->mmio.len = vcpu->mmio_size = bytes;
 	vcpu->run->mmio.is_write = vcpu->mmio_is_write = 1;
 	memcpy(vcpu->run->mmio.data, val, bytes);
 	return X86EMUL_CONTINUE;
 }
 int emulator_write_emulated(unsigned long addr,
 			    const void *val,
 			    unsigned int bytes,
 			    unsigned int *error_code,
 			    struct kvm_vcpu *vcpu)
 {
 	/* Crossing a page boundary? */
 	if (((addr + bytes - 1) ^ addr) & PAGE_MASK) {
 		int rc, now;
 		now = -addr & ~PAGE_MASK;
 		rc = emulator_write_emulated_onepage(addr, val, now, error_code,
 						     vcpu);
 		if (rc != X86EMUL_CONTINUE)
 			return rc;
 		addr += now;
 		val += now;
 		bytes -= now;
 	}
 	return emulator_write_emulated_onepage(addr, val, bytes, error_code,
 					       vcpu);
 }
 #define CMPXCHG_TYPE(t, ptr, old, new) \
 	(cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old))
 #ifdef CONFIG_X86_64
 #  define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new)
 #else
 #  define CMPXCHG64(ptr, old, new) \
 	(cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old))
 #endif
 static int emulator_cmpxchg_emulated(unsigned long addr,
 				     const void *old,
 				     const void *new,
 				     unsigned int bytes,
 				     unsigned int *error_code,
 				     struct kvm_vcpu *vcpu)
 {
 	gpa_t gpa;
 	struct page *page;
 	char *kaddr;
 	bool exchanged;
 	/* guests cmpxchg8b have to be emulated atomically */
 	if (bytes > 8 || (bytes & (bytes - 1)))
 		goto emul_write;
 	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
 	if (gpa == UNMAPPED_GVA ||
 	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
 		goto emul_write;
 	if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK))
 		goto emul_write;
 	page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
 	kaddr = kmap_atomic(page, KM_USER0);
 	kaddr += offset_in_page(gpa);
 	switch (bytes) {
 	case 1:
 		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
 		break;
 	case 2:
 		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
 		break;
 	case 4:
 		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
 		break;
 	case 8:
 		exchanged = CMPXCHG64(kaddr, old, new);
 		break;
 	default:
 		BUG();
 	}
 	kunmap_atomic(kaddr, KM_USER0);
 	kvm_release_page_dirty(page);
 	if (!exchanged)
 		return X86EMUL_CMPXCHG_FAILED;
 	kvm_mmu_pte_write(vcpu, gpa, new, bytes, 1);
 	return X86EMUL_CONTINUE;
 emul_write:
 	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");
 	return emulator_write_emulated(addr, new, bytes, error_code, vcpu);
 }
 static int kernel_pio(struct kvm_vcpu *vcpu, void *pd)
 {
 	/* TODO: String I/O for in kernel device */
 	int r;
 	if (vcpu->arch.pio.in)
 		r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
 				    vcpu->arch.pio.size, pd);
 	else
 		r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
 				     vcpu->arch.pio.port, vcpu->arch.pio.size,
 				     pd);
 	return r;
 }
 static int emulator_pio_in_emulated(int size, unsigned short port, void *val,
 			     unsigned int count, struct kvm_vcpu *vcpu)
 {
 	if (vcpu->arch.pio.count)
 		goto data_avail;
 	trace_kvm_pio(1, port, size, 1);
 	vcpu->arch.pio.port = port;
 	vcpu->arch.pio.in = 1;
 	vcpu->arch.pio.count  = count;
 	vcpu->arch.pio.size = size;
 	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
 	data_avail:
 		memcpy(val, vcpu->arch.pio_data, size * count);
 		vcpu->arch.pio.count = 0;
 		return 1;
 	}
 	vcpu->run->exit_reason = KVM_EXIT_IO;
 	vcpu->run->io.direction = KVM_EXIT_IO_IN;
 	vcpu->run->io.size = size;
 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
 	vcpu->run->io.count = count;
 	vcpu->run->io.port = port;
 	return 0;
 }
 static int emulator_pio_out_emulated(int size, unsigned short port,
 			      const void *val, unsigned int count,
 			      struct kvm_vcpu *vcpu)
 {
 	trace_kvm_pio(0, port, size, 1);
 	vcpu->arch.pio.port = port;
 	vcpu->arch.pio.in = 0;
 	vcpu->arch.pio.count = count;
 	vcpu->arch.pio.size = size;
 	memcpy(vcpu->arch.pio_data, val, size * count);
 	if (!kernel_pio(vcpu, vcpu->arch.pio_data)) {
 		vcpu->arch.pio.count = 0;
 		return 1;
 	}
 	vcpu->run->exit_reason = KVM_EXIT_IO;
 	vcpu->run->io.direction = KVM_EXIT_IO_OUT;
 	vcpu->run->io.size = size;
 	vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE;
 	vcpu->run->io.count = count;
 	vcpu->run->io.port = port;
 	return 0;
 }
 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
 {
 	return kvm_x86_ops->get_segment_base(vcpu, seg);
 }
 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
 {
 	kvm_mmu_invlpg(vcpu, address);
 	return X86EMUL_CONTINUE;
 }
 int emulate_clts(struct kvm_vcpu *vcpu)
 {
 	kvm_x86_ops->set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
 	kvm_x86_ops->fpu_activate(vcpu);
 	return X86EMUL_CONTINUE;
 }
 int emulator_get_dr(int dr, unsigned long *dest, struct kvm_vcpu *vcpu)
 {
 	return _kvm_get_dr(vcpu, dr, dest);
 }
 int emulator_set_dr(int dr, unsigned long value, struct kvm_vcpu *vcpu)
 {
 	return __kvm_set_dr(vcpu, dr, value);
 }
 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
 {
 	return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
 }
 static unsigned long emulator_get_cr(int cr, struct kvm_vcpu *vcpu)
 {
 	unsigned long value;
 	switch (cr) {
 	case 0:
 		value = kvm_read_cr0(vcpu);
 		break;
 	case 2:
 		value = vcpu->arch.cr2;
 		break;
 	case 3:
 		value = vcpu->arch.cr3;
 		break;
 	case 4:
 		value = kvm_read_cr4(vcpu);
 		break;
 	case 8:
 		value = kvm_get_cr8(vcpu);
 		break;
 	default:
 		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
 		return 0;
 	}
 	return value;
 }
 static int emulator_set_cr(int cr, unsigned long val, struct kvm_vcpu *vcpu)
 {
 	int res = 0;
 	switch (cr) {
 	case 0:
 		res = __kvm_set_cr0(vcpu, mk_cr_64(kvm_read_cr0(vcpu), val));
 		break;
 	case 2:
 		vcpu->arch.cr2 = val;
 		break;
 	case 3:
 		res = __kvm_set_cr3(vcpu, val);
 		break;
 	case 4:
 		res = __kvm_set_cr4(vcpu, mk_cr_64(kvm_read_cr4(vcpu), val));
 		break;
 	case 8:
 		res = __kvm_set_cr8(vcpu, val & 0xfUL);
 		break;
 	default:
 		vcpu_printf(vcpu, "%s: unexpected cr %u\n", __func__, cr);
 		res = -1;
 	}
 	return res;
 }
 static int emulator_get_cpl(struct kvm_vcpu *vcpu)
 {
 	return kvm_x86_ops->get_cpl(vcpu);
 }
 static void emulator_get_gdt(struct desc_ptr *dt, struct kvm_vcpu *vcpu)
 {
 	kvm_x86_ops->get_gdt(vcpu, dt);
 }
 static unsigned long emulator_get_cached_segment_base(int seg,
 						      struct kvm_vcpu *vcpu)
 {
 	return get_segment_base(vcpu, seg);
 }
 static bool emulator_get_cached_descriptor(struct desc_struct *desc, int seg,
 					   struct kvm_vcpu *vcpu)
 {
 	struct kvm_segment var;
 	kvm_get_segment(vcpu, &var, seg);
 	if (var.unusable)
 		return false;
 	if (var.g)
 		var.limit >>= 12;
 	set_desc_limit(desc, var.limit);
 	set_desc_base(desc, (unsigned long)var.base);
 	desc->type = var.type;
 	desc->s = var.s;
 	desc->dpl = var.dpl;
 	desc->p = var.present;
 	desc->avl = var.avl;
 	desc->l = var.l;
 	desc->d = var.db;
 	desc->g = var.g;
 	return true;
 }
 static void emulator_set_cached_descriptor(struct desc_struct *desc, int seg,
 					   struct kvm_vcpu *vcpu)
 {
 	struct kvm_segment var;
 	/* needed to preserve selector */
 	kvm_get_segment(vcpu, &var, seg);
 	var.base = get_desc_base(desc);
 	var.limit = get_desc_limit(desc);
 	if (desc->g)
 		var.limit = (var.limit << 12) | 0xfff;
 	var.type = desc->type;
 	var.present = desc->p;
 	var.dpl = desc->dpl;
 	var.db = desc->d;
 	var.s = desc->s;
 	var.l = desc->l;
 	var.g = desc->g;
 	var.avl = desc->avl;
 	var.present = desc->p;
 	var.unusable = !var.present;
 	var.padding = 0;
 	kvm_set_segment(vcpu, &var, seg);
 	return;
 }
 static u16 emulator_get_segment_selector(int seg, struct kvm_vcpu *vcpu)
 {
 	struct kvm_segment kvm_seg;
 	kvm_get_segment(vcpu, &kvm_seg, seg);
 	return kvm_seg.selector;
 }
 static void emulator_set_segment_selector(u16 sel, int seg,
 					  struct kvm_vcpu *vcpu)
 {
 	struct kvm_segment kvm_seg;
 	kvm_get_segment(vcpu, &kvm_seg, seg);
 	kvm_seg.selector = sel;
 	kvm_set_segment(vcpu, &kvm_seg, seg);
 }
 static struct x86_emulate_ops emulate_ops = {
 	.read_std            = kvm_read_guest_virt_system,
 	.write_std           = kvm_write_guest_virt_system,
 	.fetch               = kvm_fetch_guest_virt,
 	.read_emulated       = emulator_read_emulated,
 	.write_emulated      = emulator_write_emulated,
 	.cmpxchg_emulated    = emulator_cmpxchg_emulated,
 	.pio_in_emulated     = emulator_pio_in_emulated,
 	.pio_out_emulated    = emulator_pio_out_emulated,
 	.get_cached_descriptor = emulator_get_cached_descriptor,
 	.set_cached_descriptor = emulator_set_cached_descriptor,
 	.get_segment_selector = emulator_get_segment_selector,
 	.set_segment_selector = emulator_set_segment_selector,
 	.get_cached_segment_base = emulator_get_cached_segment_base,
 	.get_gdt             = emulator_get_gdt,
 	.get_cr              = emulator_get_cr,
 	.set_cr              = emulator_set_cr,
 	.cpl                 = emulator_get_cpl,
 	.get_dr              = emulator_get_dr,
 	.set_dr              = emulator_set_dr,
 	.set_msr             = kvm_set_msr,
 	.get_msr             = kvm_get_msr,
 };
 static void cache_all_regs(struct kvm_vcpu *vcpu)
 {
 	kvm_register_read(vcpu, VCPU_REGS_RAX);
 	kvm_register_read(vcpu, VCPU_REGS_RSP);
 	kvm_register_read(vcpu, VCPU_REGS_RIP);
 	vcpu->arch.regs_dirty = ~0;
 }
 static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
 {
 	u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
 	/*
 	 * an sti; sti; sequence only disable interrupts for the first
 	 * instruction. So, if the last instruction, be it emulated or
 	 * not, left the system with the INT_STI flag enabled, it
 	 * means that the last instruction is an sti. We should not
 	 * leave the flag on in this case. The same goes for mov ss
 	 */
 	if (!(int_shadow & mask))
 		kvm_x86_ops->set_interrupt_shadow(vcpu, mask);
 }
 static void inject_emulated_exception(struct kvm_vcpu *vcpu)
 {
 	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
 	if (ctxt->exception == PF_VECTOR)
 		kvm_inject_page_fault(vcpu, ctxt->cr2, ctxt->error_code);
 	else if (ctxt->error_code_valid)
 		kvm_queue_exception_e(vcpu, ctxt->exception, ctxt->error_code);
 	else
 		kvm_queue_exception(vcpu, ctxt->exception);
 }
 static int handle_emulation_failure(struct kvm_vcpu *vcpu)
 {
 	++vcpu->stat.insn_emulation_fail;
 	trace_kvm_emulate_insn_failed(vcpu);
 	vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 	vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
 	vcpu->run->internal.ndata = 0;
 	kvm_queue_exception(vcpu, UD_VECTOR);
 	return EMULATE_FAIL;
 }
 int emulate_instruction(struct kvm_vcpu *vcpu,
 			unsigned long cr2,
 			u16 error_code,
 			int emulation_type)
 {
 	int r;
 	struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
 	kvm_clear_exception_queue(vcpu);
 	vcpu->arch.mmio_fault_cr2 = cr2;
 	/*
 	 * TODO: fix emulate.c to use guest_read/write_register
 	 * instead of direct ->regs accesses, can save hundred cycles
 	 * on Intel for instructions that don't read/change RSP, for
 	 * for example.
 	 */
 	cache_all_regs(vcpu);
 	if (!(emulation_type & EMULTYPE_NO_DECODE)) {
 		int cs_db, cs_l;
 		kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 		vcpu->arch.emulate_ctxt.vcpu = vcpu;
 		vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
 		vcpu->arch.emulate_ctxt.eip = kvm_rip_read(vcpu);
 		vcpu->arch.emulate_ctxt.mode =
 			(!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
 			(vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
 			? X86EMUL_MODE_VM86 : cs_l
 			? X86EMUL_MODE_PROT64 :	cs_db
 			? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
 		memset(c, 0, sizeof(struct decode_cache));
 		memcpy(c->regs, vcpu->arch.regs, sizeof c->regs);
 		vcpu->arch.emulate_ctxt.interruptibility = 0;
 		vcpu->arch.emulate_ctxt.exception = -1;
 		r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
 		trace_kvm_emulate_insn_start(vcpu);
 		/* Only allow emulation of specific instructions on #UD
 		 * (namely VMMCALL, sysenter, sysexit, syscall)*/
 		if (emulation_type & EMULTYPE_TRAP_UD) {
 			if (!c->twobyte)
 				return EMULATE_FAIL;
 			switch (c->b) {
 			case 0x01: /* VMMCALL */
 				if (c->modrm_mod != 3 || c->modrm_rm != 1)
 					return EMULATE_FAIL;
 				break;
 			case 0x34: /* sysenter */
 			case 0x35: /* sysexit */
 				if (c->modrm_mod != 0 || c->modrm_rm != 0)
 					return EMULATE_FAIL;
 				break;
 			case 0x05: /* syscall */
 				if (c->modrm_mod != 0 || c->modrm_rm != 0)
 					return EMULATE_FAIL;
 				break;
 			default:
 				return EMULATE_FAIL;
 			}
 			if (!(c->modrm_reg == 0 || c->modrm_reg == 3))
 				return EMULATE_FAIL;
 		}
 		++vcpu->stat.insn_emulation;
 		if (r)  {
 			if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
 				return EMULATE_DONE;
 			if (emulation_type & EMULTYPE_SKIP)
 				return EMULATE_FAIL;
 			return handle_emulation_failure(vcpu);
 		}
 	}
 	if (emulation_type & EMULTYPE_SKIP) {
 		kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.decode.eip);
 		return EMULATE_DONE;
 	}
 	/* this is needed for vmware backdor interface to work since it
 	   changes registers values  during IO operation */
 	memcpy(c->regs, vcpu->arch.regs, sizeof c->regs);
 restart:
 	r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops);
 	if (r) { /* emulation failed */
 		/*
 		 * if emulation was due to access to shadowed page table
 		 * and it failed try to unshadow page and re-entetr the
 		 * guest to let CPU execute the instruction.
 		 */
 		if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
 			return EMULATE_DONE;
 		return handle_emulation_failure(vcpu);
 	}
 	toggle_interruptibility(vcpu, vcpu->arch.emulate_ctxt.interruptibility);
 	kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
 	memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
 	kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.eip);
 	if (vcpu->arch.emulate_ctxt.exception >= 0) {
 		inject_emulated_exception(vcpu);
 		return EMULATE_DONE;
 	}
 	if (vcpu->arch.pio.count) {
 		if (!vcpu->arch.pio.in)
 			vcpu->arch.pio.count = 0;
 		return EMULATE_DO_MMIO;
 	}
 	if (vcpu->mmio_needed) {
 		if (vcpu->mmio_is_write)
 			vcpu->mmio_needed = 0;
 		return EMULATE_DO_MMIO;
 	}
 	if (vcpu->arch.emulate_ctxt.restart)
 		goto restart;
 	return EMULATE_DONE;
 }
 EXPORT_SYMBOL_GPL(emulate_instruction);
 int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
 {
 	unsigned long val = kvm_register_read(vcpu, VCPU_REGS_RAX);
 	int ret = emulator_pio_out_emulated(size, port, &val, 1, vcpu);
 	/* do not return to emulator after return from userspace */
 	vcpu->arch.pio.count = 0;
 	return ret;
 }
 EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
 static void bounce_off(void *info)
 {
 	/* nothing */
 }
 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
 				     void *data)
 {
 	struct cpufreq_freqs *freq = data;
 	struct kvm *kvm;
 	struct kvm_vcpu *vcpu;
 	int i, send_ipi = 0;
 	if (val == CPUFREQ_PRECHANGE && freq->old > freq->new)
 		return 0;
 	if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new)
 		return 0;
 	per_cpu(cpu_tsc_khz, freq->cpu) = freq->new;
 	spin_lock(&kvm_lock);
 	list_for_each_entry(kvm, &vm_list, vm_list) {
 		kvm_for_each_vcpu(i, vcpu, kvm) {
 			if (vcpu->cpu != freq->cpu)
 				continue;
 			if (!kvm_request_guest_time_update(vcpu))
 				continue;
 			if (vcpu->cpu != smp_processor_id())
 				send_ipi++;
 		}
 	}
 	spin_unlock(&kvm_lock);
 	if (freq->old < freq->new && send_ipi) {
 		/*
 		 * We upscale the frequency.  Must make the guest
 		 * doesn't see old kvmclock values while running with
 		 * the new frequency, otherwise we risk the guest sees
 		 * time go backwards.
 		 *
 		 * In case we update the frequency for another cpu
 		 * (which might be in guest context) send an interrupt
 		 * to kick the cpu out of guest context.  Next time
 		 * guest context is entered kvmclock will be updated,
 		 * so the guest will not see stale values.
 		 */
 		smp_call_function_single(freq->cpu, bounce_off, NULL, 1);
 	}
 	return 0;
 }
 static struct notifier_block kvmclock_cpufreq_notifier_block = {
         .notifier_call  = kvmclock_cpufreq_notifier
 };
 static void kvm_timer_init(void)
 {
 	int cpu;
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 		cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block,
 					  CPUFREQ_TRANSITION_NOTIFIER);
 		for_each_online_cpu(cpu) {
 			unsigned long khz = cpufreq_get(cpu);
 			if (!khz)
 				khz = tsc_khz;
 			per_cpu(cpu_tsc_khz, cpu) = khz;
 		}
 	} else {
 		for_each_possible_cpu(cpu)
 			per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
 	}
 }
 static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
 static int kvm_is_in_guest(void)
 {
 	return percpu_read(current_vcpu) != NULL;
 }
 static int kvm_is_user_mode(void)
 {
 	int user_mode = 3;
 	if (percpu_read(current_vcpu))
 		user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
 	return user_mode != 0;
 }
 static unsigned long kvm_get_guest_ip(void)
 {
 	unsigned long ip = 0;
 	if (percpu_read(current_vcpu))
 		ip = kvm_rip_read(percpu_read(current_vcpu));
 	return ip;
 }
 static struct perf_guest_info_callbacks kvm_guest_cbs = {
 	.is_in_guest		= kvm_is_in_guest,
 	.is_user_mode		= kvm_is_user_mode,
 	.get_guest_ip		= kvm_get_guest_ip,
 };
 void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
 {
 	percpu_write(current_vcpu, vcpu);
 }
 EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
 void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
 {
 	percpu_write(current_vcpu, NULL);
 }
 EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
 int kvm_arch_init(void *opaque)
 {
 	int r;
 	struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque;
 	if (kvm_x86_ops) {
 		printk(KERN_ERR "kvm: already loaded the other module\n");
 		r = -EEXIST;
 		goto out;
 	}
 	if (!ops->cpu_has_kvm_support()) {
 		printk(KERN_ERR "kvm: no hardware support\n");
 		r = -EOPNOTSUPP;
 		goto out;
 	}
 	if (ops->disabled_by_bios()) {
 		printk(KERN_ERR "kvm: disabled by bios\n");
 		r = -EOPNOTSUPP;
 		goto out;
 	}
 	r = kvm_mmu_module_init();
 	if (r)
 		goto out;
 	kvm_init_msr_list();
 	kvm_x86_ops = ops;
 	kvm_mmu_set_nonpresent_ptes(0ull, 0ull);
 	kvm_mmu_set_base_ptes(PT_PRESENT_MASK);
 	kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK,
 			PT_DIRTY_MASK, PT64_NX_MASK, 0);
 	kvm_timer_init();
 	perf_register_guest_info_callbacks(&kvm_guest_cbs);
 	return 0;
 out:
 	return r;
 }
 void kvm_arch_exit(void)
 {
 	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
 					    CPUFREQ_TRANSITION_NOTIFIER);
 	kvm_x86_ops = NULL;
 	kvm_mmu_module_exit();
 }
 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 {
 	++vcpu->stat.halt_exits;
 	if (irqchip_in_kernel(vcpu->kvm)) {
 		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
 		return 1;
 	} else {
 		vcpu->run->exit_reason = KVM_EXIT_HLT;
 		return 0;
 	}
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 static inline gpa_t hc_gpa(struct kvm_vcpu *vcpu, unsigned long a0,
 			   unsigned long a1)
 {
 	if (is_long_mode(vcpu))
 		return a0;
 	else
 		return a0 | ((gpa_t)a1 << 32);
 }
 int kvm_hv_hypercall(struct kvm_vcpu *vcpu)
 {
 	u64 param, ingpa, outgpa, ret;
 	uint16_t code, rep_idx, rep_cnt, res = HV_STATUS_SUCCESS, rep_done = 0;
 	bool fast, longmode;
 	int cs_db, cs_l;
 	/*
 	 * hypercall generates UD from non zero cpl and real mode
 	 * per HYPER-V spec
 	 */
 	if (kvm_x86_ops->get_cpl(vcpu) != 0 || !is_protmode(vcpu)) {
 		kvm_queue_exception(vcpu, UD_VECTOR);
 		return 0;
 	}
 	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 	longmode = is_long_mode(vcpu) && cs_l == 1;
 	if (!longmode) {
 		param = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDX) << 32) |
 			(kvm_register_read(vcpu, VCPU_REGS_RAX) & 0xffffffff);
 		ingpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RBX) << 32) |
 			(kvm_register_read(vcpu, VCPU_REGS_RCX) & 0xffffffff);
 		outgpa = ((u64)kvm_register_read(vcpu, VCPU_REGS_RDI) << 32) |
 			(kvm_register_read(vcpu, VCPU_REGS_RSI) & 0xffffffff);
 	}
 #ifdef CONFIG_X86_64
 	else {
 		param = kvm_register_read(vcpu, VCPU_REGS_RCX);
 		ingpa = kvm_register_read(vcpu, VCPU_REGS_RDX);
 		outgpa = kvm_register_read(vcpu, VCPU_REGS_R8);
 	}
 #endif
 	code = param & 0xffff;
 	fast = (param >> 16) & 0x1;
 	rep_cnt = (param >> 32) & 0xfff;
 	rep_idx = (param >> 48) & 0xfff;
 	trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa);
 	switch (code) {
 	case HV_X64_HV_NOTIFY_LONG_SPIN_WAIT:
 		kvm_vcpu_on_spin(vcpu);
 		break;
 	default:
 		res = HV_STATUS_INVALID_HYPERCALL_CODE;
 		break;
 	}
 	ret = res | (((u64)rep_done & 0xfff) << 32);
 	if (longmode) {
 		kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
 	} else {
 		kvm_register_write(vcpu, VCPU_REGS_RDX, ret >> 32);
 		kvm_register_write(vcpu, VCPU_REGS_RAX, ret & 0xffffffff);
 	}
 	return 1;
 }
 int kvm_emulate_hypercall(struct kvm_vcpu *vcpu)
 {
 	unsigned long nr, a0, a1, a2, a3, ret;
 	int r = 1;
 	if (kvm_hv_hypercall_enabled(vcpu->kvm))
 		return kvm_hv_hypercall(vcpu);
 	nr = kvm_register_read(vcpu, VCPU_REGS_RAX);
 	a0 = kvm_register_read(vcpu, VCPU_REGS_RBX);
 	a1 = kvm_register_read(vcpu, VCPU_REGS_RCX);
 	a2 = kvm_register_read(vcpu, VCPU_REGS_RDX);
 	a3 = kvm_register_read(vcpu, VCPU_REGS_RSI);
 	trace_kvm_hypercall(nr, a0, a1, a2, a3);
 	if (!is_long_mode(vcpu)) {
 		nr &= 0xFFFFFFFF;
 		a0 &= 0xFFFFFFFF;
 		a1 &= 0xFFFFFFFF;
 		a2 &= 0xFFFFFFFF;
 		a3 &= 0xFFFFFFFF;
 	}
 	if (kvm_x86_ops->get_cpl(vcpu) != 0) {
 		ret = -KVM_EPERM;
 		goto out;
 	}
 	switch (nr) {
 	case KVM_HC_VAPIC_POLL_IRQ:
 		ret = 0;
 		break;
 	case KVM_HC_MMU_OP:
 		r = kvm_pv_mmu_op(vcpu, a0, hc_gpa(vcpu, a1, a2), &ret);
 		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;
 	}
 out:
 	kvm_register_write(vcpu, VCPU_REGS_RAX, ret);
 	++vcpu->stat.hypercalls;
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_hypercall);
 int kvm_fix_hypercall(struct kvm_vcpu *vcpu)
 {
 	char instruction[3];
 	unsigned long rip = kvm_rip_read(vcpu);
 	/*
 	 * Blow out the MMU to ensure that no other VCPU has an active mapping
 	 * to ensure that the updated hypercall appears atomically across all
 	 * VCPUs.
 	 */
 	kvm_mmu_zap_all(vcpu->kvm);
 	kvm_x86_ops->patch_hypercall(vcpu, instruction);
 	return emulator_write_emulated(rip, instruction, 3, NULL, vcpu);
 }
 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
 {
 	struct desc_ptr dt = { limit, base };
 	kvm_x86_ops->set_gdt(vcpu, &dt);
 }
 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
 {
 	struct desc_ptr dt = { limit, base };
 	kvm_x86_ops->set_idt(vcpu, &dt);
 }
 static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i)
 {
 	struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i];
 	int j, nent = vcpu->arch.cpuid_nent;
 	e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT;
 	/* when no next entry is found, the current entry[i] is reselected */
 	for (j = i + 1; ; j = (j + 1) % nent) {
 		struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j];
 		if (ej->function == e->function) {
 			ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT;
 			return j;
 		}
 	}
 	return 0; /* silence gcc, even though control never reaches here */
 }
 /* find an entry with matching function, matching index (if needed), and that
  * should be read next (if it's stateful) */
 static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e,
 	u32 function, u32 index)
 {
 	if (e->function != function)
 		return 0;
 	if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index)
 		return 0;
 	if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) &&
 	    !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT))
 		return 0;
 	return 1;
 }
 struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
 					      u32 function, u32 index)
 {
 	int i;
 	struct kvm_cpuid_entry2 *best = NULL;
 	for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
 		struct kvm_cpuid_entry2 *e;
 		e = &vcpu->arch.cpuid_entries[i];
 		if (is_matching_cpuid_entry(e, function, index)) {
 			if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC)
 				move_to_next_stateful_cpuid_entry(vcpu, i);
 			best = e;
 			break;
 		}
 		/*
 		 * Both basic or both extended?
 		 */
 		if (((e->function ^ function) & 0x80000000) == 0)
 			if (!best || e->function > best->function)
 				best = e;
 	}
 	return best;
 }
 EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
 int cpuid_maxphyaddr(struct kvm_vcpu *vcpu)
 {
 	struct kvm_cpuid_entry2 *best;
 	best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
 	if (!best || best->eax < 0x80000008)
 		goto not_found;
 	best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
 	if (best)
 		return best->eax & 0xff;
 not_found:
 	return 36;
 }
 void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
 {
 	u32 function, index;
 	struct kvm_cpuid_entry2 *best;
 	function = kvm_register_read(vcpu, VCPU_REGS_RAX);
 	index = kvm_register_read(vcpu, VCPU_REGS_RCX);
 	kvm_register_write(vcpu, VCPU_REGS_RAX, 0);
 	kvm_register_write(vcpu, VCPU_REGS_RBX, 0);
 	kvm_register_write(vcpu, VCPU_REGS_RCX, 0);
 	kvm_register_write(vcpu, VCPU_REGS_RDX, 0);
 	best = kvm_find_cpuid_entry(vcpu, function, index);
 	if (best) {
 		kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax);
 		kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx);
 		kvm_register_write(vcpu, VCPU_REGS_RCX, best->ecx);
 		kvm_register_write(vcpu, VCPU_REGS_RDX, best->edx);
 	}
 	kvm_x86_ops->skip_emulated_instruction(vcpu);
 	trace_kvm_cpuid(function,
 			kvm_register_read(vcpu, VCPU_REGS_RAX),
 			kvm_register_read(vcpu, VCPU_REGS_RBX),
 			kvm_register_read(vcpu, VCPU_REGS_RCX),
 			kvm_register_read(vcpu, VCPU_REGS_RDX));
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
 /*
  * Check if userspace requested an interrupt window, and that the
  * interrupt window is open.
  *
  * No need to exit to userspace if we already have an interrupt queued.
  */
 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu)
 {
 	return (!irqchip_in_kernel(vcpu->kvm) && !kvm_cpu_has_interrupt(vcpu) &&
 		vcpu->run->request_interrupt_window &&
 		kvm_arch_interrupt_allowed(vcpu));
 }
 static void post_kvm_run_save(struct kvm_vcpu *vcpu)
 {
 	struct kvm_run *kvm_run = vcpu->run;
 	kvm_run->if_flag = (kvm_get_rflags(vcpu) & X86_EFLAGS_IF) != 0;
 	kvm_run->cr8 = kvm_get_cr8(vcpu);
 	kvm_run->apic_base = kvm_get_apic_base(vcpu);
 	if (irqchip_in_kernel(vcpu->kvm))
 		kvm_run->ready_for_interrupt_injection = 1;
 	else
 		kvm_run->ready_for_interrupt_injection =
 			kvm_arch_interrupt_allowed(vcpu) &&
 			!kvm_cpu_has_interrupt(vcpu) &&
 			!kvm_event_needs_reinjection(vcpu);
 }
 static void vapic_enter(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 	struct page *page;
 	if (!apic || !apic->vapic_addr)
 		return;
 	page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
 	vcpu->arch.apic->vapic_page = page;
 }
 static void vapic_exit(struct kvm_vcpu *vcpu)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 	int idx;
 	if (!apic || !apic->vapic_addr)
 		return;
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	kvm_release_page_dirty(apic->vapic_page);
 	mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 }
 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
 {
 	int max_irr, tpr;
 	if (!kvm_x86_ops->update_cr8_intercept)
 		return;
 	if (!vcpu->arch.apic)
 		return;
 	if (!vcpu->arch.apic->vapic_addr)
 		max_irr = kvm_lapic_find_highest_irr(vcpu);
 	else
 		max_irr = -1;
 	if (max_irr != -1)
 		max_irr >>= 4;
 	tpr = kvm_lapic_get_cr8(vcpu);
 	kvm_x86_ops->update_cr8_intercept(vcpu, tpr, max_irr);
 }
 static void inject_pending_event(struct kvm_vcpu *vcpu)
 {
 	/* try to reinject previous events if any */
 	if (vcpu->arch.exception.pending) {
 		trace_kvm_inj_exception(vcpu->arch.exception.nr,
 					vcpu->arch.exception.has_error_code,
 					vcpu->arch.exception.error_code);
 		kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr,
 					  vcpu->arch.exception.has_error_code,
 					  vcpu->arch.exception.error_code,
 					  vcpu->arch.exception.reinject);
 		return;
 	}
 	if (vcpu->arch.nmi_injected) {
 		kvm_x86_ops->set_nmi(vcpu);
 		return;
 	}
 	if (vcpu->arch.interrupt.pending) {
 		kvm_x86_ops->set_irq(vcpu);
 		return;
 	}
 	/* try to inject new event if pending */
 	if (vcpu->arch.nmi_pending) {
 		if (kvm_x86_ops->nmi_allowed(vcpu)) {
 			vcpu->arch.nmi_pending = false;
 			vcpu->arch.nmi_injected = true;
 			kvm_x86_ops->set_nmi(vcpu);
 		}
 	} else if (kvm_cpu_has_interrupt(vcpu)) {
 		if (kvm_x86_ops->interrupt_allowed(vcpu)) {
 			kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu),
 					    false);
 			kvm_x86_ops->set_irq(vcpu);
 		}
 	}
 }
 static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 {
 	int r;
 	bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
 		vcpu->run->request_interrupt_window;
 	if (vcpu->requests)
 		if (test_and_clear_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
 			kvm_mmu_unload(vcpu);
 	r = kvm_mmu_reload(vcpu);
 	if (unlikely(r))
 		goto out;
 	if (vcpu->requests) {
 		if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests))
 			__kvm_migrate_timers(vcpu);
 		if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests))
 			kvm_write_guest_time(vcpu);
 		if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests))
 			kvm_mmu_sync_roots(vcpu);
 		if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
 			kvm_x86_ops->tlb_flush(vcpu);
 		if (test_and_clear_bit(KVM_REQ_REPORT_TPR_ACCESS,
 				       &vcpu->requests)) {
 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
 			r = 0;
 			goto out;
 		}
 		if (test_and_clear_bit(KVM_REQ_TRIPLE_FAULT, &vcpu->requests)) {
 			vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
 			r = 0;
 			goto out;
 		}
 		if (test_and_clear_bit(KVM_REQ_DEACTIVATE_FPU, &vcpu->requests)) {
 			vcpu->fpu_active = 0;
 			kvm_x86_ops->fpu_deactivate(vcpu);
 		}
 	}
 	preempt_disable();
 	kvm_x86_ops->prepare_guest_switch(vcpu);
 	if (vcpu->fpu_active)
 		kvm_load_guest_fpu(vcpu);
 	atomic_set(&vcpu->guest_mode, 1);
 	smp_wmb();
 	local_irq_disable();
 	if (!atomic_read(&vcpu->guest_mode) || vcpu->requests
 	    || need_resched() || signal_pending(current)) {
 		atomic_set(&vcpu->guest_mode, 0);
 		smp_wmb();
 		local_irq_enable();
 		preempt_enable();
 		r = 1;
 		goto out;
 	}
 	inject_pending_event(vcpu);
 	/* enable NMI/IRQ window open exits if needed */
 	if (vcpu->arch.nmi_pending)
 		kvm_x86_ops->enable_nmi_window(vcpu);
 	else if (kvm_cpu_has_interrupt(vcpu) || req_int_win)
 		kvm_x86_ops->enable_irq_window(vcpu);
 	if (kvm_lapic_enabled(vcpu)) {
 		update_cr8_intercept(vcpu);
 		kvm_lapic_sync_to_vapic(vcpu);
 	}
 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 	kvm_guest_enter();
 	if (unlikely(vcpu->arch.switch_db_regs)) {
 		set_debugreg(0, 7);
 		set_debugreg(vcpu->arch.eff_db[0], 0);
 		set_debugreg(vcpu->arch.eff_db[1], 1);
 		set_debugreg(vcpu->arch.eff_db[2], 2);
 		set_debugreg(vcpu->arch.eff_db[3], 3);
 	}
 	trace_kvm_entry(vcpu->vcpu_id);
 	kvm_x86_ops->run(vcpu);
 	/*
 	 * If the guest has used debug registers, at least dr7
 	 * will be disabled while returning to the host.
 	 * If we don't have active breakpoints in the host, we don't
 	 * care about the messed up debug address registers. But if
 	 * we have some of them active, restore the old state.
 	 */
 	if (hw_breakpoint_active())
 		hw_breakpoint_restore();
 	atomic_set(&vcpu->guest_mode, 0);
 	smp_wmb();
 	local_irq_enable();
 	++vcpu->stat.exits;
 	/*
 	 * We must have an instruction between local_irq_enable() and
 	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
 	 * the interrupt shadow.  The stat.exits increment will do nicely.
 	 * But we need to prevent reordering, hence this barrier():
 	 */
 	barrier();
 	kvm_guest_exit();
 	preempt_enable();
 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 	/*
 	 * Profile KVM exit RIPs:
 	 */
 	if (unlikely(prof_on == KVM_PROFILING)) {
 		unsigned long rip = kvm_rip_read(vcpu);
 		profile_hit(KVM_PROFILING, (void *)rip);
 	}
 	kvm_lapic_sync_from_vapic(vcpu);
 	r = kvm_x86_ops->handle_exit(vcpu);
 out:
 	return r;
 }
 static int __vcpu_run(struct kvm_vcpu *vcpu)
 {
 	int r;
 	struct kvm *kvm = vcpu->kvm;
 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED)) {
 		pr_debug("vcpu %d received sipi with vector # %x\n",
 			 vcpu->vcpu_id, vcpu->arch.sipi_vector);
 		kvm_lapic_reset(vcpu);
 		r = kvm_arch_vcpu_reset(vcpu);
 		if (r)
 			return r;
 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 	}
 	vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
 	vapic_enter(vcpu);
 	r = 1;
 	while (r > 0) {
 		if (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE)
 			r = vcpu_enter_guest(vcpu);
 		else {
 			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
 			kvm_vcpu_block(vcpu);
 			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
 			if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests))
 			{
 				switch(vcpu->arch.mp_state) {
 				case KVM_MP_STATE_HALTED:
 					vcpu->arch.mp_state =
 						KVM_MP_STATE_RUNNABLE;
 				case KVM_MP_STATE_RUNNABLE:
 					break;
 				case KVM_MP_STATE_SIPI_RECEIVED:
 				default:
 					r = -EINTR;
 					break;
 				}
 			}
 		}
 		if (r <= 0)
 			break;
 		clear_bit(KVM_REQ_PENDING_TIMER, &vcpu->requests);
 		if (kvm_cpu_has_pending_timer(vcpu))
 			kvm_inject_pending_timer_irqs(vcpu);
 		if (dm_request_for_irq_injection(vcpu)) {
 			r = -EINTR;
 			vcpu->run->exit_reason = KVM_EXIT_INTR;
 			++vcpu->stat.request_irq_exits;
 		}
 		if (signal_pending(current)) {
 			r = -EINTR;
 			vcpu->run->exit_reason = KVM_EXIT_INTR;
 			++vcpu->stat.signal_exits;
 		}
 		if (need_resched()) {
 			srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
 			kvm_resched(vcpu);
 			vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
 		}
 	}
 	srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
 	vapic_exit(vcpu);
 	return r;
 }
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
 	sigset_t sigsaved;
-	vcpu_load(vcpu);
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
 		kvm_vcpu_block(vcpu);
 		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 		r = -EAGAIN;
 		goto out;
 	}
 	/* re-sync apic's tpr */
 	if (!irqchip_in_kernel(vcpu->kvm))
 		kvm_set_cr8(vcpu, kvm_run->cr8);
 	if (vcpu->arch.pio.count || vcpu->mmio_needed ||
 	    vcpu->arch.emulate_ctxt.restart) {
 		if (vcpu->mmio_needed) {
 			memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
 			vcpu->mmio_read_completed = 1;
 			vcpu->mmio_needed = 0;
 		}
 		vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 		r = emulate_instruction(vcpu, 0, 0, EMULTYPE_NO_DECODE);
 		srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 		if (r != EMULATE_DONE) {
 			r = 0;
 			goto out;
 		}
 	}
 	if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL)
 		kvm_register_write(vcpu, VCPU_REGS_RAX,
 				     kvm_run->hypercall.ret);
 	r = __vcpu_run(vcpu);
 out:
 	post_kvm_run_save(vcpu);
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
-	vcpu_put(vcpu);
 	return r;
 }
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
-	vcpu_load(vcpu);
 	regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
 	regs->rbx = kvm_register_read(vcpu, VCPU_REGS_RBX);
 	regs->rcx = kvm_register_read(vcpu, VCPU_REGS_RCX);
 	regs->rdx = kvm_register_read(vcpu, VCPU_REGS_RDX);
 	regs->rsi = kvm_register_read(vcpu, VCPU_REGS_RSI);
 	regs->rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
 	regs->rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
 	regs->rbp = kvm_register_read(vcpu, VCPU_REGS_RBP);
 #ifdef CONFIG_X86_64
 	regs->r8 = kvm_register_read(vcpu, VCPU_REGS_R8);
 	regs->r9 = kvm_register_read(vcpu, VCPU_REGS_R9);
 	regs->r10 = kvm_register_read(vcpu, VCPU_REGS_R10);
 	regs->r11 = kvm_register_read(vcpu, VCPU_REGS_R11);
 	regs->r12 = kvm_register_read(vcpu, VCPU_REGS_R12);
 	regs->r13 = kvm_register_read(vcpu, VCPU_REGS_R13);
 	regs->r14 = kvm_register_read(vcpu, VCPU_REGS_R14);
 	regs->r15 = kvm_register_read(vcpu, VCPU_REGS_R15);
 #endif
 	regs->rip = kvm_rip_read(vcpu);
 	regs->rflags = kvm_get_rflags(vcpu);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
-	vcpu_load(vcpu);
 	kvm_register_write(vcpu, VCPU_REGS_RAX, regs->rax);
 	kvm_register_write(vcpu, VCPU_REGS_RBX, regs->rbx);
 	kvm_register_write(vcpu, VCPU_REGS_RCX, regs->rcx);
 	kvm_register_write(vcpu, VCPU_REGS_RDX, regs->rdx);
 	kvm_register_write(vcpu, VCPU_REGS_RSI, regs->rsi);
 	kvm_register_write(vcpu, VCPU_REGS_RDI, regs->rdi);
 	kvm_register_write(vcpu, VCPU_REGS_RSP, regs->rsp);
 	kvm_register_write(vcpu, VCPU_REGS_RBP, regs->rbp);
 #ifdef CONFIG_X86_64
 	kvm_register_write(vcpu, VCPU_REGS_R8, regs->r8);
 	kvm_register_write(vcpu, VCPU_REGS_R9, regs->r9);
 	kvm_register_write(vcpu, VCPU_REGS_R10, regs->r10);
 	kvm_register_write(vcpu, VCPU_REGS_R11, regs->r11);
 	kvm_register_write(vcpu, VCPU_REGS_R12, regs->r12);
 	kvm_register_write(vcpu, VCPU_REGS_R13, regs->r13);
 	kvm_register_write(vcpu, VCPU_REGS_R14, regs->r14);
 	kvm_register_write(vcpu, VCPU_REGS_R15, regs->r15);
 #endif
 	kvm_rip_write(vcpu, regs->rip);
 	kvm_set_rflags(vcpu, regs->rflags);
 	vcpu->arch.exception.pending = false;
-	vcpu_put(vcpu);
 	return 0;
 }
 void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
 {
 	struct kvm_segment cs;
 	kvm_get_segment(vcpu, &cs, VCPU_SREG_CS);
 	*db = cs.db;
 	*l = cs.l;
 }
 EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits);
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
 	struct desc_ptr dt;
-	vcpu_load(vcpu);
 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
 	kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
 	kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
 	kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
 	kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
 	kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
 	kvm_x86_ops->get_idt(vcpu, &dt);
 	sregs->idt.limit = dt.size;
 	sregs->idt.base = dt.address;
 	kvm_x86_ops->get_gdt(vcpu, &dt);
 	sregs->gdt.limit = dt.size;
 	sregs->gdt.base = dt.address;
 	sregs->cr0 = kvm_read_cr0(vcpu);
 	sregs->cr2 = vcpu->arch.cr2;
 	sregs->cr3 = vcpu->arch.cr3;
 	sregs->cr4 = kvm_read_cr4(vcpu);
 	sregs->cr8 = kvm_get_cr8(vcpu);
 	sregs->efer = vcpu->arch.efer;
 	sregs->apic_base = kvm_get_apic_base(vcpu);
 	memset(sregs->interrupt_bitmap, 0, sizeof sregs->interrupt_bitmap);
 	if (vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft)
 		set_bit(vcpu->arch.interrupt.nr,
 			(unsigned long *)sregs->interrupt_bitmap);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
-	vcpu_load(vcpu);
 	mp_state->mp_state = vcpu->arch.mp_state;
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
-	vcpu_load(vcpu);
 	vcpu->arch.mp_state = mp_state->mp_state;
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
 		    bool has_error_code, u32 error_code)
 {
 	struct decode_cache *c = &vcpu->arch.emulate_ctxt.decode;
 	int cs_db, cs_l, ret;
 	cache_all_regs(vcpu);
 	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
 	vcpu->arch.emulate_ctxt.vcpu = vcpu;
 	vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu);
 	vcpu->arch.emulate_ctxt.eip = kvm_rip_read(vcpu);
 	vcpu->arch.emulate_ctxt.mode =
 		(!is_protmode(vcpu)) ? X86EMUL_MODE_REAL :
 		(vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM)
 		? X86EMUL_MODE_VM86 : cs_l
 		? X86EMUL_MODE_PROT64 :	cs_db
 		? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
 	memset(c, 0, sizeof(struct decode_cache));
 	memcpy(c->regs, vcpu->arch.regs, sizeof c->regs);
 	ret = emulator_task_switch(&vcpu->arch.emulate_ctxt, &emulate_ops,
 				   tss_selector, reason, has_error_code,
 				   error_code);
 	if (ret)
 		return EMULATE_FAIL;
 	memcpy(vcpu->arch.regs, c->regs, sizeof c->regs);
 	kvm_rip_write(vcpu, vcpu->arch.emulate_ctxt.eip);
 	kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags);
 	return EMULATE_DONE;
 }
 EXPORT_SYMBOL_GPL(kvm_task_switch);
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
 	int mmu_reset_needed = 0;
 	int pending_vec, max_bits;
 	struct desc_ptr dt;
-	vcpu_load(vcpu);
 	dt.size = sregs->idt.limit;
 	dt.address = sregs->idt.base;
 	kvm_x86_ops->set_idt(vcpu, &dt);
 	dt.size = sregs->gdt.limit;
 	dt.address = sregs->gdt.base;
 	kvm_x86_ops->set_gdt(vcpu, &dt);
 	vcpu->arch.cr2 = sregs->cr2;
 	mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3;
 	vcpu->arch.cr3 = sregs->cr3;
 	kvm_set_cr8(vcpu, sregs->cr8);
 	mmu_reset_needed |= vcpu->arch.efer != sregs->efer;
 	kvm_x86_ops->set_efer(vcpu, sregs->efer);
 	kvm_set_apic_base(vcpu, sregs->apic_base);
 	mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0;
 	kvm_x86_ops->set_cr0(vcpu, sregs->cr0);
 	vcpu->arch.cr0 = sregs->cr0;
 	mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4;
 	kvm_x86_ops->set_cr4(vcpu, sregs->cr4);
 	if (!is_long_mode(vcpu) && is_pae(vcpu)) {
 		load_pdptrs(vcpu, vcpu->arch.cr3);
 		mmu_reset_needed = 1;
 	}
 	if (mmu_reset_needed)
 		kvm_mmu_reset_context(vcpu);
 	max_bits = (sizeof sregs->interrupt_bitmap) << 3;
 	pending_vec = find_first_bit(
 		(const unsigned long *)sregs->interrupt_bitmap, max_bits);
 	if (pending_vec < max_bits) {
 		kvm_queue_interrupt(vcpu, pending_vec, false);
 		pr_debug("Set back pending irq %d\n", pending_vec);
 		if (irqchip_in_kernel(vcpu->kvm))
 			kvm_pic_clear_isr_ack(vcpu->kvm);
 	}
 	kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
 	kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
 	kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
 	kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
 	kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
 	kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
 	kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
 	kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
 	update_cr8_intercept(vcpu);
 	/* Older userspace won't unhalt the vcpu on reset. */
 	if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 &&
 	    sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 &&
 	    !is_protmode(vcpu))
 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
 	unsigned long rflags;
 	int i, r;
-	vcpu_load(vcpu);
 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
 		r = -EBUSY;
 		if (vcpu->arch.exception.pending)
-			goto unlock_out;
+			goto out;
 		if (dbg->control & KVM_GUESTDBG_INJECT_DB)
 			kvm_queue_exception(vcpu, DB_VECTOR);
 		else
 			kvm_queue_exception(vcpu, BP_VECTOR);
 	}
 	/*
 	 * Read rflags as long as potentially injected trace flags are still
 	 * filtered out.
 	 */
 	rflags = kvm_get_rflags(vcpu);
 	vcpu->guest_debug = dbg->control;
 	if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE))
 		vcpu->guest_debug = 0;
 	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
 		for (i = 0; i < KVM_NR_DB_REGS; ++i)
 			vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
 		vcpu->arch.switch_db_regs =
 			(dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
 	} else {
 		for (i = 0; i < KVM_NR_DB_REGS; i++)
 			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
 		vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
 	}
 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
 		vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
 			get_segment_base(vcpu, VCPU_SREG_CS);
 	/*
 	 * Trigger an rflags update that will inject or remove the trace
 	 * flags.
 	 */
 	kvm_set_rflags(vcpu, rflags);
 	kvm_x86_ops->set_guest_debug(vcpu, dbg);
 	r = 0;
-unlock_out:
+out:
-	vcpu_put(vcpu);
 	return r;
 }
 /*
  * fxsave fpu state.  Taken from x86_64/processor.h.  To be killed when
  * we have asm/x86/processor.h
  */
 struct fxsave {
 	u16	cwd;
 	u16	swd;
 	u16	twd;
 	u16	fop;
 	u64	rip;
 	u64	rdp;
 	u32	mxcsr;
 	u32	mxcsr_mask;
 	u32	st_space[32];	/* 8*16 bytes for each FP-reg = 128 bytes */
 #ifdef CONFIG_X86_64
 	u32	xmm_space[64];	/* 16*16 bytes for each XMM-reg = 256 bytes */
 #else
 	u32	xmm_space[32];	/* 8*16 bytes for each XMM-reg = 128 bytes */
 #endif
 };
 /*
  * Translate a guest virtual address to a guest physical address.
  */
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 				    struct kvm_translation *tr)
 {
 	unsigned long vaddr = tr->linear_address;
 	gpa_t gpa;
 	int idx;
-	vcpu_load(vcpu);
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	tr->physical_address = gpa;
 	tr->valid = gpa != UNMAPPED_GVA;
 	tr->writeable = 1;
 	tr->usermode = 0;
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
-	vcpu_load(vcpu);
 	memcpy(fpu->fpr, fxsave->st_space, 128);
 	fpu->fcw = fxsave->cwd;
 	fpu->fsw = fxsave->swd;
 	fpu->ftwx = fxsave->twd;
 	fpu->last_opcode = fxsave->fop;
 	fpu->last_ip = fxsave->rip;
 	fpu->last_dp = fxsave->rdp;
 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
-	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image;
-	vcpu_load(vcpu);
 	memcpy(fxsave->st_space, fpu->fpr, 128);
 	fxsave->cwd = fpu->fcw;
 	fxsave->swd = fpu->fsw;
 	fxsave->twd = fpu->ftwx;
 	fxsave->fop = fpu->last_opcode;
 	fxsave->rip = fpu->last_ip;
 	fxsave->rdp = fpu->last_dp;
 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
-	vcpu_put(vcpu);
 	return 0;
 }
 void fx_init(struct kvm_vcpu *vcpu)
 {
 	unsigned after_mxcsr_mask;
 	/*
 	 * Touch the fpu the first time in non atomic context as if
 	 * this is the first fpu instruction the exception handler
 	 * will fire before the instruction returns and it'll have to
 	 * allocate ram with GFP_KERNEL.
 	 */
 	if (!used_math())
 		kvm_fx_save(&vcpu->arch.host_fx_image);
 	/* Initialize guest FPU by resetting ours and saving into guest's */
 	preempt_disable();
 	kvm_fx_save(&vcpu->arch.host_fx_image);
 	kvm_fx_finit();
 	kvm_fx_save(&vcpu->arch.guest_fx_image);
 	kvm_fx_restore(&vcpu->arch.host_fx_image);
 	preempt_enable();
 	vcpu->arch.cr0 |= X86_CR0_ET;
 	after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space);
 	vcpu->arch.guest_fx_image.mxcsr = 0x1f80;
 	memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask,
 	       0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask);
 }
 EXPORT_SYMBOL_GPL(fx_init);
 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
 {
 	if (vcpu->guest_fpu_loaded)
 		return;
 	vcpu->guest_fpu_loaded = 1;
 	kvm_fx_save(&vcpu->arch.host_fx_image);
 	kvm_fx_restore(&vcpu->arch.guest_fx_image);
 	trace_kvm_fpu(1);
 }
 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
 {
 	if (!vcpu->guest_fpu_loaded)
 		return;
 	vcpu->guest_fpu_loaded = 0;
 	kvm_fx_save(&vcpu->arch.guest_fx_image);
 	kvm_fx_restore(&vcpu->arch.host_fx_image);
 	++vcpu->stat.fpu_reload;
 	set_bit(KVM_REQ_DEACTIVATE_FPU, &vcpu->requests);
 	trace_kvm_fpu(0);
 }
 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
 {
 	if (vcpu->arch.time_page) {
 		kvm_release_page_dirty(vcpu->arch.time_page);
 		vcpu->arch.time_page = NULL;
 	}
 	kvm_x86_ops->vcpu_free(vcpu);
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 						unsigned int id)
 {
 	return kvm_x86_ops->vcpu_create(kvm, id);
 }
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	int r;
 	/* We do fxsave: this must be aligned. */
 	BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF);
 	vcpu->arch.mtrr_state.have_fixed = 1;
 	vcpu_load(vcpu);
 	r = kvm_arch_vcpu_reset(vcpu);
 	if (r == 0)
 		r = kvm_mmu_setup(vcpu);
 	vcpu_put(vcpu);
 	if (r < 0)
 		goto free_vcpu;
 	return 0;
 free_vcpu:
 	kvm_x86_ops->vcpu_free(vcpu);
 	return r;
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	vcpu_load(vcpu);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 	kvm_x86_ops->vcpu_free(vcpu);
 }
 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
 {
 	vcpu->arch.nmi_pending = false;
 	vcpu->arch.nmi_injected = false;
 	vcpu->arch.switch_db_regs = 0;
 	memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
 	vcpu->arch.dr6 = DR6_FIXED_1;
 	vcpu->arch.dr7 = DR7_FIXED_1;
 	return kvm_x86_ops->vcpu_reset(vcpu);
 }
 int kvm_arch_hardware_enable(void *garbage)
 {
 	/*
 	 * Since this may be called from a hotplug notifcation,
 	 * we can't get the CPU frequency directly.
 	 */
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 		int cpu = raw_smp_processor_id();
 		per_cpu(cpu_tsc_khz, cpu) = 0;
 	}
 	kvm_shared_msr_cpu_online();
 	return kvm_x86_ops->hardware_enable(garbage);
 }
 void kvm_arch_hardware_disable(void *garbage)
 {
 	kvm_x86_ops->hardware_disable(garbage);
 	drop_user_return_notifiers(garbage);
 }
 int kvm_arch_hardware_setup(void)
 {
 	return kvm_x86_ops->hardware_setup();
 }
 void kvm_arch_hardware_unsetup(void)
 {
 	kvm_x86_ops->hardware_unsetup();
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 	kvm_x86_ops->check_processor_compatibility(rtn);
 }
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	struct page *page;
 	struct kvm *kvm;
 	int r;
 	BUG_ON(vcpu->kvm == NULL);
 	kvm = vcpu->kvm;
 	vcpu->arch.mmu.root_hpa = INVALID_PAGE;
 	if (!irqchip_in_kernel(kvm) || kvm_vcpu_is_bsp(vcpu))
 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 	else
 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (!page) {
 		r = -ENOMEM;
 		goto fail;
 	}
 	vcpu->arch.pio_data = page_address(page);
 	r = kvm_mmu_create(vcpu);
 	if (r < 0)
 		goto fail_free_pio_data;
 	if (irqchip_in_kernel(kvm)) {
 		r = kvm_create_lapic(vcpu);
 		if (r < 0)
 			goto fail_mmu_destroy;
 	}
 	vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
 				       GFP_KERNEL);
 	if (!vcpu->arch.mce_banks) {
 		r = -ENOMEM;
 		goto fail_free_lapic;
 	}
 	vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
 	return 0;
 fail_free_lapic:
 	kvm_free_lapic(vcpu);
 fail_mmu_destroy:
 	kvm_mmu_destroy(vcpu);
 fail_free_pio_data:
 	free_page((unsigned long)vcpu->arch.pio_data);
 fail:
 	return r;
 }
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	int idx;
 	kfree(vcpu->arch.mce_banks);
 	kvm_free_lapic(vcpu);
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	kvm_mmu_destroy(vcpu);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 	free_page((unsigned long)vcpu->arch.pio_data);
 }
 struct  kvm *kvm_arch_create_vm(void)
 {
 	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
 	if (!kvm)
 		return ERR_PTR(-ENOMEM);
 	kvm->arch.aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
 	if (!kvm->arch.aliases) {
 		kfree(kvm);
 		return ERR_PTR(-ENOMEM);
 	}
 	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
 	rdtscll(kvm->arch.vm_init_tsc);
 	return kvm;
 }
 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 {
 	vcpu_load(vcpu);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 }
 static void kvm_free_vcpus(struct kvm *kvm)
 {
 	unsigned int i;
 	struct kvm_vcpu *vcpu;
 	/*
 	 * Unpin any mmu pages first.
 	 */
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		kvm_unload_vcpu_mmu(vcpu);
 	kvm_for_each_vcpu(i, vcpu, kvm)
 		kvm_arch_vcpu_free(vcpu);
 	mutex_lock(&kvm->lock);
 	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 		kvm->vcpus[i] = NULL;
 	atomic_set(&kvm->online_vcpus, 0);
 	mutex_unlock(&kvm->lock);
 }
 void kvm_arch_sync_events(struct kvm *kvm)
 {
 	kvm_free_all_assigned_devices(kvm);
 }
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvm_iommu_unmap_guest(kvm);
 	kvm_free_pit(kvm);
 	kfree(kvm->arch.vpic);
 	kfree(kvm->arch.vioapic);
 	kvm_free_vcpus(kvm);
 	kvm_free_physmem(kvm);
 	if (kvm->arch.apic_access_page)
 		put_page(kvm->arch.apic_access_page);
 	if (kvm->arch.ept_identity_pagetable)
 		put_page(kvm->arch.ept_identity_pagetable);
 	cleanup_srcu_struct(&kvm->srcu);
 	kfree(kvm->arch.aliases);
 	kfree(kvm);
 }
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 				struct kvm_memory_slot *memslot,
 				struct kvm_memory_slot old,
 				struct kvm_userspace_memory_region *mem,
 				int user_alloc)
 {
 	int npages = memslot->npages;
 	/*To keep backward compatibility with older userspace,
 	 *x86 needs to hanlde !user_alloc case.
 	 */
 	if (!user_alloc) {
 		if (npages && !old.rmap) {
 			unsigned long userspace_addr;
 			down_write(&current->mm->mmap_sem);
 			userspace_addr = do_mmap(NULL, 0,
 						 npages * PAGE_SIZE,
 						 PROT_READ | PROT_WRITE,
 						 MAP_PRIVATE | MAP_ANONYMOUS,
 						 0);
 			up_write(&current->mm->mmap_sem);
 			if (IS_ERR((void *)userspace_addr))
 				return PTR_ERR((void *)userspace_addr);
 			memslot->userspace_addr = userspace_addr;
 		}
 	}
 	return 0;
 }
 void kvm_arch_commit_memory_region(struct kvm *kvm,
 				struct kvm_userspace_memory_region *mem,
 				struct kvm_memory_slot old,
 				int user_alloc)
 {
 	int npages = mem->memory_size >> PAGE_SHIFT;
 	if (!user_alloc && !old.user_alloc && old.rmap && !npages) {
 		int ret;
 		down_write(&current->mm->mmap_sem);
 		ret = do_munmap(current->mm, old.userspace_addr,
 				old.npages * PAGE_SIZE);
 		up_write(&current->mm->mmap_sem);
 		if (ret < 0)
 			printk(KERN_WARNING
 			       "kvm_vm_ioctl_set_memory_region: "
 			       "failed to munmap memory\n");
 	}
 	spin_lock(&kvm->mmu_lock);
 	if (!kvm->arch.n_requested_mmu_pages) {
 		unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm);
 		kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
 	}
 	kvm_mmu_slot_remove_write_access(kvm, mem->slot);
 	spin_unlock(&kvm->mmu_lock);
 }
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
 	kvm_mmu_zap_all(kvm);
 	kvm_reload_remote_mmus(kvm);
 }
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
 		|| vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
 		|| vcpu->arch.nmi_pending ||
 		(kvm_arch_interrupt_allowed(vcpu) &&
 		 kvm_cpu_has_interrupt(vcpu));
 }
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 {
 	int me;
 	int cpu = vcpu->cpu;
 	if (waitqueue_active(&vcpu->wq)) {
 		wake_up_interruptible(&vcpu->wq);
 		++vcpu->stat.halt_wakeup;
 	}
 	me = get_cpu();
 	if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
 		if (atomic_xchg(&vcpu->guest_mode, 0))
 			smp_send_reschedule(cpu);
 	put_cpu();
 }
 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
 	return kvm_x86_ops->interrupt_allowed(vcpu);
 }
 bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip)
 {
 	unsigned long current_rip = kvm_rip_read(vcpu) +
 		get_segment_base(vcpu, VCPU_SREG_CS);
 	return current_rip == linear_rip;
 }
 EXPORT_SYMBOL_GPL(kvm_is_linear_rip);
 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu)
 {
 	unsigned long rflags;
 	rflags = kvm_x86_ops->get_rflags(vcpu);
 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
 		rflags &= ~X86_EFLAGS_TF;
 	return rflags;
 }
 EXPORT_SYMBOL_GPL(kvm_get_rflags);
 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 {
 	if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP &&
 	    kvm_is_linear_rip(vcpu, vcpu->arch.singlestep_rip))
 		rflags |= X86_EFLAGS_TF;
 	kvm_x86_ops->set_rflags(vcpu, rflags);
 }
 EXPORT_SYMBOL_GPL(kvm_set_rflags);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_exit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_invlpga);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_skinit);
 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);

virt/kvm/kvm_main.c

Diff comments View file @ 2122ff5

 /*
  * Kernel-based Virtual Machine driver for Linux
  *
  * This module enables machines with Intel VT-x extensions to run virtual
  * machines without emulation or binary translation.
  *
  * Copyright (C) 2006 Qumranet, Inc.
  *
  * Authors:
  *   Avi Kivity   <avi@qumranet.com>
  *   Yaniv Kamay  <yaniv@qumranet.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  *
  */
 #include "iodev.h"
 #include <linux/kvm_host.h>
 #include <linux/kvm.h>
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/percpu.h>
 #include <linux/mm.h>
 #include <linux/miscdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/reboot.h>
 #include <linux/debugfs.h>
 #include <linux/highmem.h>
 #include <linux/file.h>
 #include <linux/sysdev.h>
 #include <linux/cpu.h>
 #include <linux/sched.h>
 #include <linux/cpumask.h>
 #include <linux/smp.h>
 #include <linux/anon_inodes.h>
 #include <linux/profile.h>
 #include <linux/kvm_para.h>
 #include <linux/pagemap.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/bitops.h>
 #include <linux/spinlock.h>
 #include <linux/compat.h>
 #include <linux/srcu.h>
 #include <linux/hugetlb.h>
 #include <linux/slab.h>
 #include <asm/processor.h>
 #include <asm/io.h>
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
 #include <asm-generic/bitops/le.h>
 #include "coalesced_mmio.h"
 #define CREATE_TRACE_POINTS
 #include <trace/events/kvm.h>
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 /*
  * Ordering of locks:
  *
  * 		kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  */
 DEFINE_SPINLOCK(kvm_lock);
 LIST_HEAD(vm_list);
 static cpumask_var_t cpus_hardware_enabled;
 static int kvm_usage_count = 0;
 static atomic_t hardware_enable_failed;
 struct kmem_cache *kvm_vcpu_cache;
 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
 static __read_mostly struct preempt_ops kvm_preempt_ops;
 struct dentry *kvm_debugfs_dir;
 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 			   unsigned long arg);
 static int hardware_enable_all(void);
 static void hardware_disable_all(void);
 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
 static bool kvm_rebooting;
 static bool largepages_enabled = true;
 struct page *hwpoison_page;
 pfn_t hwpoison_pfn;
 inline int kvm_is_mmio_pfn(pfn_t pfn)
 {
 	if (pfn_valid(pfn)) {
 		struct page *page = compound_head(pfn_to_page(pfn));
 		return PageReserved(page);
 	}
 	return true;
 }
 /*
  * Switches to specified vcpu, until a matching vcpu_put()
  */
 void vcpu_load(struct kvm_vcpu *vcpu)
 {
 	int cpu;
 	mutex_lock(&vcpu->mutex);
 	cpu = get_cpu();
 	preempt_notifier_register(&vcpu->preempt_notifier);
 	kvm_arch_vcpu_load(vcpu, cpu);
 	put_cpu();
 }
 void vcpu_put(struct kvm_vcpu *vcpu)
 {
 	preempt_disable();
 	kvm_arch_vcpu_put(vcpu);
 	preempt_notifier_unregister(&vcpu->preempt_notifier);
 	preempt_enable();
 	mutex_unlock(&vcpu->mutex);
 }
 static void ack_flush(void *_completed)
 {
 }
 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
 {
 	int i, cpu, me;
 	cpumask_var_t cpus;
 	bool called = true;
 	struct kvm_vcpu *vcpu;
 	zalloc_cpumask_var(&cpus, GFP_ATOMIC);
 	raw_spin_lock(&kvm->requests_lock);
 	me = smp_processor_id();
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (test_and_set_bit(req, &vcpu->requests))
 			continue;
 		cpu = vcpu->cpu;
 		if (cpus != NULL && cpu != -1 && cpu != me)
 			cpumask_set_cpu(cpu, cpus);
 	}
 	if (unlikely(cpus == NULL))
 		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 	else if (!cpumask_empty(cpus))
 		smp_call_function_many(cpus, ack_flush, NULL, 1);
 	else
 		called = false;
 	raw_spin_unlock(&kvm->requests_lock);
 	free_cpumask_var(cpus);
 	return called;
 }
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
 	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 		++kvm->stat.remote_tlb_flush;
 }
 void kvm_reload_remote_mmus(struct kvm *kvm)
 {
 	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 }
 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 {
 	struct page *page;
 	int r;
 	mutex_init(&vcpu->mutex);
 	vcpu->cpu = -1;
 	vcpu->kvm = kvm;
 	vcpu->vcpu_id = id;
 	init_waitqueue_head(&vcpu->wq);
 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (!page) {
 		r = -ENOMEM;
 		goto fail;
 	}
 	vcpu->run = page_address(page);
 	r = kvm_arch_vcpu_init(vcpu);
 	if (r < 0)
 		goto fail_free_run;
 	return 0;
 fail_free_run:
 	free_page((unsigned long)vcpu->run);
 fail:
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	kvm_arch_vcpu_uninit(vcpu);
 	free_page((unsigned long)vcpu->run);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 {
 	return container_of(mn, struct kvm, mmu_notifier);
 }
 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long address)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush, idx;
 	/*
 	 * When ->invalidate_page runs, the linux pte has been zapped
 	 * already but the page is still allocated until
 	 * ->invalidate_page returns. So if we increase the sequence
 	 * here the kvm page fault will notice if the spte can't be
 	 * established because the page is going to be freed. If
 	 * instead the kvm page fault establishes the spte before
 	 * ->invalidate_page runs, kvm_unmap_hva will release it
 	 * before returning.
 	 *
 	 * The sequence increase only need to be seen at spin_unlock
 	 * time, and not at spin_lock time.
 	 *
 	 * Increasing the sequence after the spin_unlock would be
 	 * unsafe because the kvm page fault could then establish the
 	 * pte after kvm_unmap_hva returned, without noticing the page
 	 * is going to be freed.
 	 */
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	kvm->mmu_notifier_seq++;
 	need_tlb_flush = kvm_unmap_hva(kvm, address);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 	/* we've to flush the tlb before the pages can be freed */
 	if (need_tlb_flush)
 		kvm_flush_remote_tlbs(kvm);
 }
 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 					struct mm_struct *mm,
 					unsigned long address,
 					pte_t pte)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	kvm->mmu_notifier_seq++;
 	kvm_set_spte_hva(kvm, address, pte);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 						    struct mm_struct *mm,
 						    unsigned long start,
 						    unsigned long end)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush = 0, idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	/*
 	 * The count increase must become visible at unlock time as no
 	 * spte can be established without taking the mmu_lock and
 	 * count is also read inside the mmu_lock critical section.
 	 */
 	kvm->mmu_notifier_count++;
 	for (; start < end; start += PAGE_SIZE)
 		need_tlb_flush |= kvm_unmap_hva(kvm, start);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 	/* we've to flush the tlb before the pages can be freed */
 	if (need_tlb_flush)
 		kvm_flush_remote_tlbs(kvm);
 }
 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 						  struct mm_struct *mm,
 						  unsigned long start,
 						  unsigned long end)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	spin_lock(&kvm->mmu_lock);
 	/*
 	 * This sequence increase will notify the kvm page fault that
 	 * the page that is going to be mapped in the spte could have
 	 * been freed.
 	 */
 	kvm->mmu_notifier_seq++;
 	/*
 	 * The above sequence increase must be visible before the
 	 * below count decrease but both values are read by the kvm
 	 * page fault under mmu_lock spinlock so we don't need to add
 	 * a smb_wmb() here in between the two.
 	 */
 	kvm->mmu_notifier_count--;
 	spin_unlock(&kvm->mmu_lock);
 	BUG_ON(kvm->mmu_notifier_count < 0);
 }
 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 					      struct mm_struct *mm,
 					      unsigned long address)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int young, idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	spin_lock(&kvm->mmu_lock);
 	young = kvm_age_hva(kvm, address);
 	spin_unlock(&kvm->mmu_lock);
 	srcu_read_unlock(&kvm->srcu, idx);
 	if (young)
 		kvm_flush_remote_tlbs(kvm);
 	return young;
 }
 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 				     struct mm_struct *mm)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int idx;
 	idx = srcu_read_lock(&kvm->srcu);
 	kvm_arch_flush_shadow(kvm);
 	srcu_read_unlock(&kvm->srcu, idx);
 }
 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
 	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
 	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
 	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
 	.change_pte		= kvm_mmu_notifier_change_pte,
 	.release		= kvm_mmu_notifier_release,
 };
 static int kvm_init_mmu_notifier(struct kvm *kvm)
 {
 	kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 }
 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
 static int kvm_init_mmu_notifier(struct kvm *kvm)
 {
 	return 0;
 }
 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 static struct kvm *kvm_create_vm(void)
 {
 	int r = 0, i;
 	struct kvm *kvm = kvm_arch_create_vm();
 	if (IS_ERR(kvm))
 		goto out;
 	r = hardware_enable_all();
 	if (r)
 		goto out_err_nodisable;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
 	INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 #endif
 	r = -ENOMEM;
 	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 	if (!kvm->memslots)
 		goto out_err;
 	if (init_srcu_struct(&kvm->srcu))
 		goto out_err;
 	for (i = 0; i < KVM_NR_BUSES; i++) {
 		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
 					GFP_KERNEL);
 		if (!kvm->buses[i]) {
 			cleanup_srcu_struct(&kvm->srcu);
 			goto out_err;
 		}
 	}
 	r = kvm_init_mmu_notifier(kvm);
 	if (r) {
 		cleanup_srcu_struct(&kvm->srcu);
 		goto out_err;
 	}
 	kvm->mm = current->mm;
 	atomic_inc(&kvm->mm->mm_count);
 	spin_lock_init(&kvm->mmu_lock);
 	raw_spin_lock_init(&kvm->requests_lock);
 	kvm_eventfd_init(kvm);
 	mutex_init(&kvm->lock);
 	mutex_init(&kvm->irq_lock);
 	mutex_init(&kvm->slots_lock);
 	atomic_set(&kvm->users_count, 1);
 	spin_lock(&kvm_lock);
 	list_add(&kvm->vm_list, &vm_list);
 	spin_unlock(&kvm_lock);
 out:
 	return kvm;
 out_err:
 	hardware_disable_all();
 out_err_nodisable:
 	for (i = 0; i < KVM_NR_BUSES; i++)
 		kfree(kvm->buses[i]);
 	kfree(kvm->memslots);
 	kfree(kvm);
 	return ERR_PTR(r);
 }
 /*
  * Free any memory in @free but not in @dont.
  */
 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 				  struct kvm_memory_slot *dont)
 {
 	int i;
 	if (!dont || free->rmap != dont->rmap)
 		vfree(free->rmap);
 	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 		vfree(free->dirty_bitmap);
 	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
 		if (!dont || free->lpage_info[i] != dont->lpage_info[i]) {
 			vfree(free->lpage_info[i]);
 			free->lpage_info[i] = NULL;
 		}
 	}
 	free->npages = 0;
 	free->dirty_bitmap = NULL;
 	free->rmap = NULL;
 }
 void kvm_free_physmem(struct kvm *kvm)
 {
 	int i;
 	struct kvm_memslots *slots = kvm->memslots;
 	for (i = 0; i < slots->nmemslots; ++i)
 		kvm_free_physmem_slot(&slots->memslots[i], NULL);
 	kfree(kvm->memslots);
 }
 static void kvm_destroy_vm(struct kvm *kvm)
 {
 	int i;
 	struct mm_struct *mm = kvm->mm;
 	kvm_arch_sync_events(kvm);
 	spin_lock(&kvm_lock);
 	list_del(&kvm->vm_list);
 	spin_unlock(&kvm_lock);
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++)
 		kvm_io_bus_destroy(kvm->buses[i]);
 	kvm_coalesced_mmio_free(kvm);
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 #else
 	kvm_arch_flush_shadow(kvm);
 #endif
 	kvm_arch_destroy_vm(kvm);
 	hardware_disable_all();
 	mmdrop(mm);
 }
 void kvm_get_kvm(struct kvm *kvm)
 {
 	atomic_inc(&kvm->users_count);
 }
 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 void kvm_put_kvm(struct kvm *kvm)
 {
 	if (atomic_dec_and_test(&kvm->users_count))
 		kvm_destroy_vm(kvm);
 }
 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 static int kvm_vm_release(struct inode *inode, struct file *filp)
 {
 	struct kvm *kvm = filp->private_data;
 	kvm_irqfd_release(kvm);
 	kvm_put_kvm(kvm);
 	return 0;
 }
 /*
  * Allocate some memory and give it an address in the guest physical address
  * space.
  *
  * Discontiguous memory is allowed, mostly for framebuffers.
  *
  * Must be called holding mmap_sem for write.
  */
 int __kvm_set_memory_region(struct kvm *kvm,
 			    struct kvm_userspace_memory_region *mem,
 			    int user_alloc)
 {
 	int r, flush_shadow = 0;
 	gfn_t base_gfn;
 	unsigned long npages;
 	unsigned long i;
 	struct kvm_memory_slot *memslot;
 	struct kvm_memory_slot old, new;
 	struct kvm_memslots *slots, *old_memslots;
 	r = -EINVAL;
 	/* General sanity checks */
 	if (mem->memory_size & (PAGE_SIZE - 1))
 		goto out;
 	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 		goto out;
 	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
 		goto out;
 	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 		goto out;
 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 		goto out;
 	memslot = &kvm->memslots->memslots[mem->slot];
 	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 	npages = mem->memory_size >> PAGE_SHIFT;
 	r = -EINVAL;
 	if (npages > KVM_MEM_MAX_NR_PAGES)
 		goto out;
 	if (!npages)
 		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 	new = old = *memslot;
 	new.base_gfn = base_gfn;
 	new.npages = npages;
 	new.flags = mem->flags;
 	/* Disallow changing a memory slot's size. */
 	r = -EINVAL;
 	if (npages && old.npages && npages != old.npages)
 		goto out_free;
 	/* Check for overlaps */
 	r = -EEXIST;
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
 		if (s == memslot || !s->npages)
 			continue;
 		if (!((base_gfn + npages <= s->base_gfn) ||
 		      (base_gfn >= s->base_gfn + s->npages)))
 			goto out_free;
 	}
 	/* Free page dirty bitmap if unneeded */
 	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 		new.dirty_bitmap = NULL;
 	r = -ENOMEM;
 	/* Allocate if a slot is being created */
 #ifndef CONFIG_S390
 	if (npages && !new.rmap) {
 		new.rmap = vmalloc(npages * sizeof(struct page *));
 		if (!new.rmap)
 			goto out_free;
 		memset(new.rmap, 0, npages * sizeof(*new.rmap));
 		new.user_alloc = user_alloc;
 		new.userspace_addr = mem->userspace_addr;
 	}
 	if (!npages)
 		goto skip_lpage;
 	for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) {
 		unsigned long ugfn;
 		unsigned long j;
 		int lpages;
 		int level = i + 2;
 		/* Avoid unused variable warning if no large pages */
 		(void)level;
 		if (new.lpage_info[i])
 			continue;
 		lpages = 1 + (base_gfn + npages - 1) /
 			     KVM_PAGES_PER_HPAGE(level);
 		lpages -= base_gfn / KVM_PAGES_PER_HPAGE(level);
 		new.lpage_info[i] = vmalloc(lpages * sizeof(*new.lpage_info[i]));
 		if (!new.lpage_info[i])
 			goto out_free;
 		memset(new.lpage_info[i], 0,
 		       lpages * sizeof(*new.lpage_info[i]));
 		if (base_gfn % KVM_PAGES_PER_HPAGE(level))
 			new.lpage_info[i][0].write_count = 1;
 		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE(level))
 			new.lpage_info[i][lpages - 1].write_count = 1;
 		ugfn = new.userspace_addr >> PAGE_SHIFT;
 		/*
 		 * If the gfn and userspace address are not aligned wrt each
 		 * other, or if explicitly asked to, disable large page
 		 * support for this slot
 		 */
 		if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE(level) - 1) ||
 		    !largepages_enabled)
 			for (j = 0; j < lpages; ++j)
 				new.lpage_info[i][j].write_count = 1;
 	}
 skip_lpage:
 	/* Allocate page dirty bitmap if needed */
 	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 		unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(&new);
 		new.dirty_bitmap = vmalloc(dirty_bytes);
 		if (!new.dirty_bitmap)
 			goto out_free;
 		memset(new.dirty_bitmap, 0, dirty_bytes);
 		/* destroy any largepage mappings for dirty tracking */
 		if (old.npages)
 			flush_shadow = 1;
 	}
 #else  /* not defined CONFIG_S390 */
 	new.user_alloc = user_alloc;
 	if (user_alloc)
 		new.userspace_addr = mem->userspace_addr;
 #endif /* not defined CONFIG_S390 */
 	if (!npages) {
 		r = -ENOMEM;
 		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 		if (!slots)
 			goto out_free;
 		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
 		if (mem->slot >= slots->nmemslots)
 			slots->nmemslots = mem->slot + 1;
 		slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
 		old_memslots = kvm->memslots;
 		rcu_assign_pointer(kvm->memslots, slots);
 		synchronize_srcu_expedited(&kvm->srcu);
 		/* From this point no new shadow pages pointing to a deleted
 		 * memslot will be created.
 		 *
 		 * validation of sp->gfn happens in:
 		 * 	- gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 		 * 	- kvm_is_visible_gfn (mmu_check_roots)
 		 */
 		kvm_arch_flush_shadow(kvm);
 		kfree(old_memslots);
 	}
 	r = kvm_arch_prepare_memory_region(kvm, &new, old, mem, user_alloc);
 	if (r)
 		goto out_free;
 #ifdef CONFIG_DMAR
 	/* map the pages in iommu page table */
 	if (npages) {
 		r = kvm_iommu_map_pages(kvm, &new);
 		if (r)
 			goto out_free;
 	}
 #endif
 	r = -ENOMEM;
 	slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 	if (!slots)
 		goto out_free;
 	memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
 	if (mem->slot >= slots->nmemslots)
 		slots->nmemslots = mem->slot + 1;
 	/* actual memory is freed via old in kvm_free_physmem_slot below */
 	if (!npages) {
 		new.rmap = NULL;
 		new.dirty_bitmap = NULL;
 		for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i)
 			new.lpage_info[i] = NULL;
 	}
 	slots->memslots[mem->slot] = new;
 	old_memslots = kvm->memslots;
 	rcu_assign_pointer(kvm->memslots, slots);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
 	kvm_free_physmem_slot(&old, &new);
 	kfree(old_memslots);
 	if (flush_shadow)
 		kvm_arch_flush_shadow(kvm);
 	return 0;
 out_free:
 	kvm_free_physmem_slot(&new, &old);
 out:
 	return r;
 }
 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 int kvm_set_memory_region(struct kvm *kvm,
 			  struct kvm_userspace_memory_region *mem,
 			  int user_alloc)
 {
 	int r;
 	mutex_lock(&kvm->slots_lock);
 	r = __kvm_set_memory_region(kvm, mem, user_alloc);
 	mutex_unlock(&kvm->slots_lock);
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 				   struct
 				   kvm_userspace_memory_region *mem,
 				   int user_alloc)
 {
 	if (mem->slot >= KVM_MEMORY_SLOTS)
 		return -EINVAL;
 	return kvm_set_memory_region(kvm, mem, user_alloc);
 }
 int kvm_get_dirty_log(struct kvm *kvm,
 			struct kvm_dirty_log *log, int *is_dirty)
 {
 	struct kvm_memory_slot *memslot;
 	int r, i;
 	unsigned long n;
 	unsigned long any = 0;
 	r = -EINVAL;
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 	memslot = &kvm->memslots->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
 	n = kvm_dirty_bitmap_bytes(memslot);
 	for (i = 0; !any && i < n/sizeof(long); ++i)
 		any = memslot->dirty_bitmap[i];
 	r = -EFAULT;
 	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 		goto out;
 	if (any)
 		*is_dirty = 1;
 	r = 0;
 out:
 	return r;
 }
 void kvm_disable_largepages(void)
 {
 	largepages_enabled = false;
 }
 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
 int is_error_page(struct page *page)
 {
 	return page == bad_page || page == hwpoison_page;
 }
 EXPORT_SYMBOL_GPL(is_error_page);
 int is_error_pfn(pfn_t pfn)
 {
 	return pfn == bad_pfn || pfn == hwpoison_pfn;
 }
 EXPORT_SYMBOL_GPL(is_error_pfn);
 int is_hwpoison_pfn(pfn_t pfn)
 {
 	return pfn == hwpoison_pfn;
 }
 EXPORT_SYMBOL_GPL(is_hwpoison_pfn);
 static inline unsigned long bad_hva(void)
 {
 	return PAGE_OFFSET;
 }
 int kvm_is_error_hva(unsigned long addr)
 {
 	return addr == bad_hva();
 }
 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
 struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_memslots *slots = kvm_memslots(kvm);
 	for (i = 0; i < slots->nmemslots; ++i) {
 		struct kvm_memory_slot *memslot = &slots->memslots[i];
 		if (gfn >= memslot->base_gfn
 		    && gfn < memslot->base_gfn + memslot->npages)
 			return memslot;
 	}
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 {
 	gfn = unalias_gfn(kvm, gfn);
 	return gfn_to_memslot_unaliased(kvm, gfn);
 }
 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_memslots *slots = kvm_memslots(kvm);
 	gfn = unalias_gfn_instantiation(kvm, gfn);
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *memslot = &slots->memslots[i];
 		if (memslot->flags & KVM_MEMSLOT_INVALID)
 			continue;
 		if (gfn >= memslot->base_gfn
 		    && gfn < memslot->base_gfn + memslot->npages)
 			return 1;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
 {
 	struct vm_area_struct *vma;
 	unsigned long addr, size;
 	size = PAGE_SIZE;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return PAGE_SIZE;
 	down_read(&current->mm->mmap_sem);
 	vma = find_vma(current->mm, addr);
 	if (!vma)
 		goto out;
 	size = vma_kernel_pagesize(vma);
 out:
 	up_read(&current->mm->mmap_sem);
 	return size;
 }
 int memslot_id(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_memslots *slots = kvm_memslots(kvm);
 	struct kvm_memory_slot *memslot = NULL;
 	gfn = unalias_gfn(kvm, gfn);
 	for (i = 0; i < slots->nmemslots; ++i) {
 		memslot = &slots->memslots[i];
 		if (gfn >= memslot->base_gfn
 		    && gfn < memslot->base_gfn + memslot->npages)
 			break;
 	}
 	return memslot - slots->memslots;
 }
 static unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
 {
 	return slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE;
 }
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
 {
 	struct kvm_memory_slot *slot;
 	gfn = unalias_gfn_instantiation(kvm, gfn);
 	slot = gfn_to_memslot_unaliased(kvm, gfn);
 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
 		return bad_hva();
 	return gfn_to_hva_memslot(slot, gfn);
 }
 EXPORT_SYMBOL_GPL(gfn_to_hva);
 static pfn_t hva_to_pfn(struct kvm *kvm, unsigned long addr)
 {
 	struct page *page[1];
 	int npages;
 	pfn_t pfn;
 	might_sleep();
 	npages = get_user_pages_fast(addr, 1, 1, page);
 	if (unlikely(npages != 1)) {
 		struct vm_area_struct *vma;
 		if (is_hwpoison_address(addr)) {
 			get_page(hwpoison_page);
 			return page_to_pfn(hwpoison_page);
 		}
 		down_read(&current->mm->mmap_sem);
 		vma = find_vma(current->mm, addr);
 		if (vma == NULL || addr < vma->vm_start ||
 		    !(vma->vm_flags & VM_PFNMAP)) {
 			up_read(&current->mm->mmap_sem);
 			get_page(bad_page);
 			return page_to_pfn(bad_page);
 		}
 		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
 		up_read(&current->mm->mmap_sem);
 		BUG_ON(!kvm_is_mmio_pfn(pfn));
 	} else
 		pfn = page_to_pfn(page[0]);
 	return pfn;
 }
 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
 {
 	unsigned long addr;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr)) {
 		get_page(bad_page);
 		return page_to_pfn(bad_page);
 	}
 	return hva_to_pfn(kvm, addr);
 }
 EXPORT_SYMBOL_GPL(gfn_to_pfn);
 pfn_t gfn_to_pfn_memslot(struct kvm *kvm,
 			 struct kvm_memory_slot *slot, gfn_t gfn)
 {
 	unsigned long addr = gfn_to_hva_memslot(slot, gfn);
 	return hva_to_pfn(kvm, addr);
 }
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 {
 	pfn_t pfn;
 	pfn = gfn_to_pfn(kvm, gfn);
 	if (!kvm_is_mmio_pfn(pfn))
 		return pfn_to_page(pfn);
 	WARN_ON(kvm_is_mmio_pfn(pfn));
 	get_page(bad_page);
 	return bad_page;
 }
 EXPORT_SYMBOL_GPL(gfn_to_page);
 void kvm_release_page_clean(struct page *page)
 {
 	kvm_release_pfn_clean(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
 void kvm_release_pfn_clean(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		put_page(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
 void kvm_release_page_dirty(struct page *page)
 {
 	kvm_release_pfn_dirty(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
 void kvm_release_pfn_dirty(pfn_t pfn)
 {
 	kvm_set_pfn_dirty(pfn);
 	kvm_release_pfn_clean(pfn);
 }
 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
 void kvm_set_page_dirty(struct page *page)
 {
 	kvm_set_pfn_dirty(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
 void kvm_set_pfn_dirty(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn)) {
 		struct page *page = pfn_to_page(pfn);
 		if (!PageReserved(page))
 			SetPageDirty(page);
 	}
 }
 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
 void kvm_set_pfn_accessed(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		mark_page_accessed(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
 void kvm_get_pfn(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		get_page(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_get_pfn);
 static int next_segment(unsigned long len, int offset)
 {
 	if (len > PAGE_SIZE - offset)
 		return PAGE_SIZE - offset;
 	else
 		return len;
 }
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 			int len)
 {
 	int r;
 	unsigned long addr;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	r = copy_from_user(data, (void __user *)addr + offset, len);
 	if (r)
 		return -EFAULT;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
 	while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		data += seg;
 		++gfn;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_read_guest);
 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
 			  unsigned long len)
 {
 	int r;
 	unsigned long addr;
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int offset = offset_in_page(gpa);
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	pagefault_disable();
 	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
 	pagefault_enable();
 	if (r)
 		return -EFAULT;
 	return 0;
 }
 EXPORT_SYMBOL(kvm_read_guest_atomic);
 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 			 int offset, int len)
 {
 	int r;
 	unsigned long addr;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	r = copy_to_user((void __user *)addr + offset, data, len);
 	if (r)
 		return -EFAULT;
 	mark_page_dirty(kvm, gfn);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
 		    unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
 	while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		data += seg;
 		++gfn;
 	}
 	return 0;
 }
 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
 {
 	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
 }
 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
         while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		++gfn;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_clear_guest);
 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 {
 	struct kvm_memory_slot *memslot;
 	gfn = unalias_gfn(kvm, gfn);
 	memslot = gfn_to_memslot_unaliased(kvm, gfn);
 	if (memslot && memslot->dirty_bitmap) {
 		unsigned long rel_gfn = gfn - memslot->base_gfn;
 		generic___set_le_bit(rel_gfn, memslot->dirty_bitmap);
 	}
 }
 /*
  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  */
 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 {
 	DEFINE_WAIT(wait);
 	for (;;) {
 		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
 		if (kvm_arch_vcpu_runnable(vcpu)) {
 			set_bit(KVM_REQ_UNHALT, &vcpu->requests);
 			break;
 		}
 		if (kvm_cpu_has_pending_timer(vcpu))
 			break;
 		if (signal_pending(current))
 			break;
 		schedule();
 	}
 	finish_wait(&vcpu->wq, &wait);
 }
 void kvm_resched(struct kvm_vcpu *vcpu)
 {
 	if (!need_resched())
 		return;
 	cond_resched();
 }
 EXPORT_SYMBOL_GPL(kvm_resched);
 void kvm_vcpu_on_spin(struct kvm_vcpu *vcpu)
 {
 	ktime_t expires;
 	DEFINE_WAIT(wait);
 	prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
 	/* Sleep for 100 us, and hope lock-holder got scheduled */
 	expires = ktime_add_ns(ktime_get(), 100000UL);
 	schedule_hrtimeout(&expires, HRTIMER_MODE_ABS);
 	finish_wait(&vcpu->wq, &wait);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
 	struct page *page;
 	if (vmf->pgoff == 0)
 		page = virt_to_page(vcpu->run);
 #ifdef CONFIG_X86
 	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
 		page = virt_to_page(vcpu->arch.pio_data);
 #endif
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
 		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
 #endif
 	else
 		return VM_FAULT_SIGBUS;
 	get_page(page);
 	vmf->page = page;
 	return 0;
 }
 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
 	.fault = kvm_vcpu_fault,
 };
 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	vma->vm_ops = &kvm_vcpu_vm_ops;
 	return 0;
 }
 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	kvm_put_kvm(vcpu->kvm);
 	return 0;
 }
 static struct file_operations kvm_vcpu_fops = {
 	.release        = kvm_vcpu_release,
 	.unlocked_ioctl = kvm_vcpu_ioctl,
 	.compat_ioctl   = kvm_vcpu_ioctl,
 	.mmap           = kvm_vcpu_mmap,
 };
 /*
  * Allocates an inode for the vcpu.
  */
 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
 {
 	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR);
 }
 /*
  * Creates some virtual cpus.  Good luck creating more than one.
  */
 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
 {
 	int r;
 	struct kvm_vcpu *vcpu, *v;
 	vcpu = kvm_arch_vcpu_create(kvm, id);
 	if (IS_ERR(vcpu))
 		return PTR_ERR(vcpu);
 	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
 	r = kvm_arch_vcpu_setup(vcpu);
 	if (r)
 		return r;
 	mutex_lock(&kvm->lock);
 	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
 		r = -EINVAL;
 		goto vcpu_destroy;
 	}
 	kvm_for_each_vcpu(r, v, kvm)
 		if (v->vcpu_id == id) {
 			r = -EEXIST;
 			goto vcpu_destroy;
 		}
 	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
 	/* Now it's all set up, let userspace reach it */
 	kvm_get_kvm(kvm);
 	r = create_vcpu_fd(vcpu);
 	if (r < 0) {
 		kvm_put_kvm(kvm);
 		goto vcpu_destroy;
 	}
 	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
 	smp_wmb();
 	atomic_inc(&kvm->online_vcpus);
 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
 	if (kvm->bsp_vcpu_id == id)
 		kvm->bsp_vcpu = vcpu;
 #endif
 	mutex_unlock(&kvm->lock);
 	return r;
 vcpu_destroy:
 	mutex_unlock(&kvm->lock);
 	kvm_arch_vcpu_destroy(vcpu);
 	return r;
 }
 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
 {
 	if (sigset) {
 		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
 		vcpu->sigset_active = 1;
 		vcpu->sigset = *sigset;
 	} else
 		vcpu->sigset_active = 0;
 	return 0;
 }
 static long kvm_vcpu_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	struct kvm_fpu *fpu = NULL;
 	struct kvm_sregs *kvm_sregs = NULL;
 	if (vcpu->kvm->mm != current->mm)
 		return -EIO;
+#if defined(CONFIG_S390) || defined(CONFIG_PPC)
+	/*
+	 * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
+	 * so vcpu_load() would break it.
+	 */
+	if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_INTERRUPT)
+		return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
+#endif
+	vcpu_load(vcpu);
 	switch (ioctl) {
 	case KVM_RUN:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
 		break;
 	case KVM_GET_REGS: {
 		struct kvm_regs *kvm_regs;
 		r = -ENOMEM;
 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
 		if (!kvm_regs)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
 		if (r)
 			goto out_free1;
 		r = -EFAULT;
 		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
 			goto out_free1;
 		r = 0;
 out_free1:
 		kfree(kvm_regs);
 		break;
 	}
 	case KVM_SET_REGS: {
 		struct kvm_regs *kvm_regs;
 		r = -ENOMEM;
 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
 		if (!kvm_regs)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
 			goto out_free2;
 		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
 		if (r)
 			goto out_free2;
 		r = 0;
 out_free2:
 		kfree(kvm_regs);
 		break;
 	}
 	case KVM_GET_SREGS: {
 		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!kvm_sregs)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_SREGS: {
 		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!kvm_sregs)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_GET_MP_STATE: {
 		struct kvm_mp_state mp_state;
 		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &mp_state, sizeof mp_state))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_MP_STATE: {
 		struct kvm_mp_state mp_state;
 		r = -EFAULT;
 		if (copy_from_user(&mp_state, argp, sizeof mp_state))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_TRANSLATE: {
 		struct kvm_translation tr;
 		r = -EFAULT;
 		if (copy_from_user(&tr, argp, sizeof tr))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &tr, sizeof tr))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_GUEST_DEBUG: {
 		struct kvm_guest_debug dbg;
 		r = -EFAULT;
 		if (copy_from_user(&dbg, argp, sizeof dbg))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_SIGNAL_MASK: {
 		struct kvm_signal_mask __user *sigmask_arg = argp;
 		struct kvm_signal_mask kvm_sigmask;
 		sigset_t sigset, *p;
 		p = NULL;
 		if (argp) {
 			r = -EFAULT;
 			if (copy_from_user(&kvm_sigmask, argp,
 					   sizeof kvm_sigmask))
 				goto out;
 			r = -EINVAL;
 			if (kvm_sigmask.len != sizeof sigset)
 				goto out;
 			r = -EFAULT;
 			if (copy_from_user(&sigset, sigmask_arg->sigset,
 					   sizeof sigset))
 				goto out;
 			p = &sigset;
 		}
 		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
 		break;
 	}
 	case KVM_GET_FPU: {
 		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!fpu)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_FPU: {
 		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!fpu)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	default:
+		vcpu_put(vcpu);
 		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
+		vcpu_load(vcpu);
 	}
 out:
+	vcpu_put(vcpu);
 	kfree(fpu);
 	kfree(kvm_sregs);
 	return r;
 }
 static long kvm_vm_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	if (kvm->mm != current->mm)
 		return -EIO;
 	switch (ioctl) {
 	case KVM_CREATE_VCPU:
 		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
 		if (r < 0)
 			goto out;
 		break;
 	case KVM_SET_USER_MEMORY_REGION: {
 		struct kvm_userspace_memory_region kvm_userspace_mem;
 		r = -EFAULT;
 		if (copy_from_user(&kvm_userspace_mem, argp,
 						sizeof kvm_userspace_mem))
 			goto out;
 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_GET_DIRTY_LOG: {
 		struct kvm_dirty_log log;
 		r = -EFAULT;
 		if (copy_from_user(&log, argp, sizeof log))
 			goto out;
 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
 		if (r)
 			goto out;
 		break;
 	}
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	case KVM_REGISTER_COALESCED_MMIO: {
 		struct kvm_coalesced_mmio_zone zone;
 		r = -EFAULT;
 		if (copy_from_user(&zone, argp, sizeof zone))
 			goto out;
 		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_UNREGISTER_COALESCED_MMIO: {
 		struct kvm_coalesced_mmio_zone zone;
 		r = -EFAULT;
 		if (copy_from_user(&zone, argp, sizeof zone))
 			goto out;
 		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 #endif
 	case KVM_IRQFD: {
 		struct kvm_irqfd data;
 		r = -EFAULT;
 		if (copy_from_user(&data, argp, sizeof data))
 			goto out;
 		r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
 		break;
 	}
 	case KVM_IOEVENTFD: {
 		struct kvm_ioeventfd data;
 		r = -EFAULT;
 		if (copy_from_user(&data, argp, sizeof data))
 			goto out;
 		r = kvm_ioeventfd(kvm, &data);
 		break;
 	}
 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
 	case KVM_SET_BOOT_CPU_ID:
 		r = 0;
 		mutex_lock(&kvm->lock);
 		if (atomic_read(&kvm->online_vcpus) != 0)
 			r = -EBUSY;
 		else
 			kvm->bsp_vcpu_id = arg;
 		mutex_unlock(&kvm->lock);
 		break;
 #endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
 		if (r == -ENOTTY)
 			r = kvm_vm_ioctl_assigned_device(kvm, ioctl, arg);
 	}
 out:
 	return r;
 }
 #ifdef CONFIG_COMPAT
 struct compat_kvm_dirty_log {
 	__u32 slot;
 	__u32 padding1;
 	union {
 		compat_uptr_t dirty_bitmap; /* one bit per page */
 		__u64 padding2;
 	};
 };
 static long kvm_vm_compat_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	int r;
 	if (kvm->mm != current->mm)
 		return -EIO;
 	switch (ioctl) {
 	case KVM_GET_DIRTY_LOG: {
 		struct compat_kvm_dirty_log compat_log;
 		struct kvm_dirty_log log;
 		r = -EFAULT;
 		if (copy_from_user(&compat_log, (void __user *)arg,
 				   sizeof(compat_log)))
 			goto out;
 		log.slot	 = compat_log.slot;
 		log.padding1	 = compat_log.padding1;
 		log.padding2	 = compat_log.padding2;
 		log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
 		if (r)
 			goto out;
 		break;
 	}
 	default:
 		r = kvm_vm_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
 }
 #endif
 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct page *page[1];
 	unsigned long addr;
 	int npages;
 	gfn_t gfn = vmf->pgoff;
 	struct kvm *kvm = vma->vm_file->private_data;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return VM_FAULT_SIGBUS;
 	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
 				NULL);
 	if (unlikely(npages != 1))
 		return VM_FAULT_SIGBUS;
 	vmf->page = page[0];
 	return 0;
 }
 static const struct vm_operations_struct kvm_vm_vm_ops = {
 	.fault = kvm_vm_fault,
 };
 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	vma->vm_ops = &kvm_vm_vm_ops;
 	return 0;
 }
 static struct file_operations kvm_vm_fops = {
 	.release        = kvm_vm_release,
 	.unlocked_ioctl = kvm_vm_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl   = kvm_vm_compat_ioctl,
 #endif
 	.mmap           = kvm_vm_mmap,
 };
 static int kvm_dev_ioctl_create_vm(void)
 {
 	int fd, r;
 	struct kvm *kvm;
 	kvm = kvm_create_vm();
 	if (IS_ERR(kvm))
 		return PTR_ERR(kvm);
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	r = kvm_coalesced_mmio_init(kvm);
 	if (r < 0) {
 		kvm_put_kvm(kvm);
 		return r;
 	}
 #endif
 	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
 	if (fd < 0)
 		kvm_put_kvm(kvm);
 	return fd;
 }
 static long kvm_dev_ioctl_check_extension_generic(long arg)
 {
 	switch (arg) {
 	case KVM_CAP_USER_MEMORY:
 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
 	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
 	case KVM_CAP_SET_BOOT_CPU_ID:
 #endif
 	case KVM_CAP_INTERNAL_ERROR_DATA:
 		return 1;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 	case KVM_CAP_IRQ_ROUTING:
 		return KVM_MAX_IRQ_ROUTES;
 #endif
 	default:
 		break;
 	}
 	return kvm_dev_ioctl_check_extension(arg);
 }
 static long kvm_dev_ioctl(struct file *filp,
 			  unsigned int ioctl, unsigned long arg)
 {
 	long r = -EINVAL;
 	switch (ioctl) {
 	case KVM_GET_API_VERSION:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = KVM_API_VERSION;
 		break;
 	case KVM_CREATE_VM:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = kvm_dev_ioctl_create_vm();
 		break;
 	case KVM_CHECK_EXTENSION:
 		r = kvm_dev_ioctl_check_extension_generic(arg);
 		break;
 	case KVM_GET_VCPU_MMAP_SIZE:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = PAGE_SIZE;     /* struct kvm_run */
 #ifdef CONFIG_X86
 		r += PAGE_SIZE;    /* pio data page */
 #endif
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 		r += PAGE_SIZE;    /* coalesced mmio ring page */
 #endif
 		break;
 	case KVM_TRACE_ENABLE:
 	case KVM_TRACE_PAUSE:
 	case KVM_TRACE_DISABLE:
 		r = -EOPNOTSUPP;
 		break;
 	default:
 		return kvm_arch_dev_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
 }
 static struct file_operations kvm_chardev_ops = {
 	.unlocked_ioctl = kvm_dev_ioctl,
 	.compat_ioctl   = kvm_dev_ioctl,
 };
 static struct miscdevice kvm_dev = {
 	KVM_MINOR,
 	"kvm",
 	&kvm_chardev_ops,
 };
 static void hardware_enable(void *junk)
 {
 	int cpu = raw_smp_processor_id();
 	int r;
 	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
 		return;
 	cpumask_set_cpu(cpu, cpus_hardware_enabled);
 	r = kvm_arch_hardware_enable(NULL);
 	if (r) {
 		cpumask_clear_cpu(cpu, cpus_hardware_enabled);
 		atomic_inc(&hardware_enable_failed);
 		printk(KERN_INFO "kvm: enabling virtualization on "
 				 "CPU%d failed\n", cpu);
 	}
 }
 static void hardware_disable(void *junk)
 {
 	int cpu = raw_smp_processor_id();
 	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
 		return;
 	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
 	kvm_arch_hardware_disable(NULL);
 }
 static void hardware_disable_all_nolock(void)
 {
 	BUG_ON(!kvm_usage_count);
 	kvm_usage_count--;
 	if (!kvm_usage_count)
 		on_each_cpu(hardware_disable, NULL, 1);
 }
 static void hardware_disable_all(void)
 {
 	spin_lock(&kvm_lock);
 	hardware_disable_all_nolock();
 	spin_unlock(&kvm_lock);
 }
 static int hardware_enable_all(void)
 {
 	int r = 0;
 	spin_lock(&kvm_lock);
 	kvm_usage_count++;
 	if (kvm_usage_count == 1) {
 		atomic_set(&hardware_enable_failed, 0);
 		on_each_cpu(hardware_enable, NULL, 1);
 		if (atomic_read(&hardware_enable_failed)) {
 			hardware_disable_all_nolock();
 			r = -EBUSY;
 		}
 	}
 	spin_unlock(&kvm_lock);
 	return r;
 }
 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
 			   void *v)
 {
 	int cpu = (long)v;
 	if (!kvm_usage_count)
 		return NOTIFY_OK;
 	val &= ~CPU_TASKS_FROZEN;
 	switch (val) {
 	case CPU_DYING:
 		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
 		       cpu);
 		hardware_disable(NULL);
 		break;
 	case CPU_ONLINE:
 		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
 		       cpu);
 		smp_call_function_single(cpu, hardware_enable, NULL, 1);
 		break;
 	}
 	return NOTIFY_OK;
 }
 asmlinkage void kvm_handle_fault_on_reboot(void)
 {
 	if (kvm_rebooting)
 		/* spin while reset goes on */
 		while (true)
 			;
 	/* Fault while not rebooting.  We want the trace. */
 	BUG();
 }
 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
 		      void *v)
 {
 	/*
 	 * Some (well, at least mine) BIOSes hang on reboot if
 	 * in vmx root mode.
 	 *
 	 * And Intel TXT required VMX off for all cpu when system shutdown.
 	 */
 	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
 	kvm_rebooting = true;
 	on_each_cpu(hardware_disable, NULL, 1);
 	return NOTIFY_OK;
 }
 static struct notifier_block kvm_reboot_notifier = {
 	.notifier_call = kvm_reboot,
 	.priority = 0,
 };
 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
 {
 	int i;
 	for (i = 0; i < bus->dev_count; i++) {
 		struct kvm_io_device *pos = bus->devs[i];
 		kvm_iodevice_destructor(pos);
 	}
 	kfree(bus);
 }
 /* kvm_io_bus_write - called under kvm->slots_lock */
 int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
 	int i;
 	struct kvm_io_bus *bus;
 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
 			return 0;
 	return -EOPNOTSUPP;
 }
 /* kvm_io_bus_read - called under kvm->slots_lock */
 int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 		    int len, void *val)
 {
 	int i;
 	struct kvm_io_bus *bus;
 	bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
 			return 0;
 	return -EOPNOTSUPP;
 }
 /* Caller must hold slots_lock. */
 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 			    struct kvm_io_device *dev)
 {
 	struct kvm_io_bus *new_bus, *bus;
 	bus = kvm->buses[bus_idx];
 	if (bus->dev_count > NR_IOBUS_DEVS-1)
 		return -ENOSPC;
 	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
 	if (!new_bus)
 		return -ENOMEM;
 	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
 	new_bus->devs[new_bus->dev_count++] = dev;
 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kfree(bus);
 	return 0;
 }
 /* Caller must hold slots_lock. */
 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 			      struct kvm_io_device *dev)
 {
 	int i, r;
 	struct kvm_io_bus *new_bus, *bus;
 	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
 	if (!new_bus)
 		return -ENOMEM;
 	bus = kvm->buses[bus_idx];
 	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
 	r = -ENOENT;
 	for (i = 0; i < new_bus->dev_count; i++)
 		if (new_bus->devs[i] == dev) {
 			r = 0;
 			new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
 			break;
 		}
 	if (r) {
 		kfree(new_bus);
 		return r;
 	}
 	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kfree(bus);
 	return r;
 }
 static struct notifier_block kvm_cpu_notifier = {
 	.notifier_call = kvm_cpu_hotplug,
 	.priority = 20, /* must be > scheduler priority */
 };
 static int vm_stat_get(void *_offset, u64 *val)
 {
 	unsigned offset = (long)_offset;
 	struct kvm *kvm;
 	*val = 0;
 	spin_lock(&kvm_lock);
 	list_for_each_entry(kvm, &vm_list, vm_list)
 		*val += *(u32 *)((void *)kvm + offset);
 	spin_unlock(&kvm_lock);
 	return 0;
 }
 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
 static int vcpu_stat_get(void *_offset, u64 *val)
 {
 	unsigned offset = (long)_offset;
 	struct kvm *kvm;
 	struct kvm_vcpu *vcpu;
 	int i;
 	*val = 0;
 	spin_lock(&kvm_lock);
 	list_for_each_entry(kvm, &vm_list, vm_list)
 		kvm_for_each_vcpu(i, vcpu, kvm)
 			*val += *(u32 *)((void *)vcpu + offset);
 	spin_unlock(&kvm_lock);
 	return 0;
 }
 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
 static const struct file_operations *stat_fops[] = {
 	[KVM_STAT_VCPU] = &vcpu_stat_fops,
 	[KVM_STAT_VM]   = &vm_stat_fops,
 };
 static void kvm_init_debug(void)
 {
 	struct kvm_stats_debugfs_item *p;
 	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
 	for (p = debugfs_entries; p->name; ++p)
 		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
 						(void *)(long)p->offset,
 						stat_fops[p->kind]);
 }
 static void kvm_exit_debug(void)
 {
 	struct kvm_stats_debugfs_item *p;
 	for (p = debugfs_entries; p->name; ++p)
 		debugfs_remove(p->dentry);
 	debugfs_remove(kvm_debugfs_dir);
 }
 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
 {
 	if (kvm_usage_count)
 		hardware_disable(NULL);
 	return 0;
 }
 static int kvm_resume(struct sys_device *dev)
 {
 	if (kvm_usage_count)
 		hardware_enable(NULL);
 	return 0;
 }
 static struct sysdev_class kvm_sysdev_class = {
 	.name = "kvm",
 	.suspend = kvm_suspend,
 	.resume = kvm_resume,
 };
 static struct sys_device kvm_sysdev = {
 	.id = 0,
 	.cls = &kvm_sysdev_class,
 };
 struct page *bad_page;
 pfn_t bad_pfn;
 static inline
 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
 {
 	return container_of(pn, struct kvm_vcpu, preempt_notifier);
 }
 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
 {
 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 	kvm_arch_vcpu_load(vcpu, cpu);
 }
 static void kvm_sched_out(struct preempt_notifier *pn,
 			  struct task_struct *next)
 {
 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 	kvm_arch_vcpu_put(vcpu);
 }
 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
 		  struct module *module)
 {
 	int r;
 	int cpu;
 	r = kvm_arch_init(opaque);
 	if (r)
 		goto out_fail;
 	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (bad_page == NULL) {
 		r = -ENOMEM;
 		goto out;
 	}
 	bad_pfn = page_to_pfn(bad_page);
 	hwpoison_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (hwpoison_page == NULL) {
 		r = -ENOMEM;
 		goto out_free_0;
 	}
 	hwpoison_pfn = page_to_pfn(hwpoison_page);
 	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
 		r = -ENOMEM;
 		goto out_free_0;
 	}
 	r = kvm_arch_hardware_setup();
 	if (r < 0)
 		goto out_free_0a;
 	for_each_online_cpu(cpu) {
 		smp_call_function_single(cpu,
 				kvm_arch_check_processor_compat,
 				&r, 1);
 		if (r < 0)
 			goto out_free_1;
 	}
 	r = register_cpu_notifier(&kvm_cpu_notifier);
 	if (r)
 		goto out_free_2;
 	register_reboot_notifier(&kvm_reboot_notifier);
 	r = sysdev_class_register(&kvm_sysdev_class);
 	if (r)
 		goto out_free_3;
 	r = sysdev_register(&kvm_sysdev);
 	if (r)
 		goto out_free_4;
 	/* A kmem cache lets us meet the alignment requirements of fx_save. */
 	if (!vcpu_align)
 		vcpu_align = __alignof__(struct kvm_vcpu);
 	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
 					   0, NULL);
 	if (!kvm_vcpu_cache) {
 		r = -ENOMEM;
 		goto out_free_5;
 	}
 	kvm_chardev_ops.owner = module;
 	kvm_vm_fops.owner = module;
 	kvm_vcpu_fops.owner = module;
 	r = misc_register(&kvm_dev);
 	if (r) {
 		printk(KERN_ERR "kvm: misc device register failed\n");
 		goto out_free;
 	}
 	kvm_preempt_ops.sched_in = kvm_sched_in;
 	kvm_preempt_ops.sched_out = kvm_sched_out;
 	kvm_init_debug();
 	return 0;
 out_free:
 	kmem_cache_destroy(kvm_vcpu_cache);
 out_free_5:
 	sysdev_unregister(&kvm_sysdev);
 out_free_4:
 	sysdev_class_unregister(&kvm_sysdev_class);
 out_free_3:
 	unregister_reboot_notifier(&kvm_reboot_notifier);
 	unregister_cpu_notifier(&kvm_cpu_notifier);
 out_free_2:
 out_free_1:
 	kvm_arch_hardware_unsetup();
 out_free_0a:
 	free_cpumask_var(cpus_hardware_enabled);
 out_free_0:
 	if (hwpoison_page)
 		__free_page(hwpoison_page);
 	__free_page(bad_page);
 out:
 	kvm_arch_exit();
 out_fail:
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_init);
 void kvm_exit(void)
 {
 	kvm_exit_debug();
 	misc_deregister(&kvm_dev);
 	kmem_cache_destroy(kvm_vcpu_cache);
 	sysdev_unregister(&kvm_sysdev);
 	sysdev_class_unregister(&kvm_sysdev_class);
 	unregister_reboot_notifier(&kvm_reboot_notifier);
 	unregister_cpu_notifier(&kvm_cpu_notifier);
 	on_each_cpu(hardware_disable, NULL, 1);
 	kvm_arch_hardware_unsetup();
 	kvm_arch_exit();
 	free_cpumask_var(cpus_hardware_enabled);
 	__free_page(hwpoison_page);
 	__free_page(bad_page);
 }
 EXPORT_SYMBOL_GPL(kvm_exit);