Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 73880c80aa9c8dc353cd0ad26579023213cd5314

Authored by Gleb Natapov 2009-06-09 20:56:28 +0800

Committed by Avi Kivity 2009-09-10 13:32:52 +0800

Exists in master and in 7 other branches

KVM: Break dependency between vcpu index in vcpus array and vcpu_id.

Archs are free to use vcpu_id as they see fit. For x86 it is used as
vcpu's apic id. New ioctl is added to configure boot vcpu id that was
assumed to be 0 till now.

Signed-off-by: Gleb Natapov <gleb@redhat.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

Showing 9 changed files with 55 additions and 33 deletions Inline Diff

arch/ia64/include/asm/kvm_host.h
arch/ia64/kvm/Kconfig
arch/ia64/kvm/kvm-ia64.c
arch/ia64/kvm/vcpu.c
arch/x86/kvm/Kconfig
include/linux/kvm.h
include/linux/kvm_host.h
virt/kvm/Kconfig
virt/kvm/kvm_main.c

arch/ia64/include/asm/kvm_host.h

Diff comments View file @ 73880c8

 /*
  * kvm_host.h: used for kvm module, and hold ia64-specific sections.
  *
  * Copyright (C) 2007, Intel Corporation.
  *
  * Xiantao Zhang <xiantao.zhang@intel.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  * Place - Suite 330, Boston, MA 02111-1307 USA.
  *
  */
 #ifndef __ASM_KVM_HOST_H
 #define __ASM_KVM_HOST_H
 #define KVM_MEMORY_SLOTS 32
 /* memory slots that does not exposed to userspace */
 #define KVM_PRIVATE_MEM_SLOTS 4
 #define KVM_COALESCED_MMIO_PAGE_OFFSET 1
 /* define exit reasons from vmm to kvm*/
 #define EXIT_REASON_VM_PANIC		0
 #define EXIT_REASON_MMIO_INSTRUCTION	1
 #define EXIT_REASON_PAL_CALL		2
 #define EXIT_REASON_SAL_CALL		3
 #define EXIT_REASON_SWITCH_RR6		4
 #define EXIT_REASON_VM_DESTROY		5
 #define EXIT_REASON_EXTERNAL_INTERRUPT	6
 #define EXIT_REASON_IPI			7
 #define EXIT_REASON_PTC_G		8
 #define EXIT_REASON_DEBUG		20
 /*Define vmm address space and vm data space.*/
 #define KVM_VMM_SIZE (__IA64_UL_CONST(16)<<20)
 #define KVM_VMM_SHIFT 24
 #define KVM_VMM_BASE 0xD000000000000000
 #define VMM_SIZE (__IA64_UL_CONST(8)<<20)
 /*
  * Define vm_buffer, used by PAL Services, base address.
  * Note: vm_buffer is in the VMM-BLOCK, the size must be < 8M
  */
 #define KVM_VM_BUFFER_BASE (KVM_VMM_BASE + VMM_SIZE)
 #define KVM_VM_BUFFER_SIZE (__IA64_UL_CONST(8)<<20)
 /*
  * kvm guest's data area looks as follow:
  *
  *            +----------------------+	-------	KVM_VM_DATA_SIZE
  *	      |	    vcpu[n]'s data   |	 |     ___________________KVM_STK_OFFSET
  *     	      |			     |	 |    /			  |
  *     	      |	       ..........    |	 |   /vcpu's struct&stack |
  *     	      |	       ..........    |	 |  /---------------------|---- 0
  *	      |	    vcpu[5]'s data   |	 | /	   vpd		  |
  *	      |	    vcpu[4]'s data   |	 |/-----------------------|
  *	      |	    vcpu[3]'s data   |	 /	   vtlb		  |
  *	      |	    vcpu[2]'s data   |	/|------------------------|
  *	      |	    vcpu[1]'s data   |/  |	   vhpt		  |
  *	      |	    vcpu[0]'s data   |____________________________|
  *            +----------------------+	 |
  *	      |	   memory dirty log  |	 |
  *            +----------------------+	 |
  *	      |	   vm's data struct  |	 |
  *            +----------------------+	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |			     |	 |
  *	      |	  vm's p2m table  |	 |
  *	      |			     |	 |
  *            |			     |	 |
  *	      |			     |	 |  |
  * vm's data->|			     |   |  |
  *	      +----------------------+ ------- 0
  * To support large memory, needs to increase the size of p2m.
  * To support more vcpus, needs to ensure it has enough space to
  * hold vcpus' data.
  */
 #define KVM_VM_DATA_SHIFT	26
 #define KVM_VM_DATA_SIZE	(__IA64_UL_CONST(1) << KVM_VM_DATA_SHIFT)
 #define KVM_VM_DATA_BASE	(KVM_VMM_BASE + KVM_VM_DATA_SIZE)
 #define KVM_P2M_BASE		KVM_VM_DATA_BASE
 #define KVM_P2M_SIZE		(__IA64_UL_CONST(24) << 20)
 #define VHPT_SHIFT		16
 #define VHPT_SIZE		(__IA64_UL_CONST(1) << VHPT_SHIFT)
 #define VHPT_NUM_ENTRIES	(__IA64_UL_CONST(1) << (VHPT_SHIFT-5))
 #define VTLB_SHIFT		16
 #define VTLB_SIZE		(__IA64_UL_CONST(1) << VTLB_SHIFT)
 #define VTLB_NUM_ENTRIES	(1UL << (VHPT_SHIFT-5))
 #define VPD_SHIFT		16
 #define VPD_SIZE		(__IA64_UL_CONST(1) << VPD_SHIFT)
 #define VCPU_STRUCT_SHIFT	16
 #define VCPU_STRUCT_SIZE	(__IA64_UL_CONST(1) << VCPU_STRUCT_SHIFT)
 /*
  * This must match KVM_IA64_VCPU_STACK_{SHIFT,SIZE} arch/ia64/include/asm/kvm.h
  */
 #define KVM_STK_SHIFT		16
 #define KVM_STK_OFFSET		(__IA64_UL_CONST(1)<< KVM_STK_SHIFT)
 #define KVM_VM_STRUCT_SHIFT	19
 #define KVM_VM_STRUCT_SIZE	(__IA64_UL_CONST(1) << KVM_VM_STRUCT_SHIFT)
 #define KVM_MEM_DIRY_LOG_SHIFT	19
 #define KVM_MEM_DIRTY_LOG_SIZE (__IA64_UL_CONST(1) << KVM_MEM_DIRY_LOG_SHIFT)
 #ifndef __ASSEMBLY__
 /*Define the max vcpus and memory for Guests.*/
 #define KVM_MAX_VCPUS	(KVM_VM_DATA_SIZE - KVM_P2M_SIZE - KVM_VM_STRUCT_SIZE -\
 			KVM_MEM_DIRTY_LOG_SIZE) / sizeof(struct kvm_vcpu_data)
 #define KVM_MAX_MEM_SIZE (KVM_P2M_SIZE >> 3 << PAGE_SHIFT)
 #define VMM_LOG_LEN 256
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
 #include <linux/kvm_types.h>
 #include <asm/pal.h>
 #include <asm/sal.h>
 #include <asm/page.h>
 struct kvm_vcpu_data {
 	char vcpu_vhpt[VHPT_SIZE];
 	char vcpu_vtlb[VTLB_SIZE];
 	char vcpu_vpd[VPD_SIZE];
 	char vcpu_struct[VCPU_STRUCT_SIZE];
 };
 struct kvm_vm_data {
 	char kvm_p2m[KVM_P2M_SIZE];
 	char kvm_vm_struct[KVM_VM_STRUCT_SIZE];
 	char kvm_mem_dirty_log[KVM_MEM_DIRTY_LOG_SIZE];
 	struct kvm_vcpu_data vcpu_data[KVM_MAX_VCPUS];
 };
 #define VCPU_BASE(n)	(KVM_VM_DATA_BASE + \
 				offsetof(struct kvm_vm_data, vcpu_data[n]))
 #define KVM_VM_BASE	(KVM_VM_DATA_BASE + \
 				offsetof(struct kvm_vm_data, kvm_vm_struct))
 #define KVM_MEM_DIRTY_LOG_BASE	KVM_VM_DATA_BASE + \
 				offsetof(struct kvm_vm_data, kvm_mem_dirty_log)
 #define VHPT_BASE(n) (VCPU_BASE(n) + offsetof(struct kvm_vcpu_data, vcpu_vhpt))
 #define VTLB_BASE(n) (VCPU_BASE(n) + offsetof(struct kvm_vcpu_data, vcpu_vtlb))
 #define VPD_BASE(n)  (VCPU_BASE(n) + offsetof(struct kvm_vcpu_data, vcpu_vpd))
 #define VCPU_STRUCT_BASE(n)	(VCPU_BASE(n) + \
 				offsetof(struct kvm_vcpu_data, vcpu_struct))
 /*IO section definitions*/
 #define IOREQ_READ      1
 #define IOREQ_WRITE     0
 #define STATE_IOREQ_NONE        0
 #define STATE_IOREQ_READY       1
 #define STATE_IOREQ_INPROCESS   2
 #define STATE_IORESP_READY      3
 /*Guest Physical address layout.*/
 #define GPFN_MEM        (0UL << 60) /* Guest pfn is normal mem */
 #define GPFN_FRAME_BUFFER   (1UL << 60) /* VGA framebuffer */
 #define GPFN_LOW_MMIO       (2UL << 60) /* Low MMIO range */
 #define GPFN_PIB        (3UL << 60) /* PIB base */
 #define GPFN_IOSAPIC        (4UL << 60) /* IOSAPIC base */
 #define GPFN_LEGACY_IO      (5UL << 60) /* Legacy I/O base */
 #define GPFN_GFW        (6UL << 60) /* Guest Firmware */
 #define GPFN_PHYS_MMIO      (7UL << 60) /* Directed MMIO Range */
 #define GPFN_IO_MASK        (7UL << 60) /* Guest pfn is I/O type */
 #define GPFN_INV_MASK       (1UL << 63) /* Guest pfn is invalid */
 #define INVALID_MFN       (~0UL)
 #define MEM_G   (1UL << 30)
 #define MEM_M   (1UL << 20)
 #define MMIO_START       (3 * MEM_G)
 #define MMIO_SIZE        (512 * MEM_M)
 #define VGA_IO_START     0xA0000UL
 #define VGA_IO_SIZE      0x20000
 #define LEGACY_IO_START  (MMIO_START + MMIO_SIZE)
 #define LEGACY_IO_SIZE   (64 * MEM_M)
 #define IO_SAPIC_START   0xfec00000UL
 #define IO_SAPIC_SIZE    0x100000
 #define PIB_START 0xfee00000UL
 #define PIB_SIZE 0x200000
 #define GFW_START        (4 * MEM_G - 16 * MEM_M)
 #define GFW_SIZE         (16 * MEM_M)
 /*Deliver mode, defined for ioapic.c*/
 #define dest_Fixed IOSAPIC_FIXED
 #define dest_LowestPrio IOSAPIC_LOWEST_PRIORITY
 #define NMI_VECTOR      		2
 #define ExtINT_VECTOR       		0
 #define NULL_VECTOR     		(-1)
 #define IA64_SPURIOUS_INT_VECTOR    	0x0f
 #define VCPU_LID(v) (((u64)(v)->vcpu_id) << 24)
 /*
  *Delivery mode
  */
 #define SAPIC_DELIV_SHIFT      8
 #define SAPIC_FIXED            0x0
 #define SAPIC_LOWEST_PRIORITY  0x1
 #define SAPIC_PMI              0x2
 #define SAPIC_NMI              0x4
 #define SAPIC_INIT             0x5
 #define SAPIC_EXTINT           0x7
 /*
  * vcpu->requests bit members for arch
  */
 #define KVM_REQ_PTC_G		32
 #define KVM_REQ_RESUME		33
 #define KVM_PAGES_PER_HPAGE	1
 struct kvm;
 struct kvm_vcpu;
 struct kvm_mmio_req {
 	uint64_t addr;          /*  physical address		*/
 	uint64_t size;          /*  size in bytes		*/
 	uint64_t data;          /*  data (or paddr of data)     */
 	uint8_t state:4;
 	uint8_t dir:1;          /*  1=read, 0=write             */
 };
 /*Pal data struct */
 struct kvm_pal_call{
 	/*In area*/
 	uint64_t gr28;
 	uint64_t gr29;
 	uint64_t gr30;
 	uint64_t gr31;
 	/*Out area*/
 	struct ia64_pal_retval ret;
 };
 /* Sal data structure */
 struct kvm_sal_call{
 	/*In area*/
 	uint64_t in0;
 	uint64_t in1;
 	uint64_t in2;
 	uint64_t in3;
 	uint64_t in4;
 	uint64_t in5;
 	uint64_t in6;
 	uint64_t in7;
 	struct sal_ret_values ret;
 };
 /*Guest change rr6*/
 struct kvm_switch_rr6 {
 	uint64_t old_rr;
 	uint64_t new_rr;
 };
 union ia64_ipi_a{
 	unsigned long val;
 	struct {
 		unsigned long rv  : 3;
 		unsigned long ir  : 1;
 		unsigned long eid : 8;
 		unsigned long id  : 8;
 		unsigned long ib_base : 44;
 	};
 };
 union ia64_ipi_d {
 	unsigned long val;
 	struct {
 		unsigned long vector : 8;
 		unsigned long dm  : 3;
 		unsigned long ig  : 53;
 	};
 };
 /*ipi check exit data*/
 struct kvm_ipi_data{
 	union ia64_ipi_a addr;
 	union ia64_ipi_d data;
 };
 /*global purge data*/
 struct kvm_ptc_g {
 	unsigned long vaddr;
 	unsigned long rr;
 	unsigned long ps;
 	struct kvm_vcpu *vcpu;
 };
 /*Exit control data */
 struct exit_ctl_data{
 	uint32_t exit_reason;
 	uint32_t vm_status;
 	union {
 		struct kvm_mmio_req	ioreq;
 		struct kvm_pal_call	pal_data;
 		struct kvm_sal_call	sal_data;
 		struct kvm_switch_rr6	rr_data;
 		struct kvm_ipi_data	ipi_data;
 		struct kvm_ptc_g	ptc_g_data;
 	} u;
 };
 union pte_flags {
 	unsigned long val;
 	struct {
 		unsigned long p    :  1; /*0      */
 		unsigned long      :  1; /* 1     */
 		unsigned long ma   :  3; /* 2-4   */
 		unsigned long a    :  1; /* 5     */
 		unsigned long d    :  1; /* 6     */
 		unsigned long pl   :  2; /* 7-8   */
 		unsigned long ar   :  3; /* 9-11  */
 		unsigned long ppn  : 38; /* 12-49 */
 		unsigned long      :  2; /* 50-51 */
 		unsigned long ed   :  1; /* 52    */
 	};
 };
 union ia64_pta {
 	unsigned long val;
 	struct {
 		unsigned long ve : 1;
 		unsigned long reserved0 : 1;
 		unsigned long size : 6;
 		unsigned long vf : 1;
 		unsigned long reserved1 : 6;
 		unsigned long base : 49;
 	};
 };
 struct thash_cb {
 	/* THASH base information */
 	struct thash_data	*hash; /* hash table pointer */
 	union ia64_pta		pta;
 	int           num;
 };
 struct kvm_vcpu_stat {
 };
 struct kvm_vcpu_arch {
 	int launched;
 	int last_exit;
 	int last_run_cpu;
 	int vmm_tr_slot;
 	int vm_tr_slot;
 	int sn_rtc_tr_slot;
 #define KVM_MP_STATE_RUNNABLE          0
 #define KVM_MP_STATE_UNINITIALIZED     1
 #define KVM_MP_STATE_INIT_RECEIVED     2
 #define KVM_MP_STATE_HALTED            3
 	int mp_state;
 #define MAX_PTC_G_NUM			3
 	int ptc_g_count;
 	struct kvm_ptc_g ptc_g_data[MAX_PTC_G_NUM];
 	/*halt timer to wake up sleepy vcpus*/
 	struct hrtimer hlt_timer;
 	long ht_active;
 	struct kvm_lapic *apic;    /* kernel irqchip context */
 	struct vpd *vpd;
 	/* Exit data for vmm_transition*/
 	struct exit_ctl_data exit_data;
 	cpumask_t cache_coherent_map;
 	unsigned long vmm_rr;
 	unsigned long host_rr6;
 	unsigned long psbits[8];
 	unsigned long cr_iipa;
 	unsigned long cr_isr;
 	unsigned long vsa_base;
 	unsigned long dirty_log_lock_pa;
 	unsigned long __gp;
 	/* TR and TC.  */
 	struct thash_data itrs[NITRS];
 	struct thash_data dtrs[NDTRS];
 	/* Bit is set if there is a tr/tc for the region.  */
 	unsigned char itr_regions;
 	unsigned char dtr_regions;
 	unsigned char tc_regions;
 	/* purge all */
 	unsigned long ptce_base;
 	unsigned long ptce_count[2];
 	unsigned long ptce_stride[2];
 	/* itc/itm */
 	unsigned long last_itc;
 	long itc_offset;
 	unsigned long itc_check;
 	unsigned long timer_check;
 	unsigned int timer_pending;
 	unsigned int timer_fired;
 	unsigned long vrr[8];
 	unsigned long ibr[8];
 	unsigned long dbr[8];
 	unsigned long insvc[4];		/* Interrupt in service.  */
 	unsigned long xtp;
 	unsigned long metaphysical_rr0; /* from kvm_arch (so is pinned) */
 	unsigned long metaphysical_rr4;	/* from kvm_arch (so is pinned) */
 	unsigned long metaphysical_saved_rr0; /* from kvm_arch          */
 	unsigned long metaphysical_saved_rr4; /* from kvm_arch          */
 	unsigned long fp_psr;       /*used for lazy float register */
 	unsigned long saved_gp;
 	/*for phycial  emulation */
 	int mode_flags;
 	struct thash_cb vtlb;
 	struct thash_cb vhpt;
 	char irq_check;
 	char irq_new_pending;
 	unsigned long opcode;
 	unsigned long cause;
 	char log_buf[VMM_LOG_LEN];
 	union context host;
 	union context guest;
 };
 struct kvm_vm_stat {
 	u64 remote_tlb_flush;
 };
 struct kvm_sal_data {
 	unsigned long boot_ip;
 	unsigned long boot_gp;
 };
 struct kvm_arch {
 	spinlock_t dirty_log_lock;
 	unsigned long	vm_base;
 	unsigned long	metaphysical_rr0;
 	unsigned long	metaphysical_rr4;
 	unsigned long	vmm_init_rr;
-	int		online_vcpus;
 	int		is_sn2;
 	struct kvm_ioapic *vioapic;
 	struct kvm_vm_stat stat;
 	struct kvm_sal_data rdv_sal_data;
 	struct list_head assigned_dev_head;
 	struct iommu_domain *iommu_domain;
 	int iommu_flags;
 	struct hlist_head irq_ack_notifier_list;
 	unsigned long irq_sources_bitmap;
 	unsigned long irq_states[KVM_IOAPIC_NUM_PINS];
 };
 union cpuid3_t {
 	u64 value;
 	struct {
 		u64 number : 8;
 		u64 revision : 8;
 		u64 model : 8;
 		u64 family : 8;
 		u64 archrev : 8;
 		u64 rv : 24;
 	};
 };
 struct kvm_pt_regs {
 	/* The following registers are saved by SAVE_MIN: */
 	unsigned long b6;  /* scratch */
 	unsigned long b7;  /* scratch */
 	unsigned long ar_csd; /* used by cmp8xchg16 (scratch) */
 	unsigned long ar_ssd; /* reserved for future use (scratch) */
 	unsigned long r8;  /* scratch (return value register 0) */
 	unsigned long r9;  /* scratch (return value register 1) */
 	unsigned long r10; /* scratch (return value register 2) */
 	unsigned long r11; /* scratch (return value register 3) */
 	unsigned long cr_ipsr; /* interrupted task's psr */
 	unsigned long cr_iip;  /* interrupted task's instruction pointer */
 	unsigned long cr_ifs;  /* interrupted task's function state */
 	unsigned long ar_unat; /* interrupted task's NaT register (preserved) */
 	unsigned long ar_pfs;  /* prev function state  */
 	unsigned long ar_rsc;  /* RSE configuration */
 	/* The following two are valid only if cr_ipsr.cpl > 0: */
 	unsigned long ar_rnat;  /* RSE NaT */
 	unsigned long ar_bspstore; /* RSE bspstore */
 	unsigned long pr;  /* 64 predicate registers (1 bit each) */
 	unsigned long b0;  /* return pointer (bp) */
 	unsigned long loadrs;  /* size of dirty partition << 16 */
 	unsigned long r1;  /* the gp pointer */
 	unsigned long r12; /* interrupted task's memory stack pointer */
 	unsigned long r13; /* thread pointer */
 	unsigned long ar_fpsr;  /* floating point status (preserved) */
 	unsigned long r15;  /* scratch */
 	/* The remaining registers are NOT saved for system calls.  */
 	unsigned long r14;  /* scratch */
 	unsigned long r2;  /* scratch */
 	unsigned long r3;  /* scratch */
 	unsigned long r16;  /* scratch */
 	unsigned long r17;  /* scratch */
 	unsigned long r18;  /* scratch */
 	unsigned long r19;  /* scratch */
 	unsigned long r20;  /* scratch */
 	unsigned long r21;  /* scratch */
 	unsigned long r22;  /* scratch */
 	unsigned long r23;  /* scratch */
 	unsigned long r24;  /* scratch */
 	unsigned long r25;  /* scratch */
 	unsigned long r26;  /* scratch */
 	unsigned long r27;  /* scratch */
 	unsigned long r28;  /* scratch */
 	unsigned long r29;  /* scratch */
 	unsigned long r30;  /* scratch */
 	unsigned long r31;  /* scratch */
 	unsigned long ar_ccv;  /* compare/exchange value (scratch) */
 	/*
 	 * Floating point registers that the kernel considers scratch:
 	 */
 	struct ia64_fpreg f6;  /* scratch */
 	struct ia64_fpreg f7;  /* scratch */
 	struct ia64_fpreg f8;  /* scratch */
 	struct ia64_fpreg f9;  /* scratch */
 	struct ia64_fpreg f10;  /* scratch */
 	struct ia64_fpreg f11;  /* scratch */
 	unsigned long r4;  /* preserved */
 	unsigned long r5;  /* preserved */
 	unsigned long r6;  /* preserved */
 	unsigned long r7;  /* preserved */
 	unsigned long eml_unat;    /* used for emulating instruction */
 	unsigned long pad0;     /* alignment pad */
 };
 static inline struct kvm_pt_regs *vcpu_regs(struct kvm_vcpu *v)
 {
 	return (struct kvm_pt_regs *) ((unsigned long) v + KVM_STK_OFFSET) - 1;
 }
 typedef int kvm_vmm_entry(void);
 typedef void kvm_tramp_entry(union context *host, union context *guest);
 struct kvm_vmm_info{
 	struct module	*module;
 	kvm_vmm_entry 	*vmm_entry;
 	kvm_tramp_entry *tramp_entry;
 	unsigned long 	vmm_ivt;
 	unsigned long	patch_mov_ar;
 	unsigned long	patch_mov_ar_sn2;
 };
 int kvm_highest_pending_irq(struct kvm_vcpu *vcpu);
 int kvm_emulate_halt(struct kvm_vcpu *vcpu);
 int kvm_pal_emul(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
 void kvm_sal_emul(struct kvm_vcpu *vcpu);
 #endif /* __ASSEMBLY__*/
 #endif

arch/ia64/kvm/Kconfig

Diff comments View file @ 73880c8

 #
 # KVM configuration
 #
 source "virt/kvm/Kconfig"
 menuconfig VIRTUALIZATION
 	bool "Virtualization"
 	depends on HAVE_KVM || IA64
 	default y
 	---help---
 	  Say Y here to get to see options for using your Linux host to run other
 	  operating systems inside virtual machines (guests).
 	  This option alone does not add any kernel code.
 	  If you say N, all options in this submenu will be skipped and disabled.
 if VIRTUALIZATION
 config KVM
 	tristate "Kernel-based Virtual Machine (KVM) support"
 	depends on HAVE_KVM && MODULES && EXPERIMENTAL
 	# for device assignment:
 	depends on PCI
 	select PREEMPT_NOTIFIERS
 	select ANON_INODES
 	select HAVE_KVM_IRQCHIP
+	select KVM_APIC_ARCHITECTURE
 	---help---
 	  Support hosting fully virtualized guest machines using hardware
 	  virtualization extensions.  You will need a fairly recent
 	  processor equipped with virtualization extensions. You will also
 	  need to select one or more of the processor modules below.
 	  This module provides access to the hardware capabilities through
 	  a character device node named /dev/kvm.
 	  To compile this as a module, choose M here: the module
 	  will be called kvm.
 	  If unsure, say N.
 config KVM_INTEL
 	tristate "KVM for Intel Itanium 2 processors support"
 	depends on KVM && m
 	---help---
 	  Provides support for KVM on Itanium 2 processors equipped with the VT
 	  extensions.
 config KVM_TRACE
        bool
 source drivers/virtio/Kconfig
 endif # VIRTUALIZATION

arch/ia64/kvm/kvm-ia64.c

Diff comments View file @ 73880c8

 /*
  * kvm_ia64.c: Basic KVM suppport On Itanium series processors
  *
  *
  * 	Copyright (C) 2007, Intel Corporation.
  *  	Xiantao Zhang  (xiantao.zhang@intel.com)
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  * Place - Suite 330, Boston, MA 02111-1307 USA.
  *
  */
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/percpu.h>
 #include <linux/gfp.h>
 #include <linux/fs.h>
 #include <linux/smp.h>
 #include <linux/kvm_host.h>
 #include <linux/kvm.h>
 #include <linux/bitops.h>
 #include <linux/hrtimer.h>
 #include <linux/uaccess.h>
 #include <linux/iommu.h>
 #include <linux/intel-iommu.h>
 #include <asm/pgtable.h>
 #include <asm/gcc_intrin.h>
 #include <asm/pal.h>
 #include <asm/cacheflush.h>
 #include <asm/div64.h>
 #include <asm/tlb.h>
 #include <asm/elf.h>
 #include <asm/sn/addrs.h>
 #include <asm/sn/clksupport.h>
 #include <asm/sn/shub_mmr.h>
 #include "misc.h"
 #include "vti.h"
 #include "iodev.h"
 #include "ioapic.h"
 #include "lapic.h"
 #include "irq.h"
 static unsigned long kvm_vmm_base;
 static unsigned long kvm_vsa_base;
 static unsigned long kvm_vm_buffer;
 static unsigned long kvm_vm_buffer_size;
 unsigned long kvm_vmm_gp;
 static long vp_env_info;
 static struct kvm_vmm_info *kvm_vmm_info;
 static DEFINE_PER_CPU(struct kvm_vcpu *, last_vcpu);
 struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ NULL }
 };
 static unsigned long kvm_get_itc(struct kvm_vcpu *vcpu)
 {
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (vcpu->kvm->arch.is_sn2)
 		return rtc_time();
 	else
 #endif
 		return ia64_getreg(_IA64_REG_AR_ITC);
 }
 static void kvm_flush_icache(unsigned long start, unsigned long len)
 {
 	int l;
 	for (l = 0; l < (len + 32); l += 32)
 		ia64_fc((void *)(start + l));
 	ia64_sync_i();
 	ia64_srlz_i();
 }
 static void kvm_flush_tlb_all(void)
 {
 	unsigned long i, j, count0, count1, stride0, stride1, addr;
 	long flags;
 	addr    = local_cpu_data->ptce_base;
 	count0  = local_cpu_data->ptce_count[0];
 	count1  = local_cpu_data->ptce_count[1];
 	stride0 = local_cpu_data->ptce_stride[0];
 	stride1 = local_cpu_data->ptce_stride[1];
 	local_irq_save(flags);
 	for (i = 0; i < count0; ++i) {
 		for (j = 0; j < count1; ++j) {
 			ia64_ptce(addr);
 			addr += stride1;
 		}
 		addr += stride0;
 	}
 	local_irq_restore(flags);
 	ia64_srlz_i();			/* srlz.i implies srlz.d */
 }
 long ia64_pal_vp_create(u64 *vpd, u64 *host_iva, u64 *opt_handler)
 {
 	struct ia64_pal_retval iprv;
 	PAL_CALL_STK(iprv, PAL_VP_CREATE, (u64)vpd, (u64)host_iva,
 			(u64)opt_handler);
 	return iprv.status;
 }
 static  DEFINE_SPINLOCK(vp_lock);
 void kvm_arch_hardware_enable(void *garbage)
 {
 	long  status;
 	long  tmp_base;
 	unsigned long pte;
 	unsigned long saved_psr;
 	int slot;
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base), PAGE_KERNEL));
 	local_irq_save(saved_psr);
 	slot = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	local_irq_restore(saved_psr);
 	if (slot < 0)
 		return;
 	spin_lock(&vp_lock);
 	status = ia64_pal_vp_init_env(kvm_vsa_base ?
 				VP_INIT_ENV : VP_INIT_ENV_INITALIZE,
 			__pa(kvm_vm_buffer), KVM_VM_BUFFER_BASE, &tmp_base);
 	if (status != 0) {
 		printk(KERN_WARNING"kvm: Failed to Enable VT Support!!!!\n");
 		return ;
 	}
 	if (!kvm_vsa_base) {
 		kvm_vsa_base = tmp_base;
 		printk(KERN_INFO"kvm: kvm_vsa_base:0x%lx\n", kvm_vsa_base);
 	}
 	spin_unlock(&vp_lock);
 	ia64_ptr_entry(0x3, slot);
 }
 void kvm_arch_hardware_disable(void *garbage)
 {
 	long status;
 	int slot;
 	unsigned long pte;
 	unsigned long saved_psr;
 	unsigned long host_iva = ia64_getreg(_IA64_REG_CR_IVA);
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base),
 				PAGE_KERNEL));
 	local_irq_save(saved_psr);
 	slot = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	local_irq_restore(saved_psr);
 	if (slot < 0)
 		return;
 	status = ia64_pal_vp_exit_env(host_iva);
 	if (status)
 		printk(KERN_DEBUG"kvm: Failed to disable VT support! :%ld\n",
 				status);
 	ia64_ptr_entry(0x3, slot);
 }
 void kvm_arch_check_processor_compat(void *rtn)
 {
 	*(int *)rtn = 0;
 }
 int kvm_dev_ioctl_check_extension(long ext)
 {
 	int r;
 	switch (ext) {
 	case KVM_CAP_IRQCHIP:
 	case KVM_CAP_MP_STATE:
 	case KVM_CAP_IRQ_INJECT_STATUS:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
 		r = KVM_COALESCED_MMIO_PAGE_OFFSET;
 		break;
 	case KVM_CAP_IOMMU:
 		r = iommu_found();
 		break;
 	default:
 		r = 0;
 	}
 	return r;
 }
 static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu,
 					gpa_t addr, int len, int is_write)
 {
 	struct kvm_io_device *dev;
 	dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr, len, is_write);
 	return dev;
 }
 static int handle_vm_error(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 	kvm_run->hw.hardware_exit_reason = 1;
 	return 0;
 }
 static int handle_mmio(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct kvm_mmio_req *p;
 	struct kvm_io_device *mmio_dev;
 	p = kvm_get_vcpu_ioreq(vcpu);
 	if ((p->addr & PAGE_MASK) == IOAPIC_DEFAULT_BASE_ADDRESS)
 		goto mmio;
 	vcpu->mmio_needed = 1;
 	vcpu->mmio_phys_addr = kvm_run->mmio.phys_addr = p->addr;
 	vcpu->mmio_size = kvm_run->mmio.len = p->size;
 	vcpu->mmio_is_write = kvm_run->mmio.is_write = !p->dir;
 	if (vcpu->mmio_is_write)
 		memcpy(vcpu->mmio_data, &p->data, p->size);
 	memcpy(kvm_run->mmio.data, &p->data, p->size);
 	kvm_run->exit_reason = KVM_EXIT_MMIO;
 	return 0;
 mmio:
 	mmio_dev = vcpu_find_mmio_dev(vcpu, p->addr, p->size, !p->dir);
 	if (mmio_dev) {
 		if (!p->dir)
 			kvm_iodevice_write(mmio_dev, p->addr, p->size,
 						&p->data);
 		else
 			kvm_iodevice_read(mmio_dev, p->addr, p->size,
 						&p->data);
 	} else
 		printk(KERN_ERR"kvm: No iodevice found! addr:%lx\n", p->addr);
 	p->state = STATE_IORESP_READY;
 	return 1;
 }
 static int handle_pal_call(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p;
 	p = kvm_get_exit_data(vcpu);
 	if (p->exit_reason == EXIT_REASON_PAL_CALL)
 		return kvm_pal_emul(vcpu, kvm_run);
 	else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = 2;
 		return 0;
 	}
 }
 static int handle_sal_call(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p;
 	p = kvm_get_exit_data(vcpu);
 	if (p->exit_reason == EXIT_REASON_SAL_CALL) {
 		kvm_sal_emul(vcpu);
 		return 1;
 	} else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = 3;
 		return 0;
 	}
 }
 static int __apic_accept_irq(struct kvm_vcpu *vcpu, uint64_t vector)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (!test_and_set_bit(vector, &vpd->irr[0])) {
 		vcpu->arch.irq_new_pending = 1;
 		kvm_vcpu_kick(vcpu);
 		return 1;
 	}
 	return 0;
 }
 /*
  *  offset: address offset to IPI space.
  *  value:  deliver value.
  */
 static void vcpu_deliver_ipi(struct kvm_vcpu *vcpu, uint64_t dm,
 				uint64_t vector)
 {
 	switch (dm) {
 	case SAPIC_FIXED:
 		break;
 	case SAPIC_NMI:
 		vector = 2;
 		break;
 	case SAPIC_EXTINT:
 		vector = 0;
 		break;
 	case SAPIC_INIT:
 	case SAPIC_PMI:
 	default:
 		printk(KERN_ERR"kvm: Unimplemented Deliver reserved IPI!\n");
 		return;
 	}
 	__apic_accept_irq(vcpu, vector);
 }
 static struct kvm_vcpu *lid_to_vcpu(struct kvm *kvm, unsigned long id,
 			unsigned long eid)
 {
 	union ia64_lid lid;
 	int i;
-	for (i = 0; i < kvm->arch.online_vcpus; i++) {
+	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) {
 		if (kvm->vcpus[i]) {
 			lid.val = VCPU_LID(kvm->vcpus[i]);
 			if (lid.id == id && lid.eid == eid)
 				return kvm->vcpus[i];
 		}
 	}
 	return NULL;
 }
 static int handle_ipi(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p = kvm_get_exit_data(vcpu);
 	struct kvm_vcpu *target_vcpu;
 	struct kvm_pt_regs *regs;
 	union ia64_ipi_a addr = p->u.ipi_data.addr;
 	union ia64_ipi_d data = p->u.ipi_data.data;
 	target_vcpu = lid_to_vcpu(vcpu->kvm, addr.id, addr.eid);
 	if (!target_vcpu)
 		return handle_vm_error(vcpu, kvm_run);
 	if (!target_vcpu->arch.launched) {
 		regs = vcpu_regs(target_vcpu);
 		regs->cr_iip = vcpu->kvm->arch.rdv_sal_data.boot_ip;
 		regs->r1 = vcpu->kvm->arch.rdv_sal_data.boot_gp;
 		target_vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		if (waitqueue_active(&target_vcpu->wq))
 			wake_up_interruptible(&target_vcpu->wq);
 	} else {
 		vcpu_deliver_ipi(target_vcpu, data.dm, data.vector);
 		if (target_vcpu != vcpu)
 			kvm_vcpu_kick(target_vcpu);
 	}
 	return 1;
 }
 struct call_data {
 	struct kvm_ptc_g ptc_g_data;
 	struct kvm_vcpu *vcpu;
 };
 static void vcpu_global_purge(void *info)
 {
 	struct call_data *p = (struct call_data *)info;
 	struct kvm_vcpu *vcpu = p->vcpu;
 	if (test_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
 		return;
 	set_bit(KVM_REQ_PTC_G, &vcpu->requests);
 	if (vcpu->arch.ptc_g_count < MAX_PTC_G_NUM) {
 		vcpu->arch.ptc_g_data[vcpu->arch.ptc_g_count++] =
 							p->ptc_g_data;
 	} else {
 		clear_bit(KVM_REQ_PTC_G, &vcpu->requests);
 		vcpu->arch.ptc_g_count = 0;
 		set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests);
 	}
 }
 static int handle_global_purge(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	struct exit_ctl_data *p = kvm_get_exit_data(vcpu);
 	struct kvm *kvm = vcpu->kvm;
 	struct call_data call_data;
 	int i;
 	call_data.ptc_g_data = p->u.ptc_g_data;
-	for (i = 0; i < kvm->arch.online_vcpus; i++) {
+	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) {
 		if (!kvm->vcpus[i] || kvm->vcpus[i]->arch.mp_state ==
 						KVM_MP_STATE_UNINITIALIZED ||
 					vcpu == kvm->vcpus[i])
 			continue;
 		if (waitqueue_active(&kvm->vcpus[i]->wq))
 			wake_up_interruptible(&kvm->vcpus[i]->wq);
 		if (kvm->vcpus[i]->cpu != -1) {
 			call_data.vcpu = kvm->vcpus[i];
 			smp_call_function_single(kvm->vcpus[i]->cpu,
 					vcpu_global_purge, &call_data, 1);
 		} else
 			printk(KERN_WARNING"kvm: Uninit vcpu received ipi!\n");
 	}
 	return 1;
 }
 static int handle_switch_rr6(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	return 1;
 }
 static int kvm_sn2_setup_mappings(struct kvm_vcpu *vcpu)
 {
 	unsigned long pte, rtc_phys_addr, map_addr;
 	int slot;
 	map_addr = KVM_VMM_BASE + (1UL << KVM_VMM_SHIFT);
 	rtc_phys_addr = LOCAL_MMR_OFFSET | SH_RTC;
 	pte = pte_val(mk_pte_phys(rtc_phys_addr, PAGE_KERNEL_UC));
 	slot = ia64_itr_entry(0x3, map_addr, pte, PAGE_SHIFT);
 	vcpu->arch.sn_rtc_tr_slot = slot;
 	if (slot < 0) {
 		printk(KERN_ERR "Mayday mayday! RTC mapping failed!\n");
 		slot = 0;
 	}
 	return slot;
 }
 int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 {
 	ktime_t kt;
 	long itc_diff;
 	unsigned long vcpu_now_itc;
 	unsigned long expires;
 	struct hrtimer *p_ht = &vcpu->arch.hlt_timer;
 	unsigned long cyc_per_usec = local_cpu_data->cyc_per_usec;
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (irqchip_in_kernel(vcpu->kvm)) {
 		vcpu_now_itc = kvm_get_itc(vcpu) + vcpu->arch.itc_offset;
 		if (time_after(vcpu_now_itc, vpd->itm)) {
 			vcpu->arch.timer_check = 1;
 			return 1;
 		}
 		itc_diff = vpd->itm - vcpu_now_itc;
 		if (itc_diff < 0)
 			itc_diff = -itc_diff;
 		expires = div64_u64(itc_diff, cyc_per_usec);
 		kt = ktime_set(0, 1000 * expires);
 		vcpu->arch.ht_active = 1;
 		hrtimer_start(p_ht, kt, HRTIMER_MODE_ABS);
 		vcpu->arch.mp_state = KVM_MP_STATE_HALTED;
 		kvm_vcpu_block(vcpu);
 		hrtimer_cancel(p_ht);
 		vcpu->arch.ht_active = 0;
 		if (test_and_clear_bit(KVM_REQ_UNHALT, &vcpu->requests) ||
 				kvm_cpu_has_pending_timer(vcpu))
 			if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
 				vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		if (vcpu->arch.mp_state != KVM_MP_STATE_RUNNABLE)
 			return -EINTR;
 		return 1;
 	} else {
 		printk(KERN_ERR"kvm: Unsupported userspace halt!");
 		return 0;
 	}
 }
 static int handle_vm_shutdown(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run)
 {
 	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
 	return 0;
 }
 static int handle_external_interrupt(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run)
 {
 	return 1;
 }
 static int handle_vcpu_debug(struct kvm_vcpu *vcpu,
 				struct kvm_run *kvm_run)
 {
 	printk("VMM: %s", vcpu->arch.log_buf);
 	return 1;
 }
 static int (*kvm_vti_exit_handlers[])(struct kvm_vcpu *vcpu,
 		struct kvm_run *kvm_run) = {
 	[EXIT_REASON_VM_PANIC]              = handle_vm_error,
 	[EXIT_REASON_MMIO_INSTRUCTION]      = handle_mmio,
 	[EXIT_REASON_PAL_CALL]              = handle_pal_call,
 	[EXIT_REASON_SAL_CALL]              = handle_sal_call,
 	[EXIT_REASON_SWITCH_RR6]            = handle_switch_rr6,
 	[EXIT_REASON_VM_DESTROY]            = handle_vm_shutdown,
 	[EXIT_REASON_EXTERNAL_INTERRUPT]    = handle_external_interrupt,
 	[EXIT_REASON_IPI]		    = handle_ipi,
 	[EXIT_REASON_PTC_G]		    = handle_global_purge,
 	[EXIT_REASON_DEBUG]		    = handle_vcpu_debug,
 };
 static const int kvm_vti_max_exit_handlers =
 		sizeof(kvm_vti_exit_handlers)/sizeof(*kvm_vti_exit_handlers);
 static uint32_t kvm_get_exit_reason(struct kvm_vcpu *vcpu)
 {
 	struct exit_ctl_data *p_exit_data;
 	p_exit_data = kvm_get_exit_data(vcpu);
 	return p_exit_data->exit_reason;
 }
 /*
  * The guest has exited.  See if we can fix it or if we need userspace
  * assistance.
  */
 static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 {
 	u32 exit_reason = kvm_get_exit_reason(vcpu);
 	vcpu->arch.last_exit = exit_reason;
 	if (exit_reason < kvm_vti_max_exit_handlers
 			&& kvm_vti_exit_handlers[exit_reason])
 		return kvm_vti_exit_handlers[exit_reason](vcpu, kvm_run);
 	else {
 		kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
 		kvm_run->hw.hardware_exit_reason = exit_reason;
 	}
 	return 0;
 }
 static inline void vti_set_rr6(unsigned long rr6)
 {
 	ia64_set_rr(RR6, rr6);
 	ia64_srlz_i();
 }
 static int kvm_insert_vmm_mapping(struct kvm_vcpu *vcpu)
 {
 	unsigned long pte;
 	struct kvm *kvm = vcpu->kvm;
 	int r;
 	/*Insert a pair of tr to map vmm*/
 	pte = pte_val(mk_pte_phys(__pa(kvm_vmm_base), PAGE_KERNEL));
 	r = ia64_itr_entry(0x3, KVM_VMM_BASE, pte, KVM_VMM_SHIFT);
 	if (r < 0)
 		goto out;
 	vcpu->arch.vmm_tr_slot = r;
 	/*Insert a pairt of tr to map data of vm*/
 	pte = pte_val(mk_pte_phys(__pa(kvm->arch.vm_base), PAGE_KERNEL));
 	r = ia64_itr_entry(0x3, KVM_VM_DATA_BASE,
 					pte, KVM_VM_DATA_SHIFT);
 	if (r < 0)
 		goto out;
 	vcpu->arch.vm_tr_slot = r;
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (kvm->arch.is_sn2) {
 		r = kvm_sn2_setup_mappings(vcpu);
 		if (r < 0)
 			goto out;
 	}
 #endif
 	r = 0;
 out:
 	return r;
 }
 static void kvm_purge_vmm_mapping(struct kvm_vcpu *vcpu)
 {
 	struct kvm *kvm = vcpu->kvm;
 	ia64_ptr_entry(0x3, vcpu->arch.vmm_tr_slot);
 	ia64_ptr_entry(0x3, vcpu->arch.vm_tr_slot);
 #if defined(CONFIG_IA64_SGI_SN2) || defined(CONFIG_IA64_GENERIC)
 	if (kvm->arch.is_sn2)
 		ia64_ptr_entry(0x3, vcpu->arch.sn_rtc_tr_slot);
 #endif
 }
 static int kvm_vcpu_pre_transition(struct kvm_vcpu *vcpu)
 {
 	unsigned long psr;
 	int r;
 	int cpu = smp_processor_id();
 	if (vcpu->arch.last_run_cpu != cpu ||
 			per_cpu(last_vcpu, cpu) != vcpu) {
 		per_cpu(last_vcpu, cpu) = vcpu;
 		vcpu->arch.last_run_cpu = cpu;
 		kvm_flush_tlb_all();
 	}
 	vcpu->arch.host_rr6 = ia64_get_rr(RR6);
 	vti_set_rr6(vcpu->arch.vmm_rr);
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	return r;
 }
 static void kvm_vcpu_post_transition(struct kvm_vcpu *vcpu)
 {
 	kvm_purge_vmm_mapping(vcpu);
 	vti_set_rr6(vcpu->arch.host_rr6);
 }
 static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	union context *host_ctx, *guest_ctx;
 	int r;
 	/*
 	 * down_read() may sleep and return with interrupts enabled
 	 */
 	down_read(&vcpu->kvm->slots_lock);
 again:
 	if (signal_pending(current)) {
 		r = -EINTR;
 		kvm_run->exit_reason = KVM_EXIT_INTR;
 		goto out;
 	}
 	preempt_disable();
 	local_irq_disable();
 	/*Get host and guest context with guest address space.*/
 	host_ctx = kvm_get_host_context(vcpu);
 	guest_ctx = kvm_get_guest_context(vcpu);
 	clear_bit(KVM_REQ_KICK, &vcpu->requests);
 	r = kvm_vcpu_pre_transition(vcpu);
 	if (r < 0)
 		goto vcpu_run_fail;
 	up_read(&vcpu->kvm->slots_lock);
 	kvm_guest_enter();
 	/*
 	 * Transition to the guest
 	 */
 	kvm_vmm_info->tramp_entry(host_ctx, guest_ctx);
 	kvm_vcpu_post_transition(vcpu);
 	vcpu->arch.launched = 1;
 	set_bit(KVM_REQ_KICK, &vcpu->requests);
 	local_irq_enable();
 	/*
 	 * We must have an instruction between local_irq_enable() and
 	 * kvm_guest_exit(), so the timer interrupt isn't delayed by
 	 * the interrupt shadow.  The stat.exits increment will do nicely.
 	 * But we need to prevent reordering, hence this barrier():
 	 */
 	barrier();
 	kvm_guest_exit();
 	preempt_enable();
 	down_read(&vcpu->kvm->slots_lock);
 	r = kvm_handle_exit(kvm_run, vcpu);
 	if (r > 0) {
 		if (!need_resched())
 			goto again;
 	}
 out:
 	up_read(&vcpu->kvm->slots_lock);
 	if (r > 0) {
 		kvm_resched(vcpu);
 		down_read(&vcpu->kvm->slots_lock);
 		goto again;
 	}
 	return r;
 vcpu_run_fail:
 	local_irq_enable();
 	preempt_enable();
 	kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
 	goto out;
 }
 static void kvm_set_mmio_data(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmio_req *p = kvm_get_vcpu_ioreq(vcpu);
 	if (!vcpu->mmio_is_write)
 		memcpy(&p->data, vcpu->mmio_data, 8);
 	p->state = STATE_IORESP_READY;
 }
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
 	sigset_t sigsaved;
 	vcpu_load(vcpu);
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
 		kvm_vcpu_block(vcpu);
 		clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 		r = -EAGAIN;
 		goto out;
 	}
 	if (vcpu->mmio_needed) {
 		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
 		kvm_set_mmio_data(vcpu);
 		vcpu->mmio_read_completed = 1;
 		vcpu->mmio_needed = 0;
 	}
 	r = __vcpu_run(vcpu, kvm_run);
 out:
 	if (vcpu->sigset_active)
 		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
 	vcpu_put(vcpu);
 	return r;
 }
 static struct kvm *kvm_alloc_kvm(void)
 {
 	struct kvm *kvm;
 	uint64_t  vm_base;
 	BUG_ON(sizeof(struct kvm) > KVM_VM_STRUCT_SIZE);
 	vm_base = __get_free_pages(GFP_KERNEL, get_order(KVM_VM_DATA_SIZE));
 	if (!vm_base)
 		return ERR_PTR(-ENOMEM);
 	memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
 	kvm = (struct kvm *)(vm_base +
 			offsetof(struct kvm_vm_data, kvm_vm_struct));
 	kvm->arch.vm_base = vm_base;
 	printk(KERN_DEBUG"kvm: vm's data area:0x%lx\n", vm_base);
 	return kvm;
 }
 struct kvm_io_range {
 	unsigned long start;
 	unsigned long size;
 	unsigned long type;
 };
 static const struct kvm_io_range io_ranges[] = {
 	{VGA_IO_START, VGA_IO_SIZE, GPFN_FRAME_BUFFER},
 	{MMIO_START, MMIO_SIZE, GPFN_LOW_MMIO},
 	{LEGACY_IO_START, LEGACY_IO_SIZE, GPFN_LEGACY_IO},
 	{IO_SAPIC_START, IO_SAPIC_SIZE, GPFN_IOSAPIC},
 	{PIB_START, PIB_SIZE, GPFN_PIB},
 };
 static void kvm_build_io_pmt(struct kvm *kvm)
 {
 	unsigned long i, j;
 	/* Mark I/O ranges */
 	for (i = 0; i < (sizeof(io_ranges) / sizeof(struct kvm_io_range));
 							i++) {
 		for (j = io_ranges[i].start;
 				j < io_ranges[i].start + io_ranges[i].size;
 				j += PAGE_SIZE)
 			kvm_set_pmt_entry(kvm, j >> PAGE_SHIFT,
 					io_ranges[i].type, 0);
 	}
 }
 /*Use unused rids to virtualize guest rid.*/
 #define GUEST_PHYSICAL_RR0	0x1739
 #define GUEST_PHYSICAL_RR4	0x2739
 #define VMM_INIT_RR		0x1660
 static void kvm_init_vm(struct kvm *kvm)
 {
 	BUG_ON(!kvm);
 	kvm->arch.metaphysical_rr0 = GUEST_PHYSICAL_RR0;
 	kvm->arch.metaphysical_rr4 = GUEST_PHYSICAL_RR4;
 	kvm->arch.vmm_init_rr = VMM_INIT_RR;
 	/*
 	 *Fill P2M entries for MMIO/IO ranges
 	 */
 	kvm_build_io_pmt(kvm);
 	INIT_LIST_HEAD(&kvm->arch.assigned_dev_head);
 	/* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
 	set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
 }
 struct  kvm *kvm_arch_create_vm(void)
 {
 	struct kvm *kvm = kvm_alloc_kvm();
 	if (IS_ERR(kvm))
 		return ERR_PTR(-ENOMEM);
 	kvm->arch.is_sn2 = ia64_platform_is("sn2");
 	kvm_init_vm(kvm);
-	kvm->arch.online_vcpus = 0;
 	return kvm;
 }
 static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm,
 					struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_IOAPIC:
 		memcpy(&chip->chip.ioapic, ioapic_irqchip(kvm),
 				sizeof(struct kvm_ioapic_state));
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip)
 {
 	int r;
 	r = 0;
 	switch (chip->chip_id) {
 	case KVM_IRQCHIP_IOAPIC:
 		memcpy(ioapic_irqchip(kvm),
 				&chip->chip.ioapic,
 				sizeof(struct kvm_ioapic_state));
 		break;
 	default:
 		r = -EINVAL;
 		break;
 	}
 	return r;
 }
 #define RESTORE_REGS(_x) vcpu->arch._x = regs->_x
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	int i;
 	vcpu_load(vcpu);
 	for (i = 0; i < 16; i++) {
 		vpd->vgr[i] = regs->vpd.vgr[i];
 		vpd->vbgr[i] = regs->vpd.vbgr[i];
 	}
 	for (i = 0; i < 128; i++)
 		vpd->vcr[i] = regs->vpd.vcr[i];
 	vpd->vhpi = regs->vpd.vhpi;
 	vpd->vnat = regs->vpd.vnat;
 	vpd->vbnat = regs->vpd.vbnat;
 	vpd->vpsr = regs->vpd.vpsr;
 	vpd->vpr = regs->vpd.vpr;
 	memcpy(&vcpu->arch.guest, &regs->saved_guest, sizeof(union context));
 	RESTORE_REGS(mp_state);
 	RESTORE_REGS(vmm_rr);
 	memcpy(vcpu->arch.itrs, regs->itrs, sizeof(struct thash_data) * NITRS);
 	memcpy(vcpu->arch.dtrs, regs->dtrs, sizeof(struct thash_data) * NDTRS);
 	RESTORE_REGS(itr_regions);
 	RESTORE_REGS(dtr_regions);
 	RESTORE_REGS(tc_regions);
 	RESTORE_REGS(irq_check);
 	RESTORE_REGS(itc_check);
 	RESTORE_REGS(timer_check);
 	RESTORE_REGS(timer_pending);
 	RESTORE_REGS(last_itc);
 	for (i = 0; i < 8; i++) {
 		vcpu->arch.vrr[i] = regs->vrr[i];
 		vcpu->arch.ibr[i] = regs->ibr[i];
 		vcpu->arch.dbr[i] = regs->dbr[i];
 	}
 	for (i = 0; i < 4; i++)
 		vcpu->arch.insvc[i] = regs->insvc[i];
 	RESTORE_REGS(xtp);
 	RESTORE_REGS(metaphysical_rr0);
 	RESTORE_REGS(metaphysical_rr4);
 	RESTORE_REGS(metaphysical_saved_rr0);
 	RESTORE_REGS(metaphysical_saved_rr4);
 	RESTORE_REGS(fp_psr);
 	RESTORE_REGS(saved_gp);
 	vcpu->arch.irq_new_pending = 1;
 	vcpu->arch.itc_offset = regs->saved_itc - kvm_get_itc(vcpu);
 	set_bit(KVM_REQ_RESUME, &vcpu->requests);
 	vcpu_put(vcpu);
 	return 0;
 }
 long kvm_arch_vm_ioctl(struct file *filp,
 		unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r = -EINVAL;
 	switch (ioctl) {
 	case KVM_SET_MEMORY_REGION: {
 		struct kvm_memory_region kvm_mem;
 		struct kvm_userspace_memory_region kvm_userspace_mem;
 		r = -EFAULT;
 		if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem))
 			goto out;
 		kvm_userspace_mem.slot = kvm_mem.slot;
 		kvm_userspace_mem.flags = kvm_mem.flags;
 		kvm_userspace_mem.guest_phys_addr =
 					kvm_mem.guest_phys_addr;
 		kvm_userspace_mem.memory_size = kvm_mem.memory_size;
 		r = kvm_vm_ioctl_set_memory_region(kvm,
 					&kvm_userspace_mem, 0);
 		if (r)
 			goto out;
 		break;
 		}
 	case KVM_CREATE_IRQCHIP:
 		r = -EFAULT;
 		r = kvm_ioapic_init(kvm);
 		if (r)
 			goto out;
 		r = kvm_setup_default_irq_routing(kvm);
 		if (r) {
 			kfree(kvm->arch.vioapic);
 			goto out;
 		}
 		break;
 	case KVM_IRQ_LINE_STATUS:
 	case KVM_IRQ_LINE: {
 		struct kvm_irq_level irq_event;
 		r = -EFAULT;
 		if (copy_from_user(&irq_event, argp, sizeof irq_event))
 			goto out;
 		if (irqchip_in_kernel(kvm)) {
 			__s32 status;
 			mutex_lock(&kvm->irq_lock);
 			status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
 				    irq_event.irq, irq_event.level);
 			mutex_unlock(&kvm->irq_lock);
 			if (ioctl == KVM_IRQ_LINE_STATUS) {
 				irq_event.status = status;
 				if (copy_to_user(argp, &irq_event,
 							sizeof irq_event))
 					goto out;
 			}
 			r = 0;
 		}
 		break;
 		}
 	case KVM_GET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip chip;
 		r = -EFAULT;
 		if (copy_from_user(&chip, argp, sizeof chip))
 				goto out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto out;
 		r = kvm_vm_ioctl_get_irqchip(kvm, &chip);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &chip, sizeof chip))
 				goto out;
 		r = 0;
 		break;
 		}
 	case KVM_SET_IRQCHIP: {
 		/* 0: PIC master, 1: PIC slave, 2: IOAPIC */
 		struct kvm_irqchip chip;
 		r = -EFAULT;
 		if (copy_from_user(&chip, argp, sizeof chip))
 				goto out;
 		r = -ENXIO;
 		if (!irqchip_in_kernel(kvm))
 			goto out;
 		r = kvm_vm_ioctl_set_irqchip(kvm, &chip);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 		}
 	default:
 		;
 	}
 out:
 	return r;
 }
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 		struct kvm_sregs *sregs)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 		struct kvm_sregs *sregs)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 		struct kvm_translation *tr)
 {
 	return -EINVAL;
 }
 static int kvm_alloc_vmm_area(void)
 {
 	if (!kvm_vmm_base && (kvm_vm_buffer_size < KVM_VM_BUFFER_SIZE)) {
 		kvm_vmm_base = __get_free_pages(GFP_KERNEL,
 				get_order(KVM_VMM_SIZE));
 		if (!kvm_vmm_base)
 			return -ENOMEM;
 		memset((void *)kvm_vmm_base, 0, KVM_VMM_SIZE);
 		kvm_vm_buffer = kvm_vmm_base + VMM_SIZE;
 		printk(KERN_DEBUG"kvm:VMM's Base Addr:0x%lx, vm_buffer:0x%lx\n",
 				kvm_vmm_base, kvm_vm_buffer);
 	}
 	return 0;
 }
 static void kvm_free_vmm_area(void)
 {
 	if (kvm_vmm_base) {
 		/*Zero this area before free to avoid bits leak!!*/
 		memset((void *)kvm_vmm_base, 0, KVM_VMM_SIZE);
 		free_pages(kvm_vmm_base, get_order(KVM_VMM_SIZE));
 		kvm_vmm_base  = 0;
 		kvm_vm_buffer = 0;
 		kvm_vsa_base = 0;
 	}
 }
 static int vti_init_vpd(struct kvm_vcpu *vcpu)
 {
 	int i;
 	union cpuid3_t cpuid3;
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	if (IS_ERR(vpd))
 		return PTR_ERR(vpd);
 	/* CPUID init */
 	for (i = 0; i < 5; i++)
 		vpd->vcpuid[i] = ia64_get_cpuid(i);
 	/* Limit the CPUID number to 5 */
 	cpuid3.value = vpd->vcpuid[3];
 	cpuid3.number = 4;	/* 5 - 1 */
 	vpd->vcpuid[3] = cpuid3.value;
 	/*Set vac and vdc fields*/
 	vpd->vac.a_from_int_cr = 1;
 	vpd->vac.a_to_int_cr = 1;
 	vpd->vac.a_from_psr = 1;
 	vpd->vac.a_from_cpuid = 1;
 	vpd->vac.a_cover = 1;
 	vpd->vac.a_bsw = 1;
 	vpd->vac.a_int = 1;
 	vpd->vdc.d_vmsw = 1;
 	/*Set virtual buffer*/
 	vpd->virt_env_vaddr = KVM_VM_BUFFER_BASE;
 	return 0;
 }
 static int vti_create_vp(struct kvm_vcpu *vcpu)
 {
 	long ret;
 	struct vpd *vpd = vcpu->arch.vpd;
 	unsigned long  vmm_ivt;
 	vmm_ivt = kvm_vmm_info->vmm_ivt;
 	printk(KERN_DEBUG "kvm: vcpu:%p,ivt: 0x%lx\n", vcpu, vmm_ivt);
 	ret = ia64_pal_vp_create((u64 *)vpd, (u64 *)vmm_ivt, 0);
 	if (ret) {
 		printk(KERN_ERR"kvm: ia64_pal_vp_create failed!\n");
 		return -EINVAL;
 	}
 	return 0;
 }
 static void init_ptce_info(struct kvm_vcpu *vcpu)
 {
 	ia64_ptce_info_t ptce = {0};
 	ia64_get_ptce(&ptce);
 	vcpu->arch.ptce_base = ptce.base;
 	vcpu->arch.ptce_count[0] = ptce.count[0];
 	vcpu->arch.ptce_count[1] = ptce.count[1];
 	vcpu->arch.ptce_stride[0] = ptce.stride[0];
 	vcpu->arch.ptce_stride[1] = ptce.stride[1];
 }
 static void kvm_migrate_hlt_timer(struct kvm_vcpu *vcpu)
 {
 	struct hrtimer *p_ht = &vcpu->arch.hlt_timer;
 	if (hrtimer_cancel(p_ht))
 		hrtimer_start_expires(p_ht, HRTIMER_MODE_ABS);
 }
 static enum hrtimer_restart hlt_timer_fn(struct hrtimer *data)
 {
 	struct kvm_vcpu *vcpu;
 	wait_queue_head_t *q;
 	vcpu  = container_of(data, struct kvm_vcpu, arch.hlt_timer);
 	q = &vcpu->wq;
 	if (vcpu->arch.mp_state != KVM_MP_STATE_HALTED)
 		goto out;
 	if (waitqueue_active(q))
 		wake_up_interruptible(q);
 out:
 	vcpu->arch.timer_fired = 1;
 	vcpu->arch.timer_check = 1;
 	return HRTIMER_NORESTART;
 }
 #define PALE_RESET_ENTRY    0x80000000ffffffb0UL
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 {
 	struct kvm_vcpu *v;
 	int r;
 	int i;
 	long itc_offset;
 	struct kvm *kvm = vcpu->kvm;
 	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
 	union context *p_ctx = &vcpu->arch.guest;
 	struct kvm_vcpu *vmm_vcpu = to_guest(vcpu->kvm, vcpu);
 	/*Init vcpu context for first run.*/
 	if (IS_ERR(vmm_vcpu))
 		return PTR_ERR(vmm_vcpu);
 	if (kvm_vcpu_is_bsp(vcpu)) {
 		vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
 		/*Set entry address for first run.*/
 		regs->cr_iip = PALE_RESET_ENTRY;
 		/*Initialize itc offset for vcpus*/
 		itc_offset = 0UL - kvm_get_itc(vcpu);
 		for (i = 0; i < KVM_MAX_VCPUS; i++) {
 			v = (struct kvm_vcpu *)((char *)vcpu +
 					sizeof(struct kvm_vcpu_data) * i);
 			v->arch.itc_offset = itc_offset;
 			v->arch.last_itc = 0;
 		}
 	} else
 		vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED;
 	r = -ENOMEM;
 	vcpu->arch.apic = kzalloc(sizeof(struct kvm_lapic), GFP_KERNEL);
 	if (!vcpu->arch.apic)
 		goto out;
 	vcpu->arch.apic->vcpu = vcpu;
 	p_ctx->gr[1] = 0;
 	p_ctx->gr[12] = (unsigned long)((char *)vmm_vcpu + KVM_STK_OFFSET);
 	p_ctx->gr[13] = (unsigned long)vmm_vcpu;
 	p_ctx->psr = 0x1008522000UL;
 	p_ctx->ar[40] = FPSR_DEFAULT; /*fpsr*/
 	p_ctx->caller_unat = 0;
 	p_ctx->pr = 0x0;
 	p_ctx->ar[36] = 0x0; /*unat*/
 	p_ctx->ar[19] = 0x0; /*rnat*/
 	p_ctx->ar[18] = (unsigned long)vmm_vcpu +
 				((sizeof(struct kvm_vcpu)+15) & ~15);
 	p_ctx->ar[64] = 0x0; /*pfs*/
 	p_ctx->cr[0] = 0x7e04UL;
 	p_ctx->cr[2] = (unsigned long)kvm_vmm_info->vmm_ivt;
 	p_ctx->cr[8] = 0x3c;
 	/*Initilize region register*/
 	p_ctx->rr[0] = 0x30;
 	p_ctx->rr[1] = 0x30;
 	p_ctx->rr[2] = 0x30;
 	p_ctx->rr[3] = 0x30;
 	p_ctx->rr[4] = 0x30;
 	p_ctx->rr[5] = 0x30;
 	p_ctx->rr[7] = 0x30;
 	/*Initilize branch register 0*/
 	p_ctx->br[0] = *(unsigned long *)kvm_vmm_info->vmm_entry;
 	vcpu->arch.vmm_rr = kvm->arch.vmm_init_rr;
 	vcpu->arch.metaphysical_rr0 = kvm->arch.metaphysical_rr0;
 	vcpu->arch.metaphysical_rr4 = kvm->arch.metaphysical_rr4;
 	hrtimer_init(&vcpu->arch.hlt_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 	vcpu->arch.hlt_timer.function = hlt_timer_fn;
 	vcpu->arch.last_run_cpu = -1;
 	vcpu->arch.vpd = (struct vpd *)VPD_BASE(vcpu->vcpu_id);
 	vcpu->arch.vsa_base = kvm_vsa_base;
 	vcpu->arch.__gp = kvm_vmm_gp;
 	vcpu->arch.dirty_log_lock_pa = __pa(&kvm->arch.dirty_log_lock);
 	vcpu->arch.vhpt.hash = (struct thash_data *)VHPT_BASE(vcpu->vcpu_id);
 	vcpu->arch.vtlb.hash = (struct thash_data *)VTLB_BASE(vcpu->vcpu_id);
 	init_ptce_info(vcpu);
 	r = 0;
 out:
 	return r;
 }
 static int vti_vcpu_setup(struct kvm_vcpu *vcpu, int id)
 {
 	unsigned long psr;
 	int r;
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	if (r)
 		goto fail;
 	r = kvm_vcpu_init(vcpu, vcpu->kvm, id);
 	if (r)
 		goto fail;
 	r = vti_init_vpd(vcpu);
 	if (r) {
 		printk(KERN_DEBUG"kvm: vpd init error!!\n");
 		goto uninit;
 	}
 	r = vti_create_vp(vcpu);
 	if (r)
 		goto uninit;
 	kvm_purge_vmm_mapping(vcpu);
 	return 0;
 uninit:
 	kvm_vcpu_uninit(vcpu);
 fail:
 	return r;
 }
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 		unsigned int id)
 {
 	struct kvm_vcpu *vcpu;
 	unsigned long vm_base = kvm->arch.vm_base;
 	int r;
 	int cpu;
 	BUG_ON(sizeof(struct kvm_vcpu) > VCPU_STRUCT_SIZE/2);
 	r = -EINVAL;
 	if (id >= KVM_MAX_VCPUS) {
 		printk(KERN_ERR"kvm: Can't configure vcpus > %ld",
 				KVM_MAX_VCPUS);
 		goto fail;
 	}
 	r = -ENOMEM;
 	if (!vm_base) {
 		printk(KERN_ERR"kvm: Create vcpu[%d] error!\n", id);
 		goto fail;
 	}
 	vcpu = (struct kvm_vcpu *)(vm_base + offsetof(struct kvm_vm_data,
 					vcpu_data[id].vcpu_struct));
 	vcpu->kvm = kvm;
 	cpu = get_cpu();
 	r = vti_vcpu_setup(vcpu, id);
 	put_cpu();
 	if (r) {
 		printk(KERN_DEBUG"kvm: vcpu_setup error!!\n");
 		goto fail;
 	}
-	kvm->arch.online_vcpus++;
 	return vcpu;
 fail:
 	return ERR_PTR(r);
 }
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
 	return -EINVAL;
 }
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
 	return -EINVAL;
 }
 static void free_kvm(struct kvm *kvm)
 {
 	unsigned long vm_base = kvm->arch.vm_base;
 	if (vm_base) {
 		memset((void *)vm_base, 0, KVM_VM_DATA_SIZE);
 		free_pages(vm_base, get_order(KVM_VM_DATA_SIZE));
 	}
 }
 static void kvm_release_vm_pages(struct kvm *kvm)
 {
 	struct kvm_memory_slot *memslot;
 	int i, j;
 	unsigned long base_gfn;
 	for (i = 0; i < kvm->nmemslots; i++) {
 		memslot = &kvm->memslots[i];
 		base_gfn = memslot->base_gfn;
 		for (j = 0; j < memslot->npages; j++) {
 			if (memslot->rmap[j])
 				put_page((struct page *)memslot->rmap[j]);
 		}
 	}
 }
 void kvm_arch_sync_events(struct kvm *kvm)
 {
 }
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	kvm_iommu_unmap_guest(kvm);
 #ifdef  KVM_CAP_DEVICE_ASSIGNMENT
 	kvm_free_all_assigned_devices(kvm);
 #endif
 	kfree(kvm->arch.vioapic);
 	kvm_release_vm_pages(kvm);
 	kvm_free_physmem(kvm);
 	free_kvm(kvm);
 }
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 {
 }
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	if (cpu != vcpu->cpu) {
 		vcpu->cpu = cpu;
 		if (vcpu->arch.ht_active)
 			kvm_migrate_hlt_timer(vcpu);
 	}
 }
 #define SAVE_REGS(_x) 	regs->_x = vcpu->arch._x
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
 	int i;
 	vcpu_load(vcpu);
 	for (i = 0; i < 16; i++) {
 		regs->vpd.vgr[i] = vpd->vgr[i];
 		regs->vpd.vbgr[i] = vpd->vbgr[i];
 	}
 	for (i = 0; i < 128; i++)
 		regs->vpd.vcr[i] = vpd->vcr[i];
 	regs->vpd.vhpi = vpd->vhpi;
 	regs->vpd.vnat = vpd->vnat;
 	regs->vpd.vbnat = vpd->vbnat;
 	regs->vpd.vpsr = vpd->vpsr;
 	regs->vpd.vpr = vpd->vpr;
 	memcpy(&regs->saved_guest, &vcpu->arch.guest, sizeof(union context));
 	SAVE_REGS(mp_state);
 	SAVE_REGS(vmm_rr);
 	memcpy(regs->itrs, vcpu->arch.itrs, sizeof(struct thash_data) * NITRS);
 	memcpy(regs->dtrs, vcpu->arch.dtrs, sizeof(struct thash_data) * NDTRS);
 	SAVE_REGS(itr_regions);
 	SAVE_REGS(dtr_regions);
 	SAVE_REGS(tc_regions);
 	SAVE_REGS(irq_check);
 	SAVE_REGS(itc_check);
 	SAVE_REGS(timer_check);
 	SAVE_REGS(timer_pending);
 	SAVE_REGS(last_itc);
 	for (i = 0; i < 8; i++) {
 		regs->vrr[i] = vcpu->arch.vrr[i];
 		regs->ibr[i] = vcpu->arch.ibr[i];
 		regs->dbr[i] = vcpu->arch.dbr[i];
 	}
 	for (i = 0; i < 4; i++)
 		regs->insvc[i] = vcpu->arch.insvc[i];
 	regs->saved_itc = vcpu->arch.itc_offset + kvm_get_itc(vcpu);
 	SAVE_REGS(xtp);
 	SAVE_REGS(metaphysical_rr0);
 	SAVE_REGS(metaphysical_rr4);
 	SAVE_REGS(metaphysical_saved_rr0);
 	SAVE_REGS(metaphysical_saved_rr4);
 	SAVE_REGS(fp_psr);
 	SAVE_REGS(saved_gp);
 	vcpu_put(vcpu);
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_get_stack(struct kvm_vcpu *vcpu,
 				  struct kvm_ia64_vcpu_stack *stack)
 {
 	memcpy(stack, vcpu, sizeof(struct kvm_ia64_vcpu_stack));
 	return 0;
 }
 int kvm_arch_vcpu_ioctl_set_stack(struct kvm_vcpu *vcpu,
 				  struct kvm_ia64_vcpu_stack *stack)
 {
 	memcpy(vcpu + 1, &stack->stack[0] + sizeof(struct kvm_vcpu),
 	       sizeof(struct kvm_ia64_vcpu_stack) - sizeof(struct kvm_vcpu));
 	vcpu->arch.exit_data = ((struct kvm_vcpu *)stack)->arch.exit_data;
 	return 0;
 }
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	hrtimer_cancel(&vcpu->arch.hlt_timer);
 	kfree(vcpu->arch.apic);
 }
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	struct kvm_ia64_vcpu_stack *stack = NULL;
 	long r;
 	switch (ioctl) {
 	case KVM_IA64_VCPU_GET_STACK: {
 		struct kvm_ia64_vcpu_stack __user *user_stack;
 	        void __user *first_p = argp;
 		r = -EFAULT;
 		if (copy_from_user(&user_stack, first_p, sizeof(void *)))
 			goto out;
 		if (!access_ok(VERIFY_WRITE, user_stack,
 			       sizeof(struct kvm_ia64_vcpu_stack))) {
 			printk(KERN_INFO "KVM_IA64_VCPU_GET_STACK: "
 			       "Illegal user destination address for stack\n");
 			goto out;
 		}
 		stack = kzalloc(sizeof(struct kvm_ia64_vcpu_stack), GFP_KERNEL);
 		if (!stack) {
 			r = -ENOMEM;
 			goto out;
 		}
 		r = kvm_arch_vcpu_ioctl_get_stack(vcpu, stack);
 		if (r)
 			goto out;
 		if (copy_to_user(user_stack, stack,
 				 sizeof(struct kvm_ia64_vcpu_stack)))
 			goto out;
 		break;
 	}
 	case KVM_IA64_VCPU_SET_STACK: {
 		struct kvm_ia64_vcpu_stack __user *user_stack;
 	        void __user *first_p = argp;
 		r = -EFAULT;
 		if (copy_from_user(&user_stack, first_p, sizeof(void *)))
 			goto out;
 		if (!access_ok(VERIFY_READ, user_stack,
 			    sizeof(struct kvm_ia64_vcpu_stack))) {
 			printk(KERN_INFO "KVM_IA64_VCPU_SET_STACK: "
 			       "Illegal user address for stack\n");
 			goto out;
 		}
 		stack = kmalloc(sizeof(struct kvm_ia64_vcpu_stack), GFP_KERNEL);
 		if (!stack) {
 			r = -ENOMEM;
 			goto out;
 		}
 		if (copy_from_user(stack, user_stack,
 				   sizeof(struct kvm_ia64_vcpu_stack)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_stack(vcpu, stack);
 		break;
 	}
 	default:
 		r = -EINVAL;
 	}
 out:
 	kfree(stack);
 	return r;
 }
 int kvm_arch_set_memory_region(struct kvm *kvm,
 		struct kvm_userspace_memory_region *mem,
 		struct kvm_memory_slot old,
 		int user_alloc)
 {
 	unsigned long i;
 	unsigned long pfn;
 	int npages = mem->memory_size >> PAGE_SHIFT;
 	struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot];
 	unsigned long base_gfn = memslot->base_gfn;
 	if (base_gfn + npages > (KVM_MAX_MEM_SIZE >> PAGE_SHIFT))
 		return -ENOMEM;
 	for (i = 0; i < npages; i++) {
 		pfn = gfn_to_pfn(kvm, base_gfn + i);
 		if (!kvm_is_mmio_pfn(pfn)) {
 			kvm_set_pmt_entry(kvm, base_gfn + i,
 					pfn << PAGE_SHIFT,
 				_PAGE_AR_RWX | _PAGE_MA_WB);
 			memslot->rmap[i] = (unsigned long)pfn_to_page(pfn);
 		} else {
 			kvm_set_pmt_entry(kvm, base_gfn + i,
 					GPFN_PHYS_MMIO | (pfn << PAGE_SHIFT),
 					_PAGE_MA_UC);
 			memslot->rmap[i] = 0;
 			}
 	}
 	return 0;
 }
 void kvm_arch_flush_shadow(struct kvm *kvm)
 {
 	kvm_flush_remote_tlbs(kvm);
 }
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg)
 {
 	return -EINVAL;
 }
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	kvm_vcpu_uninit(vcpu);
 }
 static int vti_cpu_has_kvm_support(void)
 {
 	long  avail = 1, status = 1, control = 1;
 	long ret;
 	ret = ia64_pal_proc_get_features(&avail, &status, &control, 0);
 	if (ret)
 		goto out;
 	if (!(avail & PAL_PROC_VM_BIT))
 		goto out;
 	printk(KERN_DEBUG"kvm: Hardware Supports VT\n");
 	ret = ia64_pal_vp_env_info(&kvm_vm_buffer_size, &vp_env_info);
 	if (ret)
 		goto out;
 	printk(KERN_DEBUG"kvm: VM Buffer Size:0x%lx\n", kvm_vm_buffer_size);
 	if (!(vp_env_info & VP_OPCODE)) {
 		printk(KERN_WARNING"kvm: No opcode ability on hardware, "
 				"vm_env_info:0x%lx\n", vp_env_info);
 	}
 	return 1;
 out:
 	return 0;
 }
 /*
  * On SN2, the ITC isn't stable, so copy in fast path code to use the
  * SN2 RTC, replacing the ITC based default verion.
  */
 static void kvm_patch_vmm(struct kvm_vmm_info *vmm_info,
 			  struct module *module)
 {
 	unsigned long new_ar, new_ar_sn2;
 	unsigned long module_base;
 	if (!ia64_platform_is("sn2"))
 		return;
 	module_base = (unsigned long)module->module_core;
 	new_ar = kvm_vmm_base + vmm_info->patch_mov_ar - module_base;
 	new_ar_sn2 = kvm_vmm_base + vmm_info->patch_mov_ar_sn2 - module_base;
 	printk(KERN_INFO "kvm: Patching ITC emulation to use SGI SN2 RTC "
 	       "as source\n");
 	/*
 	 * Copy the SN2 version of mov_ar into place. They are both
 	 * the same size, so 6 bundles is sufficient (6 * 0x10).
 	 */
 	memcpy((void *)new_ar, (void *)new_ar_sn2, 0x60);
 }
 static int kvm_relocate_vmm(struct kvm_vmm_info *vmm_info,
 			    struct module *module)
 {
 	unsigned long module_base;
 	unsigned long vmm_size;
 	unsigned long vmm_offset, func_offset, fdesc_offset;
 	struct fdesc *p_fdesc;
 	BUG_ON(!module);
 	if (!kvm_vmm_base) {
 		printk("kvm: kvm area hasn't been initilized yet!!\n");
 		return -EFAULT;
 	}
 	/*Calculate new position of relocated vmm module.*/
 	module_base = (unsigned long)module->module_core;
 	vmm_size = module->core_size;
 	if (unlikely(vmm_size > KVM_VMM_SIZE))
 		return -EFAULT;
 	memcpy((void *)kvm_vmm_base, (void *)module_base, vmm_size);
 	kvm_patch_vmm(vmm_info, module);
 	kvm_flush_icache(kvm_vmm_base, vmm_size);
 	/*Recalculate kvm_vmm_info based on new VMM*/
 	vmm_offset = vmm_info->vmm_ivt - module_base;
 	kvm_vmm_info->vmm_ivt = KVM_VMM_BASE + vmm_offset;
 	printk(KERN_DEBUG"kvm: Relocated VMM's IVT Base Addr:%lx\n",
 			kvm_vmm_info->vmm_ivt);
 	fdesc_offset = (unsigned long)vmm_info->vmm_entry - module_base;
 	kvm_vmm_info->vmm_entry = (kvm_vmm_entry *)(KVM_VMM_BASE +
 							fdesc_offset);
 	func_offset = *(unsigned long *)vmm_info->vmm_entry - module_base;
 	p_fdesc = (struct fdesc *)(kvm_vmm_base + fdesc_offset);
 	p_fdesc->ip = KVM_VMM_BASE + func_offset;
 	p_fdesc->gp = KVM_VMM_BASE+(p_fdesc->gp - module_base);
 	printk(KERN_DEBUG"kvm: Relocated VMM's Init Entry Addr:%lx\n",
 			KVM_VMM_BASE+func_offset);
 	fdesc_offset = (unsigned long)vmm_info->tramp_entry - module_base;
 	kvm_vmm_info->tramp_entry = (kvm_tramp_entry *)(KVM_VMM_BASE +
 			fdesc_offset);
 	func_offset = *(unsigned long *)vmm_info->tramp_entry - module_base;
 	p_fdesc = (struct fdesc *)(kvm_vmm_base + fdesc_offset);
 	p_fdesc->ip = KVM_VMM_BASE + func_offset;
 	p_fdesc->gp = KVM_VMM_BASE + (p_fdesc->gp - module_base);
 	kvm_vmm_gp = p_fdesc->gp;
 	printk(KERN_DEBUG"kvm: Relocated VMM's Entry IP:%p\n",
 						kvm_vmm_info->vmm_entry);
 	printk(KERN_DEBUG"kvm: Relocated VMM's Trampoline Entry IP:0x%lx\n",
 						KVM_VMM_BASE + func_offset);
 	return 0;
 }
 int kvm_arch_init(void *opaque)
 {
 	int r;
 	struct kvm_vmm_info *vmm_info = (struct kvm_vmm_info *)opaque;
 	if (!vti_cpu_has_kvm_support()) {
 		printk(KERN_ERR "kvm: No Hardware Virtualization Support!\n");
 		r = -EOPNOTSUPP;
 		goto out;
 	}
 	if (kvm_vmm_info) {
 		printk(KERN_ERR "kvm: Already loaded VMM module!\n");
 		r = -EEXIST;
 		goto out;
 	}
 	r = -ENOMEM;
 	kvm_vmm_info = kzalloc(sizeof(struct kvm_vmm_info), GFP_KERNEL);
 	if (!kvm_vmm_info)
 		goto out;
 	if (kvm_alloc_vmm_area())
 		goto out_free0;
 	r = kvm_relocate_vmm(vmm_info, vmm_info->module);
 	if (r)
 		goto out_free1;
 	return 0;
 out_free1:
 	kvm_free_vmm_area();
 out_free0:
 	kfree(kvm_vmm_info);
 out:
 	return r;
 }
 void kvm_arch_exit(void)
 {
 	kvm_free_vmm_area();
 	kfree(kvm_vmm_info);
 	kvm_vmm_info = NULL;
 }
 static int kvm_ia64_sync_dirty_log(struct kvm *kvm,
 		struct kvm_dirty_log *log)
 {
 	struct kvm_memory_slot *memslot;
 	int r, i;
 	long n, base;
 	unsigned long *dirty_bitmap = (unsigned long *)(kvm->arch.vm_base +
 			offsetof(struct kvm_vm_data, kvm_mem_dirty_log));
 	r = -EINVAL;
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 	memslot = &kvm->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
 	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 	base = memslot->base_gfn / BITS_PER_LONG;
 	for (i = 0; i < n/sizeof(long); ++i) {
 		memslot->dirty_bitmap[i] = dirty_bitmap[base + i];
 		dirty_bitmap[base + i] = 0;
 	}
 	r = 0;
 out:
 	return r;
 }
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 		struct kvm_dirty_log *log)
 {
 	int r;
 	int n;
 	struct kvm_memory_slot *memslot;
 	int is_dirty = 0;
 	spin_lock(&kvm->arch.dirty_log_lock);
 	r = kvm_ia64_sync_dirty_log(kvm, log);
 	if (r)
 		goto out;
 	r = kvm_get_dirty_log(kvm, log, &is_dirty);
 	if (r)
 		goto out;
 	/* If nothing is dirty, don't bother messing with page tables. */
 	if (is_dirty) {
 		kvm_flush_remote_tlbs(kvm);
 		memslot = &kvm->memslots[log->slot];
 		n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 		memset(memslot->dirty_bitmap, 0, n);
 	}
 	r = 0;
 out:
 	spin_unlock(&kvm->arch.dirty_log_lock);
 	return r;
 }
 int kvm_arch_hardware_setup(void)
 {
 	return 0;
 }
 void kvm_arch_hardware_unsetup(void)
 {
 }
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
 {
 	int me;
 	int cpu = vcpu->cpu;
 	if (waitqueue_active(&vcpu->wq))
 		wake_up_interruptible(&vcpu->wq);
 	me = get_cpu();
 	if (cpu != me && (unsigned) cpu < nr_cpu_ids && cpu_online(cpu))
 		if (!test_and_set_bit(KVM_REQ_KICK, &vcpu->requests))
 			smp_send_reschedule(cpu);
 	put_cpu();
 }
 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq)
 {
 	return __apic_accept_irq(vcpu, irq->vector);
 }
 int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest)
 {
 	return apic->vcpu->vcpu_id == dest;
 }
 int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
 {
 	return 0;
 }
 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
 {
 	return vcpu1->arch.xtp - vcpu2->arch.xtp;
 }
 int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 		int short_hand, int dest, int dest_mode)
 {
 	struct kvm_lapic *target = vcpu->arch.apic;
 	return (dest_mode == 0) ?
 		kvm_apic_match_physical_addr(target, dest) :
 		kvm_apic_match_logical_addr(target, dest);
 }
 static int find_highest_bits(int *dat)
 {
 	u32  bits, bitnum;
 	int i;
 	/* loop for all 256 bits */
 	for (i = 7; i >= 0 ; i--) {
 		bits = dat[i];
 		if (bits) {
 			bitnum = fls(bits);
 			return i * 32 + bitnum - 1;
 		}
 	}
 	return -1;
 }
 int kvm_highest_pending_irq(struct kvm_vcpu *vcpu)
 {
     struct vpd *vpd = to_host(vcpu->kvm, vcpu->arch.vpd);
     if (vpd->irr[0] & (1UL << NMI_VECTOR))
 		return NMI_VECTOR;
     if (vpd->irr[0] & (1UL << ExtINT_VECTOR))
 		return ExtINT_VECTOR;
     return find_highest_bits((int *)&vpd->irr[0]);
 }
 int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu)
 {
 	if (kvm_highest_pending_irq(vcpu) != -1)
 		return 1;
 	return 0;
 }
 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu)
 {
 	/* do real check here */
 	return 1;
 }
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.timer_fired;
 }
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	return gfn;
 }
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE;
 }
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	vcpu_load(vcpu);
 	mp_state->mp_state = vcpu->arch.mp_state;
 	vcpu_put(vcpu);
 	return 0;
 }
 static int vcpu_reset(struct kvm_vcpu *vcpu)
 {
 	int r;
 	long psr;
 	local_irq_save(psr);
 	r = kvm_insert_vmm_mapping(vcpu);
 	local_irq_restore(psr);
 	if (r)
 		goto fail;
 	vcpu->arch.launched = 0;
 	kvm_arch_vcpu_uninit(vcpu);
 	r = kvm_arch_vcpu_init(vcpu);
 	if (r)
 		goto fail;
 	kvm_purge_vmm_mapping(vcpu);
 	r = 0;
 fail:
 	return r;
 }
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
 	int r = 0;
 	vcpu_load(vcpu);
 	vcpu->arch.mp_state = mp_state->mp_state;
 	if (vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)
 		r = vcpu_reset(vcpu);
 	vcpu_put(vcpu);
 	return r;
 }

arch/ia64/kvm/vcpu.c

Diff comments View file @ 73880c8

1	/*	1	/*
2	* kvm_vcpu.c: handling all virtual cpu related thing.	2	* kvm_vcpu.c: handling all virtual cpu related thing.
3	* Copyright (c) 2005, Intel Corporation.	3	* Copyright (c) 2005, Intel Corporation.
4	*	4	*
5	* This program is free software; you can redistribute it and/or modify it	5	* This program is free software; you can redistribute it and/or modify it
6	* under the terms and conditions of the GNU General Public License,	6	* under the terms and conditions of the GNU General Public License,
7	* version 2, as published by the Free Software Foundation.	7	* version 2, as published by the Free Software Foundation.
8	*	8	*
9	* This program is distributed in the hope it will be useful, but WITHOUT	9	* This program is distributed in the hope it will be useful, but WITHOUT
10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12	* more details.	12	* more details.
13	*	13	*
14	* You should have received a copy of the GNU General Public License along with	14	* You should have received a copy of the GNU General Public License along with
15	* this program; if not, write to the Free Software Foundation, Inc., 59 Temple	15	* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16	* Place - Suite 330, Boston, MA 02111-1307 USA.	16	* Place - Suite 330, Boston, MA 02111-1307 USA.
17	*	17	*
18	* Shaofan Li (Susue Li) <susie.li@intel.com>	18	* Shaofan Li (Susue Li) <susie.li@intel.com>
19	* Yaozu Dong (Eddie Dong) (Eddie.dong@intel.com)	19	* Yaozu Dong (Eddie Dong) (Eddie.dong@intel.com)
20	* Xuefei Xu (Anthony Xu) (Anthony.xu@intel.com)	20	* Xuefei Xu (Anthony Xu) (Anthony.xu@intel.com)
21	* Xiantao Zhang <xiantao.zhang@intel.com>	21	* Xiantao Zhang <xiantao.zhang@intel.com>
22	*/	22	*/
23		23
24	#include <linux/kvm_host.h>	24	#include <linux/kvm_host.h>
25	#include <linux/types.h>	25	#include <linux/types.h>
26		26
27	#include <asm/processor.h>	27	#include <asm/processor.h>
28	#include <asm/ia64regs.h>	28	#include <asm/ia64regs.h>
29	#include <asm/gcc_intrin.h>	29	#include <asm/gcc_intrin.h>
30	#include <asm/kregs.h>	30	#include <asm/kregs.h>
31	#include <asm/pgtable.h>	31	#include <asm/pgtable.h>
32	#include <asm/tlb.h>	32	#include <asm/tlb.h>
33		33
34	#include "asm-offsets.h"	34	#include "asm-offsets.h"
35	#include "vcpu.h"	35	#include "vcpu.h"
36		36
37	/*	37	/*
38	* Special notes:	38	* Special notes:
39	* - Index by it/dt/rt sequence	39	* - Index by it/dt/rt sequence
40	* - Only existing mode transitions are allowed in this table	40	* - Only existing mode transitions are allowed in this table
41	* - RSE is placed at lazy mode when emulating guest partial mode	41	* - RSE is placed at lazy mode when emulating guest partial mode
42	* - If gva happens to be rr0 and rr4, only allowed case is identity	42	* - If gva happens to be rr0 and rr4, only allowed case is identity
43	* mapping (gva=gpa), or panic! (How?)	43	* mapping (gva=gpa), or panic! (How?)
44	*/	44	*/
45	int mm_switch_table[8][8] = {	45	int mm_switch_table[8][8] = {
46	/* 2004/09/12(Kevin): Allow switch to self */	46	/* 2004/09/12(Kevin): Allow switch to self */
47	/*	47	/*
48	* (it,dt,rt): (0,0,0) -> (1,1,1)	48	* (it,dt,rt): (0,0,0) -> (1,1,1)
49	* This kind of transition usually occurs in the very early	49	* This kind of transition usually occurs in the very early
50	* stage of Linux boot up procedure. Another case is in efi	50	* stage of Linux boot up procedure. Another case is in efi
51	* and pal calls. (see "arch/ia64/kernel/head.S")	51	* and pal calls. (see "arch/ia64/kernel/head.S")
52	*	52	*
53	* (it,dt,rt): (0,0,0) -> (0,1,1)	53	* (it,dt,rt): (0,0,0) -> (0,1,1)
54	* This kind of transition is found when OSYa exits efi boot	54	* This kind of transition is found when OSYa exits efi boot
55	* service. Due to gva = gpa in this case (Same region),	55	* service. Due to gva = gpa in this case (Same region),
56	* data access can be satisfied though itlb entry for physical	56	* data access can be satisfied though itlb entry for physical
57	* emulation is hit.	57	* emulation is hit.
58	*/	58	*/
59	{SW_SELF, 0, 0, SW_NOP, 0, 0, 0, SW_P2V},	59	{SW_SELF, 0, 0, SW_NOP, 0, 0, 0, SW_P2V},
60	{0, 0, 0, 0, 0, 0, 0, 0},	60	{0, 0, 0, 0, 0, 0, 0, 0},
61	{0, 0, 0, 0, 0, 0, 0, 0},	61	{0, 0, 0, 0, 0, 0, 0, 0},
62	/*	62	/*
63	* (it,dt,rt): (0,1,1) -> (1,1,1)	63	* (it,dt,rt): (0,1,1) -> (1,1,1)
64	* This kind of transition is found in OSYa.	64	* This kind of transition is found in OSYa.
65	*	65	*
66	* (it,dt,rt): (0,1,1) -> (0,0,0)	66	* (it,dt,rt): (0,1,1) -> (0,0,0)
67	* This kind of transition is found in OSYa	67	* This kind of transition is found in OSYa
68	*/	68	*/
69	{SW_NOP, 0, 0, SW_SELF, 0, 0, 0, SW_P2V},	69	{SW_NOP, 0, 0, SW_SELF, 0, 0, 0, SW_P2V},
70	/* (1,0,0)->(1,1,1) */	70	/* (1,0,0)->(1,1,1) */
71	{0, 0, 0, 0, 0, 0, 0, SW_P2V},	71	{0, 0, 0, 0, 0, 0, 0, SW_P2V},
72	/*	72	/*
73	* (it,dt,rt): (1,0,1) -> (1,1,1)	73	* (it,dt,rt): (1,0,1) -> (1,1,1)
74	* This kind of transition usually occurs when Linux returns	74	* This kind of transition usually occurs when Linux returns
75	* from the low level TLB miss handlers.	75	* from the low level TLB miss handlers.
76	* (see "arch/ia64/kernel/ivt.S")	76	* (see "arch/ia64/kernel/ivt.S")
77	*/	77	*/
78	{0, 0, 0, 0, 0, SW_SELF, 0, SW_P2V},	78	{0, 0, 0, 0, 0, SW_SELF, 0, SW_P2V},
79	{0, 0, 0, 0, 0, 0, 0, 0},	79	{0, 0, 0, 0, 0, 0, 0, 0},
80	/*	80	/*
81	* (it,dt,rt): (1,1,1) -> (1,0,1)	81	* (it,dt,rt): (1,1,1) -> (1,0,1)
82	* This kind of transition usually occurs in Linux low level	82	* This kind of transition usually occurs in Linux low level
83	* TLB miss handler. (see "arch/ia64/kernel/ivt.S")	83	* TLB miss handler. (see "arch/ia64/kernel/ivt.S")
84	*	84	*
85	* (it,dt,rt): (1,1,1) -> (0,0,0)	85	* (it,dt,rt): (1,1,1) -> (0,0,0)
86	* This kind of transition usually occurs in pal and efi calls,	86	* This kind of transition usually occurs in pal and efi calls,
87	* which requires running in physical mode.	87	* which requires running in physical mode.
88	* (see "arch/ia64/kernel/head.S")	88	* (see "arch/ia64/kernel/head.S")
89	* (1,1,1)->(1,0,0)	89	* (1,1,1)->(1,0,0)
90	*/	90	*/
91		91
92	{SW_V2P, 0, 0, 0, SW_V2P, SW_V2P, 0, SW_SELF},	92	{SW_V2P, 0, 0, 0, SW_V2P, SW_V2P, 0, SW_SELF},
93	};	93	};
94		94
95	void physical_mode_init(struct kvm_vcpu *vcpu)	95	void physical_mode_init(struct kvm_vcpu *vcpu)
96	{	96	{
97	vcpu->arch.mode_flags = GUEST_IN_PHY;	97	vcpu->arch.mode_flags = GUEST_IN_PHY;
98	}	98	}
99		99
100	void switch_to_physical_rid(struct kvm_vcpu *vcpu)	100	void switch_to_physical_rid(struct kvm_vcpu *vcpu)
101	{	101	{
102	unsigned long psr;	102	unsigned long psr;
103		103
104	/* Save original virtual mode rr[0] and rr[4] */	104	/* Save original virtual mode rr[0] and rr[4] */
105	psr = ia64_clear_ic();	105	psr = ia64_clear_ic();
106	ia64_set_rr(VRN0<<VRN_SHIFT, vcpu->arch.metaphysical_rr0);	106	ia64_set_rr(VRN0<<VRN_SHIFT, vcpu->arch.metaphysical_rr0);
107	ia64_srlz_d();	107	ia64_srlz_d();
108	ia64_set_rr(VRN4<<VRN_SHIFT, vcpu->arch.metaphysical_rr4);	108	ia64_set_rr(VRN4<<VRN_SHIFT, vcpu->arch.metaphysical_rr4);
109	ia64_srlz_d();	109	ia64_srlz_d();
110		110
111	ia64_set_psr(psr);	111	ia64_set_psr(psr);
112	return;	112	return;
113	}	113	}
114		114
115	void switch_to_virtual_rid(struct kvm_vcpu *vcpu)	115	void switch_to_virtual_rid(struct kvm_vcpu *vcpu)
116	{	116	{
117	unsigned long psr;	117	unsigned long psr;
118		118
119	psr = ia64_clear_ic();	119	psr = ia64_clear_ic();
120	ia64_set_rr(VRN0 << VRN_SHIFT, vcpu->arch.metaphysical_saved_rr0);	120	ia64_set_rr(VRN0 << VRN_SHIFT, vcpu->arch.metaphysical_saved_rr0);
121	ia64_srlz_d();	121	ia64_srlz_d();
122	ia64_set_rr(VRN4 << VRN_SHIFT, vcpu->arch.metaphysical_saved_rr4);	122	ia64_set_rr(VRN4 << VRN_SHIFT, vcpu->arch.metaphysical_saved_rr4);
123	ia64_srlz_d();	123	ia64_srlz_d();
124	ia64_set_psr(psr);	124	ia64_set_psr(psr);
125	return;	125	return;
126	}	126	}
127		127
128	static int mm_switch_action(struct ia64_psr opsr, struct ia64_psr npsr)	128	static int mm_switch_action(struct ia64_psr opsr, struct ia64_psr npsr)
129	{	129	{
130	return mm_switch_table[MODE_IND(opsr)][MODE_IND(npsr)];	130	return mm_switch_table[MODE_IND(opsr)][MODE_IND(npsr)];
131	}	131	}
132		132
133	void switch_mm_mode(struct kvm_vcpu *vcpu, struct ia64_psr old_psr,	133	void switch_mm_mode(struct kvm_vcpu *vcpu, struct ia64_psr old_psr,
134	struct ia64_psr new_psr)	134	struct ia64_psr new_psr)
135	{	135	{
136	int act;	136	int act;
137	act = mm_switch_action(old_psr, new_psr);	137	act = mm_switch_action(old_psr, new_psr);
138	switch (act) {	138	switch (act) {
139	case SW_V2P:	139	case SW_V2P:
140	/*printk("V -> P mode transition: (0x%lx -> 0x%lx)\n",	140	/*printk("V -> P mode transition: (0x%lx -> 0x%lx)\n",
141	old_psr.val, new_psr.val);*/	141	old_psr.val, new_psr.val);*/
142	switch_to_physical_rid(vcpu);	142	switch_to_physical_rid(vcpu);
143	/*	143	/*
144	* Set rse to enforced lazy, to prevent active rse	144	* Set rse to enforced lazy, to prevent active rse
145	*save/restor when guest physical mode.	145	*save/restor when guest physical mode.
146	*/	146	*/
147	vcpu->arch.mode_flags \|= GUEST_IN_PHY;	147	vcpu->arch.mode_flags \|= GUEST_IN_PHY;
148	break;	148	break;
149	case SW_P2V:	149	case SW_P2V:
150	switch_to_virtual_rid(vcpu);	150	switch_to_virtual_rid(vcpu);
151	/*	151	/*
152	* recover old mode which is saved when entering	152	* recover old mode which is saved when entering
153	* guest physical mode	153	* guest physical mode
154	*/	154	*/
155	vcpu->arch.mode_flags &= ~GUEST_IN_PHY;	155	vcpu->arch.mode_flags &= ~GUEST_IN_PHY;
156	break;	156	break;
157	case SW_SELF:	157	case SW_SELF:
158	break;	158	break;
159	case SW_NOP:	159	case SW_NOP:
160	break;	160	break;
161	default:	161	default:
162	/* Sanity check */	162	/* Sanity check */
163	break;	163	break;
164	}	164	}
165	return;	165	return;
166	}	166	}
167		167
168	/*	168	/*
169	* In physical mode, insert tc/tr for region 0 and 4 uses	169	* In physical mode, insert tc/tr for region 0 and 4 uses
170	* RID[0] and RID[4] which is for physical mode emulation.	170	* RID[0] and RID[4] which is for physical mode emulation.
171	* However what those inserted tc/tr wants is rid for	171	* However what those inserted tc/tr wants is rid for
172	* virtual mode. So original virtual rid needs to be restored	172	* virtual mode. So original virtual rid needs to be restored
173	* before insert.	173	* before insert.
174	*	174	*
175	* Operations which required such switch include:	175	* Operations which required such switch include:
176	* - insertions (itc., itr.)	176	* - insertions (itc., itr.)
177	* - purges (ptc.* and ptr.*)	177	* - purges (ptc.* and ptr.*)
178	* - tpa	178	* - tpa
179	* - tak	179	* - tak
180	* - thash?, ttag?	180	* - thash?, ttag?
181	* All above needs actual virtual rid for destination entry.	181	* All above needs actual virtual rid for destination entry.
182	*/	182	*/
183		183
184	void check_mm_mode_switch(struct kvm_vcpu *vcpu, struct ia64_psr old_psr,	184	void check_mm_mode_switch(struct kvm_vcpu *vcpu, struct ia64_psr old_psr,
185	struct ia64_psr new_psr)	185	struct ia64_psr new_psr)
186	{	186	{
187		187
188	if ((old_psr.dt != new_psr.dt)	188	if ((old_psr.dt != new_psr.dt)
189	\|\| (old_psr.it != new_psr.it)	189	\|\| (old_psr.it != new_psr.it)
190	\|\| (old_psr.rt != new_psr.rt))	190	\|\| (old_psr.rt != new_psr.rt))
191	switch_mm_mode(vcpu, old_psr, new_psr);	191	switch_mm_mode(vcpu, old_psr, new_psr);
192		192
193	return;	193	return;
194	}	194	}
195		195
196		196
197	/*	197	/*
198	* In physical mode, insert tc/tr for region 0 and 4 uses	198	* In physical mode, insert tc/tr for region 0 and 4 uses
199	* RID[0] and RID[4] which is for physical mode emulation.	199	* RID[0] and RID[4] which is for physical mode emulation.
200	* However what those inserted tc/tr wants is rid for	200	* However what those inserted tc/tr wants is rid for
201	* virtual mode. So original virtual rid needs to be restored	201	* virtual mode. So original virtual rid needs to be restored
202	* before insert.	202	* before insert.
203	*	203	*
204	* Operations which required such switch include:	204	* Operations which required such switch include:
205	* - insertions (itc., itr.)	205	* - insertions (itc., itr.)
206	* - purges (ptc.* and ptr.*)	206	* - purges (ptc.* and ptr.*)
207	* - tpa	207	* - tpa
208	* - tak	208	* - tak
209	* - thash?, ttag?	209	* - thash?, ttag?
210	* All above needs actual virtual rid for destination entry.	210	* All above needs actual virtual rid for destination entry.
211	*/	211	*/
212		212
213	void prepare_if_physical_mode(struct kvm_vcpu *vcpu)	213	void prepare_if_physical_mode(struct kvm_vcpu *vcpu)
214	{	214	{
215	if (is_physical_mode(vcpu)) {	215	if (is_physical_mode(vcpu)) {
216	vcpu->arch.mode_flags \|= GUEST_PHY_EMUL;	216	vcpu->arch.mode_flags \|= GUEST_PHY_EMUL;
217	switch_to_virtual_rid(vcpu);	217	switch_to_virtual_rid(vcpu);
218	}	218	}
219	return;	219	return;
220	}	220	}
221		221
222	/* Recover always follows prepare */	222	/* Recover always follows prepare */
223	void recover_if_physical_mode(struct kvm_vcpu *vcpu)	223	void recover_if_physical_mode(struct kvm_vcpu *vcpu)
224	{	224	{
225	if (is_physical_mode(vcpu))	225	if (is_physical_mode(vcpu))
226	switch_to_physical_rid(vcpu);	226	switch_to_physical_rid(vcpu);
227	vcpu->arch.mode_flags &= ~GUEST_PHY_EMUL;	227	vcpu->arch.mode_flags &= ~GUEST_PHY_EMUL;
228	return;	228	return;
229	}	229	}
230		230
231	#define RPT(x) ((u16) &((struct kvm_pt_regs *)0)->x)	231	#define RPT(x) ((u16) &((struct kvm_pt_regs *)0)->x)
232		232
233	static u16 gr_info[32] = {	233	static u16 gr_info[32] = {
234	0, /* r0 is read-only : WE SHOULD NEVER GET THIS */	234	0, /* r0 is read-only : WE SHOULD NEVER GET THIS */
235	RPT(r1), RPT(r2), RPT(r3),	235	RPT(r1), RPT(r2), RPT(r3),
236	RPT(r4), RPT(r5), RPT(r6), RPT(r7),	236	RPT(r4), RPT(r5), RPT(r6), RPT(r7),
237	RPT(r8), RPT(r9), RPT(r10), RPT(r11),	237	RPT(r8), RPT(r9), RPT(r10), RPT(r11),
238	RPT(r12), RPT(r13), RPT(r14), RPT(r15),	238	RPT(r12), RPT(r13), RPT(r14), RPT(r15),
239	RPT(r16), RPT(r17), RPT(r18), RPT(r19),	239	RPT(r16), RPT(r17), RPT(r18), RPT(r19),
240	RPT(r20), RPT(r21), RPT(r22), RPT(r23),	240	RPT(r20), RPT(r21), RPT(r22), RPT(r23),
241	RPT(r24), RPT(r25), RPT(r26), RPT(r27),	241	RPT(r24), RPT(r25), RPT(r26), RPT(r27),
242	RPT(r28), RPT(r29), RPT(r30), RPT(r31)	242	RPT(r28), RPT(r29), RPT(r30), RPT(r31)
243	};	243	};
244		244
245	#define IA64_FIRST_STACKED_GR 32	245	#define IA64_FIRST_STACKED_GR 32
246	#define IA64_FIRST_ROTATING_FR 32	246	#define IA64_FIRST_ROTATING_FR 32
247		247
248	static inline unsigned long	248	static inline unsigned long
249	rotate_reg(unsigned long sor, unsigned long rrb, unsigned long reg)	249	rotate_reg(unsigned long sor, unsigned long rrb, unsigned long reg)
250	{	250	{
251	reg += rrb;	251	reg += rrb;
252	if (reg >= sor)	252	if (reg >= sor)
253	reg -= sor;	253	reg -= sor;
254	return reg;	254	return reg;
255	}	255	}
256		256
257	/*	257	/*
258	* Return the (rotated) index for floating point register	258	* Return the (rotated) index for floating point register
259	* be in the REGNUM (REGNUM must range from 32-127,	259	* be in the REGNUM (REGNUM must range from 32-127,
260	* result is in the range from 0-95.	260	* result is in the range from 0-95.
261	*/	261	*/
262	static inline unsigned long fph_index(struct kvm_pt_regs *regs,	262	static inline unsigned long fph_index(struct kvm_pt_regs *regs,
263	long regnum)	263	long regnum)
264	{	264	{
265	unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;	265	unsigned long rrb_fr = (regs->cr_ifs >> 25) & 0x7f;
266	return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));	266	return rotate_reg(96, rrb_fr, (regnum - IA64_FIRST_ROTATING_FR));
267	}	267	}
268		268
269	/*	269	/*
270	* The inverse of the above: given bspstore and the number of	270	* The inverse of the above: given bspstore and the number of
271	* registers, calculate ar.bsp.	271	* registers, calculate ar.bsp.
272	*/	272	*/
273	static inline unsigned long kvm_rse_skip_regs(unsigned long addr,	273	static inline unsigned long kvm_rse_skip_regs(unsigned long addr,
274	long num_regs)	274	long num_regs)
275	{	275	{
276	long delta = ia64_rse_slot_num(addr) + num_regs;	276	long delta = ia64_rse_slot_num(addr) + num_regs;
277	int i = 0;	277	int i = 0;
278		278
279	if (num_regs < 0)	279	if (num_regs < 0)
280	delta -= 0x3e;	280	delta -= 0x3e;
281	if (delta < 0) {	281	if (delta < 0) {
282	while (delta <= -0x3f) {	282	while (delta <= -0x3f) {
283	i--;	283	i--;
284	delta += 0x3f;	284	delta += 0x3f;
285	}	285	}
286	} else {	286	} else {
287	while (delta >= 0x3f) {	287	while (delta >= 0x3f) {
288	i++;	288	i++;
289	delta -= 0x3f;	289	delta -= 0x3f;
290	}	290	}
291	}	291	}
292		292
293	return addr + num_regs + i;	293	return addr + num_regs + i;
294	}	294	}
295		295
296	static void get_rse_reg(struct kvm_pt_regs *regs, unsigned long r1,	296	static void get_rse_reg(struct kvm_pt_regs *regs, unsigned long r1,
297	unsigned long val, int nat)	297	unsigned long val, int nat)
298	{	298	{
299	unsigned long bsp, addr, rnat_addr, bspstore;	299	unsigned long bsp, addr, rnat_addr, bspstore;
300	unsigned long kbs = (void ) current_vcpu + VMM_RBS_OFFSET;	300	unsigned long kbs = (void ) current_vcpu + VMM_RBS_OFFSET;
301	unsigned long nat_mask;	301	unsigned long nat_mask;
302	unsigned long old_rsc, new_rsc;	302	unsigned long old_rsc, new_rsc;
303	long sof = (regs->cr_ifs) & 0x7f;	303	long sof = (regs->cr_ifs) & 0x7f;
304	long sor = (((regs->cr_ifs >> 14) & 0xf) << 3);	304	long sor = (((regs->cr_ifs >> 14) & 0xf) << 3);
305	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;	305	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
306	long ridx = r1 - 32;	306	long ridx = r1 - 32;
307		307
308	if (ridx < sor)	308	if (ridx < sor)
309	ridx = rotate_reg(sor, rrb_gr, ridx);	309	ridx = rotate_reg(sor, rrb_gr, ridx);
310		310
311	old_rsc = ia64_getreg(_IA64_REG_AR_RSC);	311	old_rsc = ia64_getreg(_IA64_REG_AR_RSC);
312	new_rsc = old_rsc&(~(0x3));	312	new_rsc = old_rsc&(~(0x3));
313	ia64_setreg(_IA64_REG_AR_RSC, new_rsc);	313	ia64_setreg(_IA64_REG_AR_RSC, new_rsc);
314		314
315	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);	315	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);
316	bsp = kbs + (regs->loadrs >> 19);	316	bsp = kbs + (regs->loadrs >> 19);
317		317
318	addr = kvm_rse_skip_regs(bsp, -sof + ridx);	318	addr = kvm_rse_skip_regs(bsp, -sof + ridx);
319	nat_mask = 1UL << ia64_rse_slot_num(addr);	319	nat_mask = 1UL << ia64_rse_slot_num(addr);
320	rnat_addr = ia64_rse_rnat_addr(addr);	320	rnat_addr = ia64_rse_rnat_addr(addr);
321		321
322	if (addr >= bspstore) {	322	if (addr >= bspstore) {
323	ia64_flushrs();	323	ia64_flushrs();
324	ia64_mf();	324	ia64_mf();
325	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);	325	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);
326	}	326	}
327	val = addr;	327	val = addr;
328	if (nat) {	328	if (nat) {
329	if (bspstore < rnat_addr)	329	if (bspstore < rnat_addr)
330	*nat = (int)!!(ia64_getreg(_IA64_REG_AR_RNAT)	330	*nat = (int)!!(ia64_getreg(_IA64_REG_AR_RNAT)
331	& nat_mask);	331	& nat_mask);
332	else	332	else
333	nat = (int)!!((rnat_addr) & nat_mask);	333	nat = (int)!!((rnat_addr) & nat_mask);
334	ia64_setreg(_IA64_REG_AR_RSC, old_rsc);	334	ia64_setreg(_IA64_REG_AR_RSC, old_rsc);
335	}	335	}
336	}	336	}
337		337
338	void set_rse_reg(struct kvm_pt_regs *regs, unsigned long r1,	338	void set_rse_reg(struct kvm_pt_regs *regs, unsigned long r1,
339	unsigned long val, unsigned long nat)	339	unsigned long val, unsigned long nat)
340	{	340	{
341	unsigned long bsp, bspstore, addr, rnat_addr;	341	unsigned long bsp, bspstore, addr, rnat_addr;
342	unsigned long kbs = (void ) current_vcpu + VMM_RBS_OFFSET;	342	unsigned long kbs = (void ) current_vcpu + VMM_RBS_OFFSET;
343	unsigned long nat_mask;	343	unsigned long nat_mask;
344	unsigned long old_rsc, new_rsc, psr;	344	unsigned long old_rsc, new_rsc, psr;
345	unsigned long rnat;	345	unsigned long rnat;
346	long sof = (regs->cr_ifs) & 0x7f;	346	long sof = (regs->cr_ifs) & 0x7f;
347	long sor = (((regs->cr_ifs >> 14) & 0xf) << 3);	347	long sor = (((regs->cr_ifs >> 14) & 0xf) << 3);
348	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;	348	long rrb_gr = (regs->cr_ifs >> 18) & 0x7f;
349	long ridx = r1 - 32;	349	long ridx = r1 - 32;
350		350
351	if (ridx < sor)	351	if (ridx < sor)
352	ridx = rotate_reg(sor, rrb_gr, ridx);	352	ridx = rotate_reg(sor, rrb_gr, ridx);
353		353
354	old_rsc = ia64_getreg(_IA64_REG_AR_RSC);	354	old_rsc = ia64_getreg(_IA64_REG_AR_RSC);
355	/* put RSC to lazy mode, and set loadrs 0 */	355	/* put RSC to lazy mode, and set loadrs 0 */
356	new_rsc = old_rsc & (~0x3fff0003);	356	new_rsc = old_rsc & (~0x3fff0003);
357	ia64_setreg(_IA64_REG_AR_RSC, new_rsc);	357	ia64_setreg(_IA64_REG_AR_RSC, new_rsc);
358	bsp = kbs + (regs->loadrs >> 19); /* 16 + 3 */	358	bsp = kbs + (regs->loadrs >> 19); /* 16 + 3 */
359		359
360	addr = kvm_rse_skip_regs(bsp, -sof + ridx);	360	addr = kvm_rse_skip_regs(bsp, -sof + ridx);
361	nat_mask = 1UL << ia64_rse_slot_num(addr);	361	nat_mask = 1UL << ia64_rse_slot_num(addr);
362	rnat_addr = ia64_rse_rnat_addr(addr);	362	rnat_addr = ia64_rse_rnat_addr(addr);
363		363
364	local_irq_save(psr);	364	local_irq_save(psr);
365	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);	365	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);
366	if (addr >= bspstore) {	366	if (addr >= bspstore) {
367		367
368	ia64_flushrs();	368	ia64_flushrs();
369	ia64_mf();	369	ia64_mf();
370	*addr = val;	370	*addr = val;
371	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);	371	bspstore = (unsigned long *)ia64_getreg(_IA64_REG_AR_BSPSTORE);
372	rnat = ia64_getreg(_IA64_REG_AR_RNAT);	372	rnat = ia64_getreg(_IA64_REG_AR_RNAT);
373	if (bspstore < rnat_addr)	373	if (bspstore < rnat_addr)
374	rnat = rnat & (~nat_mask);	374	rnat = rnat & (~nat_mask);
375	else	375	else
376	rnat_addr = (rnat_addr)&(~nat_mask);	376	rnat_addr = (rnat_addr)&(~nat_mask);
377		377
378	ia64_mf();	378	ia64_mf();
379	ia64_loadrs();	379	ia64_loadrs();
380	ia64_setreg(_IA64_REG_AR_RNAT, rnat);	380	ia64_setreg(_IA64_REG_AR_RNAT, rnat);
381	} else {	381	} else {
382	rnat = ia64_getreg(_IA64_REG_AR_RNAT);	382	rnat = ia64_getreg(_IA64_REG_AR_RNAT);
383	*addr = val;	383	*addr = val;
384	if (bspstore < rnat_addr)	384	if (bspstore < rnat_addr)
385	rnat = rnat&(~nat_mask);	385	rnat = rnat&(~nat_mask);
386	else	386	else
387	rnat_addr = (rnat_addr) & (~nat_mask);	387	rnat_addr = (rnat_addr) & (~nat_mask);
388		388
389	ia64_setreg(_IA64_REG_AR_BSPSTORE, (unsigned long)bspstore);	389	ia64_setreg(_IA64_REG_AR_BSPSTORE, (unsigned long)bspstore);
390	ia64_setreg(_IA64_REG_AR_RNAT, rnat);	390	ia64_setreg(_IA64_REG_AR_RNAT, rnat);
391	}	391	}
392	local_irq_restore(psr);	392	local_irq_restore(psr);
393	ia64_setreg(_IA64_REG_AR_RSC, old_rsc);	393	ia64_setreg(_IA64_REG_AR_RSC, old_rsc);
394	}	394	}
395		395
396	void getreg(unsigned long regnum, unsigned long *val,	396	void getreg(unsigned long regnum, unsigned long *val,
397	int nat, struct kvm_pt_regs regs)	397	int nat, struct kvm_pt_regs regs)
398	{	398	{
399	unsigned long addr, *unat;	399	unsigned long addr, *unat;
400	if (regnum >= IA64_FIRST_STACKED_GR) {	400	if (regnum >= IA64_FIRST_STACKED_GR) {
401	get_rse_reg(regs, regnum, val, nat);	401	get_rse_reg(regs, regnum, val, nat);
402	return;	402	return;
403	}	403	}
404		404
405	/*	405	/*
406	* Now look at registers in [0-31] range and init correct UNAT	406	* Now look at registers in [0-31] range and init correct UNAT
407	*/	407	*/
408	addr = (unsigned long)regs;	408	addr = (unsigned long)regs;
409	unat = &regs->eml_unat;	409	unat = &regs->eml_unat;
410		410
411	addr += gr_info[regnum];	411	addr += gr_info[regnum];
412		412
413	val = (unsigned long *)addr;	413	val = (unsigned long *)addr;
414	/*	414	/*
415	* do it only when requested	415	* do it only when requested
416	*/	416	*/
417	if (nat)	417	if (nat)
418	nat = (unat >> ((addr >> 3) & 0x3f)) & 0x1UL;	418	nat = (unat >> ((addr >> 3) & 0x3f)) & 0x1UL;
419	}	419	}
420		420
421	void setreg(unsigned long regnum, unsigned long val,	421	void setreg(unsigned long regnum, unsigned long val,
422	int nat, struct kvm_pt_regs *regs)	422	int nat, struct kvm_pt_regs *regs)
423	{	423	{
424	unsigned long addr;	424	unsigned long addr;
425	unsigned long bitmask;	425	unsigned long bitmask;
426	unsigned long *unat;	426	unsigned long *unat;
427		427
428	/*	428	/*
429	* First takes care of stacked registers	429	* First takes care of stacked registers
430	*/	430	*/
431	if (regnum >= IA64_FIRST_STACKED_GR) {	431	if (regnum >= IA64_FIRST_STACKED_GR) {
432	set_rse_reg(regs, regnum, val, nat);	432	set_rse_reg(regs, regnum, val, nat);
433	return;	433	return;
434	}	434	}
435		435
436	/*	436	/*
437	* Now look at registers in [0-31] range and init correct UNAT	437	* Now look at registers in [0-31] range and init correct UNAT
438	*/	438	*/
439	addr = (unsigned long)regs;	439	addr = (unsigned long)regs;
440	unat = &regs->eml_unat;	440	unat = &regs->eml_unat;
441	/*	441	/*
442	* add offset from base of struct	442	* add offset from base of struct
443	* and do it !	443	* and do it !
444	*/	444	*/
445	addr += gr_info[regnum];	445	addr += gr_info[regnum];
446		446
447	(unsigned long )addr = val;	447	(unsigned long )addr = val;
448		448
449	/*	449	/*
450	* We need to clear the corresponding UNAT bit to fully emulate the load	450	* We need to clear the corresponding UNAT bit to fully emulate the load
451	* UNAT bit_pos = GR[r3]{8:3} form EAS-2.4	451	* UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
452	*/	452	*/
453	bitmask = 1UL << ((addr >> 3) & 0x3f);	453	bitmask = 1UL << ((addr >> 3) & 0x3f);
454	if (nat)	454	if (nat)
455	*unat \|= bitmask;	455	*unat \|= bitmask;
456	else	456	else
457	*unat &= ~bitmask;	457	*unat &= ~bitmask;
458		458
459	}	459	}
460		460
461	u64 vcpu_get_gr(struct kvm_vcpu *vcpu, unsigned long reg)	461	u64 vcpu_get_gr(struct kvm_vcpu *vcpu, unsigned long reg)
462	{	462	{
463	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	463	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
464	unsigned long val;	464	unsigned long val;
465		465
466	if (!reg)	466	if (!reg)
467	return 0;	467	return 0;
468	getreg(reg, &val, 0, regs);	468	getreg(reg, &val, 0, regs);
469	return val;	469	return val;
470	}	470	}
471		471
472	void vcpu_set_gr(struct kvm_vcpu *vcpu, unsigned long reg, u64 value, int nat)	472	void vcpu_set_gr(struct kvm_vcpu *vcpu, unsigned long reg, u64 value, int nat)
473	{	473	{
474	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	474	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
475	long sof = (regs->cr_ifs) & 0x7f;	475	long sof = (regs->cr_ifs) & 0x7f;
476		476
477	if (!reg)	477	if (!reg)
478	return;	478	return;
479	if (reg >= sof + 32)	479	if (reg >= sof + 32)
480	return;	480	return;
481	setreg(reg, value, nat, regs); /* FIXME: handle NATs later*/	481	setreg(reg, value, nat, regs); /* FIXME: handle NATs later*/
482	}	482	}
483		483
484	void getfpreg(unsigned long regnum, struct ia64_fpreg *fpval,	484	void getfpreg(unsigned long regnum, struct ia64_fpreg *fpval,
485	struct kvm_pt_regs *regs)	485	struct kvm_pt_regs *regs)
486	{	486	{
487	/* Take floating register rotation into consideration*/	487	/* Take floating register rotation into consideration*/
488	if (regnum >= IA64_FIRST_ROTATING_FR)	488	if (regnum >= IA64_FIRST_ROTATING_FR)
489	regnum = IA64_FIRST_ROTATING_FR + fph_index(regs, regnum);	489	regnum = IA64_FIRST_ROTATING_FR + fph_index(regs, regnum);
490	#define CASE_FIXED_FP(reg) \	490	#define CASE_FIXED_FP(reg) \
491	case (reg) : \	491	case (reg) : \
492	ia64_stf_spill(fpval, reg); \	492	ia64_stf_spill(fpval, reg); \
493	break	493	break
494		494
495	switch (regnum) {	495	switch (regnum) {
496	CASE_FIXED_FP(0);	496	CASE_FIXED_FP(0);
497	CASE_FIXED_FP(1);	497	CASE_FIXED_FP(1);
498	CASE_FIXED_FP(2);	498	CASE_FIXED_FP(2);
499	CASE_FIXED_FP(3);	499	CASE_FIXED_FP(3);
500	CASE_FIXED_FP(4);	500	CASE_FIXED_FP(4);
501	CASE_FIXED_FP(5);	501	CASE_FIXED_FP(5);
502		502
503	CASE_FIXED_FP(6);	503	CASE_FIXED_FP(6);
504	CASE_FIXED_FP(7);	504	CASE_FIXED_FP(7);
505	CASE_FIXED_FP(8);	505	CASE_FIXED_FP(8);
506	CASE_FIXED_FP(9);	506	CASE_FIXED_FP(9);
507	CASE_FIXED_FP(10);	507	CASE_FIXED_FP(10);
508	CASE_FIXED_FP(11);	508	CASE_FIXED_FP(11);
509		509
510	CASE_FIXED_FP(12);	510	CASE_FIXED_FP(12);
511	CASE_FIXED_FP(13);	511	CASE_FIXED_FP(13);
512	CASE_FIXED_FP(14);	512	CASE_FIXED_FP(14);
513	CASE_FIXED_FP(15);	513	CASE_FIXED_FP(15);
514	CASE_FIXED_FP(16);	514	CASE_FIXED_FP(16);
515	CASE_FIXED_FP(17);	515	CASE_FIXED_FP(17);
516	CASE_FIXED_FP(18);	516	CASE_FIXED_FP(18);
517	CASE_FIXED_FP(19);	517	CASE_FIXED_FP(19);
518	CASE_FIXED_FP(20);	518	CASE_FIXED_FP(20);
519	CASE_FIXED_FP(21);	519	CASE_FIXED_FP(21);
520	CASE_FIXED_FP(22);	520	CASE_FIXED_FP(22);
521	CASE_FIXED_FP(23);	521	CASE_FIXED_FP(23);
522	CASE_FIXED_FP(24);	522	CASE_FIXED_FP(24);
523	CASE_FIXED_FP(25);	523	CASE_FIXED_FP(25);
524	CASE_FIXED_FP(26);	524	CASE_FIXED_FP(26);
525	CASE_FIXED_FP(27);	525	CASE_FIXED_FP(27);
526	CASE_FIXED_FP(28);	526	CASE_FIXED_FP(28);
527	CASE_FIXED_FP(29);	527	CASE_FIXED_FP(29);
528	CASE_FIXED_FP(30);	528	CASE_FIXED_FP(30);
529	CASE_FIXED_FP(31);	529	CASE_FIXED_FP(31);
530	CASE_FIXED_FP(32);	530	CASE_FIXED_FP(32);
531	CASE_FIXED_FP(33);	531	CASE_FIXED_FP(33);
532	CASE_FIXED_FP(34);	532	CASE_FIXED_FP(34);
533	CASE_FIXED_FP(35);	533	CASE_FIXED_FP(35);
534	CASE_FIXED_FP(36);	534	CASE_FIXED_FP(36);
535	CASE_FIXED_FP(37);	535	CASE_FIXED_FP(37);
536	CASE_FIXED_FP(38);	536	CASE_FIXED_FP(38);
537	CASE_FIXED_FP(39);	537	CASE_FIXED_FP(39);
538	CASE_FIXED_FP(40);	538	CASE_FIXED_FP(40);
539	CASE_FIXED_FP(41);	539	CASE_FIXED_FP(41);
540	CASE_FIXED_FP(42);	540	CASE_FIXED_FP(42);
541	CASE_FIXED_FP(43);	541	CASE_FIXED_FP(43);
542	CASE_FIXED_FP(44);	542	CASE_FIXED_FP(44);
543	CASE_FIXED_FP(45);	543	CASE_FIXED_FP(45);
544	CASE_FIXED_FP(46);	544	CASE_FIXED_FP(46);
545	CASE_FIXED_FP(47);	545	CASE_FIXED_FP(47);
546	CASE_FIXED_FP(48);	546	CASE_FIXED_FP(48);
547	CASE_FIXED_FP(49);	547	CASE_FIXED_FP(49);
548	CASE_FIXED_FP(50);	548	CASE_FIXED_FP(50);
549	CASE_FIXED_FP(51);	549	CASE_FIXED_FP(51);
550	CASE_FIXED_FP(52);	550	CASE_FIXED_FP(52);
551	CASE_FIXED_FP(53);	551	CASE_FIXED_FP(53);
552	CASE_FIXED_FP(54);	552	CASE_FIXED_FP(54);
553	CASE_FIXED_FP(55);	553	CASE_FIXED_FP(55);
554	CASE_FIXED_FP(56);	554	CASE_FIXED_FP(56);
555	CASE_FIXED_FP(57);	555	CASE_FIXED_FP(57);
556	CASE_FIXED_FP(58);	556	CASE_FIXED_FP(58);
557	CASE_FIXED_FP(59);	557	CASE_FIXED_FP(59);
558	CASE_FIXED_FP(60);	558	CASE_FIXED_FP(60);
559	CASE_FIXED_FP(61);	559	CASE_FIXED_FP(61);
560	CASE_FIXED_FP(62);	560	CASE_FIXED_FP(62);
561	CASE_FIXED_FP(63);	561	CASE_FIXED_FP(63);
562	CASE_FIXED_FP(64);	562	CASE_FIXED_FP(64);
563	CASE_FIXED_FP(65);	563	CASE_FIXED_FP(65);
564	CASE_FIXED_FP(66);	564	CASE_FIXED_FP(66);
565	CASE_FIXED_FP(67);	565	CASE_FIXED_FP(67);
566	CASE_FIXED_FP(68);	566	CASE_FIXED_FP(68);
567	CASE_FIXED_FP(69);	567	CASE_FIXED_FP(69);
568	CASE_FIXED_FP(70);	568	CASE_FIXED_FP(70);
569	CASE_FIXED_FP(71);	569	CASE_FIXED_FP(71);
570	CASE_FIXED_FP(72);	570	CASE_FIXED_FP(72);
571	CASE_FIXED_FP(73);	571	CASE_FIXED_FP(73);
572	CASE_FIXED_FP(74);	572	CASE_FIXED_FP(74);
573	CASE_FIXED_FP(75);	573	CASE_FIXED_FP(75);
574	CASE_FIXED_FP(76);	574	CASE_FIXED_FP(76);
575	CASE_FIXED_FP(77);	575	CASE_FIXED_FP(77);
576	CASE_FIXED_FP(78);	576	CASE_FIXED_FP(78);
577	CASE_FIXED_FP(79);	577	CASE_FIXED_FP(79);
578	CASE_FIXED_FP(80);	578	CASE_FIXED_FP(80);
579	CASE_FIXED_FP(81);	579	CASE_FIXED_FP(81);
580	CASE_FIXED_FP(82);	580	CASE_FIXED_FP(82);
581	CASE_FIXED_FP(83);	581	CASE_FIXED_FP(83);
582	CASE_FIXED_FP(84);	582	CASE_FIXED_FP(84);
583	CASE_FIXED_FP(85);	583	CASE_FIXED_FP(85);
584	CASE_FIXED_FP(86);	584	CASE_FIXED_FP(86);
585	CASE_FIXED_FP(87);	585	CASE_FIXED_FP(87);
586	CASE_FIXED_FP(88);	586	CASE_FIXED_FP(88);
587	CASE_FIXED_FP(89);	587	CASE_FIXED_FP(89);
588	CASE_FIXED_FP(90);	588	CASE_FIXED_FP(90);
589	CASE_FIXED_FP(91);	589	CASE_FIXED_FP(91);
590	CASE_FIXED_FP(92);	590	CASE_FIXED_FP(92);
591	CASE_FIXED_FP(93);	591	CASE_FIXED_FP(93);
592	CASE_FIXED_FP(94);	592	CASE_FIXED_FP(94);
593	CASE_FIXED_FP(95);	593	CASE_FIXED_FP(95);
594	CASE_FIXED_FP(96);	594	CASE_FIXED_FP(96);
595	CASE_FIXED_FP(97);	595	CASE_FIXED_FP(97);
596	CASE_FIXED_FP(98);	596	CASE_FIXED_FP(98);
597	CASE_FIXED_FP(99);	597	CASE_FIXED_FP(99);
598	CASE_FIXED_FP(100);	598	CASE_FIXED_FP(100);
599	CASE_FIXED_FP(101);	599	CASE_FIXED_FP(101);
600	CASE_FIXED_FP(102);	600	CASE_FIXED_FP(102);
601	CASE_FIXED_FP(103);	601	CASE_FIXED_FP(103);
602	CASE_FIXED_FP(104);	602	CASE_FIXED_FP(104);
603	CASE_FIXED_FP(105);	603	CASE_FIXED_FP(105);
604	CASE_FIXED_FP(106);	604	CASE_FIXED_FP(106);
605	CASE_FIXED_FP(107);	605	CASE_FIXED_FP(107);
606	CASE_FIXED_FP(108);	606	CASE_FIXED_FP(108);
607	CASE_FIXED_FP(109);	607	CASE_FIXED_FP(109);
608	CASE_FIXED_FP(110);	608	CASE_FIXED_FP(110);
609	CASE_FIXED_FP(111);	609	CASE_FIXED_FP(111);
610	CASE_FIXED_FP(112);	610	CASE_FIXED_FP(112);
611	CASE_FIXED_FP(113);	611	CASE_FIXED_FP(113);
612	CASE_FIXED_FP(114);	612	CASE_FIXED_FP(114);
613	CASE_FIXED_FP(115);	613	CASE_FIXED_FP(115);
614	CASE_FIXED_FP(116);	614	CASE_FIXED_FP(116);
615	CASE_FIXED_FP(117);	615	CASE_FIXED_FP(117);
616	CASE_FIXED_FP(118);	616	CASE_FIXED_FP(118);
617	CASE_FIXED_FP(119);	617	CASE_FIXED_FP(119);
618	CASE_FIXED_FP(120);	618	CASE_FIXED_FP(120);
619	CASE_FIXED_FP(121);	619	CASE_FIXED_FP(121);
620	CASE_FIXED_FP(122);	620	CASE_FIXED_FP(122);
621	CASE_FIXED_FP(123);	621	CASE_FIXED_FP(123);
622	CASE_FIXED_FP(124);	622	CASE_FIXED_FP(124);
623	CASE_FIXED_FP(125);	623	CASE_FIXED_FP(125);
624	CASE_FIXED_FP(126);	624	CASE_FIXED_FP(126);
625	CASE_FIXED_FP(127);	625	CASE_FIXED_FP(127);
626	}	626	}
627	#undef CASE_FIXED_FP	627	#undef CASE_FIXED_FP
628	}	628	}
629		629
630	void setfpreg(unsigned long regnum, struct ia64_fpreg *fpval,	630	void setfpreg(unsigned long regnum, struct ia64_fpreg *fpval,
631	struct kvm_pt_regs *regs)	631	struct kvm_pt_regs *regs)
632	{	632	{
633	/* Take floating register rotation into consideration*/	633	/* Take floating register rotation into consideration*/
634	if (regnum >= IA64_FIRST_ROTATING_FR)	634	if (regnum >= IA64_FIRST_ROTATING_FR)
635	regnum = IA64_FIRST_ROTATING_FR + fph_index(regs, regnum);	635	regnum = IA64_FIRST_ROTATING_FR + fph_index(regs, regnum);
636		636
637	#define CASE_FIXED_FP(reg) \	637	#define CASE_FIXED_FP(reg) \
638	case (reg) : \	638	case (reg) : \
639	ia64_ldf_fill(reg, fpval); \	639	ia64_ldf_fill(reg, fpval); \
640	break	640	break
641		641
642	switch (regnum) {	642	switch (regnum) {
643	CASE_FIXED_FP(2);	643	CASE_FIXED_FP(2);
644	CASE_FIXED_FP(3);	644	CASE_FIXED_FP(3);
645	CASE_FIXED_FP(4);	645	CASE_FIXED_FP(4);
646	CASE_FIXED_FP(5);	646	CASE_FIXED_FP(5);
647		647
648	CASE_FIXED_FP(6);	648	CASE_FIXED_FP(6);
649	CASE_FIXED_FP(7);	649	CASE_FIXED_FP(7);
650	CASE_FIXED_FP(8);	650	CASE_FIXED_FP(8);
651	CASE_FIXED_FP(9);	651	CASE_FIXED_FP(9);
652	CASE_FIXED_FP(10);	652	CASE_FIXED_FP(10);
653	CASE_FIXED_FP(11);	653	CASE_FIXED_FP(11);
654		654
655	CASE_FIXED_FP(12);	655	CASE_FIXED_FP(12);
656	CASE_FIXED_FP(13);	656	CASE_FIXED_FP(13);
657	CASE_FIXED_FP(14);	657	CASE_FIXED_FP(14);
658	CASE_FIXED_FP(15);	658	CASE_FIXED_FP(15);
659	CASE_FIXED_FP(16);	659	CASE_FIXED_FP(16);
660	CASE_FIXED_FP(17);	660	CASE_FIXED_FP(17);
661	CASE_FIXED_FP(18);	661	CASE_FIXED_FP(18);
662	CASE_FIXED_FP(19);	662	CASE_FIXED_FP(19);
663	CASE_FIXED_FP(20);	663	CASE_FIXED_FP(20);
664	CASE_FIXED_FP(21);	664	CASE_FIXED_FP(21);
665	CASE_FIXED_FP(22);	665	CASE_FIXED_FP(22);
666	CASE_FIXED_FP(23);	666	CASE_FIXED_FP(23);
667	CASE_FIXED_FP(24);	667	CASE_FIXED_FP(24);
668	CASE_FIXED_FP(25);	668	CASE_FIXED_FP(25);
669	CASE_FIXED_FP(26);	669	CASE_FIXED_FP(26);
670	CASE_FIXED_FP(27);	670	CASE_FIXED_FP(27);
671	CASE_FIXED_FP(28);	671	CASE_FIXED_FP(28);
672	CASE_FIXED_FP(29);	672	CASE_FIXED_FP(29);
673	CASE_FIXED_FP(30);	673	CASE_FIXED_FP(30);
674	CASE_FIXED_FP(31);	674	CASE_FIXED_FP(31);
675	CASE_FIXED_FP(32);	675	CASE_FIXED_FP(32);
676	CASE_FIXED_FP(33);	676	CASE_FIXED_FP(33);
677	CASE_FIXED_FP(34);	677	CASE_FIXED_FP(34);
678	CASE_FIXED_FP(35);	678	CASE_FIXED_FP(35);
679	CASE_FIXED_FP(36);	679	CASE_FIXED_FP(36);
680	CASE_FIXED_FP(37);	680	CASE_FIXED_FP(37);
681	CASE_FIXED_FP(38);	681	CASE_FIXED_FP(38);
682	CASE_FIXED_FP(39);	682	CASE_FIXED_FP(39);
683	CASE_FIXED_FP(40);	683	CASE_FIXED_FP(40);
684	CASE_FIXED_FP(41);	684	CASE_FIXED_FP(41);
685	CASE_FIXED_FP(42);	685	CASE_FIXED_FP(42);
686	CASE_FIXED_FP(43);	686	CASE_FIXED_FP(43);
687	CASE_FIXED_FP(44);	687	CASE_FIXED_FP(44);
688	CASE_FIXED_FP(45);	688	CASE_FIXED_FP(45);
689	CASE_FIXED_FP(46);	689	CASE_FIXED_FP(46);
690	CASE_FIXED_FP(47);	690	CASE_FIXED_FP(47);
691	CASE_FIXED_FP(48);	691	CASE_FIXED_FP(48);
692	CASE_FIXED_FP(49);	692	CASE_FIXED_FP(49);
693	CASE_FIXED_FP(50);	693	CASE_FIXED_FP(50);
694	CASE_FIXED_FP(51);	694	CASE_FIXED_FP(51);
695	CASE_FIXED_FP(52);	695	CASE_FIXED_FP(52);
696	CASE_FIXED_FP(53);	696	CASE_FIXED_FP(53);
697	CASE_FIXED_FP(54);	697	CASE_FIXED_FP(54);
698	CASE_FIXED_FP(55);	698	CASE_FIXED_FP(55);
699	CASE_FIXED_FP(56);	699	CASE_FIXED_FP(56);
700	CASE_FIXED_FP(57);	700	CASE_FIXED_FP(57);
701	CASE_FIXED_FP(58);	701	CASE_FIXED_FP(58);
702	CASE_FIXED_FP(59);	702	CASE_FIXED_FP(59);
703	CASE_FIXED_FP(60);	703	CASE_FIXED_FP(60);
704	CASE_FIXED_FP(61);	704	CASE_FIXED_FP(61);
705	CASE_FIXED_FP(62);	705	CASE_FIXED_FP(62);
706	CASE_FIXED_FP(63);	706	CASE_FIXED_FP(63);
707	CASE_FIXED_FP(64);	707	CASE_FIXED_FP(64);
708	CASE_FIXED_FP(65);	708	CASE_FIXED_FP(65);
709	CASE_FIXED_FP(66);	709	CASE_FIXED_FP(66);
710	CASE_FIXED_FP(67);	710	CASE_FIXED_FP(67);
711	CASE_FIXED_FP(68);	711	CASE_FIXED_FP(68);
712	CASE_FIXED_FP(69);	712	CASE_FIXED_FP(69);
713	CASE_FIXED_FP(70);	713	CASE_FIXED_FP(70);
714	CASE_FIXED_FP(71);	714	CASE_FIXED_FP(71);
715	CASE_FIXED_FP(72);	715	CASE_FIXED_FP(72);
716	CASE_FIXED_FP(73);	716	CASE_FIXED_FP(73);
717	CASE_FIXED_FP(74);	717	CASE_FIXED_FP(74);
718	CASE_FIXED_FP(75);	718	CASE_FIXED_FP(75);
719	CASE_FIXED_FP(76);	719	CASE_FIXED_FP(76);
720	CASE_FIXED_FP(77);	720	CASE_FIXED_FP(77);
721	CASE_FIXED_FP(78);	721	CASE_FIXED_FP(78);
722	CASE_FIXED_FP(79);	722	CASE_FIXED_FP(79);
723	CASE_FIXED_FP(80);	723	CASE_FIXED_FP(80);
724	CASE_FIXED_FP(81);	724	CASE_FIXED_FP(81);
725	CASE_FIXED_FP(82);	725	CASE_FIXED_FP(82);
726	CASE_FIXED_FP(83);	726	CASE_FIXED_FP(83);
727	CASE_FIXED_FP(84);	727	CASE_FIXED_FP(84);
728	CASE_FIXED_FP(85);	728	CASE_FIXED_FP(85);
729	CASE_FIXED_FP(86);	729	CASE_FIXED_FP(86);
730	CASE_FIXED_FP(87);	730	CASE_FIXED_FP(87);
731	CASE_FIXED_FP(88);	731	CASE_FIXED_FP(88);
732	CASE_FIXED_FP(89);	732	CASE_FIXED_FP(89);
733	CASE_FIXED_FP(90);	733	CASE_FIXED_FP(90);
734	CASE_FIXED_FP(91);	734	CASE_FIXED_FP(91);
735	CASE_FIXED_FP(92);	735	CASE_FIXED_FP(92);
736	CASE_FIXED_FP(93);	736	CASE_FIXED_FP(93);
737	CASE_FIXED_FP(94);	737	CASE_FIXED_FP(94);
738	CASE_FIXED_FP(95);	738	CASE_FIXED_FP(95);
739	CASE_FIXED_FP(96);	739	CASE_FIXED_FP(96);
740	CASE_FIXED_FP(97);	740	CASE_FIXED_FP(97);
741	CASE_FIXED_FP(98);	741	CASE_FIXED_FP(98);
742	CASE_FIXED_FP(99);	742	CASE_FIXED_FP(99);
743	CASE_FIXED_FP(100);	743	CASE_FIXED_FP(100);
744	CASE_FIXED_FP(101);	744	CASE_FIXED_FP(101);
745	CASE_FIXED_FP(102);	745	CASE_FIXED_FP(102);
746	CASE_FIXED_FP(103);	746	CASE_FIXED_FP(103);
747	CASE_FIXED_FP(104);	747	CASE_FIXED_FP(104);
748	CASE_FIXED_FP(105);	748	CASE_FIXED_FP(105);
749	CASE_FIXED_FP(106);	749	CASE_FIXED_FP(106);
750	CASE_FIXED_FP(107);	750	CASE_FIXED_FP(107);
751	CASE_FIXED_FP(108);	751	CASE_FIXED_FP(108);
752	CASE_FIXED_FP(109);	752	CASE_FIXED_FP(109);
753	CASE_FIXED_FP(110);	753	CASE_FIXED_FP(110);
754	CASE_FIXED_FP(111);	754	CASE_FIXED_FP(111);
755	CASE_FIXED_FP(112);	755	CASE_FIXED_FP(112);
756	CASE_FIXED_FP(113);	756	CASE_FIXED_FP(113);
757	CASE_FIXED_FP(114);	757	CASE_FIXED_FP(114);
758	CASE_FIXED_FP(115);	758	CASE_FIXED_FP(115);
759	CASE_FIXED_FP(116);	759	CASE_FIXED_FP(116);
760	CASE_FIXED_FP(117);	760	CASE_FIXED_FP(117);
761	CASE_FIXED_FP(118);	761	CASE_FIXED_FP(118);
762	CASE_FIXED_FP(119);	762	CASE_FIXED_FP(119);
763	CASE_FIXED_FP(120);	763	CASE_FIXED_FP(120);
764	CASE_FIXED_FP(121);	764	CASE_FIXED_FP(121);
765	CASE_FIXED_FP(122);	765	CASE_FIXED_FP(122);
766	CASE_FIXED_FP(123);	766	CASE_FIXED_FP(123);
767	CASE_FIXED_FP(124);	767	CASE_FIXED_FP(124);
768	CASE_FIXED_FP(125);	768	CASE_FIXED_FP(125);
769	CASE_FIXED_FP(126);	769	CASE_FIXED_FP(126);
770	CASE_FIXED_FP(127);	770	CASE_FIXED_FP(127);
771	}	771	}
772	}	772	}
773		773
774	void vcpu_get_fpreg(struct kvm_vcpu *vcpu, unsigned long reg,	774	void vcpu_get_fpreg(struct kvm_vcpu *vcpu, unsigned long reg,
775	struct ia64_fpreg *val)	775	struct ia64_fpreg *val)
776	{	776	{
777	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	777	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
778		778
779	getfpreg(reg, val, regs); /* FIXME: handle NATs later*/	779	getfpreg(reg, val, regs); /* FIXME: handle NATs later*/
780	}	780	}
781		781
782	void vcpu_set_fpreg(struct kvm_vcpu *vcpu, unsigned long reg,	782	void vcpu_set_fpreg(struct kvm_vcpu *vcpu, unsigned long reg,
783	struct ia64_fpreg *val)	783	struct ia64_fpreg *val)
784	{	784	{
785	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	785	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
786		786
787	if (reg > 1)	787	if (reg > 1)
788	setfpreg(reg, val, regs); /* FIXME: handle NATs later*/	788	setfpreg(reg, val, regs); /* FIXME: handle NATs later*/
789	}	789	}
790		790
791	/*	791	/*
792	* The Altix RTC is mapped specially here for the vmm module	792	* The Altix RTC is mapped specially here for the vmm module
793	*/	793	*/
794	#define SN_RTC_BASE (u64 *)(KVM_VMM_BASE+(1UL<<KVM_VMM_SHIFT))	794	#define SN_RTC_BASE (u64 *)(KVM_VMM_BASE+(1UL<<KVM_VMM_SHIFT))
795	static long kvm_get_itc(struct kvm_vcpu *vcpu)	795	static long kvm_get_itc(struct kvm_vcpu *vcpu)
796	{	796	{
797	#if defined(CONFIG_IA64_SGI_SN2) \|\| defined(CONFIG_IA64_GENERIC)	797	#if defined(CONFIG_IA64_SGI_SN2) \|\| defined(CONFIG_IA64_GENERIC)
798	struct kvm kvm = (struct kvm )KVM_VM_BASE;	798	struct kvm kvm = (struct kvm )KVM_VM_BASE;
799		799
800	if (kvm->arch.is_sn2)	800	if (kvm->arch.is_sn2)
801	return (*SN_RTC_BASE);	801	return (*SN_RTC_BASE);
802	else	802	else
803	#endif	803	#endif
804	return ia64_getreg(_IA64_REG_AR_ITC);	804	return ia64_getreg(_IA64_REG_AR_ITC);
805	}	805	}
806		806
807	/************************************************************************	807	/************************************************************************
808	* lsapic timer	808	* lsapic timer
809	***********************************************************************/	809	***********************************************************************/
810	u64 vcpu_get_itc(struct kvm_vcpu *vcpu)	810	u64 vcpu_get_itc(struct kvm_vcpu *vcpu)
811	{	811	{
812	unsigned long guest_itc;	812	unsigned long guest_itc;
813	guest_itc = VMX(vcpu, itc_offset) + kvm_get_itc(vcpu);	813	guest_itc = VMX(vcpu, itc_offset) + kvm_get_itc(vcpu);
814		814
815	if (guest_itc >= VMX(vcpu, last_itc)) {	815	if (guest_itc >= VMX(vcpu, last_itc)) {
816	VMX(vcpu, last_itc) = guest_itc;	816	VMX(vcpu, last_itc) = guest_itc;
817	return guest_itc;	817	return guest_itc;
818	} else	818	} else
819	return VMX(vcpu, last_itc);	819	return VMX(vcpu, last_itc);
820	}	820	}
821		821
822	static inline void vcpu_set_itm(struct kvm_vcpu *vcpu, u64 val);	822	static inline void vcpu_set_itm(struct kvm_vcpu *vcpu, u64 val);
823	static void vcpu_set_itc(struct kvm_vcpu *vcpu, u64 val)	823	static void vcpu_set_itc(struct kvm_vcpu *vcpu, u64 val)
824	{	824	{
825	struct kvm_vcpu *v;	825	struct kvm_vcpu *v;
826	struct kvm *kvm;	826	struct kvm *kvm;
827	int i;	827	int i;
828	long itc_offset = val - kvm_get_itc(vcpu);	828	long itc_offset = val - kvm_get_itc(vcpu);
829	unsigned long vitv = VCPU(vcpu, itv);	829	unsigned long vitv = VCPU(vcpu, itv);
830		830
831	kvm = (struct kvm *)KVM_VM_BASE;	831	kvm = (struct kvm *)KVM_VM_BASE;
832		832
833	if (kvm_vcpu_is_bsp(vcpu)) {	833	if (kvm_vcpu_is_bsp(vcpu)) {
834	for (i = 0; i < kvm->arch.online_vcpus; i++) {	834	for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) {
835	v = (struct kvm_vcpu )((char )vcpu +	835	v = (struct kvm_vcpu )((char )vcpu +
836	sizeof(struct kvm_vcpu_data) * i);	836	sizeof(struct kvm_vcpu_data) * i);
837	VMX(v, itc_offset) = itc_offset;	837	VMX(v, itc_offset) = itc_offset;
838	VMX(v, last_itc) = 0;	838	VMX(v, last_itc) = 0;
839	}	839	}
840	}	840	}
841	VMX(vcpu, last_itc) = 0;	841	VMX(vcpu, last_itc) = 0;
842	if (VCPU(vcpu, itm) <= val) {	842	if (VCPU(vcpu, itm) <= val) {
843	VMX(vcpu, itc_check) = 0;	843	VMX(vcpu, itc_check) = 0;
844	vcpu_unpend_interrupt(vcpu, vitv);	844	vcpu_unpend_interrupt(vcpu, vitv);
845	} else {	845	} else {
846	VMX(vcpu, itc_check) = 1;	846	VMX(vcpu, itc_check) = 1;
847	vcpu_set_itm(vcpu, VCPU(vcpu, itm));	847	vcpu_set_itm(vcpu, VCPU(vcpu, itm));
848	}	848	}
849		849
850	}	850	}
851		851
852	static inline u64 vcpu_get_itm(struct kvm_vcpu *vcpu)	852	static inline u64 vcpu_get_itm(struct kvm_vcpu *vcpu)
853	{	853	{
854	return ((u64)VCPU(vcpu, itm));	854	return ((u64)VCPU(vcpu, itm));
855	}	855	}
856		856
857	static inline void vcpu_set_itm(struct kvm_vcpu *vcpu, u64 val)	857	static inline void vcpu_set_itm(struct kvm_vcpu *vcpu, u64 val)
858	{	858	{
859	unsigned long vitv = VCPU(vcpu, itv);	859	unsigned long vitv = VCPU(vcpu, itv);
860	VCPU(vcpu, itm) = val;	860	VCPU(vcpu, itm) = val;
861		861
862	if (val > vcpu_get_itc(vcpu)) {	862	if (val > vcpu_get_itc(vcpu)) {
863	VMX(vcpu, itc_check) = 1;	863	VMX(vcpu, itc_check) = 1;
864	vcpu_unpend_interrupt(vcpu, vitv);	864	vcpu_unpend_interrupt(vcpu, vitv);
865	VMX(vcpu, timer_pending) = 0;	865	VMX(vcpu, timer_pending) = 0;
866	} else	866	} else
867	VMX(vcpu, itc_check) = 0;	867	VMX(vcpu, itc_check) = 0;
868	}	868	}
869		869
870	#define ITV_VECTOR(itv) (itv&0xff)	870	#define ITV_VECTOR(itv) (itv&0xff)
871	#define ITV_IRQ_MASK(itv) (itv&(1<<16))	871	#define ITV_IRQ_MASK(itv) (itv&(1<<16))
872		872
873	static inline void vcpu_set_itv(struct kvm_vcpu *vcpu, u64 val)	873	static inline void vcpu_set_itv(struct kvm_vcpu *vcpu, u64 val)
874	{	874	{
875	VCPU(vcpu, itv) = val;	875	VCPU(vcpu, itv) = val;
876	if (!ITV_IRQ_MASK(val) && vcpu->arch.timer_pending) {	876	if (!ITV_IRQ_MASK(val) && vcpu->arch.timer_pending) {
877	vcpu_pend_interrupt(vcpu, ITV_VECTOR(val));	877	vcpu_pend_interrupt(vcpu, ITV_VECTOR(val));
878	vcpu->arch.timer_pending = 0;	878	vcpu->arch.timer_pending = 0;
879	}	879	}
880	}	880	}
881		881
882	static inline void vcpu_set_eoi(struct kvm_vcpu *vcpu, u64 val)	882	static inline void vcpu_set_eoi(struct kvm_vcpu *vcpu, u64 val)
883	{	883	{
884	int vec;	884	int vec;
885		885
886	vec = highest_inservice_irq(vcpu);	886	vec = highest_inservice_irq(vcpu);
887	if (vec == NULL_VECTOR)	887	if (vec == NULL_VECTOR)
888	return;	888	return;
889	VMX(vcpu, insvc[vec >> 6]) &= ~(1UL << (vec & 63));	889	VMX(vcpu, insvc[vec >> 6]) &= ~(1UL << (vec & 63));
890	VCPU(vcpu, eoi) = 0;	890	VCPU(vcpu, eoi) = 0;
891	vcpu->arch.irq_new_pending = 1;	891	vcpu->arch.irq_new_pending = 1;
892		892
893	}	893	}
894		894
895	/* See Table 5-8 in SDM vol2 for the definition */	895	/* See Table 5-8 in SDM vol2 for the definition */
896	int irq_masked(struct kvm_vcpu *vcpu, int h_pending, int h_inservice)	896	int irq_masked(struct kvm_vcpu *vcpu, int h_pending, int h_inservice)
897	{	897	{
898	union ia64_tpr vtpr;	898	union ia64_tpr vtpr;
899		899
900	vtpr.val = VCPU(vcpu, tpr);	900	vtpr.val = VCPU(vcpu, tpr);
901		901
902	if (h_inservice == NMI_VECTOR)	902	if (h_inservice == NMI_VECTOR)
903	return IRQ_MASKED_BY_INSVC;	903	return IRQ_MASKED_BY_INSVC;
904		904
905	if (h_pending == NMI_VECTOR) {	905	if (h_pending == NMI_VECTOR) {
906	/* Non Maskable Interrupt */	906	/* Non Maskable Interrupt */
907	return IRQ_NO_MASKED;	907	return IRQ_NO_MASKED;
908	}	908	}
909		909
910	if (h_inservice == ExtINT_VECTOR)	910	if (h_inservice == ExtINT_VECTOR)
911	return IRQ_MASKED_BY_INSVC;	911	return IRQ_MASKED_BY_INSVC;
912		912
913	if (h_pending == ExtINT_VECTOR) {	913	if (h_pending == ExtINT_VECTOR) {
914	if (vtpr.mmi) {	914	if (vtpr.mmi) {
915	/* mask all external IRQ */	915	/* mask all external IRQ */
916	return IRQ_MASKED_BY_VTPR;	916	return IRQ_MASKED_BY_VTPR;
917	} else	917	} else
918	return IRQ_NO_MASKED;	918	return IRQ_NO_MASKED;
919	}	919	}
920		920
921	if (is_higher_irq(h_pending, h_inservice)) {	921	if (is_higher_irq(h_pending, h_inservice)) {
922	if (is_higher_class(h_pending, vtpr.mic + (vtpr.mmi << 4)))	922	if (is_higher_class(h_pending, vtpr.mic + (vtpr.mmi << 4)))
923	return IRQ_NO_MASKED;	923	return IRQ_NO_MASKED;
924	else	924	else
925	return IRQ_MASKED_BY_VTPR;	925	return IRQ_MASKED_BY_VTPR;
926	} else {	926	} else {
927	return IRQ_MASKED_BY_INSVC;	927	return IRQ_MASKED_BY_INSVC;
928	}	928	}
929	}	929	}
930		930
931	void vcpu_pend_interrupt(struct kvm_vcpu *vcpu, u8 vec)	931	void vcpu_pend_interrupt(struct kvm_vcpu *vcpu, u8 vec)
932	{	932	{
933	long spsr;	933	long spsr;
934	int ret;	934	int ret;
935		935
936	local_irq_save(spsr);	936	local_irq_save(spsr);
937	ret = test_and_set_bit(vec, &VCPU(vcpu, irr[0]));	937	ret = test_and_set_bit(vec, &VCPU(vcpu, irr[0]));
938	local_irq_restore(spsr);	938	local_irq_restore(spsr);
939		939
940	vcpu->arch.irq_new_pending = 1;	940	vcpu->arch.irq_new_pending = 1;
941	}	941	}
942		942
943	void vcpu_unpend_interrupt(struct kvm_vcpu *vcpu, u8 vec)	943	void vcpu_unpend_interrupt(struct kvm_vcpu *vcpu, u8 vec)
944	{	944	{
945	long spsr;	945	long spsr;
946	int ret;	946	int ret;
947		947
948	local_irq_save(spsr);	948	local_irq_save(spsr);
949	ret = test_and_clear_bit(vec, &VCPU(vcpu, irr[0]));	949	ret = test_and_clear_bit(vec, &VCPU(vcpu, irr[0]));
950	local_irq_restore(spsr);	950	local_irq_restore(spsr);
951	if (ret) {	951	if (ret) {
952	vcpu->arch.irq_new_pending = 1;	952	vcpu->arch.irq_new_pending = 1;
953	wmb();	953	wmb();
954	}	954	}
955	}	955	}
956		956
957	void update_vhpi(struct kvm_vcpu *vcpu, int vec)	957	void update_vhpi(struct kvm_vcpu *vcpu, int vec)
958	{	958	{
959	u64 vhpi;	959	u64 vhpi;
960		960
961	if (vec == NULL_VECTOR)	961	if (vec == NULL_VECTOR)
962	vhpi = 0;	962	vhpi = 0;
963	else if (vec == NMI_VECTOR)	963	else if (vec == NMI_VECTOR)
964	vhpi = 32;	964	vhpi = 32;
965	else if (vec == ExtINT_VECTOR)	965	else if (vec == ExtINT_VECTOR)
966	vhpi = 16;	966	vhpi = 16;
967	else	967	else
968	vhpi = vec >> 4;	968	vhpi = vec >> 4;
969		969
970	VCPU(vcpu, vhpi) = vhpi;	970	VCPU(vcpu, vhpi) = vhpi;
971	if (VCPU(vcpu, vac).a_int)	971	if (VCPU(vcpu, vac).a_int)
972	ia64_call_vsa(PAL_VPS_SET_PENDING_INTERRUPT,	972	ia64_call_vsa(PAL_VPS_SET_PENDING_INTERRUPT,
973	(u64)vcpu->arch.vpd, 0, 0, 0, 0, 0, 0);	973	(u64)vcpu->arch.vpd, 0, 0, 0, 0, 0, 0);
974	}	974	}
975		975
976	u64 vcpu_get_ivr(struct kvm_vcpu *vcpu)	976	u64 vcpu_get_ivr(struct kvm_vcpu *vcpu)
977	{	977	{
978	int vec, h_inservice, mask;	978	int vec, h_inservice, mask;
979		979
980	vec = highest_pending_irq(vcpu);	980	vec = highest_pending_irq(vcpu);
981	h_inservice = highest_inservice_irq(vcpu);	981	h_inservice = highest_inservice_irq(vcpu);
982	mask = irq_masked(vcpu, vec, h_inservice);	982	mask = irq_masked(vcpu, vec, h_inservice);
983	if (vec == NULL_VECTOR \|\| mask == IRQ_MASKED_BY_INSVC) {	983	if (vec == NULL_VECTOR \|\| mask == IRQ_MASKED_BY_INSVC) {
984	if (VCPU(vcpu, vhpi))	984	if (VCPU(vcpu, vhpi))
985	update_vhpi(vcpu, NULL_VECTOR);	985	update_vhpi(vcpu, NULL_VECTOR);
986	return IA64_SPURIOUS_INT_VECTOR;	986	return IA64_SPURIOUS_INT_VECTOR;
987	}	987	}
988	if (mask == IRQ_MASKED_BY_VTPR) {	988	if (mask == IRQ_MASKED_BY_VTPR) {
989	update_vhpi(vcpu, vec);	989	update_vhpi(vcpu, vec);
990	return IA64_SPURIOUS_INT_VECTOR;	990	return IA64_SPURIOUS_INT_VECTOR;
991	}	991	}
992	VMX(vcpu, insvc[vec >> 6]) \|= (1UL << (vec & 63));	992	VMX(vcpu, insvc[vec >> 6]) \|= (1UL << (vec & 63));
993	vcpu_unpend_interrupt(vcpu, vec);	993	vcpu_unpend_interrupt(vcpu, vec);
994	return (u64)vec;	994	return (u64)vec;
995	}	995	}
996		996
997	/**************************************************************************	997	/**************************************************************************
998	Privileged operation emulation routines	998	Privileged operation emulation routines
999	**************************************************************************/	999	**************************************************************************/
1000	u64 vcpu_thash(struct kvm_vcpu *vcpu, u64 vadr)	1000	u64 vcpu_thash(struct kvm_vcpu *vcpu, u64 vadr)
1001	{	1001	{
1002	union ia64_pta vpta;	1002	union ia64_pta vpta;
1003	union ia64_rr vrr;	1003	union ia64_rr vrr;
1004	u64 pval;	1004	u64 pval;
1005	u64 vhpt_offset;	1005	u64 vhpt_offset;
1006		1006
1007	vpta.val = vcpu_get_pta(vcpu);	1007	vpta.val = vcpu_get_pta(vcpu);
1008	vrr.val = vcpu_get_rr(vcpu, vadr);	1008	vrr.val = vcpu_get_rr(vcpu, vadr);
1009	vhpt_offset = ((vadr >> vrr.ps) << 3) & ((1UL << (vpta.size)) - 1);	1009	vhpt_offset = ((vadr >> vrr.ps) << 3) & ((1UL << (vpta.size)) - 1);
1010	if (vpta.vf) {	1010	if (vpta.vf) {
1011	pval = ia64_call_vsa(PAL_VPS_THASH, vadr, vrr.val,	1011	pval = ia64_call_vsa(PAL_VPS_THASH, vadr, vrr.val,
1012	vpta.val, 0, 0, 0, 0);	1012	vpta.val, 0, 0, 0, 0);
1013	} else {	1013	} else {
1014	pval = (vadr & VRN_MASK) \| vhpt_offset \|	1014	pval = (vadr & VRN_MASK) \| vhpt_offset \|
1015	(vpta.val << 3 >> (vpta.size + 3) << (vpta.size));	1015	(vpta.val << 3 >> (vpta.size + 3) << (vpta.size));
1016	}	1016	}
1017	return pval;	1017	return pval;
1018	}	1018	}
1019		1019
1020	u64 vcpu_ttag(struct kvm_vcpu *vcpu, u64 vadr)	1020	u64 vcpu_ttag(struct kvm_vcpu *vcpu, u64 vadr)
1021	{	1021	{
1022	union ia64_rr vrr;	1022	union ia64_rr vrr;
1023	union ia64_pta vpta;	1023	union ia64_pta vpta;
1024	u64 pval;	1024	u64 pval;
1025		1025
1026	vpta.val = vcpu_get_pta(vcpu);	1026	vpta.val = vcpu_get_pta(vcpu);
1027	vrr.val = vcpu_get_rr(vcpu, vadr);	1027	vrr.val = vcpu_get_rr(vcpu, vadr);
1028	if (vpta.vf) {	1028	if (vpta.vf) {
1029	pval = ia64_call_vsa(PAL_VPS_TTAG, vadr, vrr.val,	1029	pval = ia64_call_vsa(PAL_VPS_TTAG, vadr, vrr.val,
1030	0, 0, 0, 0, 0);	1030	0, 0, 0, 0, 0);
1031	} else	1031	} else
1032	pval = 1;	1032	pval = 1;
1033		1033
1034	return pval;	1034	return pval;
1035	}	1035	}
1036		1036
1037	u64 vcpu_tak(struct kvm_vcpu *vcpu, u64 vadr)	1037	u64 vcpu_tak(struct kvm_vcpu *vcpu, u64 vadr)
1038	{	1038	{
1039	struct thash_data *data;	1039	struct thash_data *data;
1040	union ia64_pta vpta;	1040	union ia64_pta vpta;
1041	u64 key;	1041	u64 key;
1042		1042
1043	vpta.val = vcpu_get_pta(vcpu);	1043	vpta.val = vcpu_get_pta(vcpu);
1044	if (vpta.vf == 0) {	1044	if (vpta.vf == 0) {
1045	key = 1;	1045	key = 1;
1046	return key;	1046	return key;
1047	}	1047	}
1048	data = vtlb_lookup(vcpu, vadr, D_TLB);	1048	data = vtlb_lookup(vcpu, vadr, D_TLB);
1049	if (!data \|\| !data->p)	1049	if (!data \|\| !data->p)
1050	key = 1;	1050	key = 1;
1051	else	1051	else
1052	key = data->key;	1052	key = data->key;
1053		1053
1054	return key;	1054	return key;
1055	}	1055	}
1056		1056
1057	void kvm_thash(struct kvm_vcpu *vcpu, INST64 inst)	1057	void kvm_thash(struct kvm_vcpu *vcpu, INST64 inst)
1058	{	1058	{
1059	unsigned long thash, vadr;	1059	unsigned long thash, vadr;
1060		1060
1061	vadr = vcpu_get_gr(vcpu, inst.M46.r3);	1061	vadr = vcpu_get_gr(vcpu, inst.M46.r3);
1062	thash = vcpu_thash(vcpu, vadr);	1062	thash = vcpu_thash(vcpu, vadr);
1063	vcpu_set_gr(vcpu, inst.M46.r1, thash, 0);	1063	vcpu_set_gr(vcpu, inst.M46.r1, thash, 0);
1064	}	1064	}
1065		1065
1066	void kvm_ttag(struct kvm_vcpu *vcpu, INST64 inst)	1066	void kvm_ttag(struct kvm_vcpu *vcpu, INST64 inst)
1067	{	1067	{
1068	unsigned long tag, vadr;	1068	unsigned long tag, vadr;
1069		1069
1070	vadr = vcpu_get_gr(vcpu, inst.M46.r3);	1070	vadr = vcpu_get_gr(vcpu, inst.M46.r3);
1071	tag = vcpu_ttag(vcpu, vadr);	1071	tag = vcpu_ttag(vcpu, vadr);
1072	vcpu_set_gr(vcpu, inst.M46.r1, tag, 0);	1072	vcpu_set_gr(vcpu, inst.M46.r1, tag, 0);
1073	}	1073	}
1074		1074
1075	int vcpu_tpa(struct kvm_vcpu vcpu, u64 vadr, unsigned long padr)	1075	int vcpu_tpa(struct kvm_vcpu vcpu, u64 vadr, unsigned long padr)
1076	{	1076	{
1077	struct thash_data *data;	1077	struct thash_data *data;
1078	union ia64_isr visr, pt_isr;	1078	union ia64_isr visr, pt_isr;
1079	struct kvm_pt_regs *regs;	1079	struct kvm_pt_regs *regs;
1080	struct ia64_psr vpsr;	1080	struct ia64_psr vpsr;
1081		1081
1082	regs = vcpu_regs(vcpu);	1082	regs = vcpu_regs(vcpu);
1083	pt_isr.val = VMX(vcpu, cr_isr);	1083	pt_isr.val = VMX(vcpu, cr_isr);
1084	visr.val = 0;	1084	visr.val = 0;
1085	visr.ei = pt_isr.ei;	1085	visr.ei = pt_isr.ei;
1086	visr.ir = pt_isr.ir;	1086	visr.ir = pt_isr.ir;
1087	vpsr = (struct ia64_psr )&VCPU(vcpu, vpsr);	1087	vpsr = (struct ia64_psr )&VCPU(vcpu, vpsr);
1088	visr.na = 1;	1088	visr.na = 1;
1089		1089
1090	data = vhpt_lookup(vadr);	1090	data = vhpt_lookup(vadr);
1091	if (data) {	1091	if (data) {
1092	if (data->p == 0) {	1092	if (data->p == 0) {
1093	vcpu_set_isr(vcpu, visr.val);	1093	vcpu_set_isr(vcpu, visr.val);
1094	data_page_not_present(vcpu, vadr);	1094	data_page_not_present(vcpu, vadr);
1095	return IA64_FAULT;	1095	return IA64_FAULT;
1096	} else if (data->ma == VA_MATTR_NATPAGE) {	1096	} else if (data->ma == VA_MATTR_NATPAGE) {
1097	vcpu_set_isr(vcpu, visr.val);	1097	vcpu_set_isr(vcpu, visr.val);
1098	dnat_page_consumption(vcpu, vadr);	1098	dnat_page_consumption(vcpu, vadr);
1099	return IA64_FAULT;	1099	return IA64_FAULT;
1100	} else {	1100	} else {
1101	*padr = (data->gpaddr >> data->ps << data->ps) \|	1101	*padr = (data->gpaddr >> data->ps << data->ps) \|
1102	(vadr & (PSIZE(data->ps) - 1));	1102	(vadr & (PSIZE(data->ps) - 1));
1103	return IA64_NO_FAULT;	1103	return IA64_NO_FAULT;
1104	}	1104	}
1105	}	1105	}
1106		1106
1107	data = vtlb_lookup(vcpu, vadr, D_TLB);	1107	data = vtlb_lookup(vcpu, vadr, D_TLB);
1108	if (data) {	1108	if (data) {
1109	if (data->p == 0) {	1109	if (data->p == 0) {
1110	vcpu_set_isr(vcpu, visr.val);	1110	vcpu_set_isr(vcpu, visr.val);
1111	data_page_not_present(vcpu, vadr);	1111	data_page_not_present(vcpu, vadr);
1112	return IA64_FAULT;	1112	return IA64_FAULT;
1113	} else if (data->ma == VA_MATTR_NATPAGE) {	1113	} else if (data->ma == VA_MATTR_NATPAGE) {
1114	vcpu_set_isr(vcpu, visr.val);	1114	vcpu_set_isr(vcpu, visr.val);
1115	dnat_page_consumption(vcpu, vadr);	1115	dnat_page_consumption(vcpu, vadr);
1116	return IA64_FAULT;	1116	return IA64_FAULT;
1117	} else{	1117	} else{
1118	*padr = ((data->ppn >> (data->ps - 12)) << data->ps)	1118	*padr = ((data->ppn >> (data->ps - 12)) << data->ps)
1119	\| (vadr & (PSIZE(data->ps) - 1));	1119	\| (vadr & (PSIZE(data->ps) - 1));
1120	return IA64_NO_FAULT;	1120	return IA64_NO_FAULT;
1121	}	1121	}
1122	}	1122	}
1123	if (!vhpt_enabled(vcpu, vadr, NA_REF)) {	1123	if (!vhpt_enabled(vcpu, vadr, NA_REF)) {
1124	if (vpsr.ic) {	1124	if (vpsr.ic) {
1125	vcpu_set_isr(vcpu, visr.val);	1125	vcpu_set_isr(vcpu, visr.val);
1126	alt_dtlb(vcpu, vadr);	1126	alt_dtlb(vcpu, vadr);
1127	return IA64_FAULT;	1127	return IA64_FAULT;
1128	} else {	1128	} else {
1129	nested_dtlb(vcpu);	1129	nested_dtlb(vcpu);
1130	return IA64_FAULT;	1130	return IA64_FAULT;
1131	}	1131	}
1132	} else {	1132	} else {
1133	if (vpsr.ic) {	1133	if (vpsr.ic) {
1134	vcpu_set_isr(vcpu, visr.val);	1134	vcpu_set_isr(vcpu, visr.val);
1135	dvhpt_fault(vcpu, vadr);	1135	dvhpt_fault(vcpu, vadr);
1136	return IA64_FAULT;	1136	return IA64_FAULT;
1137	} else{	1137	} else{
1138	nested_dtlb(vcpu);	1138	nested_dtlb(vcpu);
1139	return IA64_FAULT;	1139	return IA64_FAULT;
1140	}	1140	}
1141	}	1141	}
1142		1142
1143	return IA64_NO_FAULT;	1143	return IA64_NO_FAULT;
1144	}	1144	}
1145		1145
1146	int kvm_tpa(struct kvm_vcpu *vcpu, INST64 inst)	1146	int kvm_tpa(struct kvm_vcpu *vcpu, INST64 inst)
1147	{	1147	{
1148	unsigned long r1, r3;	1148	unsigned long r1, r3;
1149		1149
1150	r3 = vcpu_get_gr(vcpu, inst.M46.r3);	1150	r3 = vcpu_get_gr(vcpu, inst.M46.r3);
1151		1151
1152	if (vcpu_tpa(vcpu, r3, &r1))	1152	if (vcpu_tpa(vcpu, r3, &r1))
1153	return IA64_FAULT;	1153	return IA64_FAULT;
1154		1154
1155	vcpu_set_gr(vcpu, inst.M46.r1, r1, 0);	1155	vcpu_set_gr(vcpu, inst.M46.r1, r1, 0);
1156	return(IA64_NO_FAULT);	1156	return(IA64_NO_FAULT);
1157	}	1157	}
1158		1158
1159	void kvm_tak(struct kvm_vcpu *vcpu, INST64 inst)	1159	void kvm_tak(struct kvm_vcpu *vcpu, INST64 inst)
1160	{	1160	{
1161	unsigned long r1, r3;	1161	unsigned long r1, r3;
1162		1162
1163	r3 = vcpu_get_gr(vcpu, inst.M46.r3);	1163	r3 = vcpu_get_gr(vcpu, inst.M46.r3);
1164	r1 = vcpu_tak(vcpu, r3);	1164	r1 = vcpu_tak(vcpu, r3);
1165	vcpu_set_gr(vcpu, inst.M46.r1, r1, 0);	1165	vcpu_set_gr(vcpu, inst.M46.r1, r1, 0);
1166	}	1166	}
1167		1167
1168	/************************************	1168	/************************************
1169	* Insert/Purge translation register/cache	1169	* Insert/Purge translation register/cache
1170	************************************/	1170	************************************/
1171	void vcpu_itc_i(struct kvm_vcpu *vcpu, u64 pte, u64 itir, u64 ifa)	1171	void vcpu_itc_i(struct kvm_vcpu *vcpu, u64 pte, u64 itir, u64 ifa)
1172	{	1172	{
1173	thash_purge_and_insert(vcpu, pte, itir, ifa, I_TLB);	1173	thash_purge_and_insert(vcpu, pte, itir, ifa, I_TLB);
1174	}	1174	}
1175		1175
1176	void vcpu_itc_d(struct kvm_vcpu *vcpu, u64 pte, u64 itir, u64 ifa)	1176	void vcpu_itc_d(struct kvm_vcpu *vcpu, u64 pte, u64 itir, u64 ifa)
1177	{	1177	{
1178	thash_purge_and_insert(vcpu, pte, itir, ifa, D_TLB);	1178	thash_purge_and_insert(vcpu, pte, itir, ifa, D_TLB);
1179	}	1179	}
1180		1180
1181	void vcpu_itr_i(struct kvm_vcpu *vcpu, u64 slot, u64 pte, u64 itir, u64 ifa)	1181	void vcpu_itr_i(struct kvm_vcpu *vcpu, u64 slot, u64 pte, u64 itir, u64 ifa)
1182	{	1182	{
1183	u64 ps, va, rid;	1183	u64 ps, va, rid;
1184	struct thash_data *p_itr;	1184	struct thash_data *p_itr;
1185		1185
1186	ps = itir_ps(itir);	1186	ps = itir_ps(itir);
1187	va = PAGEALIGN(ifa, ps);	1187	va = PAGEALIGN(ifa, ps);
1188	pte &= ~PAGE_FLAGS_RV_MASK;	1188	pte &= ~PAGE_FLAGS_RV_MASK;
1189	rid = vcpu_get_rr(vcpu, ifa);	1189	rid = vcpu_get_rr(vcpu, ifa);
1190	rid = rid & RR_RID_MASK;	1190	rid = rid & RR_RID_MASK;
1191	p_itr = (struct thash_data *)&vcpu->arch.itrs[slot];	1191	p_itr = (struct thash_data *)&vcpu->arch.itrs[slot];
1192	vcpu_set_tr(p_itr, pte, itir, va, rid);	1192	vcpu_set_tr(p_itr, pte, itir, va, rid);
1193	vcpu_quick_region_set(VMX(vcpu, itr_regions), va);	1193	vcpu_quick_region_set(VMX(vcpu, itr_regions), va);
1194	}	1194	}
1195		1195
1196		1196
1197	void vcpu_itr_d(struct kvm_vcpu *vcpu, u64 slot, u64 pte, u64 itir, u64 ifa)	1197	void vcpu_itr_d(struct kvm_vcpu *vcpu, u64 slot, u64 pte, u64 itir, u64 ifa)
1198	{	1198	{
1199	u64 gpfn;	1199	u64 gpfn;
1200	u64 ps, va, rid;	1200	u64 ps, va, rid;
1201	struct thash_data *p_dtr;	1201	struct thash_data *p_dtr;
1202		1202
1203	ps = itir_ps(itir);	1203	ps = itir_ps(itir);
1204	va = PAGEALIGN(ifa, ps);	1204	va = PAGEALIGN(ifa, ps);
1205	pte &= ~PAGE_FLAGS_RV_MASK;	1205	pte &= ~PAGE_FLAGS_RV_MASK;
1206		1206
1207	if (ps != _PAGE_SIZE_16M)	1207	if (ps != _PAGE_SIZE_16M)
1208	thash_purge_entries(vcpu, va, ps);	1208	thash_purge_entries(vcpu, va, ps);
1209	gpfn = (pte & _PAGE_PPN_MASK) >> PAGE_SHIFT;	1209	gpfn = (pte & _PAGE_PPN_MASK) >> PAGE_SHIFT;
1210	if (__gpfn_is_io(gpfn))	1210	if (__gpfn_is_io(gpfn))
1211	pte \|= VTLB_PTE_IO;	1211	pte \|= VTLB_PTE_IO;
1212	rid = vcpu_get_rr(vcpu, va);	1212	rid = vcpu_get_rr(vcpu, va);
1213	rid = rid & RR_RID_MASK;	1213	rid = rid & RR_RID_MASK;
1214	p_dtr = (struct thash_data *)&vcpu->arch.dtrs[slot];	1214	p_dtr = (struct thash_data *)&vcpu->arch.dtrs[slot];
1215	vcpu_set_tr((struct thash_data *)&vcpu->arch.dtrs[slot],	1215	vcpu_set_tr((struct thash_data *)&vcpu->arch.dtrs[slot],
1216	pte, itir, va, rid);	1216	pte, itir, va, rid);
1217	vcpu_quick_region_set(VMX(vcpu, dtr_regions), va);	1217	vcpu_quick_region_set(VMX(vcpu, dtr_regions), va);
1218	}	1218	}
1219		1219
1220	void vcpu_ptr_d(struct kvm_vcpu *vcpu, u64 ifa, u64 ps)	1220	void vcpu_ptr_d(struct kvm_vcpu *vcpu, u64 ifa, u64 ps)
1221	{	1221	{
1222	int index;	1222	int index;
1223	u64 va;	1223	u64 va;
1224		1224
1225	va = PAGEALIGN(ifa, ps);	1225	va = PAGEALIGN(ifa, ps);
1226	while ((index = vtr_find_overlap(vcpu, va, ps, D_TLB)) >= 0)	1226	while ((index = vtr_find_overlap(vcpu, va, ps, D_TLB)) >= 0)
1227	vcpu->arch.dtrs[index].page_flags = 0;	1227	vcpu->arch.dtrs[index].page_flags = 0;
1228		1228
1229	thash_purge_entries(vcpu, va, ps);	1229	thash_purge_entries(vcpu, va, ps);
1230	}	1230	}
1231		1231
1232	void vcpu_ptr_i(struct kvm_vcpu *vcpu, u64 ifa, u64 ps)	1232	void vcpu_ptr_i(struct kvm_vcpu *vcpu, u64 ifa, u64 ps)
1233	{	1233	{
1234	int index;	1234	int index;
1235	u64 va;	1235	u64 va;
1236		1236
1237	va = PAGEALIGN(ifa, ps);	1237	va = PAGEALIGN(ifa, ps);
1238	while ((index = vtr_find_overlap(vcpu, va, ps, I_TLB)) >= 0)	1238	while ((index = vtr_find_overlap(vcpu, va, ps, I_TLB)) >= 0)
1239	vcpu->arch.itrs[index].page_flags = 0;	1239	vcpu->arch.itrs[index].page_flags = 0;
1240		1240
1241	thash_purge_entries(vcpu, va, ps);	1241	thash_purge_entries(vcpu, va, ps);
1242	}	1242	}
1243		1243
1244	void vcpu_ptc_l(struct kvm_vcpu *vcpu, u64 va, u64 ps)	1244	void vcpu_ptc_l(struct kvm_vcpu *vcpu, u64 va, u64 ps)
1245	{	1245	{
1246	va = PAGEALIGN(va, ps);	1246	va = PAGEALIGN(va, ps);
1247	thash_purge_entries(vcpu, va, ps);	1247	thash_purge_entries(vcpu, va, ps);
1248	}	1248	}
1249		1249
1250	void vcpu_ptc_e(struct kvm_vcpu *vcpu, u64 va)	1250	void vcpu_ptc_e(struct kvm_vcpu *vcpu, u64 va)
1251	{	1251	{
1252	thash_purge_all(vcpu);	1252	thash_purge_all(vcpu);
1253	}	1253	}
1254		1254
1255	void vcpu_ptc_ga(struct kvm_vcpu *vcpu, u64 va, u64 ps)	1255	void vcpu_ptc_ga(struct kvm_vcpu *vcpu, u64 va, u64 ps)
1256	{	1256	{
1257	struct exit_ctl_data *p = &vcpu->arch.exit_data;	1257	struct exit_ctl_data *p = &vcpu->arch.exit_data;
1258	long psr;	1258	long psr;
1259	local_irq_save(psr);	1259	local_irq_save(psr);
1260	p->exit_reason = EXIT_REASON_PTC_G;	1260	p->exit_reason = EXIT_REASON_PTC_G;
1261		1261
1262	p->u.ptc_g_data.rr = vcpu_get_rr(vcpu, va);	1262	p->u.ptc_g_data.rr = vcpu_get_rr(vcpu, va);
1263	p->u.ptc_g_data.vaddr = va;	1263	p->u.ptc_g_data.vaddr = va;
1264	p->u.ptc_g_data.ps = ps;	1264	p->u.ptc_g_data.ps = ps;
1265	vmm_transition(vcpu);	1265	vmm_transition(vcpu);
1266	/* Do Local Purge Here*/	1266	/* Do Local Purge Here*/
1267	vcpu_ptc_l(vcpu, va, ps);	1267	vcpu_ptc_l(vcpu, va, ps);
1268	local_irq_restore(psr);	1268	local_irq_restore(psr);
1269	}	1269	}
1270		1270
1271		1271
1272	void vcpu_ptc_g(struct kvm_vcpu *vcpu, u64 va, u64 ps)	1272	void vcpu_ptc_g(struct kvm_vcpu *vcpu, u64 va, u64 ps)
1273	{	1273	{
1274	vcpu_ptc_ga(vcpu, va, ps);	1274	vcpu_ptc_ga(vcpu, va, ps);
1275	}	1275	}
1276		1276
1277	void kvm_ptc_e(struct kvm_vcpu *vcpu, INST64 inst)	1277	void kvm_ptc_e(struct kvm_vcpu *vcpu, INST64 inst)
1278	{	1278	{
1279	unsigned long ifa;	1279	unsigned long ifa;
1280		1280
1281	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1281	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1282	vcpu_ptc_e(vcpu, ifa);	1282	vcpu_ptc_e(vcpu, ifa);
1283	}	1283	}
1284		1284
1285	void kvm_ptc_g(struct kvm_vcpu *vcpu, INST64 inst)	1285	void kvm_ptc_g(struct kvm_vcpu *vcpu, INST64 inst)
1286	{	1286	{
1287	unsigned long ifa, itir;	1287	unsigned long ifa, itir;
1288		1288
1289	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1289	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1290	itir = vcpu_get_gr(vcpu, inst.M45.r2);	1290	itir = vcpu_get_gr(vcpu, inst.M45.r2);
1291	vcpu_ptc_g(vcpu, ifa, itir_ps(itir));	1291	vcpu_ptc_g(vcpu, ifa, itir_ps(itir));
1292	}	1292	}
1293		1293
1294	void kvm_ptc_ga(struct kvm_vcpu *vcpu, INST64 inst)	1294	void kvm_ptc_ga(struct kvm_vcpu *vcpu, INST64 inst)
1295	{	1295	{
1296	unsigned long ifa, itir;	1296	unsigned long ifa, itir;
1297		1297
1298	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1298	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1299	itir = vcpu_get_gr(vcpu, inst.M45.r2);	1299	itir = vcpu_get_gr(vcpu, inst.M45.r2);
1300	vcpu_ptc_ga(vcpu, ifa, itir_ps(itir));	1300	vcpu_ptc_ga(vcpu, ifa, itir_ps(itir));
1301	}	1301	}
1302		1302
1303	void kvm_ptc_l(struct kvm_vcpu *vcpu, INST64 inst)	1303	void kvm_ptc_l(struct kvm_vcpu *vcpu, INST64 inst)
1304	{	1304	{
1305	unsigned long ifa, itir;	1305	unsigned long ifa, itir;
1306		1306
1307	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1307	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1308	itir = vcpu_get_gr(vcpu, inst.M45.r2);	1308	itir = vcpu_get_gr(vcpu, inst.M45.r2);
1309	vcpu_ptc_l(vcpu, ifa, itir_ps(itir));	1309	vcpu_ptc_l(vcpu, ifa, itir_ps(itir));
1310	}	1310	}
1311		1311
1312	void kvm_ptr_d(struct kvm_vcpu *vcpu, INST64 inst)	1312	void kvm_ptr_d(struct kvm_vcpu *vcpu, INST64 inst)
1313	{	1313	{
1314	unsigned long ifa, itir;	1314	unsigned long ifa, itir;
1315		1315
1316	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1316	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1317	itir = vcpu_get_gr(vcpu, inst.M45.r2);	1317	itir = vcpu_get_gr(vcpu, inst.M45.r2);
1318	vcpu_ptr_d(vcpu, ifa, itir_ps(itir));	1318	vcpu_ptr_d(vcpu, ifa, itir_ps(itir));
1319	}	1319	}
1320		1320
1321	void kvm_ptr_i(struct kvm_vcpu *vcpu, INST64 inst)	1321	void kvm_ptr_i(struct kvm_vcpu *vcpu, INST64 inst)
1322	{	1322	{
1323	unsigned long ifa, itir;	1323	unsigned long ifa, itir;
1324		1324
1325	ifa = vcpu_get_gr(vcpu, inst.M45.r3);	1325	ifa = vcpu_get_gr(vcpu, inst.M45.r3);
1326	itir = vcpu_get_gr(vcpu, inst.M45.r2);	1326	itir = vcpu_get_gr(vcpu, inst.M45.r2);
1327	vcpu_ptr_i(vcpu, ifa, itir_ps(itir));	1327	vcpu_ptr_i(vcpu, ifa, itir_ps(itir));
1328	}	1328	}
1329		1329
1330	void kvm_itr_d(struct kvm_vcpu *vcpu, INST64 inst)	1330	void kvm_itr_d(struct kvm_vcpu *vcpu, INST64 inst)
1331	{	1331	{
1332	unsigned long itir, ifa, pte, slot;	1332	unsigned long itir, ifa, pte, slot;
1333		1333
1334	slot = vcpu_get_gr(vcpu, inst.M45.r3);	1334	slot = vcpu_get_gr(vcpu, inst.M45.r3);
1335	pte = vcpu_get_gr(vcpu, inst.M45.r2);	1335	pte = vcpu_get_gr(vcpu, inst.M45.r2);
1336	itir = vcpu_get_itir(vcpu);	1336	itir = vcpu_get_itir(vcpu);
1337	ifa = vcpu_get_ifa(vcpu);	1337	ifa = vcpu_get_ifa(vcpu);
1338	vcpu_itr_d(vcpu, slot, pte, itir, ifa);	1338	vcpu_itr_d(vcpu, slot, pte, itir, ifa);
1339	}	1339	}
1340		1340
1341		1341
1342		1342
1343	void kvm_itr_i(struct kvm_vcpu *vcpu, INST64 inst)	1343	void kvm_itr_i(struct kvm_vcpu *vcpu, INST64 inst)
1344	{	1344	{
1345	unsigned long itir, ifa, pte, slot;	1345	unsigned long itir, ifa, pte, slot;
1346		1346
1347	slot = vcpu_get_gr(vcpu, inst.M45.r3);	1347	slot = vcpu_get_gr(vcpu, inst.M45.r3);
1348	pte = vcpu_get_gr(vcpu, inst.M45.r2);	1348	pte = vcpu_get_gr(vcpu, inst.M45.r2);
1349	itir = vcpu_get_itir(vcpu);	1349	itir = vcpu_get_itir(vcpu);
1350	ifa = vcpu_get_ifa(vcpu);	1350	ifa = vcpu_get_ifa(vcpu);
1351	vcpu_itr_i(vcpu, slot, pte, itir, ifa);	1351	vcpu_itr_i(vcpu, slot, pte, itir, ifa);
1352	}	1352	}
1353		1353
1354	void kvm_itc_d(struct kvm_vcpu *vcpu, INST64 inst)	1354	void kvm_itc_d(struct kvm_vcpu *vcpu, INST64 inst)
1355	{	1355	{
1356	unsigned long itir, ifa, pte;	1356	unsigned long itir, ifa, pte;
1357		1357
1358	itir = vcpu_get_itir(vcpu);	1358	itir = vcpu_get_itir(vcpu);
1359	ifa = vcpu_get_ifa(vcpu);	1359	ifa = vcpu_get_ifa(vcpu);
1360	pte = vcpu_get_gr(vcpu, inst.M45.r2);	1360	pte = vcpu_get_gr(vcpu, inst.M45.r2);
1361	vcpu_itc_d(vcpu, pte, itir, ifa);	1361	vcpu_itc_d(vcpu, pte, itir, ifa);
1362	}	1362	}
1363		1363
1364	void kvm_itc_i(struct kvm_vcpu *vcpu, INST64 inst)	1364	void kvm_itc_i(struct kvm_vcpu *vcpu, INST64 inst)
1365	{	1365	{
1366	unsigned long itir, ifa, pte;	1366	unsigned long itir, ifa, pte;
1367		1367
1368	itir = vcpu_get_itir(vcpu);	1368	itir = vcpu_get_itir(vcpu);
1369	ifa = vcpu_get_ifa(vcpu);	1369	ifa = vcpu_get_ifa(vcpu);
1370	pte = vcpu_get_gr(vcpu, inst.M45.r2);	1370	pte = vcpu_get_gr(vcpu, inst.M45.r2);
1371	vcpu_itc_i(vcpu, pte, itir, ifa);	1371	vcpu_itc_i(vcpu, pte, itir, ifa);
1372	}	1372	}
1373		1373
1374	/*************************************	1374	/*************************************
1375	* Moves to semi-privileged registers	1375	* Moves to semi-privileged registers
1376	*************************************/	1376	*************************************/
1377		1377
1378	void kvm_mov_to_ar_imm(struct kvm_vcpu *vcpu, INST64 inst)	1378	void kvm_mov_to_ar_imm(struct kvm_vcpu *vcpu, INST64 inst)
1379	{	1379	{
1380	unsigned long imm;	1380	unsigned long imm;
1381		1381
1382	if (inst.M30.s)	1382	if (inst.M30.s)
1383	imm = -inst.M30.imm;	1383	imm = -inst.M30.imm;
1384	else	1384	else
1385	imm = inst.M30.imm;	1385	imm = inst.M30.imm;
1386		1386
1387	vcpu_set_itc(vcpu, imm);	1387	vcpu_set_itc(vcpu, imm);
1388	}	1388	}
1389		1389
1390	void kvm_mov_to_ar_reg(struct kvm_vcpu *vcpu, INST64 inst)	1390	void kvm_mov_to_ar_reg(struct kvm_vcpu *vcpu, INST64 inst)
1391	{	1391	{
1392	unsigned long r2;	1392	unsigned long r2;
1393		1393
1394	r2 = vcpu_get_gr(vcpu, inst.M29.r2);	1394	r2 = vcpu_get_gr(vcpu, inst.M29.r2);
1395	vcpu_set_itc(vcpu, r2);	1395	vcpu_set_itc(vcpu, r2);
1396	}	1396	}
1397		1397
1398	void kvm_mov_from_ar_reg(struct kvm_vcpu *vcpu, INST64 inst)	1398	void kvm_mov_from_ar_reg(struct kvm_vcpu *vcpu, INST64 inst)
1399	{	1399	{
1400	unsigned long r1;	1400	unsigned long r1;
1401		1401
1402	r1 = vcpu_get_itc(vcpu);	1402	r1 = vcpu_get_itc(vcpu);
1403	vcpu_set_gr(vcpu, inst.M31.r1, r1, 0);	1403	vcpu_set_gr(vcpu, inst.M31.r1, r1, 0);
1404	}	1404	}
1405		1405
1406	/**************************************************************************	1406	/**************************************************************************
1407	struct kvm_vcpu protection key register access routines	1407	struct kvm_vcpu protection key register access routines
1408	**************************************************************************/	1408	**************************************************************************/
1409		1409
1410	unsigned long vcpu_get_pkr(struct kvm_vcpu *vcpu, unsigned long reg)	1410	unsigned long vcpu_get_pkr(struct kvm_vcpu *vcpu, unsigned long reg)
1411	{	1411	{
1412	return ((unsigned long)ia64_get_pkr(reg));	1412	return ((unsigned long)ia64_get_pkr(reg));
1413	}	1413	}
1414		1414
1415	void vcpu_set_pkr(struct kvm_vcpu *vcpu, unsigned long reg, unsigned long val)	1415	void vcpu_set_pkr(struct kvm_vcpu *vcpu, unsigned long reg, unsigned long val)
1416	{	1416	{
1417	ia64_set_pkr(reg, val);	1417	ia64_set_pkr(reg, val);
1418	}	1418	}
1419		1419
1420	/********************************	1420	/********************************
1421	* Moves to privileged registers	1421	* Moves to privileged registers
1422	********************************/	1422	********************************/
1423	unsigned long vcpu_set_rr(struct kvm_vcpu *vcpu, unsigned long reg,	1423	unsigned long vcpu_set_rr(struct kvm_vcpu *vcpu, unsigned long reg,
1424	unsigned long val)	1424	unsigned long val)
1425	{	1425	{
1426	union ia64_rr oldrr, newrr;	1426	union ia64_rr oldrr, newrr;
1427	unsigned long rrval;	1427	unsigned long rrval;
1428	struct exit_ctl_data *p = &vcpu->arch.exit_data;	1428	struct exit_ctl_data *p = &vcpu->arch.exit_data;
1429	unsigned long psr;	1429	unsigned long psr;
1430		1430
1431	oldrr.val = vcpu_get_rr(vcpu, reg);	1431	oldrr.val = vcpu_get_rr(vcpu, reg);
1432	newrr.val = val;	1432	newrr.val = val;
1433	vcpu->arch.vrr[reg >> VRN_SHIFT] = val;	1433	vcpu->arch.vrr[reg >> VRN_SHIFT] = val;
1434		1434
1435	switch ((unsigned long)(reg >> VRN_SHIFT)) {	1435	switch ((unsigned long)(reg >> VRN_SHIFT)) {
1436	case VRN6:	1436	case VRN6:
1437	vcpu->arch.vmm_rr = vrrtomrr(val);	1437	vcpu->arch.vmm_rr = vrrtomrr(val);
1438	local_irq_save(psr);	1438	local_irq_save(psr);
1439	p->exit_reason = EXIT_REASON_SWITCH_RR6;	1439	p->exit_reason = EXIT_REASON_SWITCH_RR6;
1440	vmm_transition(vcpu);	1440	vmm_transition(vcpu);
1441	local_irq_restore(psr);	1441	local_irq_restore(psr);
1442	break;	1442	break;
1443	case VRN4:	1443	case VRN4:
1444	rrval = vrrtomrr(val);	1444	rrval = vrrtomrr(val);
1445	vcpu->arch.metaphysical_saved_rr4 = rrval;	1445	vcpu->arch.metaphysical_saved_rr4 = rrval;
1446	if (!is_physical_mode(vcpu))	1446	if (!is_physical_mode(vcpu))
1447	ia64_set_rr(reg, rrval);	1447	ia64_set_rr(reg, rrval);
1448	break;	1448	break;
1449	case VRN0:	1449	case VRN0:
1450	rrval = vrrtomrr(val);	1450	rrval = vrrtomrr(val);
1451	vcpu->arch.metaphysical_saved_rr0 = rrval;	1451	vcpu->arch.metaphysical_saved_rr0 = rrval;
1452	if (!is_physical_mode(vcpu))	1452	if (!is_physical_mode(vcpu))
1453	ia64_set_rr(reg, rrval);	1453	ia64_set_rr(reg, rrval);
1454	break;	1454	break;
1455	default:	1455	default:
1456	ia64_set_rr(reg, vrrtomrr(val));	1456	ia64_set_rr(reg, vrrtomrr(val));
1457	break;	1457	break;
1458	}	1458	}
1459		1459
1460	return (IA64_NO_FAULT);	1460	return (IA64_NO_FAULT);
1461	}	1461	}
1462		1462
1463	void kvm_mov_to_rr(struct kvm_vcpu *vcpu, INST64 inst)	1463	void kvm_mov_to_rr(struct kvm_vcpu *vcpu, INST64 inst)
1464	{	1464	{
1465	unsigned long r3, r2;	1465	unsigned long r3, r2;
1466		1466
1467	r3 = vcpu_get_gr(vcpu, inst.M42.r3);	1467	r3 = vcpu_get_gr(vcpu, inst.M42.r3);
1468	r2 = vcpu_get_gr(vcpu, inst.M42.r2);	1468	r2 = vcpu_get_gr(vcpu, inst.M42.r2);
1469	vcpu_set_rr(vcpu, r3, r2);	1469	vcpu_set_rr(vcpu, r3, r2);
1470	}	1470	}
1471		1471
1472	void kvm_mov_to_dbr(struct kvm_vcpu *vcpu, INST64 inst)	1472	void kvm_mov_to_dbr(struct kvm_vcpu *vcpu, INST64 inst)
1473	{	1473	{
1474	}	1474	}
1475		1475
1476	void kvm_mov_to_ibr(struct kvm_vcpu *vcpu, INST64 inst)	1476	void kvm_mov_to_ibr(struct kvm_vcpu *vcpu, INST64 inst)
1477	{	1477	{
1478	}	1478	}
1479		1479
1480	void kvm_mov_to_pmc(struct kvm_vcpu *vcpu, INST64 inst)	1480	void kvm_mov_to_pmc(struct kvm_vcpu *vcpu, INST64 inst)
1481	{	1481	{
1482	unsigned long r3, r2;	1482	unsigned long r3, r2;
1483		1483
1484	r3 = vcpu_get_gr(vcpu, inst.M42.r3);	1484	r3 = vcpu_get_gr(vcpu, inst.M42.r3);
1485	r2 = vcpu_get_gr(vcpu, inst.M42.r2);	1485	r2 = vcpu_get_gr(vcpu, inst.M42.r2);
1486	vcpu_set_pmc(vcpu, r3, r2);	1486	vcpu_set_pmc(vcpu, r3, r2);
1487	}	1487	}
1488		1488
1489	void kvm_mov_to_pmd(struct kvm_vcpu *vcpu, INST64 inst)	1489	void kvm_mov_to_pmd(struct kvm_vcpu *vcpu, INST64 inst)
1490	{	1490	{
1491	unsigned long r3, r2;	1491	unsigned long r3, r2;
1492		1492
1493	r3 = vcpu_get_gr(vcpu, inst.M42.r3);	1493	r3 = vcpu_get_gr(vcpu, inst.M42.r3);
1494	r2 = vcpu_get_gr(vcpu, inst.M42.r2);	1494	r2 = vcpu_get_gr(vcpu, inst.M42.r2);
1495	vcpu_set_pmd(vcpu, r3, r2);	1495	vcpu_set_pmd(vcpu, r3, r2);
1496	}	1496	}
1497		1497
1498	void kvm_mov_to_pkr(struct kvm_vcpu *vcpu, INST64 inst)	1498	void kvm_mov_to_pkr(struct kvm_vcpu *vcpu, INST64 inst)
1499	{	1499	{
1500	u64 r3, r2;	1500	u64 r3, r2;
1501		1501
1502	r3 = vcpu_get_gr(vcpu, inst.M42.r3);	1502	r3 = vcpu_get_gr(vcpu, inst.M42.r3);
1503	r2 = vcpu_get_gr(vcpu, inst.M42.r2);	1503	r2 = vcpu_get_gr(vcpu, inst.M42.r2);
1504	vcpu_set_pkr(vcpu, r3, r2);	1504	vcpu_set_pkr(vcpu, r3, r2);
1505	}	1505	}
1506		1506
1507	void kvm_mov_from_rr(struct kvm_vcpu *vcpu, INST64 inst)	1507	void kvm_mov_from_rr(struct kvm_vcpu *vcpu, INST64 inst)
1508	{	1508	{
1509	unsigned long r3, r1;	1509	unsigned long r3, r1;
1510		1510
1511	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1511	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1512	r1 = vcpu_get_rr(vcpu, r3);	1512	r1 = vcpu_get_rr(vcpu, r3);
1513	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1513	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1514	}	1514	}
1515		1515
1516	void kvm_mov_from_pkr(struct kvm_vcpu *vcpu, INST64 inst)	1516	void kvm_mov_from_pkr(struct kvm_vcpu *vcpu, INST64 inst)
1517	{	1517	{
1518	unsigned long r3, r1;	1518	unsigned long r3, r1;
1519		1519
1520	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1520	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1521	r1 = vcpu_get_pkr(vcpu, r3);	1521	r1 = vcpu_get_pkr(vcpu, r3);
1522	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1522	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1523	}	1523	}
1524		1524
1525	void kvm_mov_from_dbr(struct kvm_vcpu *vcpu, INST64 inst)	1525	void kvm_mov_from_dbr(struct kvm_vcpu *vcpu, INST64 inst)
1526	{	1526	{
1527	unsigned long r3, r1;	1527	unsigned long r3, r1;
1528		1528
1529	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1529	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1530	r1 = vcpu_get_dbr(vcpu, r3);	1530	r1 = vcpu_get_dbr(vcpu, r3);
1531	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1531	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1532	}	1532	}
1533		1533
1534	void kvm_mov_from_ibr(struct kvm_vcpu *vcpu, INST64 inst)	1534	void kvm_mov_from_ibr(struct kvm_vcpu *vcpu, INST64 inst)
1535	{	1535	{
1536	unsigned long r3, r1;	1536	unsigned long r3, r1;
1537		1537
1538	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1538	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1539	r1 = vcpu_get_ibr(vcpu, r3);	1539	r1 = vcpu_get_ibr(vcpu, r3);
1540	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1540	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1541	}	1541	}
1542		1542
1543	void kvm_mov_from_pmc(struct kvm_vcpu *vcpu, INST64 inst)	1543	void kvm_mov_from_pmc(struct kvm_vcpu *vcpu, INST64 inst)
1544	{	1544	{
1545	unsigned long r3, r1;	1545	unsigned long r3, r1;
1546		1546
1547	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1547	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1548	r1 = vcpu_get_pmc(vcpu, r3);	1548	r1 = vcpu_get_pmc(vcpu, r3);
1549	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1549	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1550	}	1550	}
1551		1551
1552	unsigned long vcpu_get_cpuid(struct kvm_vcpu *vcpu, unsigned long reg)	1552	unsigned long vcpu_get_cpuid(struct kvm_vcpu *vcpu, unsigned long reg)
1553	{	1553	{
1554	/* FIXME: This could get called as a result of a rsvd-reg fault */	1554	/* FIXME: This could get called as a result of a rsvd-reg fault */
1555	if (reg > (ia64_get_cpuid(3) & 0xff))	1555	if (reg > (ia64_get_cpuid(3) & 0xff))
1556	return 0;	1556	return 0;
1557	else	1557	else
1558	return ia64_get_cpuid(reg);	1558	return ia64_get_cpuid(reg);
1559	}	1559	}
1560		1560
1561	void kvm_mov_from_cpuid(struct kvm_vcpu *vcpu, INST64 inst)	1561	void kvm_mov_from_cpuid(struct kvm_vcpu *vcpu, INST64 inst)
1562	{	1562	{
1563	unsigned long r3, r1;	1563	unsigned long r3, r1;
1564		1564
1565	r3 = vcpu_get_gr(vcpu, inst.M43.r3);	1565	r3 = vcpu_get_gr(vcpu, inst.M43.r3);
1566	r1 = vcpu_get_cpuid(vcpu, r3);	1566	r1 = vcpu_get_cpuid(vcpu, r3);
1567	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);	1567	vcpu_set_gr(vcpu, inst.M43.r1, r1, 0);
1568	}	1568	}
1569		1569
1570	void vcpu_set_tpr(struct kvm_vcpu *vcpu, unsigned long val)	1570	void vcpu_set_tpr(struct kvm_vcpu *vcpu, unsigned long val)
1571	{	1571	{
1572	VCPU(vcpu, tpr) = val;	1572	VCPU(vcpu, tpr) = val;
1573	vcpu->arch.irq_check = 1;	1573	vcpu->arch.irq_check = 1;
1574	}	1574	}
1575		1575
1576	unsigned long kvm_mov_to_cr(struct kvm_vcpu *vcpu, INST64 inst)	1576	unsigned long kvm_mov_to_cr(struct kvm_vcpu *vcpu, INST64 inst)
1577	{	1577	{
1578	unsigned long r2;	1578	unsigned long r2;
1579		1579
1580	r2 = vcpu_get_gr(vcpu, inst.M32.r2);	1580	r2 = vcpu_get_gr(vcpu, inst.M32.r2);
1581	VCPU(vcpu, vcr[inst.M32.cr3]) = r2;	1581	VCPU(vcpu, vcr[inst.M32.cr3]) = r2;
1582		1582
1583	switch (inst.M32.cr3) {	1583	switch (inst.M32.cr3) {
1584	case 0:	1584	case 0:
1585	vcpu_set_dcr(vcpu, r2);	1585	vcpu_set_dcr(vcpu, r2);
1586	break;	1586	break;
1587	case 1:	1587	case 1:
1588	vcpu_set_itm(vcpu, r2);	1588	vcpu_set_itm(vcpu, r2);
1589	break;	1589	break;
1590	case 66:	1590	case 66:
1591	vcpu_set_tpr(vcpu, r2);	1591	vcpu_set_tpr(vcpu, r2);
1592	break;	1592	break;
1593	case 67:	1593	case 67:
1594	vcpu_set_eoi(vcpu, r2);	1594	vcpu_set_eoi(vcpu, r2);
1595	break;	1595	break;
1596	default:	1596	default:
1597	break;	1597	break;
1598	}	1598	}
1599		1599
1600	return 0;	1600	return 0;
1601	}	1601	}
1602		1602
1603	unsigned long kvm_mov_from_cr(struct kvm_vcpu *vcpu, INST64 inst)	1603	unsigned long kvm_mov_from_cr(struct kvm_vcpu *vcpu, INST64 inst)
1604	{	1604	{
1605	unsigned long tgt = inst.M33.r1;	1605	unsigned long tgt = inst.M33.r1;
1606	unsigned long val;	1606	unsigned long val;
1607		1607
1608	switch (inst.M33.cr3) {	1608	switch (inst.M33.cr3) {
1609	case 65:	1609	case 65:
1610	val = vcpu_get_ivr(vcpu);	1610	val = vcpu_get_ivr(vcpu);
1611	vcpu_set_gr(vcpu, tgt, val, 0);	1611	vcpu_set_gr(vcpu, tgt, val, 0);
1612	break;	1612	break;
1613		1613
1614	case 67:	1614	case 67:
1615	vcpu_set_gr(vcpu, tgt, 0L, 0);	1615	vcpu_set_gr(vcpu, tgt, 0L, 0);
1616	break;	1616	break;
1617	default:	1617	default:
1618	val = VCPU(vcpu, vcr[inst.M33.cr3]);	1618	val = VCPU(vcpu, vcr[inst.M33.cr3]);
1619	vcpu_set_gr(vcpu, tgt, val, 0);	1619	vcpu_set_gr(vcpu, tgt, val, 0);
1620	break;	1620	break;
1621	}	1621	}
1622		1622
1623	return 0;	1623	return 0;
1624	}	1624	}
1625		1625
1626	void vcpu_set_psr(struct kvm_vcpu *vcpu, unsigned long val)	1626	void vcpu_set_psr(struct kvm_vcpu *vcpu, unsigned long val)
1627	{	1627	{
1628		1628
1629	unsigned long mask;	1629	unsigned long mask;
1630	struct kvm_pt_regs *regs;	1630	struct kvm_pt_regs *regs;
1631	struct ia64_psr old_psr, new_psr;	1631	struct ia64_psr old_psr, new_psr;
1632		1632
1633	old_psr = (struct ia64_psr )&VCPU(vcpu, vpsr);	1633	old_psr = (struct ia64_psr )&VCPU(vcpu, vpsr);
1634		1634
1635	regs = vcpu_regs(vcpu);	1635	regs = vcpu_regs(vcpu);
1636	/* We only support guest as:	1636	/* We only support guest as:
1637	* vpsr.pk = 0	1637	* vpsr.pk = 0
1638	* vpsr.is = 0	1638	* vpsr.is = 0
1639	* Otherwise panic	1639	* Otherwise panic
1640	*/	1640	*/
1641	if (val & (IA64_PSR_PK \| IA64_PSR_IS \| IA64_PSR_VM))	1641	if (val & (IA64_PSR_PK \| IA64_PSR_IS \| IA64_PSR_VM))
1642	panic_vm(vcpu, "Only support guests with vpsr.pk =0 \	1642	panic_vm(vcpu, "Only support guests with vpsr.pk =0 \
1643	& vpsr.is=0\n");	1643	& vpsr.is=0\n");
1644		1644
1645	/*	1645	/*
1646	* For those IA64_PSR bits: id/da/dd/ss/ed/ia	1646	* For those IA64_PSR bits: id/da/dd/ss/ed/ia
1647	* Since these bits will become 0, after success execution of each	1647	* Since these bits will become 0, after success execution of each
1648	* instruction, we will change set them to mIA64_PSR	1648	* instruction, we will change set them to mIA64_PSR
1649	*/	1649	*/
1650	VCPU(vcpu, vpsr) = val	1650	VCPU(vcpu, vpsr) = val
1651	& (~(IA64_PSR_ID \| IA64_PSR_DA \| IA64_PSR_DD \|	1651	& (~(IA64_PSR_ID \| IA64_PSR_DA \| IA64_PSR_DD \|
1652	IA64_PSR_SS \| IA64_PSR_ED \| IA64_PSR_IA));	1652	IA64_PSR_SS \| IA64_PSR_ED \| IA64_PSR_IA));
1653		1653
1654	if (!old_psr.i && (val & IA64_PSR_I)) {	1654	if (!old_psr.i && (val & IA64_PSR_I)) {
1655	/* vpsr.i 0->1 */	1655	/* vpsr.i 0->1 */
1656	vcpu->arch.irq_check = 1;	1656	vcpu->arch.irq_check = 1;
1657	}	1657	}
1658	new_psr = (struct ia64_psr )&VCPU(vcpu, vpsr);	1658	new_psr = (struct ia64_psr )&VCPU(vcpu, vpsr);
1659		1659
1660	/*	1660	/*
1661	* All vIA64_PSR bits shall go to mPSR (v->tf->tf_special.psr)	1661	* All vIA64_PSR bits shall go to mPSR (v->tf->tf_special.psr)
1662	* , except for the following bits:	1662	* , except for the following bits:
1663	* ic/i/dt/si/rt/mc/it/bn/vm	1663	* ic/i/dt/si/rt/mc/it/bn/vm
1664	*/	1664	*/
1665	mask = IA64_PSR_IC + IA64_PSR_I + IA64_PSR_DT + IA64_PSR_SI +	1665	mask = IA64_PSR_IC + IA64_PSR_I + IA64_PSR_DT + IA64_PSR_SI +
1666	IA64_PSR_RT + IA64_PSR_MC + IA64_PSR_IT + IA64_PSR_BN +	1666	IA64_PSR_RT + IA64_PSR_MC + IA64_PSR_IT + IA64_PSR_BN +
1667	IA64_PSR_VM;	1667	IA64_PSR_VM;
1668		1668
1669	regs->cr_ipsr = (regs->cr_ipsr & mask) \| (val & (~mask));	1669	regs->cr_ipsr = (regs->cr_ipsr & mask) \| (val & (~mask));
1670		1670
1671	check_mm_mode_switch(vcpu, old_psr, new_psr);	1671	check_mm_mode_switch(vcpu, old_psr, new_psr);
1672		1672
1673	return ;	1673	return ;
1674	}	1674	}
1675		1675
1676	unsigned long vcpu_cover(struct kvm_vcpu *vcpu)	1676	unsigned long vcpu_cover(struct kvm_vcpu *vcpu)
1677	{	1677	{
1678	struct ia64_psr vpsr;	1678	struct ia64_psr vpsr;
1679		1679
1680	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1680	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1681	vpsr = (struct ia64_psr )&VCPU(vcpu, vpsr);	1681	vpsr = (struct ia64_psr )&VCPU(vcpu, vpsr);
1682		1682
1683	if (!vpsr.ic)	1683	if (!vpsr.ic)
1684	VCPU(vcpu, ifs) = regs->cr_ifs;	1684	VCPU(vcpu, ifs) = regs->cr_ifs;
1685	regs->cr_ifs = IA64_IFS_V;	1685	regs->cr_ifs = IA64_IFS_V;
1686	return (IA64_NO_FAULT);	1686	return (IA64_NO_FAULT);
1687	}	1687	}
1688		1688
1689		1689
1690		1690
1691	/**************************************************************************	1691	/**************************************************************************
1692	VCPU banked general register access routines	1692	VCPU banked general register access routines
1693	**************************************************************************/	1693	**************************************************************************/
1694	#define vcpu_bsw0_unat(i, b0unat, b1unat, runat, VMM_PT_REGS_R16_SLOT) \	1694	#define vcpu_bsw0_unat(i, b0unat, b1unat, runat, VMM_PT_REGS_R16_SLOT) \
1695	do { \	1695	do { \
1696	__asm__ __volatile__ ( \	1696	__asm__ __volatile__ ( \
1697	";;extr.u %0 = %3,%6,16;;\n" \	1697	";;extr.u %0 = %3,%6,16;;\n" \
1698	"dep %1 = %0, %1, 0, 16;;\n" \	1698	"dep %1 = %0, %1, 0, 16;;\n" \
1699	"st8 [%4] = %1\n" \	1699	"st8 [%4] = %1\n" \
1700	"extr.u %0 = %2, 16, 16;;\n" \	1700	"extr.u %0 = %2, 16, 16;;\n" \
1701	"dep %3 = %0, %3, %6, 16;;\n" \	1701	"dep %3 = %0, %3, %6, 16;;\n" \
1702	"st8 [%5] = %3\n" \	1702	"st8 [%5] = %3\n" \
1703	::"r"(i), "r"(b1unat), "r"(b0unat), \	1703	::"r"(i), "r"(b1unat), "r"(b0unat), \
1704	"r"(*runat), "r"(b1unat), "r"(runat), \	1704	"r"(*runat), "r"(b1unat), "r"(runat), \
1705	"i"(VMM_PT_REGS_R16_SLOT) : "memory"); \	1705	"i"(VMM_PT_REGS_R16_SLOT) : "memory"); \
1706	} while (0)	1706	} while (0)
1707		1707
1708	void vcpu_bsw0(struct kvm_vcpu *vcpu)	1708	void vcpu_bsw0(struct kvm_vcpu *vcpu)
1709	{	1709	{
1710	unsigned long i;	1710	unsigned long i;
1711		1711
1712	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1712	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1713	unsigned long *r = &regs->r16;	1713	unsigned long *r = &regs->r16;
1714	unsigned long *b0 = &VCPU(vcpu, vbgr[0]);	1714	unsigned long *b0 = &VCPU(vcpu, vbgr[0]);
1715	unsigned long *b1 = &VCPU(vcpu, vgr[0]);	1715	unsigned long *b1 = &VCPU(vcpu, vgr[0]);
1716	unsigned long *runat = &regs->eml_unat;	1716	unsigned long *runat = &regs->eml_unat;
1717	unsigned long *b0unat = &VCPU(vcpu, vbnat);	1717	unsigned long *b0unat = &VCPU(vcpu, vbnat);
1718	unsigned long *b1unat = &VCPU(vcpu, vnat);	1718	unsigned long *b1unat = &VCPU(vcpu, vnat);
1719		1719
1720		1720
1721	if (VCPU(vcpu, vpsr) & IA64_PSR_BN) {	1721	if (VCPU(vcpu, vpsr) & IA64_PSR_BN) {
1722	for (i = 0; i < 16; i++) {	1722	for (i = 0; i < 16; i++) {
1723	b1++ = r;	1723	b1++ = r;
1724	r++ = b0++;	1724	r++ = b0++;
1725	}	1725	}
1726	vcpu_bsw0_unat(i, b0unat, b1unat, runat,	1726	vcpu_bsw0_unat(i, b0unat, b1unat, runat,
1727	VMM_PT_REGS_R16_SLOT);	1727	VMM_PT_REGS_R16_SLOT);
1728	VCPU(vcpu, vpsr) &= ~IA64_PSR_BN;	1728	VCPU(vcpu, vpsr) &= ~IA64_PSR_BN;
1729	}	1729	}
1730	}	1730	}
1731		1731
1732	#define vcpu_bsw1_unat(i, b0unat, b1unat, runat, VMM_PT_REGS_R16_SLOT) \	1732	#define vcpu_bsw1_unat(i, b0unat, b1unat, runat, VMM_PT_REGS_R16_SLOT) \
1733	do { \	1733	do { \
1734	__asm__ __volatile__ (";;extr.u %0 = %3, %6, 16;;\n" \	1734	__asm__ __volatile__ (";;extr.u %0 = %3, %6, 16;;\n" \
1735	"dep %1 = %0, %1, 16, 16;;\n" \	1735	"dep %1 = %0, %1, 16, 16;;\n" \
1736	"st8 [%4] = %1\n" \	1736	"st8 [%4] = %1\n" \
1737	"extr.u %0 = %2, 0, 16;;\n" \	1737	"extr.u %0 = %2, 0, 16;;\n" \
1738	"dep %3 = %0, %3, %6, 16;;\n" \	1738	"dep %3 = %0, %3, %6, 16;;\n" \
1739	"st8 [%5] = %3\n" \	1739	"st8 [%5] = %3\n" \
1740	::"r"(i), "r"(b0unat), "r"(b1unat), \	1740	::"r"(i), "r"(b0unat), "r"(b1unat), \
1741	"r"(*runat), "r"(b0unat), "r"(runat), \	1741	"r"(*runat), "r"(b0unat), "r"(runat), \
1742	"i"(VMM_PT_REGS_R16_SLOT) : "memory"); \	1742	"i"(VMM_PT_REGS_R16_SLOT) : "memory"); \
1743	} while (0)	1743	} while (0)
1744		1744
1745	void vcpu_bsw1(struct kvm_vcpu *vcpu)	1745	void vcpu_bsw1(struct kvm_vcpu *vcpu)
1746	{	1746	{
1747	unsigned long i;	1747	unsigned long i;
1748	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1748	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1749	unsigned long *r = &regs->r16;	1749	unsigned long *r = &regs->r16;
1750	unsigned long *b0 = &VCPU(vcpu, vbgr[0]);	1750	unsigned long *b0 = &VCPU(vcpu, vbgr[0]);
1751	unsigned long *b1 = &VCPU(vcpu, vgr[0]);	1751	unsigned long *b1 = &VCPU(vcpu, vgr[0]);
1752	unsigned long *runat = &regs->eml_unat;	1752	unsigned long *runat = &regs->eml_unat;
1753	unsigned long *b0unat = &VCPU(vcpu, vbnat);	1753	unsigned long *b0unat = &VCPU(vcpu, vbnat);
1754	unsigned long *b1unat = &VCPU(vcpu, vnat);	1754	unsigned long *b1unat = &VCPU(vcpu, vnat);
1755		1755
1756	if (!(VCPU(vcpu, vpsr) & IA64_PSR_BN)) {	1756	if (!(VCPU(vcpu, vpsr) & IA64_PSR_BN)) {
1757	for (i = 0; i < 16; i++) {	1757	for (i = 0; i < 16; i++) {
1758	b0++ = r;	1758	b0++ = r;
1759	r++ = b1++;	1759	r++ = b1++;
1760	}	1760	}
1761	vcpu_bsw1_unat(i, b0unat, b1unat, runat,	1761	vcpu_bsw1_unat(i, b0unat, b1unat, runat,
1762	VMM_PT_REGS_R16_SLOT);	1762	VMM_PT_REGS_R16_SLOT);
1763	VCPU(vcpu, vpsr) \|= IA64_PSR_BN;	1763	VCPU(vcpu, vpsr) \|= IA64_PSR_BN;
1764	}	1764	}
1765	}	1765	}
1766		1766
1767	void vcpu_rfi(struct kvm_vcpu *vcpu)	1767	void vcpu_rfi(struct kvm_vcpu *vcpu)
1768	{	1768	{
1769	unsigned long ifs, psr;	1769	unsigned long ifs, psr;
1770	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1770	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1771		1771
1772	psr = VCPU(vcpu, ipsr);	1772	psr = VCPU(vcpu, ipsr);
1773	if (psr & IA64_PSR_BN)	1773	if (psr & IA64_PSR_BN)
1774	vcpu_bsw1(vcpu);	1774	vcpu_bsw1(vcpu);
1775	else	1775	else
1776	vcpu_bsw0(vcpu);	1776	vcpu_bsw0(vcpu);
1777	vcpu_set_psr(vcpu, psr);	1777	vcpu_set_psr(vcpu, psr);
1778	ifs = VCPU(vcpu, ifs);	1778	ifs = VCPU(vcpu, ifs);
1779	if (ifs >> 63)	1779	if (ifs >> 63)
1780	regs->cr_ifs = ifs;	1780	regs->cr_ifs = ifs;
1781	regs->cr_iip = VCPU(vcpu, iip);	1781	regs->cr_iip = VCPU(vcpu, iip);
1782	}	1782	}
1783		1783
1784	/*	1784	/*
1785	VPSR can't keep track of below bits of guest PSR	1785	VPSR can't keep track of below bits of guest PSR
1786	This function gets guest PSR	1786	This function gets guest PSR
1787	*/	1787	*/
1788		1788
1789	unsigned long vcpu_get_psr(struct kvm_vcpu *vcpu)	1789	unsigned long vcpu_get_psr(struct kvm_vcpu *vcpu)
1790	{	1790	{
1791	unsigned long mask;	1791	unsigned long mask;
1792	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1792	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1793		1793
1794	mask = IA64_PSR_BE \| IA64_PSR_UP \| IA64_PSR_AC \| IA64_PSR_MFL \|	1794	mask = IA64_PSR_BE \| IA64_PSR_UP \| IA64_PSR_AC \| IA64_PSR_MFL \|
1795	IA64_PSR_MFH \| IA64_PSR_CPL \| IA64_PSR_RI;	1795	IA64_PSR_MFH \| IA64_PSR_CPL \| IA64_PSR_RI;
1796	return (VCPU(vcpu, vpsr) & ~mask) \| (regs->cr_ipsr & mask);	1796	return (VCPU(vcpu, vpsr) & ~mask) \| (regs->cr_ipsr & mask);
1797	}	1797	}
1798		1798
1799	void kvm_rsm(struct kvm_vcpu *vcpu, INST64 inst)	1799	void kvm_rsm(struct kvm_vcpu *vcpu, INST64 inst)
1800	{	1800	{
1801	unsigned long vpsr;	1801	unsigned long vpsr;
1802	unsigned long imm24 = (inst.M44.i<<23) \| (inst.M44.i2<<21)	1802	unsigned long imm24 = (inst.M44.i<<23) \| (inst.M44.i2<<21)
1803	\| inst.M44.imm;	1803	\| inst.M44.imm;
1804		1804
1805	vpsr = vcpu_get_psr(vcpu);	1805	vpsr = vcpu_get_psr(vcpu);
1806	vpsr &= (~imm24);	1806	vpsr &= (~imm24);
1807	vcpu_set_psr(vcpu, vpsr);	1807	vcpu_set_psr(vcpu, vpsr);
1808	}	1808	}
1809		1809
1810	void kvm_ssm(struct kvm_vcpu *vcpu, INST64 inst)	1810	void kvm_ssm(struct kvm_vcpu *vcpu, INST64 inst)
1811	{	1811	{
1812	unsigned long vpsr;	1812	unsigned long vpsr;
1813	unsigned long imm24 = (inst.M44.i << 23) \| (inst.M44.i2 << 21)	1813	unsigned long imm24 = (inst.M44.i << 23) \| (inst.M44.i2 << 21)
1814	\| inst.M44.imm;	1814	\| inst.M44.imm;
1815		1815
1816	vpsr = vcpu_get_psr(vcpu);	1816	vpsr = vcpu_get_psr(vcpu);
1817	vpsr \|= imm24;	1817	vpsr \|= imm24;
1818	vcpu_set_psr(vcpu, vpsr);	1818	vcpu_set_psr(vcpu, vpsr);
1819	}	1819	}
1820		1820
1821	/* Generate Mask	1821	/* Generate Mask
1822	* Parameter:	1822	* Parameter:
1823	* bit -- starting bit	1823	* bit -- starting bit
1824	* len -- how many bits	1824	* len -- how many bits
1825	*/	1825	*/
1826	#define MASK(bit,len) \	1826	#define MASK(bit,len) \
1827	({ \	1827	({ \
1828	__u64 ret; \	1828	__u64 ret; \
1829	\	1829	\
1830	__asm __volatile("dep %0=-1, r0, %1, %2"\	1830	__asm __volatile("dep %0=-1, r0, %1, %2"\
1831	: "=r" (ret): \	1831	: "=r" (ret): \
1832	"M" (bit), \	1832	"M" (bit), \
1833	"M" (len)); \	1833	"M" (len)); \
1834	ret; \	1834	ret; \
1835	})	1835	})
1836		1836
1837	void vcpu_set_psr_l(struct kvm_vcpu *vcpu, unsigned long val)	1837	void vcpu_set_psr_l(struct kvm_vcpu *vcpu, unsigned long val)
1838	{	1838	{
1839	val = (val & MASK(0, 32)) \| (vcpu_get_psr(vcpu) & MASK(32, 32));	1839	val = (val & MASK(0, 32)) \| (vcpu_get_psr(vcpu) & MASK(32, 32));
1840	vcpu_set_psr(vcpu, val);	1840	vcpu_set_psr(vcpu, val);
1841	}	1841	}
1842		1842
1843	void kvm_mov_to_psr(struct kvm_vcpu *vcpu, INST64 inst)	1843	void kvm_mov_to_psr(struct kvm_vcpu *vcpu, INST64 inst)
1844	{	1844	{
1845	unsigned long val;	1845	unsigned long val;
1846		1846
1847	val = vcpu_get_gr(vcpu, inst.M35.r2);	1847	val = vcpu_get_gr(vcpu, inst.M35.r2);
1848	vcpu_set_psr_l(vcpu, val);	1848	vcpu_set_psr_l(vcpu, val);
1849	}	1849	}
1850		1850
1851	void kvm_mov_from_psr(struct kvm_vcpu *vcpu, INST64 inst)	1851	void kvm_mov_from_psr(struct kvm_vcpu *vcpu, INST64 inst)
1852	{	1852	{
1853	unsigned long val;	1853	unsigned long val;
1854		1854
1855	val = vcpu_get_psr(vcpu);	1855	val = vcpu_get_psr(vcpu);
1856	val = (val & MASK(0, 32)) \| (val & MASK(35, 2));	1856	val = (val & MASK(0, 32)) \| (val & MASK(35, 2));
1857	vcpu_set_gr(vcpu, inst.M33.r1, val, 0);	1857	vcpu_set_gr(vcpu, inst.M33.r1, val, 0);
1858	}	1858	}
1859		1859
1860	void vcpu_increment_iip(struct kvm_vcpu *vcpu)	1860	void vcpu_increment_iip(struct kvm_vcpu *vcpu)
1861	{	1861	{
1862	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1862	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1863	struct ia64_psr ipsr = (struct ia64_psr )&regs->cr_ipsr;	1863	struct ia64_psr ipsr = (struct ia64_psr )&regs->cr_ipsr;
1864	if (ipsr->ri == 2) {	1864	if (ipsr->ri == 2) {
1865	ipsr->ri = 0;	1865	ipsr->ri = 0;
1866	regs->cr_iip += 16;	1866	regs->cr_iip += 16;
1867	} else	1867	} else
1868	ipsr->ri++;	1868	ipsr->ri++;
1869	}	1869	}
1870		1870
1871	void vcpu_decrement_iip(struct kvm_vcpu *vcpu)	1871	void vcpu_decrement_iip(struct kvm_vcpu *vcpu)
1872	{	1872	{
1873	struct kvm_pt_regs *regs = vcpu_regs(vcpu);	1873	struct kvm_pt_regs *regs = vcpu_regs(vcpu);
1874	struct ia64_psr ipsr = (struct ia64_psr )&regs->cr_ipsr;	1874	struct ia64_psr ipsr = (struct ia64_psr )&regs->cr_ipsr;
1875		1875
1876	if (ipsr->ri == 0) {	1876	if (ipsr->ri == 0) {
1877	ipsr->ri = 2;	1877	ipsr->ri = 2;
1878	regs->cr_iip -= 16;	1878	regs->cr_iip -= 16;
1879	} else	1879	} else
1880	ipsr->ri--;	1880	ipsr->ri--;
1881	}	1881	}
1882		1882
1883	/** Emulate a privileged operation.	1883	/** Emulate a privileged operation.
1884	*	1884	*
1885	*	1885	*
1886	* @param vcpu virtual cpu	1886	* @param vcpu virtual cpu
1887	* @cause the reason cause virtualization fault	1887	* @cause the reason cause virtualization fault
1888	* @opcode the instruction code which cause virtualization fault	1888	* @opcode the instruction code which cause virtualization fault
1889	*/	1889	*/
1890		1890
1891	void kvm_emulate(struct kvm_vcpu vcpu, struct kvm_pt_regs regs)	1891	void kvm_emulate(struct kvm_vcpu vcpu, struct kvm_pt_regs regs)
1892	{	1892	{
1893	unsigned long status, cause, opcode ;	1893	unsigned long status, cause, opcode ;
1894	INST64 inst;	1894	INST64 inst;
1895		1895
1896	status = IA64_NO_FAULT;	1896	status = IA64_NO_FAULT;
1897	cause = VMX(vcpu, cause);	1897	cause = VMX(vcpu, cause);
1898	opcode = VMX(vcpu, opcode);	1898	opcode = VMX(vcpu, opcode);
1899	inst.inst = opcode;	1899	inst.inst = opcode;
1900	/*	1900	/*
1901	* Switch to actual virtual rid in rr0 and rr4,	1901	* Switch to actual virtual rid in rr0 and rr4,
1902	* which is required by some tlb related instructions.	1902	* which is required by some tlb related instructions.
1903	*/	1903	*/
1904	prepare_if_physical_mode(vcpu);	1904	prepare_if_physical_mode(vcpu);
1905		1905
1906	switch (cause) {	1906	switch (cause) {
1907	case EVENT_RSM:	1907	case EVENT_RSM:
1908	kvm_rsm(vcpu, inst);	1908	kvm_rsm(vcpu, inst);
1909	break;	1909	break;
1910	case EVENT_SSM:	1910	case EVENT_SSM:
1911	kvm_ssm(vcpu, inst);	1911	kvm_ssm(vcpu, inst);
1912	break;	1912	break;
1913	case EVENT_MOV_TO_PSR:	1913	case EVENT_MOV_TO_PSR:
1914	kvm_mov_to_psr(vcpu, inst);	1914	kvm_mov_to_psr(vcpu, inst);
1915	break;	1915	break;
1916	case EVENT_MOV_FROM_PSR:	1916	case EVENT_MOV_FROM_PSR:
1917	kvm_mov_from_psr(vcpu, inst);	1917	kvm_mov_from_psr(vcpu, inst);
1918	break;	1918	break;
1919	case EVENT_MOV_FROM_CR:	1919	case EVENT_MOV_FROM_CR:
1920	kvm_mov_from_cr(vcpu, inst);	1920	kvm_mov_from_cr(vcpu, inst);
1921	break;	1921	break;
1922	case EVENT_MOV_TO_CR:	1922	case EVENT_MOV_TO_CR:
1923	kvm_mov_to_cr(vcpu, inst);	1923	kvm_mov_to_cr(vcpu, inst);
1924	break;	1924	break;
1925	case EVENT_BSW_0:	1925	case EVENT_BSW_0:
1926	vcpu_bsw0(vcpu);	1926	vcpu_bsw0(vcpu);
1927	break;	1927	break;
1928	case EVENT_BSW_1:	1928	case EVENT_BSW_1:
1929	vcpu_bsw1(vcpu);	1929	vcpu_bsw1(vcpu);
1930	break;	1930	break;
1931	case EVENT_COVER:	1931	case EVENT_COVER:
1932	vcpu_cover(vcpu);	1932	vcpu_cover(vcpu);
1933	break;	1933	break;
1934	case EVENT_RFI:	1934	case EVENT_RFI:
1935	vcpu_rfi(vcpu);	1935	vcpu_rfi(vcpu);
1936	break;	1936	break;
1937	case EVENT_ITR_D:	1937	case EVENT_ITR_D:
1938	kvm_itr_d(vcpu, inst);	1938	kvm_itr_d(vcpu, inst);
1939	break;	1939	break;
1940	case EVENT_ITR_I:	1940	case EVENT_ITR_I:
1941	kvm_itr_i(vcpu, inst);	1941	kvm_itr_i(vcpu, inst);
1942	break;	1942	break;
1943	case EVENT_PTR_D:	1943	case EVENT_PTR_D:
1944	kvm_ptr_d(vcpu, inst);	1944	kvm_ptr_d(vcpu, inst);
1945	break;	1945	break;
1946	case EVENT_PTR_I:	1946	case EVENT_PTR_I:
1947	kvm_ptr_i(vcpu, inst);	1947	kvm_ptr_i(vcpu, inst);
1948	break;	1948	break;
1949	case EVENT_ITC_D:	1949	case EVENT_ITC_D:
1950	kvm_itc_d(vcpu, inst);	1950	kvm_itc_d(vcpu, inst);
1951	break;	1951	break;
1952	case EVENT_ITC_I:	1952	case EVENT_ITC_I:
1953	kvm_itc_i(vcpu, inst);	1953	kvm_itc_i(vcpu, inst);
1954	break;	1954	break;
1955	case EVENT_PTC_L:	1955	case EVENT_PTC_L:
1956	kvm_ptc_l(vcpu, inst);	1956	kvm_ptc_l(vcpu, inst);
1957	break;	1957	break;
1958	case EVENT_PTC_G:	1958	case EVENT_PTC_G:
1959	kvm_ptc_g(vcpu, inst);	1959	kvm_ptc_g(vcpu, inst);
1960	break;	1960	break;
1961	case EVENT_PTC_GA:	1961	case EVENT_PTC_GA:
1962	kvm_ptc_ga(vcpu, inst);	1962	kvm_ptc_ga(vcpu, inst);
1963	break;	1963	break;
1964	case EVENT_PTC_E:	1964	case EVENT_PTC_E:
1965	kvm_ptc_e(vcpu, inst);	1965	kvm_ptc_e(vcpu, inst);
1966	break;	1966	break;
1967	case EVENT_MOV_TO_RR:	1967	case EVENT_MOV_TO_RR:
1968	kvm_mov_to_rr(vcpu, inst);	1968	kvm_mov_to_rr(vcpu, inst);
1969	break;	1969	break;
1970	case EVENT_MOV_FROM_RR:	1970	case EVENT_MOV_FROM_RR:
1971	kvm_mov_from_rr(vcpu, inst);	1971	kvm_mov_from_rr(vcpu, inst);
1972	break;	1972	break;
1973	case EVENT_THASH:	1973	case EVENT_THASH:
1974	kvm_thash(vcpu, inst);	1974	kvm_thash(vcpu, inst);
1975	break;	1975	break;
1976	case EVENT_TTAG:	1976	case EVENT_TTAG:
1977	kvm_ttag(vcpu, inst);	1977	kvm_ttag(vcpu, inst);
1978	break;	1978	break;
1979	case EVENT_TPA:	1979	case EVENT_TPA:
1980	status = kvm_tpa(vcpu, inst);	1980	status = kvm_tpa(vcpu, inst);
1981	break;	1981	break;
1982	case EVENT_TAK:	1982	case EVENT_TAK:
1983	kvm_tak(vcpu, inst);	1983	kvm_tak(vcpu, inst);
1984	break;	1984	break;
1985	case EVENT_MOV_TO_AR_IMM:	1985	case EVENT_MOV_TO_AR_IMM:
1986	kvm_mov_to_ar_imm(vcpu, inst);	1986	kvm_mov_to_ar_imm(vcpu, inst);
1987	break;	1987	break;
1988	case EVENT_MOV_TO_AR:	1988	case EVENT_MOV_TO_AR:
1989	kvm_mov_to_ar_reg(vcpu, inst);	1989	kvm_mov_to_ar_reg(vcpu, inst);
1990	break;	1990	break;
1991	case EVENT_MOV_FROM_AR:	1991	case EVENT_MOV_FROM_AR:
1992	kvm_mov_from_ar_reg(vcpu, inst);	1992	kvm_mov_from_ar_reg(vcpu, inst);
1993	break;	1993	break;
1994	case EVENT_MOV_TO_DBR:	1994	case EVENT_MOV_TO_DBR:
1995	kvm_mov_to_dbr(vcpu, inst);	1995	kvm_mov_to_dbr(vcpu, inst);
1996	break;	1996	break;
1997	case EVENT_MOV_TO_IBR:	1997	case EVENT_MOV_TO_IBR:
1998	kvm_mov_to_ibr(vcpu, inst);	1998	kvm_mov_to_ibr(vcpu, inst);
1999	break;	1999	break;
2000	case EVENT_MOV_TO_PMC:	2000	case EVENT_MOV_TO_PMC:
2001	kvm_mov_to_pmc(vcpu, inst);	2001	kvm_mov_to_pmc(vcpu, inst);
2002	break;	2002	break;
2003	case EVENT_MOV_TO_PMD:	2003	case EVENT_MOV_TO_PMD:
2004	kvm_mov_to_pmd(vcpu, inst);	2004	kvm_mov_to_pmd(vcpu, inst);
2005	break;	2005	break;
2006	case EVENT_MOV_TO_PKR:	2006	case EVENT_MOV_TO_PKR:
2007	kvm_mov_to_pkr(vcpu, inst);	2007	kvm_mov_to_pkr(vcpu, inst);
2008	break;	2008	break;
2009	case EVENT_MOV_FROM_DBR:	2009	case EVENT_MOV_FROM_DBR:
2010	kvm_mov_from_dbr(vcpu, inst);	2010	kvm_mov_from_dbr(vcpu, inst);
2011	break;	2011	break;
2012	case EVENT_MOV_FROM_IBR:	2012	case EVENT_MOV_FROM_IBR:
2013	kvm_mov_from_ibr(vcpu, inst);	2013	kvm_mov_from_ibr(vcpu, inst);
2014	break;	2014	break;
2015	case EVENT_MOV_FROM_PMC:	2015	case EVENT_MOV_FROM_PMC:
2016	kvm_mov_from_pmc(vcpu, inst);	2016	kvm_mov_from_pmc(vcpu, inst);
2017	break;	2017	break;
2018	case EVENT_MOV_FROM_PKR:	2018	case EVENT_MOV_FROM_PKR:
2019	kvm_mov_from_pkr(vcpu, inst);	2019	kvm_mov_from_pkr(vcpu, inst);
2020	break;	2020	break;
2021	case EVENT_MOV_FROM_CPUID:	2021	case EVENT_MOV_FROM_CPUID:
2022	kvm_mov_from_cpuid(vcpu, inst);	2022	kvm_mov_from_cpuid(vcpu, inst);
2023	break;	2023	break;
2024	case EVENT_VMSW:	2024	case EVENT_VMSW:
2025	status = IA64_FAULT;	2025	status = IA64_FAULT;
2026	break;	2026	break;
2027	default:	2027	default:
2028	break;	2028	break;
2029	};	2029	};
2030	/Assume all status is NO_FAULT ?/	2030	/Assume all status is NO_FAULT ?/
2031	if (status == IA64_NO_FAULT && cause != EVENT_RFI)	2031	if (status == IA64_NO_FAULT && cause != EVENT_RFI)
2032	vcpu_increment_iip(vcpu);	2032	vcpu_increment_iip(vcpu);
2033		2033
2034	recover_if_physical_mode(vcpu);	2034	recover_if_physical_mode(vcpu);
2035	}	2035	}
2036		2036
2037	void init_vcpu(struct kvm_vcpu *vcpu)	2037	void init_vcpu(struct kvm_vcpu *vcpu)
2038	{	2038	{
2039	int i;	2039	int i;
2040		2040
2041	vcpu->arch.mode_flags = GUEST_IN_PHY;	2041	vcpu->arch.mode_flags = GUEST_IN_PHY;
2042	VMX(vcpu, vrr[0]) = 0x38;	2042	VMX(vcpu, vrr[0]) = 0x38;
2043	VMX(vcpu, vrr[1]) = 0x38;	2043	VMX(vcpu, vrr[1]) = 0x38;
2044	VMX(vcpu, vrr[2]) = 0x38;	2044	VMX(vcpu, vrr[2]) = 0x38;
2045	VMX(vcpu, vrr[3]) = 0x38;	2045	VMX(vcpu, vrr[3]) = 0x38;
2046	VMX(vcpu, vrr[4]) = 0x38;	2046	VMX(vcpu, vrr[4]) = 0x38;
2047	VMX(vcpu, vrr[5]) = 0x38;	2047	VMX(vcpu, vrr[5]) = 0x38;
2048	VMX(vcpu, vrr[6]) = 0x38;	2048	VMX(vcpu, vrr[6]) = 0x38;
2049	VMX(vcpu, vrr[7]) = 0x38;	2049	VMX(vcpu, vrr[7]) = 0x38;
2050	VCPU(vcpu, vpsr) = IA64_PSR_BN;	2050	VCPU(vcpu, vpsr) = IA64_PSR_BN;
2051	VCPU(vcpu, dcr) = 0;	2051	VCPU(vcpu, dcr) = 0;
2052	/* pta.size must not be 0. The minimum is 15 (32k) */	2052	/* pta.size must not be 0. The minimum is 15 (32k) */
2053	VCPU(vcpu, pta) = 15 << 2;	2053	VCPU(vcpu, pta) = 15 << 2;
2054	VCPU(vcpu, itv) = 0x10000;	2054	VCPU(vcpu, itv) = 0x10000;
2055	VCPU(vcpu, itm) = 0;	2055	VCPU(vcpu, itm) = 0;
2056	VMX(vcpu, last_itc) = 0;	2056	VMX(vcpu, last_itc) = 0;
2057		2057
2058	VCPU(vcpu, lid) = VCPU_LID(vcpu);	2058	VCPU(vcpu, lid) = VCPU_LID(vcpu);
2059	VCPU(vcpu, ivr) = 0;	2059	VCPU(vcpu, ivr) = 0;
2060	VCPU(vcpu, tpr) = 0x10000;	2060	VCPU(vcpu, tpr) = 0x10000;
2061	VCPU(vcpu, eoi) = 0;	2061	VCPU(vcpu, eoi) = 0;
2062	VCPU(vcpu, irr[0]) = 0;	2062	VCPU(vcpu, irr[0]) = 0;
2063	VCPU(vcpu, irr[1]) = 0;	2063	VCPU(vcpu, irr[1]) = 0;
2064	VCPU(vcpu, irr[2]) = 0;	2064	VCPU(vcpu, irr[2]) = 0;
2065	VCPU(vcpu, irr[3]) = 0;	2065	VCPU(vcpu, irr[3]) = 0;
2066	VCPU(vcpu, pmv) = 0x10000;	2066	VCPU(vcpu, pmv) = 0x10000;
2067	VCPU(vcpu, cmcv) = 0x10000;	2067	VCPU(vcpu, cmcv) = 0x10000;
2068	VCPU(vcpu, lrr0) = 0x10000; /* default reset value? */	2068	VCPU(vcpu, lrr0) = 0x10000; /* default reset value? */
2069	VCPU(vcpu, lrr1) = 0x10000; /* default reset value? */	2069	VCPU(vcpu, lrr1) = 0x10000; /* default reset value? */
2070	update_vhpi(vcpu, NULL_VECTOR);	2070	update_vhpi(vcpu, NULL_VECTOR);
2071	VLSAPIC_XTP(vcpu) = 0x80; /* disabled */	2071	VLSAPIC_XTP(vcpu) = 0x80; /* disabled */
2072		2072
2073	for (i = 0; i < 4; i++)	2073	for (i = 0; i < 4; i++)
2074	VLSAPIC_INSVC(vcpu, i) = 0;	2074	VLSAPIC_INSVC(vcpu, i) = 0;
2075	}	2075	}
2076		2076
2077	void kvm_init_all_rr(struct kvm_vcpu *vcpu)	2077	void kvm_init_all_rr(struct kvm_vcpu *vcpu)
2078	{	2078	{
2079	unsigned long psr;	2079	unsigned long psr;
2080		2080
2081	local_irq_save(psr);	2081	local_irq_save(psr);
2082		2082
2083	/* WARNING: not allow co-exist of both virtual mode and physical	2083	/* WARNING: not allow co-exist of both virtual mode and physical
2084	* mode in same region	2084	* mode in same region
2085	*/	2085	*/
2086		2086
2087	vcpu->arch.metaphysical_saved_rr0 = vrrtomrr(VMX(vcpu, vrr[VRN0]));	2087	vcpu->arch.metaphysical_saved_rr0 = vrrtomrr(VMX(vcpu, vrr[VRN0]));
2088	vcpu->arch.metaphysical_saved_rr4 = vrrtomrr(VMX(vcpu, vrr[VRN4]));	2088	vcpu->arch.metaphysical_saved_rr4 = vrrtomrr(VMX(vcpu, vrr[VRN4]));
2089		2089
2090	if (is_physical_mode(vcpu)) {	2090	if (is_physical_mode(vcpu)) {
2091	if (vcpu->arch.mode_flags & GUEST_PHY_EMUL)	2091	if (vcpu->arch.mode_flags & GUEST_PHY_EMUL)
2092	panic_vm(vcpu, "Machine Status conflicts!\n");	2092	panic_vm(vcpu, "Machine Status conflicts!\n");
2093		2093
2094	ia64_set_rr((VRN0 << VRN_SHIFT), vcpu->arch.metaphysical_rr0);	2094	ia64_set_rr((VRN0 << VRN_SHIFT), vcpu->arch.metaphysical_rr0);
2095	ia64_dv_serialize_data();	2095	ia64_dv_serialize_data();
2096	ia64_set_rr((VRN4 << VRN_SHIFT), vcpu->arch.metaphysical_rr4);	2096	ia64_set_rr((VRN4 << VRN_SHIFT), vcpu->arch.metaphysical_rr4);
2097	ia64_dv_serialize_data();	2097	ia64_dv_serialize_data();
2098	} else {	2098	} else {
2099	ia64_set_rr((VRN0 << VRN_SHIFT),	2099	ia64_set_rr((VRN0 << VRN_SHIFT),
2100	vcpu->arch.metaphysical_saved_rr0);	2100	vcpu->arch.metaphysical_saved_rr0);
2101	ia64_dv_serialize_data();	2101	ia64_dv_serialize_data();
2102	ia64_set_rr((VRN4 << VRN_SHIFT),	2102	ia64_set_rr((VRN4 << VRN_SHIFT),
2103	vcpu->arch.metaphysical_saved_rr4);	2103	vcpu->arch.metaphysical_saved_rr4);
2104	ia64_dv_serialize_data();	2104	ia64_dv_serialize_data();
2105	}	2105	}
2106	ia64_set_rr((VRN1 << VRN_SHIFT),	2106	ia64_set_rr((VRN1 << VRN_SHIFT),
2107	vrrtomrr(VMX(vcpu, vrr[VRN1])));	2107	vrrtomrr(VMX(vcpu, vrr[VRN1])));
2108	ia64_dv_serialize_data();	2108	ia64_dv_serialize_data();
2109	ia64_set_rr((VRN2 << VRN_SHIFT),	2109	ia64_set_rr((VRN2 << VRN_SHIFT),
2110	vrrtomrr(VMX(vcpu, vrr[VRN2])));	2110	vrrtomrr(VMX(vcpu, vrr[VRN2])));
2111	ia64_dv_serialize_data();	2111	ia64_dv_serialize_data();
2112	ia64_set_rr((VRN3 << VRN_SHIFT),	2112	ia64_set_rr((VRN3 << VRN_SHIFT),
2113	vrrtomrr(VMX(vcpu, vrr[VRN3])));	2113	vrrtomrr(VMX(vcpu, vrr[VRN3])));
2114	ia64_dv_serialize_data();	2114	ia64_dv_serialize_data();
2115	ia64_set_rr((VRN5 << VRN_SHIFT),	2115	ia64_set_rr((VRN5 << VRN_SHIFT),
2116	vrrtomrr(VMX(vcpu, vrr[VRN5])));	2116	vrrtomrr(VMX(vcpu, vrr[VRN5])));
2117	ia64_dv_serialize_data();	2117	ia64_dv_serialize_data();
2118	ia64_set_rr((VRN7 << VRN_SHIFT),	2118	ia64_set_rr((VRN7 << VRN_SHIFT),
2119	vrrtomrr(VMX(vcpu, vrr[VRN7])));	2119	vrrtomrr(VMX(vcpu, vrr[VRN7])));
2120	ia64_dv_serialize_data();	2120	ia64_dv_serialize_data();
2121	ia64_srlz_d();	2121	ia64_srlz_d();
2122	ia64_set_psr(psr);	2122	ia64_set_psr(psr);
2123	}	2123	}
2124		2124
2125	int vmm_entry(void)	2125	int vmm_entry(void)
2126	{	2126	{
2127	struct kvm_vcpu *v;	2127	struct kvm_vcpu *v;
2128	v = current_vcpu;	2128	v = current_vcpu;
2129		2129
2130	ia64_call_vsa(PAL_VPS_RESTORE, (unsigned long)v->arch.vpd,	2130	ia64_call_vsa(PAL_VPS_RESTORE, (unsigned long)v->arch.vpd,
2131	0, 0, 0, 0, 0, 0);	2131	0, 0, 0, 0, 0, 0);
2132	kvm_init_vtlb(v);	2132	kvm_init_vtlb(v);
2133	kvm_init_vhpt(v);	2133	kvm_init_vhpt(v);
2134	init_vcpu(v);	2134	init_vcpu(v);
2135	kvm_init_all_rr(v);	2135	kvm_init_all_rr(v);
2136	vmm_reset_entry();	2136	vmm_reset_entry();
2137		2137
2138	return 0;	2138	return 0;
2139	}	2139	}
2140		2140
2141	static void kvm_show_registers(struct kvm_pt_regs *regs)	2141	static void kvm_show_registers(struct kvm_pt_regs *regs)
2142	{	2142	{
2143	unsigned long ip = regs->cr_iip + ia64_psr(regs)->ri;	2143	unsigned long ip = regs->cr_iip + ia64_psr(regs)->ri;
2144		2144
2145	struct kvm_vcpu *vcpu = current_vcpu;	2145	struct kvm_vcpu *vcpu = current_vcpu;
2146	if (vcpu != NULL)	2146	if (vcpu != NULL)
2147	printk("vcpu 0x%p vcpu %d\n",	2147	printk("vcpu 0x%p vcpu %d\n",
2148	vcpu, vcpu->vcpu_id);	2148	vcpu, vcpu->vcpu_id);
2149		2149
2150	printk("psr : %016lx ifs : %016lx ip : [<%016lx>]\n",	2150	printk("psr : %016lx ifs : %016lx ip : [<%016lx>]\n",
2151	regs->cr_ipsr, regs->cr_ifs, ip);	2151	regs->cr_ipsr, regs->cr_ifs, ip);
2152		2152
2153	printk("unat: %016lx pfs : %016lx rsc : %016lx\n",	2153	printk("unat: %016lx pfs : %016lx rsc : %016lx\n",
2154	regs->ar_unat, regs->ar_pfs, regs->ar_rsc);	2154	regs->ar_unat, regs->ar_pfs, regs->ar_rsc);
2155	printk("rnat: %016lx bspstore: %016lx pr : %016lx\n",	2155	printk("rnat: %016lx bspstore: %016lx pr : %016lx\n",
2156	regs->ar_rnat, regs->ar_bspstore, regs->pr);	2156	regs->ar_rnat, regs->ar_bspstore, regs->pr);
2157	printk("ldrs: %016lx ccv : %016lx fpsr: %016lx\n",	2157	printk("ldrs: %016lx ccv : %016lx fpsr: %016lx\n",
2158	regs->loadrs, regs->ar_ccv, regs->ar_fpsr);	2158	regs->loadrs, regs->ar_ccv, regs->ar_fpsr);
2159	printk("csd : %016lx ssd : %016lx\n", regs->ar_csd, regs->ar_ssd);	2159	printk("csd : %016lx ssd : %016lx\n", regs->ar_csd, regs->ar_ssd);
2160	printk("b0 : %016lx b6 : %016lx b7 : %016lx\n", regs->b0,	2160	printk("b0 : %016lx b6 : %016lx b7 : %016lx\n", regs->b0,
2161	regs->b6, regs->b7);	2161	regs->b6, regs->b7);
2162	printk("f6 : %05lx%016lx f7 : %05lx%016lx\n",	2162	printk("f6 : %05lx%016lx f7 : %05lx%016lx\n",
2163	regs->f6.u.bits[1], regs->f6.u.bits[0],	2163	regs->f6.u.bits[1], regs->f6.u.bits[0],
2164	regs->f7.u.bits[1], regs->f7.u.bits[0]);	2164	regs->f7.u.bits[1], regs->f7.u.bits[0]);
2165	printk("f8 : %05lx%016lx f9 : %05lx%016lx\n",	2165	printk("f8 : %05lx%016lx f9 : %05lx%016lx\n",
2166	regs->f8.u.bits[1], regs->f8.u.bits[0],	2166	regs->f8.u.bits[1], regs->f8.u.bits[0],
2167	regs->f9.u.bits[1], regs->f9.u.bits[0]);	2167	regs->f9.u.bits[1], regs->f9.u.bits[0]);
2168	printk("f10 : %05lx%016lx f11 : %05lx%016lx\n",	2168	printk("f10 : %05lx%016lx f11 : %05lx%016lx\n",
2169	regs->f10.u.bits[1], regs->f10.u.bits[0],	2169	regs->f10.u.bits[1], regs->f10.u.bits[0],
2170	regs->f11.u.bits[1], regs->f11.u.bits[0]);	2170	regs->f11.u.bits[1], regs->f11.u.bits[0]);
2171		2171
2172	printk("r1 : %016lx r2 : %016lx r3 : %016lx\n", regs->r1,	2172	printk("r1 : %016lx r2 : %016lx r3 : %016lx\n", regs->r1,
2173	regs->r2, regs->r3);	2173	regs->r2, regs->r3);
2174	printk("r8 : %016lx r9 : %016lx r10 : %016lx\n", regs->r8,	2174	printk("r8 : %016lx r9 : %016lx r10 : %016lx\n", regs->r8,
2175	regs->r9, regs->r10);	2175	regs->r9, regs->r10);
2176	printk("r11 : %016lx r12 : %016lx r13 : %016lx\n", regs->r11,	2176	printk("r11 : %016lx r12 : %016lx r13 : %016lx\n", regs->r11,
2177	regs->r12, regs->r13);	2177	regs->r12, regs->r13);
2178	printk("r14 : %016lx r15 : %016lx r16 : %016lx\n", regs->r14,	2178	printk("r14 : %016lx r15 : %016lx r16 : %016lx\n", regs->r14,
2179	regs->r15, regs->r16);	2179	regs->r15, regs->r16);
2180	printk("r17 : %016lx r18 : %016lx r19 : %016lx\n", regs->r17,	2180	printk("r17 : %016lx r18 : %016lx r19 : %016lx\n", regs->r17,
2181	regs->r18, regs->r19);	2181	regs->r18, regs->r19);
2182	printk("r20 : %016lx r21 : %016lx r22 : %016lx\n", regs->r20,	2182	printk("r20 : %016lx r21 : %016lx r22 : %016lx\n", regs->r20,
2183	regs->r21, regs->r22);	2183	regs->r21, regs->r22);
2184	printk("r23 : %016lx r24 : %016lx r25 : %016lx\n", regs->r23,	2184	printk("r23 : %016lx r24 : %016lx r25 : %016lx\n", regs->r23,
2185	regs->r24, regs->r25);	2185	regs->r24, regs->r25);
2186	printk("r26 : %016lx r27 : %016lx r28 : %016lx\n", regs->r26,	2186	printk("r26 : %016lx r27 : %016lx r28 : %016lx\n", regs->r26,
2187	regs->r27, regs->r28);	2187	regs->r27, regs->r28);
2188	printk("r29 : %016lx r30 : %016lx r31 : %016lx\n", regs->r29,	2188	printk("r29 : %016lx r30 : %016lx r31 : %016lx\n", regs->r29,
2189	regs->r30, regs->r31);	2189	regs->r30, regs->r31);
2190		2190
2191	}	2191	}
2192		2192
2193	void panic_vm(struct kvm_vcpu v, const char fmt, ...)	2193	void panic_vm(struct kvm_vcpu v, const char fmt, ...)
2194	{	2194	{
2195	va_list args;	2195	va_list args;
2196	char buf[256];	2196	char buf[256];
2197		2197
2198	struct kvm_pt_regs *regs = vcpu_regs(v);	2198	struct kvm_pt_regs *regs = vcpu_regs(v);
2199	struct exit_ctl_data *p = &v->arch.exit_data;	2199	struct exit_ctl_data *p = &v->arch.exit_data;
2200	va_start(args, fmt);	2200	va_start(args, fmt);
2201	vsnprintf(buf, sizeof(buf), fmt, args);	2201	vsnprintf(buf, sizeof(buf), fmt, args);
2202	va_end(args);	2202	va_end(args);
2203	printk(buf);	2203	printk(buf);
2204	kvm_show_registers(regs);	2204	kvm_show_registers(regs);
2205	p->exit_reason = EXIT_REASON_VM_PANIC;	2205	p->exit_reason = EXIT_REASON_VM_PANIC;
2206	vmm_transition(v);	2206	vmm_transition(v);
2207	/Never to return/	2207	/Never to return/
2208	while (1);	2208	while (1);
2209	}	2209	}
2210		2210

arch/x86/kvm/Kconfig

Diff comments View file @ 73880c8

 #
 # KVM configuration
 #
 source "virt/kvm/Kconfig"
 menuconfig VIRTUALIZATION
 	bool "Virtualization"
 	depends on HAVE_KVM || X86
 	default y
 	---help---
 	  Say Y here to get to see options for using your Linux host to run other
 	  operating systems inside virtual machines (guests).
 	  This option alone does not add any kernel code.
 	  If you say N, all options in this submenu will be skipped and disabled.
 if VIRTUALIZATION
 config KVM
 	tristate "Kernel-based Virtual Machine (KVM) support"
 	depends on HAVE_KVM
 	# for device assignment:
 	depends on PCI
 	select PREEMPT_NOTIFIERS
 	select MMU_NOTIFIER
 	select ANON_INODES
 	select HAVE_KVM_IRQCHIP
 	select HAVE_KVM_EVENTFD
+	select KVM_APIC_ARCHITECTURE
 	---help---
 	  Support hosting fully virtualized guest machines using hardware
 	  virtualization extensions.  You will need a fairly recent
 	  processor equipped with virtualization extensions. You will also
 	  need to select one or more of the processor modules below.
 	  This module provides access to the hardware capabilities through
 	  a character device node named /dev/kvm.
 	  To compile this as a module, choose M here: the module
 	  will be called kvm.
 	  If unsure, say N.
 config KVM_INTEL
 	tristate "KVM for Intel processors support"
 	depends on KVM
 	---help---
 	  Provides support for KVM on Intel processors equipped with the VT
 	  extensions.
 	  To compile this as a module, choose M here: the module
 	  will be called kvm-intel.
 config KVM_AMD
 	tristate "KVM for AMD processors support"
 	depends on KVM
 	---help---
 	  Provides support for KVM on AMD processors equipped with the AMD-V
 	  (SVM) extensions.
 	  To compile this as a module, choose M here: the module
 	  will be called kvm-amd.
 config KVM_TRACE
 	bool "KVM trace support"
 	depends on KVM && SYSFS
 	select MARKERS
 	select RELAY
 	select DEBUG_FS
 	default n
 	---help---
 	  This option allows reading a trace of kvm-related events through
 	  relayfs.  Note the ABI is not considered stable and will be
 	  modified in future updates.
 # OK, it's a little counter-intuitive to do this, but it puts it neatly under
 # the virtualization menu.
 source drivers/lguest/Kconfig
 source drivers/virtio/Kconfig
 endif # VIRTUALIZATION

include/linux/kvm.h

Diff comments View file @ 73880c8

 #ifndef __LINUX_KVM_H
 #define __LINUX_KVM_H
 /*
  * Userspace interface for /dev/kvm - kernel based virtual machine
  *
  * Note: you must update KVM_API_VERSION if you change this interface.
  */
 #include <linux/types.h>
 #include <linux/compiler.h>
 #include <linux/ioctl.h>
 #include <asm/kvm.h>
 #define KVM_API_VERSION 12
 /* for KVM_TRACE_ENABLE */
 struct kvm_user_trace_setup {
 	__u32 buf_size; /* sub_buffer size of each per-cpu */
 	__u32 buf_nr; /* the number of sub_buffers of each per-cpu */
 };
 /* for KVM_CREATE_MEMORY_REGION */
 struct kvm_memory_region {
 	__u32 slot;
 	__u32 flags;
 	__u64 guest_phys_addr;
 	__u64 memory_size; /* bytes */
 };
 /* for KVM_SET_USER_MEMORY_REGION */
 struct kvm_userspace_memory_region {
 	__u32 slot;
 	__u32 flags;
 	__u64 guest_phys_addr;
 	__u64 memory_size; /* bytes */
 	__u64 userspace_addr; /* start of the userspace allocated memory */
 };
 /* for kvm_memory_region::flags */
 #define KVM_MEM_LOG_DIRTY_PAGES  1UL
 /* for KVM_IRQ_LINE */
 struct kvm_irq_level {
 	/*
 	 * ACPI gsi notion of irq.
 	 * For IA-64 (APIC model) IOAPIC0: irq 0-23; IOAPIC1: irq 24-47..
 	 * For X86 (standard AT mode) PIC0/1: irq 0-15. IOAPIC0: 0-23..
 	 */
 	union {
 		__u32 irq;
 		__s32 status;
 	};
 	__u32 level;
 };
 struct kvm_irqchip {
 	__u32 chip_id;
 	__u32 pad;
         union {
 		char dummy[512];  /* reserving space */
 #ifdef __KVM_HAVE_PIT
 		struct kvm_pic_state pic;
 #endif
 #ifdef __KVM_HAVE_IOAPIC
 		struct kvm_ioapic_state ioapic;
 #endif
 	} chip;
 };
 /* for KVM_CREATE_PIT2 */
 struct kvm_pit_config {
 	__u32 flags;
 	__u32 pad[15];
 };
 #define KVM_PIT_SPEAKER_DUMMY     1
 #define KVM_EXIT_UNKNOWN          0
 #define KVM_EXIT_EXCEPTION        1
 #define KVM_EXIT_IO               2
 #define KVM_EXIT_HYPERCALL        3
 #define KVM_EXIT_DEBUG            4
 #define KVM_EXIT_HLT              5
 #define KVM_EXIT_MMIO             6
 #define KVM_EXIT_IRQ_WINDOW_OPEN  7
 #define KVM_EXIT_SHUTDOWN         8
 #define KVM_EXIT_FAIL_ENTRY       9
 #define KVM_EXIT_INTR             10
 #define KVM_EXIT_SET_TPR          11
 #define KVM_EXIT_TPR_ACCESS       12
 #define KVM_EXIT_S390_SIEIC       13
 #define KVM_EXIT_S390_RESET       14
 #define KVM_EXIT_DCR              15
 #define KVM_EXIT_NMI              16
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
 	/* in */
 	__u8 request_interrupt_window;
 	__u8 padding1[7];
 	/* out */
 	__u32 exit_reason;
 	__u8 ready_for_interrupt_injection;
 	__u8 if_flag;
 	__u8 padding2[2];
 	/* in (pre_kvm_run), out (post_kvm_run) */
 	__u64 cr8;
 	__u64 apic_base;
 	union {
 		/* KVM_EXIT_UNKNOWN */
 		struct {
 			__u64 hardware_exit_reason;
 		} hw;
 		/* KVM_EXIT_FAIL_ENTRY */
 		struct {
 			__u64 hardware_entry_failure_reason;
 		} fail_entry;
 		/* KVM_EXIT_EXCEPTION */
 		struct {
 			__u32 exception;
 			__u32 error_code;
 		} ex;
 		/* KVM_EXIT_IO */
 		struct {
 #define KVM_EXIT_IO_IN  0
 #define KVM_EXIT_IO_OUT 1
 			__u8 direction;
 			__u8 size; /* bytes */
 			__u16 port;
 			__u32 count;
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
 		struct {
 			struct kvm_debug_exit_arch arch;
 		} debug;
 		/* KVM_EXIT_MMIO */
 		struct {
 			__u64 phys_addr;
 			__u8  data[8];
 			__u32 len;
 			__u8  is_write;
 		} mmio;
 		/* KVM_EXIT_HYPERCALL */
 		struct {
 			__u64 nr;
 			__u64 args[6];
 			__u64 ret;
 			__u32 longmode;
 			__u32 pad;
 		} hypercall;
 		/* KVM_EXIT_TPR_ACCESS */
 		struct {
 			__u64 rip;
 			__u32 is_write;
 			__u32 pad;
 		} tpr_access;
 		/* KVM_EXIT_S390_SIEIC */
 		struct {
 			__u8 icptcode;
 			__u64 mask; /* psw upper half */
 			__u64 addr; /* psw lower half */
 			__u16 ipa;
 			__u32 ipb;
 		} s390_sieic;
 		/* KVM_EXIT_S390_RESET */
 #define KVM_S390_RESET_POR       1
 #define KVM_S390_RESET_CLEAR     2
 #define KVM_S390_RESET_SUBSYSTEM 4
 #define KVM_S390_RESET_CPU_INIT  8
 #define KVM_S390_RESET_IPL       16
 		__u64 s390_reset_flags;
 		/* KVM_EXIT_DCR */
 		struct {
 			__u32 dcrn;
 			__u32 data;
 			__u8  is_write;
 		} dcr;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
 };
 /* for KVM_REGISTER_COALESCED_MMIO / KVM_UNREGISTER_COALESCED_MMIO */
 struct kvm_coalesced_mmio_zone {
 	__u64 addr;
 	__u32 size;
 	__u32 pad;
 };
 struct kvm_coalesced_mmio {
 	__u64 phys_addr;
 	__u32 len;
 	__u32 pad;
 	__u8  data[8];
 };
 struct kvm_coalesced_mmio_ring {
 	__u32 first, last;
 	struct kvm_coalesced_mmio coalesced_mmio[0];
 };
 #define KVM_COALESCED_MMIO_MAX \
 	((PAGE_SIZE - sizeof(struct kvm_coalesced_mmio_ring)) / \
 	 sizeof(struct kvm_coalesced_mmio))
 /* for KVM_TRANSLATE */
 struct kvm_translation {
 	/* in */
 	__u64 linear_address;
 	/* out */
 	__u64 physical_address;
 	__u8  valid;
 	__u8  writeable;
 	__u8  usermode;
 	__u8  pad[5];
 };
 /* for KVM_INTERRUPT */
 struct kvm_interrupt {
 	/* in */
 	__u32 irq;
 };
 /* for KVM_GET_DIRTY_LOG */
 struct kvm_dirty_log {
 	__u32 slot;
 	__u32 padding1;
 	union {
 		void __user *dirty_bitmap; /* one bit per page */
 		__u64 padding2;
 	};
 };
 /* for KVM_SET_SIGNAL_MASK */
 struct kvm_signal_mask {
 	__u32 len;
 	__u8  sigset[0];
 };
 /* for KVM_TPR_ACCESS_REPORTING */
 struct kvm_tpr_access_ctl {
 	__u32 enabled;
 	__u32 flags;
 	__u32 reserved[8];
 };
 /* for KVM_SET_VAPIC_ADDR */
 struct kvm_vapic_addr {
 	__u64 vapic_addr;
 };
 /* for KVM_SET_MPSTATE */
 #define KVM_MP_STATE_RUNNABLE          0
 #define KVM_MP_STATE_UNINITIALIZED     1
 #define KVM_MP_STATE_INIT_RECEIVED     2
 #define KVM_MP_STATE_HALTED            3
 #define KVM_MP_STATE_SIPI_RECEIVED     4
 struct kvm_mp_state {
 	__u32 mp_state;
 };
 struct kvm_s390_psw {
 	__u64 mask;
 	__u64 addr;
 };
 /* valid values for type in kvm_s390_interrupt */
 #define KVM_S390_SIGP_STOP		0xfffe0000u
 #define KVM_S390_PROGRAM_INT		0xfffe0001u
 #define KVM_S390_SIGP_SET_PREFIX	0xfffe0002u
 #define KVM_S390_RESTART		0xfffe0003u
 #define KVM_S390_INT_VIRTIO		0xffff2603u
 #define KVM_S390_INT_SERVICE		0xffff2401u
 #define KVM_S390_INT_EMERGENCY		0xffff1201u
 struct kvm_s390_interrupt {
 	__u32 type;
 	__u32 parm;
 	__u64 parm64;
 };
 /* for KVM_SET_GUEST_DEBUG */
 #define KVM_GUESTDBG_ENABLE		0x00000001
 #define KVM_GUESTDBG_SINGLESTEP		0x00000002
 struct kvm_guest_debug {
 	__u32 control;
 	__u32 pad;
 	struct kvm_guest_debug_arch arch;
 };
 #define KVM_TRC_SHIFT           16
 /*
  * kvm trace categories
  */
 #define KVM_TRC_ENTRYEXIT       (1 << KVM_TRC_SHIFT)
 #define KVM_TRC_HANDLER         (1 << (KVM_TRC_SHIFT + 1)) /* only 12 bits */
 /*
  * kvm trace action
  */
 #define KVM_TRC_VMENTRY         (KVM_TRC_ENTRYEXIT + 0x01)
 #define KVM_TRC_VMEXIT          (KVM_TRC_ENTRYEXIT + 0x02)
 #define KVM_TRC_PAGE_FAULT      (KVM_TRC_HANDLER + 0x01)
 #define KVM_TRC_HEAD_SIZE       12
 #define KVM_TRC_CYCLE_SIZE      8
 #define KVM_TRC_EXTRA_MAX       7
 /* This structure represents a single trace buffer record. */
 struct kvm_trace_rec {
 	/* variable rec_val
 	 * is split into:
 	 * bits 0 - 27  -> event id
 	 * bits 28 -30  -> number of extra data args of size u32
 	 * bits 31      -> binary indicator for if tsc is in record
 	 */
 	__u32 rec_val;
 	__u32 pid;
 	__u32 vcpu_id;
 	union {
 		struct {
 			__u64 timestamp;
 			__u32 extra_u32[KVM_TRC_EXTRA_MAX];
 		} __attribute__((packed)) timestamp;
 		struct {
 			__u32 extra_u32[KVM_TRC_EXTRA_MAX];
 		} notimestamp;
 	} u;
 };
 #define TRACE_REC_EVENT_ID(val) \
 		(0x0fffffff & (val))
 #define TRACE_REC_NUM_DATA_ARGS(val) \
 		(0x70000000 & ((val) << 28))
 #define TRACE_REC_TCS(val) \
 		(0x80000000 & ((val) << 31))
 #define KVMIO 0xAE
 /*
  * ioctls for /dev/kvm fds:
  */
 #define KVM_GET_API_VERSION       _IO(KVMIO,   0x00)
 #define KVM_CREATE_VM             _IO(KVMIO,   0x01) /* returns a VM fd */
 #define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 0x02, struct kvm_msr_list)
 #define KVM_S390_ENABLE_SIE       _IO(KVMIO,   0x06)
 /*
  * Check if a kvm extension is available.  Argument is extension number,
  * return is 1 (yes) or 0 (no, sorry).
  */
 #define KVM_CHECK_EXTENSION       _IO(KVMIO,   0x03)
 /*
  * Get size for mmap(vcpu_fd)
  */
 #define KVM_GET_VCPU_MMAP_SIZE    _IO(KVMIO,   0x04) /* in bytes */
 #define KVM_GET_SUPPORTED_CPUID   _IOWR(KVMIO, 0x05, struct kvm_cpuid2)
 /*
  * ioctls for kvm trace
  */
 #define KVM_TRACE_ENABLE          _IOW(KVMIO, 0x06, struct kvm_user_trace_setup)
 #define KVM_TRACE_PAUSE           _IO(KVMIO,  0x07)
 #define KVM_TRACE_DISABLE         _IO(KVMIO,  0x08)
 /*
  * Extension capability list.
  */
 #define KVM_CAP_IRQCHIP	  0
 #define KVM_CAP_HLT	  1
 #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2
 #define KVM_CAP_USER_MEMORY 3
 #define KVM_CAP_SET_TSS_ADDR 4
 #define KVM_CAP_VAPIC 6
 #define KVM_CAP_EXT_CPUID 7
 #define KVM_CAP_CLOCKSOURCE 8
 #define KVM_CAP_NR_VCPUS 9       /* returns max vcpus per vm */
 #define KVM_CAP_NR_MEMSLOTS 10   /* returns max memory slots per vm */
 #define KVM_CAP_PIT 11
 #define KVM_CAP_NOP_IO_DELAY 12
 #define KVM_CAP_PV_MMU 13
 #define KVM_CAP_MP_STATE 14
 #define KVM_CAP_COALESCED_MMIO 15
 #define KVM_CAP_SYNC_MMU 16  /* Changes to host mmap are reflected in guest */
 #ifdef __KVM_HAVE_DEVICE_ASSIGNMENT
 #define KVM_CAP_DEVICE_ASSIGNMENT 17
 #endif
 #define KVM_CAP_IOMMU 18
 #ifdef __KVM_HAVE_MSI
 #define KVM_CAP_DEVICE_MSI 20
 #endif
 /* Bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_DESTROY_MEMORY_REGION_WORKS 21
 #ifdef __KVM_HAVE_USER_NMI
 #define KVM_CAP_USER_NMI 22
 #endif
 #ifdef __KVM_HAVE_GUEST_DEBUG
 #define KVM_CAP_SET_GUEST_DEBUG 23
 #endif
 #ifdef __KVM_HAVE_PIT
 #define KVM_CAP_REINJECT_CONTROL 24
 #endif
 #ifdef __KVM_HAVE_IOAPIC
 #define KVM_CAP_IRQ_ROUTING 25
 #endif
 #define KVM_CAP_IRQ_INJECT_STATUS 26
 #ifdef __KVM_HAVE_DEVICE_ASSIGNMENT
 #define KVM_CAP_DEVICE_DEASSIGNMENT 27
 #endif
 #ifdef __KVM_HAVE_MSIX
 #define KVM_CAP_DEVICE_MSIX 28
 #endif
 #define KVM_CAP_ASSIGN_DEV_IRQ 29
 /* Another bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_JOIN_MEMORY_REGIONS_WORKS 30
 #ifdef __KVM_HAVE_MCE
 #define KVM_CAP_MCE 31
 #endif
 #define KVM_CAP_IRQFD 32
 #ifdef __KVM_HAVE_PIT
 #define KVM_CAP_PIT2 33
 #endif
+#define KVM_CAP_SET_BOOT_CPU_ID 34
 #ifdef KVM_CAP_IRQ_ROUTING
 struct kvm_irq_routing_irqchip {
 	__u32 irqchip;
 	__u32 pin;
 };
 struct kvm_irq_routing_msi {
 	__u32 address_lo;
 	__u32 address_hi;
 	__u32 data;
 	__u32 pad;
 };
 /* gsi routing entry types */
 #define KVM_IRQ_ROUTING_IRQCHIP 1
 #define KVM_IRQ_ROUTING_MSI 2
 struct kvm_irq_routing_entry {
 	__u32 gsi;
 	__u32 type;
 	__u32 flags;
 	__u32 pad;
 	union {
 		struct kvm_irq_routing_irqchip irqchip;
 		struct kvm_irq_routing_msi msi;
 		__u32 pad[8];
 	} u;
 };
 struct kvm_irq_routing {
 	__u32 nr;
 	__u32 flags;
 	struct kvm_irq_routing_entry entries[0];
 };
 #endif
 #ifdef KVM_CAP_MCE
 /* x86 MCE */
 struct kvm_x86_mce {
 	__u64 status;
 	__u64 addr;
 	__u64 misc;
 	__u64 mcg_status;
 	__u8 bank;
 	__u8 pad1[7];
 	__u64 pad2[3];
 };
 #endif
 #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)
 struct kvm_irqfd {
 	__u32 fd;
 	__u32 gsi;
 	__u32 flags;
 	__u8  pad[20];
 };
 /*
  * ioctls for VM fds
  */
 #define KVM_SET_MEMORY_REGION     _IOW(KVMIO, 0x40, struct kvm_memory_region)
 /*
  * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns
  * a vcpu fd.
  */
 #define KVM_CREATE_VCPU           _IO(KVMIO,  0x41)
 #define KVM_GET_DIRTY_LOG         _IOW(KVMIO, 0x42, struct kvm_dirty_log)
 #define KVM_SET_MEMORY_ALIAS      _IOW(KVMIO, 0x43, struct kvm_memory_alias)
 #define KVM_SET_NR_MMU_PAGES      _IO(KVMIO, 0x44)
 #define KVM_GET_NR_MMU_PAGES      _IO(KVMIO, 0x45)
 #define KVM_SET_USER_MEMORY_REGION _IOW(KVMIO, 0x46,\
 					struct kvm_userspace_memory_region)
 #define KVM_SET_TSS_ADDR          _IO(KVMIO, 0x47)
 /* Device model IOC */
 #define KVM_CREATE_IRQCHIP	  _IO(KVMIO,  0x60)
 #define KVM_IRQ_LINE		  _IOW(KVMIO, 0x61, struct kvm_irq_level)
 #define KVM_GET_IRQCHIP		  _IOWR(KVMIO, 0x62, struct kvm_irqchip)
 #define KVM_SET_IRQCHIP		  _IOR(KVMIO,  0x63, struct kvm_irqchip)
 #define KVM_CREATE_PIT		  _IO(KVMIO,  0x64)
 #define KVM_GET_PIT		  _IOWR(KVMIO, 0x65, struct kvm_pit_state)
 #define KVM_SET_PIT		  _IOR(KVMIO,  0x66, struct kvm_pit_state)
 #define KVM_IRQ_LINE_STATUS	  _IOWR(KVMIO, 0x67, struct kvm_irq_level)
 #define KVM_REGISTER_COALESCED_MMIO \
 			_IOW(KVMIO,  0x67, struct kvm_coalesced_mmio_zone)
 #define KVM_UNREGISTER_COALESCED_MMIO \
 			_IOW(KVMIO,  0x68, struct kvm_coalesced_mmio_zone)
 #define KVM_ASSIGN_PCI_DEVICE _IOR(KVMIO, 0x69, \
 				   struct kvm_assigned_pci_dev)
 #define KVM_SET_GSI_ROUTING       _IOW(KVMIO, 0x6a, struct kvm_irq_routing)
 /* deprecated, replaced by KVM_ASSIGN_DEV_IRQ */
 #define KVM_ASSIGN_IRQ _IOR(KVMIO, 0x70, \
 			    struct kvm_assigned_irq)
 #define KVM_ASSIGN_DEV_IRQ        _IOW(KVMIO, 0x70, struct kvm_assigned_irq)
 #define KVM_REINJECT_CONTROL      _IO(KVMIO, 0x71)
 #define KVM_DEASSIGN_PCI_DEVICE _IOW(KVMIO, 0x72, \
 				     struct kvm_assigned_pci_dev)
 #define KVM_ASSIGN_SET_MSIX_NR \
 			_IOW(KVMIO, 0x73, struct kvm_assigned_msix_nr)
 #define KVM_ASSIGN_SET_MSIX_ENTRY \
 			_IOW(KVMIO, 0x74, struct kvm_assigned_msix_entry)
 #define KVM_DEASSIGN_DEV_IRQ       _IOW(KVMIO, 0x75, struct kvm_assigned_irq)
 #define KVM_IRQFD                  _IOW(KVMIO, 0x76, struct kvm_irqfd)
 #define KVM_CREATE_PIT2		   _IOW(KVMIO, 0x77, struct kvm_pit_config)
+#define KVM_SET_BOOT_CPU_ID        _IO(KVMIO, 0x78)
 /*
  * ioctls for vcpu fds
  */
 #define KVM_RUN                   _IO(KVMIO,   0x80)
 #define KVM_GET_REGS              _IOR(KVMIO,  0x81, struct kvm_regs)
 #define KVM_SET_REGS              _IOW(KVMIO,  0x82, struct kvm_regs)
 #define KVM_GET_SREGS             _IOR(KVMIO,  0x83, struct kvm_sregs)
 #define KVM_SET_SREGS             _IOW(KVMIO,  0x84, struct kvm_sregs)
 #define KVM_TRANSLATE             _IOWR(KVMIO, 0x85, struct kvm_translation)
 #define KVM_INTERRUPT             _IOW(KVMIO,  0x86, struct kvm_interrupt)
 /* KVM_DEBUG_GUEST is no longer supported, use KVM_SET_GUEST_DEBUG instead */
 #define KVM_DEBUG_GUEST           __KVM_DEPRECATED_DEBUG_GUEST
 #define KVM_GET_MSRS              _IOWR(KVMIO, 0x88, struct kvm_msrs)
 #define KVM_SET_MSRS              _IOW(KVMIO,  0x89, struct kvm_msrs)
 #define KVM_SET_CPUID             _IOW(KVMIO,  0x8a, struct kvm_cpuid)
 #define KVM_SET_SIGNAL_MASK       _IOW(KVMIO,  0x8b, struct kvm_signal_mask)
 #define KVM_GET_FPU               _IOR(KVMIO,  0x8c, struct kvm_fpu)
 #define KVM_SET_FPU               _IOW(KVMIO,  0x8d, struct kvm_fpu)
 #define KVM_GET_LAPIC             _IOR(KVMIO,  0x8e, struct kvm_lapic_state)
 #define KVM_SET_LAPIC             _IOW(KVMIO,  0x8f, struct kvm_lapic_state)
 #define KVM_SET_CPUID2            _IOW(KVMIO,  0x90, struct kvm_cpuid2)
 #define KVM_GET_CPUID2            _IOWR(KVMIO, 0x91, struct kvm_cpuid2)
 /* Available with KVM_CAP_VAPIC */
 #define KVM_TPR_ACCESS_REPORTING  _IOWR(KVMIO,  0x92, struct kvm_tpr_access_ctl)
 /* Available with KVM_CAP_VAPIC */
 #define KVM_SET_VAPIC_ADDR        _IOW(KVMIO,  0x93, struct kvm_vapic_addr)
 /* valid for virtual machine (for floating interrupt)_and_ vcpu */
 #define KVM_S390_INTERRUPT        _IOW(KVMIO,  0x94, struct kvm_s390_interrupt)
 /* store status for s390 */
 #define KVM_S390_STORE_STATUS_NOADDR    (-1ul)
 #define KVM_S390_STORE_STATUS_PREFIXED  (-2ul)
 #define KVM_S390_STORE_STATUS	  _IOW(KVMIO,  0x95, unsigned long)
 /* initial ipl psw for s390 */
 #define KVM_S390_SET_INITIAL_PSW  _IOW(KVMIO,  0x96, struct kvm_s390_psw)
 /* initial reset for s390 */
 #define KVM_S390_INITIAL_RESET    _IO(KVMIO,  0x97)
 #define KVM_GET_MP_STATE          _IOR(KVMIO,  0x98, struct kvm_mp_state)
 #define KVM_SET_MP_STATE          _IOW(KVMIO,  0x99, struct kvm_mp_state)
 /* Available with KVM_CAP_NMI */
 #define KVM_NMI                   _IO(KVMIO,  0x9a)
 /* Available with KVM_CAP_SET_GUEST_DEBUG */
 #define KVM_SET_GUEST_DEBUG       _IOW(KVMIO,  0x9b, struct kvm_guest_debug)
 /* MCE for x86 */
 #define KVM_X86_SETUP_MCE         _IOW(KVMIO,  0x9c, __u64)
 #define KVM_X86_GET_MCE_CAP_SUPPORTED _IOR(KVMIO,  0x9d, __u64)
 #define KVM_X86_SET_MCE           _IOW(KVMIO,  0x9e, struct kvm_x86_mce)
 /*
  * Deprecated interfaces
  */
 struct kvm_breakpoint {
 	__u32 enabled;
 	__u32 padding;
 	__u64 address;
 };
 struct kvm_debug_guest {
 	__u32 enabled;
 	__u32 pad;
 	struct kvm_breakpoint breakpoints[4];
 	__u32 singlestep;
 };
 #define __KVM_DEPRECATED_DEBUG_GUEST _IOW(KVMIO,  0x87, struct kvm_debug_guest)
 #define KVM_IA64_VCPU_GET_STACK   _IOR(KVMIO,  0x9a, void *)
 #define KVM_IA64_VCPU_SET_STACK   _IOW(KVMIO,  0x9b, void *)
 #define KVM_TRC_INJ_VIRQ         (KVM_TRC_HANDLER + 0x02)
 #define KVM_TRC_REDELIVER_EVT    (KVM_TRC_HANDLER + 0x03)
 #define KVM_TRC_PEND_INTR        (KVM_TRC_HANDLER + 0x04)
 #define KVM_TRC_IO_READ          (KVM_TRC_HANDLER + 0x05)
 #define KVM_TRC_IO_WRITE         (KVM_TRC_HANDLER + 0x06)
 #define KVM_TRC_CR_READ          (KVM_TRC_HANDLER + 0x07)
 #define KVM_TRC_CR_WRITE         (KVM_TRC_HANDLER + 0x08)
 #define KVM_TRC_DR_READ          (KVM_TRC_HANDLER + 0x09)
 #define KVM_TRC_DR_WRITE         (KVM_TRC_HANDLER + 0x0A)
 #define KVM_TRC_MSR_READ         (KVM_TRC_HANDLER + 0x0B)
 #define KVM_TRC_MSR_WRITE        (KVM_TRC_HANDLER + 0x0C)
 #define KVM_TRC_CPUID            (KVM_TRC_HANDLER + 0x0D)
 #define KVM_TRC_INTR             (KVM_TRC_HANDLER + 0x0E)
 #define KVM_TRC_NMI              (KVM_TRC_HANDLER + 0x0F)
 #define KVM_TRC_VMMCALL          (KVM_TRC_HANDLER + 0x10)
 #define KVM_TRC_HLT              (KVM_TRC_HANDLER + 0x11)
 #define KVM_TRC_CLTS             (KVM_TRC_HANDLER + 0x12)
 #define KVM_TRC_LMSW             (KVM_TRC_HANDLER + 0x13)
 #define KVM_TRC_APIC_ACCESS      (KVM_TRC_HANDLER + 0x14)
 #define KVM_TRC_TDP_FAULT        (KVM_TRC_HANDLER + 0x15)
 #define KVM_TRC_GTLB_WRITE       (KVM_TRC_HANDLER + 0x16)
 #define KVM_TRC_STLB_WRITE       (KVM_TRC_HANDLER + 0x17)
 #define KVM_TRC_STLB_INVAL       (KVM_TRC_HANDLER + 0x18)
 #define KVM_TRC_PPC_INSTR        (KVM_TRC_HANDLER + 0x19)
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 struct kvm_assigned_pci_dev {
 	__u32 assigned_dev_id;
 	__u32 busnr;
 	__u32 devfn;
 	__u32 flags;
 	union {
 		__u32 reserved[12];
 	};
 };
 #define KVM_DEV_IRQ_HOST_INTX    (1 << 0)
 #define KVM_DEV_IRQ_HOST_MSI     (1 << 1)
 #define KVM_DEV_IRQ_HOST_MSIX    (1 << 2)
 #define KVM_DEV_IRQ_GUEST_INTX   (1 << 8)
 #define KVM_DEV_IRQ_GUEST_MSI    (1 << 9)
 #define KVM_DEV_IRQ_GUEST_MSIX   (1 << 10)
 #define KVM_DEV_IRQ_HOST_MASK	 0x00ff
 #define KVM_DEV_IRQ_GUEST_MASK   0xff00
 struct kvm_assigned_irq {
 	__u32 assigned_dev_id;
 	__u32 host_irq;
 	__u32 guest_irq;
 	__u32 flags;
 	union {
 		struct {
 			__u32 addr_lo;
 			__u32 addr_hi;
 			__u32 data;
 		} guest_msi;
 		__u32 reserved[12];
 	};
 };
 struct kvm_assigned_msix_nr {
 	__u32 assigned_dev_id;
 	__u16 entry_nr;
 	__u16 padding;
 };
 #define KVM_MAX_MSIX_PER_DEV		256
 struct kvm_assigned_msix_entry {
 	__u32 assigned_dev_id;
 	__u32 gsi;
 	__u16 entry; /* The index of entry in the MSI-X table */
 	__u16 padding[3];
 };
 #endif

include/linux/kvm_host.h

Diff comments View file @ 73880c8

 #ifndef __KVM_HOST_H
 #define __KVM_HOST_H
 /*
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  */
 #include <linux/types.h>
 #include <linux/hardirq.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/preempt.h>
 #include <linux/marker.h>
 #include <linux/msi.h>
 #include <asm/signal.h>
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
 #include <linux/kvm_types.h>
 #include <asm/kvm_host.h>
 /*
  * vcpu->requests bit members
  */
 #define KVM_REQ_TLB_FLUSH          0
 #define KVM_REQ_MIGRATE_TIMER      1
 #define KVM_REQ_REPORT_TPR_ACCESS  2
 #define KVM_REQ_MMU_RELOAD         3
 #define KVM_REQ_TRIPLE_FAULT       4
 #define KVM_REQ_PENDING_TIMER      5
 #define KVM_REQ_UNHALT             6
 #define KVM_REQ_MMU_SYNC           7
 #define KVM_REQ_KVMCLOCK_UPDATE    8
 #define KVM_REQ_KICK               9
 #define KVM_USERSPACE_IRQ_SOURCE_ID	0
 struct kvm_vcpu;
 extern struct kmem_cache *kvm_vcpu_cache;
 /*
  * It would be nice to use something smarter than a linear search, TBD...
  * Thankfully we dont expect many devices to register (famous last words :),
  * so until then it will suffice.  At least its abstracted so we can change
  * in one place.
  */
 struct kvm_io_bus {
 	int                   dev_count;
 #define NR_IOBUS_DEVS 6
 	struct kvm_io_device *devs[NR_IOBUS_DEVS];
 };
 void kvm_io_bus_init(struct kvm_io_bus *bus);
 void kvm_io_bus_destroy(struct kvm_io_bus *bus);
 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
 					  gpa_t addr, int len, int is_write);
 void kvm_io_bus_register_dev(struct kvm_io_bus *bus,
 			     struct kvm_io_device *dev);
 struct kvm_vcpu {
 	struct kvm *kvm;
 #ifdef CONFIG_PREEMPT_NOTIFIERS
 	struct preempt_notifier preempt_notifier;
 #endif
 	int vcpu_id;
 	struct mutex mutex;
 	int   cpu;
 	struct kvm_run *run;
 	unsigned long requests;
 	unsigned long guest_debug;
 	int fpu_active;
 	int guest_fpu_loaded;
 	wait_queue_head_t wq;
 	int sigset_active;
 	sigset_t sigset;
 	struct kvm_vcpu_stat stat;
 #ifdef CONFIG_HAS_IOMEM
 	int mmio_needed;
 	int mmio_read_completed;
 	int mmio_is_write;
 	int mmio_size;
 	unsigned char mmio_data[8];
 	gpa_t mmio_phys_addr;
 #endif
 	struct kvm_vcpu_arch arch;
 };
 struct kvm_memory_slot {
 	gfn_t base_gfn;
 	unsigned long npages;
 	unsigned long flags;
 	unsigned long *rmap;
 	unsigned long *dirty_bitmap;
 	struct {
 		unsigned long rmap_pde;
 		int write_count;
 	} *lpage_info;
 	unsigned long userspace_addr;
 	int user_alloc;
 };
 struct kvm_kernel_irq_routing_entry {
 	u32 gsi;
 	u32 type;
 	int (*set)(struct kvm_kernel_irq_routing_entry *e,
 		    struct kvm *kvm, int level);
 	union {
 		struct {
 			unsigned irqchip;
 			unsigned pin;
 		} irqchip;
 		struct msi_msg msi;
 	};
 	struct list_head link;
 };
 struct kvm {
 	spinlock_t mmu_lock;
 	spinlock_t requests_lock;
 	struct rw_semaphore slots_lock;
 	struct mm_struct *mm; /* userspace tied to this vm */
 	int nmemslots;
 	struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS +
 					KVM_PRIVATE_MEM_SLOTS];
+#ifdef CONFIG_KVM_APIC_ARCHITECTURE
+	u32 bsp_vcpu_id;
 	struct kvm_vcpu *bsp_vcpu;
+#endif
 	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
+	atomic_t online_vcpus;
 	struct list_head vm_list;
 	struct mutex lock;
 	struct kvm_io_bus mmio_bus;
 	struct kvm_io_bus pio_bus;
 #ifdef CONFIG_HAVE_KVM_EVENTFD
 	struct {
 		spinlock_t        lock;
 		struct list_head  items;
 	} irqfds;
 #endif
 	struct kvm_vm_stat stat;
 	struct kvm_arch arch;
 	atomic_t users_count;
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	struct kvm_coalesced_mmio_dev *coalesced_mmio_dev;
 	struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
 #endif
 	struct mutex irq_lock;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 	struct list_head irq_routing; /* of kvm_kernel_irq_routing_entry */
 	struct hlist_head mask_notifier_list;
 #endif
 #ifdef KVM_ARCH_WANT_MMU_NOTIFIER
 	struct mmu_notifier mmu_notifier;
 	unsigned long mmu_notifier_seq;
 	long mmu_notifier_count;
 #endif
 };
 /* The guest did something we don't support. */
 #define pr_unimpl(vcpu, fmt, ...)					\
  do {									\
 	if (printk_ratelimit())						\
 		printk(KERN_ERR "kvm: %i: cpu%i " fmt,			\
 		       current->tgid, (vcpu)->vcpu_id , ## __VA_ARGS__); \
  } while (0)
 #define kvm_printf(kvm, fmt ...) printk(KERN_DEBUG fmt)
 #define vcpu_printf(vcpu, fmt...) kvm_printf(vcpu->kvm, fmt)
 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id);
 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu);
 void vcpu_load(struct kvm_vcpu *vcpu);
 void vcpu_put(struct kvm_vcpu *vcpu);
 int kvm_init(void *opaque, unsigned int vcpu_size,
 		  struct module *module);
 void kvm_exit(void);
 void kvm_get_kvm(struct kvm *kvm);
 void kvm_put_kvm(struct kvm *kvm);
 #define HPA_MSB ((sizeof(hpa_t) * 8) - 1)
 #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB)
 static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; }
 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva);
 extern struct page *bad_page;
 extern pfn_t bad_pfn;
 int is_error_page(struct page *page);
 int is_error_pfn(pfn_t pfn);
 int kvm_is_error_hva(unsigned long addr);
 int kvm_set_memory_region(struct kvm *kvm,
 			  struct kvm_userspace_memory_region *mem,
 			  int user_alloc);
 int __kvm_set_memory_region(struct kvm *kvm,
 			    struct kvm_userspace_memory_region *mem,
 			    int user_alloc);
 int kvm_arch_set_memory_region(struct kvm *kvm,
 				struct kvm_userspace_memory_region *mem,
 				struct kvm_memory_slot old,
 				int user_alloc);
 void kvm_arch_flush_shadow(struct kvm *kvm);
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
 void kvm_release_page_clean(struct page *page);
 void kvm_release_page_dirty(struct page *page);
 void kvm_set_page_dirty(struct page *page);
 void kvm_set_page_accessed(struct page *page);
 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn);
 void kvm_release_pfn_dirty(pfn_t);
 void kvm_release_pfn_clean(pfn_t pfn);
 void kvm_set_pfn_dirty(pfn_t pfn);
 void kvm_set_pfn_accessed(pfn_t pfn);
 void kvm_get_pfn(pfn_t pfn);
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 			int len);
 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
 			  unsigned long len);
 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len);
 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 			 int offset, int len);
 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
 		    unsigned long len);
 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len);
 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len);
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn);
 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn);
 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
 void kvm_vcpu_block(struct kvm_vcpu *vcpu);
 void kvm_resched(struct kvm_vcpu *vcpu);
 void kvm_load_guest_fpu(struct kvm_vcpu *vcpu);
 void kvm_put_guest_fpu(struct kvm_vcpu *vcpu);
 void kvm_flush_remote_tlbs(struct kvm *kvm);
 void kvm_reload_remote_mmus(struct kvm *kvm);
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg);
 long kvm_arch_vcpu_ioctl(struct file *filp,
 			 unsigned int ioctl, unsigned long arg);
 int kvm_dev_ioctl_check_extension(long ext);
 int kvm_get_dirty_log(struct kvm *kvm,
 			struct kvm_dirty_log *log, int *is_dirty);
 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 				struct kvm_dirty_log *log);
 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 				   struct
 				   kvm_userspace_memory_region *mem,
 				   int user_alloc);
 long kvm_arch_vm_ioctl(struct file *filp,
 		       unsigned int ioctl, unsigned long arg);
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu);
 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 				    struct kvm_translation *tr);
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs);
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs);
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs);
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state);
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state);
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg);
 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run);
 int kvm_arch_init(void *opaque);
 void kvm_arch_exit(void);
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu);
 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id);
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
 int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu);
 void kvm_arch_hardware_enable(void *garbage);
 void kvm_arch_hardware_disable(void *garbage);
 int kvm_arch_hardware_setup(void);
 void kvm_arch_hardware_unsetup(void);
 void kvm_arch_check_processor_compat(void *rtn);
 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu);
 int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu);
 void kvm_free_physmem(struct kvm *kvm);
 struct  kvm *kvm_arch_create_vm(void);
 void kvm_arch_destroy_vm(struct kvm *kvm);
 void kvm_free_all_assigned_devices(struct kvm *kvm);
 void kvm_arch_sync_events(struct kvm *kvm);
 int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_interrupt(struct kvm_vcpu *v);
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu);
 void kvm_vcpu_kick(struct kvm_vcpu *vcpu);
 int kvm_is_mmio_pfn(pfn_t pfn);
 struct kvm_irq_ack_notifier {
 	struct hlist_node link;
 	unsigned gsi;
 	void (*irq_acked)(struct kvm_irq_ack_notifier *kian);
 };
 #define KVM_ASSIGNED_MSIX_PENDING		0x1
 struct kvm_guest_msix_entry {
 	u32 vector;
 	u16 entry;
 	u16 flags;
 };
 struct kvm_assigned_dev_kernel {
 	struct kvm_irq_ack_notifier ack_notifier;
 	struct work_struct interrupt_work;
 	struct list_head list;
 	int assigned_dev_id;
 	int host_busnr;
 	int host_devfn;
 	unsigned int entries_nr;
 	int host_irq;
 	bool host_irq_disabled;
 	struct msix_entry *host_msix_entries;
 	int guest_irq;
 	struct kvm_guest_msix_entry *guest_msix_entries;
 	unsigned long irq_requested_type;
 	int irq_source_id;
 	int flags;
 	struct pci_dev *dev;
 	struct kvm *kvm;
 	spinlock_t assigned_dev_lock;
 };
 struct kvm_irq_mask_notifier {
 	void (*func)(struct kvm_irq_mask_notifier *kimn, bool masked);
 	int irq;
 	struct hlist_node link;
 };
 void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq,
 				    struct kvm_irq_mask_notifier *kimn);
 void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq,
 				      struct kvm_irq_mask_notifier *kimn);
 void kvm_fire_mask_notifiers(struct kvm *kvm, int irq, bool mask);
 int kvm_set_irq(struct kvm *kvm, int irq_source_id, int irq, int level);
 void kvm_notify_acked_irq(struct kvm *kvm, unsigned irqchip, unsigned pin);
 void kvm_register_irq_ack_notifier(struct kvm *kvm,
 				   struct kvm_irq_ack_notifier *kian);
 void kvm_unregister_irq_ack_notifier(struct kvm *kvm,
 				   struct kvm_irq_ack_notifier *kian);
 int kvm_request_irq_source_id(struct kvm *kvm);
 void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
 /* For vcpu->arch.iommu_flags */
 #define KVM_IOMMU_CACHE_COHERENCY	0x1
 #ifdef CONFIG_IOMMU_API
 int kvm_iommu_map_pages(struct kvm *kvm, gfn_t base_gfn,
 			unsigned long npages);
 int kvm_iommu_map_guest(struct kvm *kvm);
 int kvm_iommu_unmap_guest(struct kvm *kvm);
 int kvm_assign_device(struct kvm *kvm,
 		      struct kvm_assigned_dev_kernel *assigned_dev);
 int kvm_deassign_device(struct kvm *kvm,
 			struct kvm_assigned_dev_kernel *assigned_dev);
 #else /* CONFIG_IOMMU_API */
 static inline int kvm_iommu_map_pages(struct kvm *kvm,
 				      gfn_t base_gfn,
 				      unsigned long npages)
 {
 	return 0;
 }
 static inline int kvm_iommu_map_guest(struct kvm *kvm)
 {
 	return -ENODEV;
 }
 static inline int kvm_iommu_unmap_guest(struct kvm *kvm)
 {
 	return 0;
 }
 static inline int kvm_assign_device(struct kvm *kvm,
 		struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	return 0;
 }
 static inline int kvm_deassign_device(struct kvm *kvm,
 		struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	return 0;
 }
 #endif /* CONFIG_IOMMU_API */
 static inline void kvm_guest_enter(void)
 {
 	account_system_vtime(current);
 	current->flags |= PF_VCPU;
 }
 static inline void kvm_guest_exit(void)
 {
 	account_system_vtime(current);
 	current->flags &= ~PF_VCPU;
 }
 static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot)
 {
 	return slot - kvm->memslots;
 }
 static inline gpa_t gfn_to_gpa(gfn_t gfn)
 {
 	return (gpa_t)gfn << PAGE_SHIFT;
 }
 static inline hpa_t pfn_to_hpa(pfn_t pfn)
 {
 	return (hpa_t)pfn << PAGE_SHIFT;
 }
 static inline void kvm_migrate_timers(struct kvm_vcpu *vcpu)
 {
 	set_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests);
 }
 enum kvm_stat_kind {
 	KVM_STAT_VM,
 	KVM_STAT_VCPU,
 };
 struct kvm_stats_debugfs_item {
 	const char *name;
 	int offset;
 	enum kvm_stat_kind kind;
 	struct dentry *dentry;
 };
 extern struct kvm_stats_debugfs_item debugfs_entries[];
 extern struct dentry *kvm_debugfs_dir;
 #define KVMTRACE_5D(evt, vcpu, d1, d2, d3, d4, d5, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 5, d1, d2, d3, d4, d5)
 #define KVMTRACE_4D(evt, vcpu, d1, d2, d3, d4, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 4, d1, d2, d3, d4, 0)
 #define KVMTRACE_3D(evt, vcpu, d1, d2, d3, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 3, d1, d2, d3, 0, 0)
 #define KVMTRACE_2D(evt, vcpu, d1, d2, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 2, d1, d2, 0, 0, 0)
 #define KVMTRACE_1D(evt, vcpu, d1, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 1, d1, 0, 0, 0, 0)
 #define KVMTRACE_0D(evt, vcpu, name) \
 	trace_mark(kvm_trace_##name, "%u %p %u %u %u %u %u %u", KVM_TRC_##evt, \
 						vcpu, 0, 0, 0, 0, 0, 0)
 #ifdef CONFIG_KVM_TRACE
 int kvm_trace_ioctl(unsigned int ioctl, unsigned long arg);
 void kvm_trace_cleanup(void);
 #else
 static inline
 int kvm_trace_ioctl(unsigned int ioctl, unsigned long arg)
 {
 	return -EINVAL;
 }
 #define kvm_trace_cleanup() ((void)0)
 #endif
 #ifdef KVM_ARCH_WANT_MMU_NOTIFIER
 static inline int mmu_notifier_retry(struct kvm_vcpu *vcpu, unsigned long mmu_seq)
 {
 	if (unlikely(vcpu->kvm->mmu_notifier_count))
 		return 1;
 	/*
 	 * Both reads happen under the mmu_lock and both values are
 	 * modified under mmu_lock, so there's no need of smb_rmb()
 	 * here in between, otherwise mmu_notifier_count should be
 	 * read before mmu_notifier_seq, see
 	 * mmu_notifier_invalidate_range_end write side.
 	 */
 	if (vcpu->kvm->mmu_notifier_seq != mmu_seq)
 		return 1;
 	return 0;
 }
 #endif
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 #define KVM_MAX_IRQ_ROUTES 1024
 int kvm_setup_default_irq_routing(struct kvm *kvm);
 int kvm_set_irq_routing(struct kvm *kvm,
 			const struct kvm_irq_routing_entry *entries,
 			unsigned nr,
 			unsigned flags);
 void kvm_free_irq_routing(struct kvm *kvm);
 #else
 static inline void kvm_free_irq_routing(struct kvm *kvm) {}
 #endif
 #ifdef CONFIG_HAVE_KVM_EVENTFD
 void kvm_irqfd_init(struct kvm *kvm);
 int kvm_irqfd(struct kvm *kvm, int fd, int gsi, int flags);
 void kvm_irqfd_release(struct kvm *kvm);
 #else
 static inline void kvm_irqfd_init(struct kvm *kvm) {}
 static inline int kvm_irqfd(struct kvm *kvm, int fd, int gsi, int flags)
 {
 	return -EINVAL;
 }
 static inline void kvm_irqfd_release(struct kvm *kvm) {}
 #endif /* CONFIG_HAVE_KVM_EVENTFD */
+#ifdef CONFIG_KVM_APIC_ARCHITECTURE
 static inline bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
 {
 	return vcpu->kvm->bsp_vcpu == vcpu;
 }
+#endif
 #endif

virt/kvm/Kconfig

Diff comments View file @ 73880c8

 # KVM common configuration items and defaults
 config HAVE_KVM
        bool
 config HAVE_KVM_IRQCHIP
        bool
 config HAVE_KVM_EVENTFD
        bool
        select EVENTFD
+config KVM_APIC_ARCHITECTURE
+       bool

virt/kvm/kvm_main.c

Diff comments View file @ 73880c8

 /*
  * Kernel-based Virtual Machine driver for Linux
  *
  * This module enables machines with Intel VT-x extensions to run virtual
  * machines without emulation or binary translation.
  *
  * Copyright (C) 2006 Qumranet, Inc.
  *
  * Authors:
  *   Avi Kivity   <avi@qumranet.com>
  *   Yaniv Kamay  <yaniv@qumranet.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  *
  */
 #include "iodev.h"
 #include <linux/kvm_host.h>
 #include <linux/kvm.h>
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/percpu.h>
 #include <linux/gfp.h>
 #include <linux/mm.h>
 #include <linux/miscdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/reboot.h>
 #include <linux/debugfs.h>
 #include <linux/highmem.h>
 #include <linux/file.h>
 #include <linux/sysdev.h>
 #include <linux/cpu.h>
 #include <linux/sched.h>
 #include <linux/cpumask.h>
 #include <linux/smp.h>
 #include <linux/anon_inodes.h>
 #include <linux/profile.h>
 #include <linux/kvm_para.h>
 #include <linux/pagemap.h>
 #include <linux/mman.h>
 #include <linux/swap.h>
 #include <linux/bitops.h>
 #include <linux/spinlock.h>
 #include <asm/processor.h>
 #include <asm/io.h>
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 #include "coalesced_mmio.h"
 #endif
 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 #include "irq.h"
 #endif
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 /*
  * Ordering of locks:
  *
  * 		kvm->lock --> kvm->irq_lock
  */
 DEFINE_SPINLOCK(kvm_lock);
 LIST_HEAD(vm_list);
 static cpumask_var_t cpus_hardware_enabled;
 struct kmem_cache *kvm_vcpu_cache;
 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
 static __read_mostly struct preempt_ops kvm_preempt_ops;
 struct dentry *kvm_debugfs_dir;
 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 			   unsigned long arg);
 static bool kvm_rebooting;
 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
 static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
 						      int assigned_dev_id)
 {
 	struct list_head *ptr;
 	struct kvm_assigned_dev_kernel *match;
 	list_for_each(ptr, head) {
 		match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
 		if (match->assigned_dev_id == assigned_dev_id)
 			return match;
 	}
 	return NULL;
 }
 static int find_index_from_host_irq(struct kvm_assigned_dev_kernel
 				    *assigned_dev, int irq)
 {
 	int i, index;
 	struct msix_entry *host_msix_entries;
 	host_msix_entries = assigned_dev->host_msix_entries;
 	index = -1;
 	for (i = 0; i < assigned_dev->entries_nr; i++)
 		if (irq == host_msix_entries[i].vector) {
 			index = i;
 			break;
 		}
 	if (index < 0) {
 		printk(KERN_WARNING "Fail to find correlated MSI-X entry!\n");
 		return 0;
 	}
 	return index;
 }
 static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
 {
 	struct kvm_assigned_dev_kernel *assigned_dev;
 	struct kvm *kvm;
 	int i;
 	assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
 				    interrupt_work);
 	kvm = assigned_dev->kvm;
 	mutex_lock(&kvm->irq_lock);
 	spin_lock_irq(&assigned_dev->assigned_dev_lock);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		struct kvm_guest_msix_entry *guest_entries =
 			assigned_dev->guest_msix_entries;
 		for (i = 0; i < assigned_dev->entries_nr; i++) {
 			if (!(guest_entries[i].flags &
 					KVM_ASSIGNED_MSIX_PENDING))
 				continue;
 			guest_entries[i].flags &= ~KVM_ASSIGNED_MSIX_PENDING;
 			kvm_set_irq(assigned_dev->kvm,
 				    assigned_dev->irq_source_id,
 				    guest_entries[i].vector, 1);
 		}
 	} else
 		kvm_set_irq(assigned_dev->kvm, assigned_dev->irq_source_id,
 			    assigned_dev->guest_irq, 1);
 	spin_unlock_irq(&assigned_dev->assigned_dev_lock);
 	mutex_unlock(&assigned_dev->kvm->irq_lock);
 }
 static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
 {
 	unsigned long flags;
 	struct kvm_assigned_dev_kernel *assigned_dev =
 		(struct kvm_assigned_dev_kernel *) dev_id;
 	spin_lock_irqsave(&assigned_dev->assigned_dev_lock, flags);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		int index = find_index_from_host_irq(assigned_dev, irq);
 		if (index < 0)
 			goto out;
 		assigned_dev->guest_msix_entries[index].flags |=
 			KVM_ASSIGNED_MSIX_PENDING;
 	}
 	schedule_work(&assigned_dev->interrupt_work);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_GUEST_INTX) {
 		disable_irq_nosync(irq);
 		assigned_dev->host_irq_disabled = true;
 	}
 out:
 	spin_unlock_irqrestore(&assigned_dev->assigned_dev_lock, flags);
 	return IRQ_HANDLED;
 }
 /* Ack the irq line for an assigned device */
 static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
 {
 	struct kvm_assigned_dev_kernel *dev;
 	unsigned long flags;
 	if (kian->gsi == -1)
 		return;
 	dev = container_of(kian, struct kvm_assigned_dev_kernel,
 			   ack_notifier);
 	kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
 	/* The guest irq may be shared so this ack may be
 	 * from another device.
 	 */
 	spin_lock_irqsave(&dev->assigned_dev_lock, flags);
 	if (dev->host_irq_disabled) {
 		enable_irq(dev->host_irq);
 		dev->host_irq_disabled = false;
 	}
 	spin_unlock_irqrestore(&dev->assigned_dev_lock, flags);
 }
 static void deassign_guest_irq(struct kvm *kvm,
 			       struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	kvm_unregister_irq_ack_notifier(kvm, &assigned_dev->ack_notifier);
 	assigned_dev->ack_notifier.gsi = -1;
 	if (assigned_dev->irq_source_id != -1)
 		kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
 	assigned_dev->irq_source_id = -1;
 	assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_GUEST_MASK);
 }
 /* The function implicit hold kvm->lock mutex due to cancel_work_sync() */
 static void deassign_host_irq(struct kvm *kvm,
 			      struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	/*
 	 * In kvm_free_device_irq, cancel_work_sync return true if:
 	 * 1. work is scheduled, and then cancelled.
 	 * 2. work callback is executed.
 	 *
 	 * The first one ensured that the irq is disabled and no more events
 	 * would happen. But for the second one, the irq may be enabled (e.g.
 	 * for MSI). So we disable irq here to prevent further events.
 	 *
 	 * Notice this maybe result in nested disable if the interrupt type is
 	 * INTx, but it's OK for we are going to free it.
 	 *
 	 * If this function is a part of VM destroy, please ensure that till
 	 * now, the kvm state is still legal for probably we also have to wait
 	 * interrupt_work done.
 	 */
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		int i;
 		for (i = 0; i < assigned_dev->entries_nr; i++)
 			disable_irq_nosync(assigned_dev->
 					   host_msix_entries[i].vector);
 		cancel_work_sync(&assigned_dev->interrupt_work);
 		for (i = 0; i < assigned_dev->entries_nr; i++)
 			free_irq(assigned_dev->host_msix_entries[i].vector,
 				 (void *)assigned_dev);
 		assigned_dev->entries_nr = 0;
 		kfree(assigned_dev->host_msix_entries);
 		kfree(assigned_dev->guest_msix_entries);
 		pci_disable_msix(assigned_dev->dev);
 	} else {
 		/* Deal with MSI and INTx */
 		disable_irq_nosync(assigned_dev->host_irq);
 		cancel_work_sync(&assigned_dev->interrupt_work);
 		free_irq(assigned_dev->host_irq, (void *)assigned_dev);
 		if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSI)
 			pci_disable_msi(assigned_dev->dev);
 	}
 	assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_HOST_MASK);
 }
 static int kvm_deassign_irq(struct kvm *kvm,
 			    struct kvm_assigned_dev_kernel *assigned_dev,
 			    unsigned long irq_requested_type)
 {
 	unsigned long guest_irq_type, host_irq_type;
 	if (!irqchip_in_kernel(kvm))
 		return -EINVAL;
 	/* no irq assignment to deassign */
 	if (!assigned_dev->irq_requested_type)
 		return -ENXIO;
 	host_irq_type = irq_requested_type & KVM_DEV_IRQ_HOST_MASK;
 	guest_irq_type = irq_requested_type & KVM_DEV_IRQ_GUEST_MASK;
 	if (host_irq_type)
 		deassign_host_irq(kvm, assigned_dev);
 	if (guest_irq_type)
 		deassign_guest_irq(kvm, assigned_dev);
 	return 0;
 }
 static void kvm_free_assigned_irq(struct kvm *kvm,
 				  struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
 }
 static void kvm_free_assigned_device(struct kvm *kvm,
 				     struct kvm_assigned_dev_kernel
 				     *assigned_dev)
 {
 	kvm_free_assigned_irq(kvm, assigned_dev);
 	pci_reset_function(assigned_dev->dev);
 	pci_release_regions(assigned_dev->dev);
 	pci_disable_device(assigned_dev->dev);
 	pci_dev_put(assigned_dev->dev);
 	list_del(&assigned_dev->list);
 	kfree(assigned_dev);
 }
 void kvm_free_all_assigned_devices(struct kvm *kvm)
 {
 	struct list_head *ptr, *ptr2;
 	struct kvm_assigned_dev_kernel *assigned_dev;
 	list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
 		assigned_dev = list_entry(ptr,
 					  struct kvm_assigned_dev_kernel,
 					  list);
 		kvm_free_assigned_device(kvm, assigned_dev);
 	}
 }
 static int assigned_device_enable_host_intx(struct kvm *kvm,
 					    struct kvm_assigned_dev_kernel *dev)
 {
 	dev->host_irq = dev->dev->irq;
 	/* Even though this is PCI, we don't want to use shared
 	 * interrupts. Sharing host devices with guest-assigned devices
 	 * on the same interrupt line is not a happy situation: there
 	 * are going to be long delays in accepting, acking, etc.
 	 */
 	if (request_irq(dev->host_irq, kvm_assigned_dev_intr,
 			0, "kvm_assigned_intx_device", (void *)dev))
 		return -EIO;
 	return 0;
 }
 #ifdef __KVM_HAVE_MSI
 static int assigned_device_enable_host_msi(struct kvm *kvm,
 					   struct kvm_assigned_dev_kernel *dev)
 {
 	int r;
 	if (!dev->dev->msi_enabled) {
 		r = pci_enable_msi(dev->dev);
 		if (r)
 			return r;
 	}
 	dev->host_irq = dev->dev->irq;
 	if (request_irq(dev->host_irq, kvm_assigned_dev_intr, 0,
 			"kvm_assigned_msi_device", (void *)dev)) {
 		pci_disable_msi(dev->dev);
 		return -EIO;
 	}
 	return 0;
 }
 #endif
 #ifdef __KVM_HAVE_MSIX
 static int assigned_device_enable_host_msix(struct kvm *kvm,
 					    struct kvm_assigned_dev_kernel *dev)
 {
 	int i, r = -EINVAL;
 	/* host_msix_entries and guest_msix_entries should have been
 	 * initialized */
 	if (dev->entries_nr == 0)
 		return r;
 	r = pci_enable_msix(dev->dev, dev->host_msix_entries, dev->entries_nr);
 	if (r)
 		return r;
 	for (i = 0; i < dev->entries_nr; i++) {
 		r = request_irq(dev->host_msix_entries[i].vector,
 				kvm_assigned_dev_intr, 0,
 				"kvm_assigned_msix_device",
 				(void *)dev);
 		/* FIXME: free requested_irq's on failure */
 		if (r)
 			return r;
 	}
 	return 0;
 }
 #endif
 static int assigned_device_enable_guest_intx(struct kvm *kvm,
 				struct kvm_assigned_dev_kernel *dev,
 				struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = irq->guest_irq;
 	return 0;
 }
 #ifdef __KVM_HAVE_MSI
 static int assigned_device_enable_guest_msi(struct kvm *kvm,
 			struct kvm_assigned_dev_kernel *dev,
 			struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = -1;
 	dev->host_irq_disabled = false;
 	return 0;
 }
 #endif
 #ifdef __KVM_HAVE_MSIX
 static int assigned_device_enable_guest_msix(struct kvm *kvm,
 			struct kvm_assigned_dev_kernel *dev,
 			struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = -1;
 	dev->host_irq_disabled = false;
 	return 0;
 }
 #endif
 static int assign_host_irq(struct kvm *kvm,
 			   struct kvm_assigned_dev_kernel *dev,
 			   __u32 host_irq_type)
 {
 	int r = -EEXIST;
 	if (dev->irq_requested_type & KVM_DEV_IRQ_HOST_MASK)
 		return r;
 	switch (host_irq_type) {
 	case KVM_DEV_IRQ_HOST_INTX:
 		r = assigned_device_enable_host_intx(kvm, dev);
 		break;
 #ifdef __KVM_HAVE_MSI
 	case KVM_DEV_IRQ_HOST_MSI:
 		r = assigned_device_enable_host_msi(kvm, dev);
 		break;
 #endif
 #ifdef __KVM_HAVE_MSIX
 	case KVM_DEV_IRQ_HOST_MSIX:
 		r = assigned_device_enable_host_msix(kvm, dev);
 		break;
 #endif
 	default:
 		r = -EINVAL;
 	}
 	if (!r)
 		dev->irq_requested_type |= host_irq_type;
 	return r;
 }
 static int assign_guest_irq(struct kvm *kvm,
 			    struct kvm_assigned_dev_kernel *dev,
 			    struct kvm_assigned_irq *irq,
 			    unsigned long guest_irq_type)
 {
 	int id;
 	int r = -EEXIST;
 	if (dev->irq_requested_type & KVM_DEV_IRQ_GUEST_MASK)
 		return r;
 	id = kvm_request_irq_source_id(kvm);
 	if (id < 0)
 		return id;
 	dev->irq_source_id = id;
 	switch (guest_irq_type) {
 	case KVM_DEV_IRQ_GUEST_INTX:
 		r = assigned_device_enable_guest_intx(kvm, dev, irq);
 		break;
 #ifdef __KVM_HAVE_MSI
 	case KVM_DEV_IRQ_GUEST_MSI:
 		r = assigned_device_enable_guest_msi(kvm, dev, irq);
 		break;
 #endif
 #ifdef __KVM_HAVE_MSIX
 	case KVM_DEV_IRQ_GUEST_MSIX:
 		r = assigned_device_enable_guest_msix(kvm, dev, irq);
 		break;
 #endif
 	default:
 		r = -EINVAL;
 	}
 	if (!r) {
 		dev->irq_requested_type |= guest_irq_type;
 		kvm_register_irq_ack_notifier(kvm, &dev->ack_notifier);
 	} else
 		kvm_free_irq_source_id(kvm, dev->irq_source_id);
 	return r;
 }
 /* TODO Deal with KVM_DEV_IRQ_ASSIGNED_MASK_MSIX */
 static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
 				   struct kvm_assigned_irq *assigned_irq)
 {
 	int r = -EINVAL;
 	struct kvm_assigned_dev_kernel *match;
 	unsigned long host_irq_type, guest_irq_type;
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
 	if (!irqchip_in_kernel(kvm))
 		return r;
 	mutex_lock(&kvm->lock);
 	r = -ENODEV;
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_irq->assigned_dev_id);
 	if (!match)
 		goto out;
 	host_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_HOST_MASK);
 	guest_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_GUEST_MASK);
 	r = -EINVAL;
 	/* can only assign one type at a time */
 	if (hweight_long(host_irq_type) > 1)
 		goto out;
 	if (hweight_long(guest_irq_type) > 1)
 		goto out;
 	if (host_irq_type == 0 && guest_irq_type == 0)
 		goto out;
 	r = 0;
 	if (host_irq_type)
 		r = assign_host_irq(kvm, match, host_irq_type);
 	if (r)
 		goto out;
 	if (guest_irq_type)
 		r = assign_guest_irq(kvm, match, assigned_irq, guest_irq_type);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_deassign_dev_irq(struct kvm *kvm,
 					 struct kvm_assigned_irq
 					 *assigned_irq)
 {
 	int r = -ENODEV;
 	struct kvm_assigned_dev_kernel *match;
 	mutex_lock(&kvm->lock);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_irq->assigned_dev_id);
 	if (!match)
 		goto out;
 	r = kvm_deassign_irq(kvm, match, assigned_irq->flags);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
 				      struct kvm_assigned_pci_dev *assigned_dev)
 {
 	int r = 0;
 	struct kvm_assigned_dev_kernel *match;
 	struct pci_dev *dev;
 	down_read(&kvm->slots_lock);
 	mutex_lock(&kvm->lock);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_dev->assigned_dev_id);
 	if (match) {
 		/* device already assigned */
 		r = -EEXIST;
 		goto out;
 	}
 	match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
 	if (match == NULL) {
 		printk(KERN_INFO "%s: Couldn't allocate memory\n",
 		       __func__);
 		r = -ENOMEM;
 		goto out;
 	}
 	dev = pci_get_bus_and_slot(assigned_dev->busnr,
 				   assigned_dev->devfn);
 	if (!dev) {
 		printk(KERN_INFO "%s: host device not found\n", __func__);
 		r = -EINVAL;
 		goto out_free;
 	}
 	if (pci_enable_device(dev)) {
 		printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
 		r = -EBUSY;
 		goto out_put;
 	}
 	r = pci_request_regions(dev, "kvm_assigned_device");
 	if (r) {
 		printk(KERN_INFO "%s: Could not get access to device regions\n",
 		       __func__);
 		goto out_disable;
 	}
 	pci_reset_function(dev);
 	match->assigned_dev_id = assigned_dev->assigned_dev_id;
 	match->host_busnr = assigned_dev->busnr;
 	match->host_devfn = assigned_dev->devfn;
 	match->flags = assigned_dev->flags;
 	match->dev = dev;
 	spin_lock_init(&match->assigned_dev_lock);
 	match->irq_source_id = -1;
 	match->kvm = kvm;
 	match->ack_notifier.irq_acked = kvm_assigned_dev_ack_irq;
 	INIT_WORK(&match->interrupt_work,
 		  kvm_assigned_dev_interrupt_work_handler);
 	list_add(&match->list, &kvm->arch.assigned_dev_head);
 	if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
 		if (!kvm->arch.iommu_domain) {
 			r = kvm_iommu_map_guest(kvm);
 			if (r)
 				goto out_list_del;
 		}
 		r = kvm_assign_device(kvm, match);
 		if (r)
 			goto out_list_del;
 	}
 out:
 	mutex_unlock(&kvm->lock);
 	up_read(&kvm->slots_lock);
 	return r;
 out_list_del:
 	list_del(&match->list);
 	pci_release_regions(dev);
 out_disable:
 	pci_disable_device(dev);
 out_put:
 	pci_dev_put(dev);
 out_free:
 	kfree(match);
 	mutex_unlock(&kvm->lock);
 	up_read(&kvm->slots_lock);
 	return r;
 }
 #endif
 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
 static int kvm_vm_ioctl_deassign_device(struct kvm *kvm,
 		struct kvm_assigned_pci_dev *assigned_dev)
 {
 	int r = 0;
 	struct kvm_assigned_dev_kernel *match;
 	mutex_lock(&kvm->lock);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_dev->assigned_dev_id);
 	if (!match) {
 		printk(KERN_INFO "%s: device hasn't been assigned before, "
 		  "so cannot be deassigned\n", __func__);
 		r = -EINVAL;
 		goto out;
 	}
 	if (match->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU)
 		kvm_deassign_device(kvm, match);
 	kvm_free_assigned_device(kvm, match);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 #endif
-static inline int valid_vcpu(int n)
-{
-	return likely(n >= 0 && n < KVM_MAX_VCPUS);
-}
 inline int kvm_is_mmio_pfn(pfn_t pfn)
 {
 	if (pfn_valid(pfn)) {
 		struct page *page = compound_head(pfn_to_page(pfn));
 		return PageReserved(page);
 	}
 	return true;
 }
 /*
  * Switches to specified vcpu, until a matching vcpu_put()
  */
 void vcpu_load(struct kvm_vcpu *vcpu)
 {
 	int cpu;
 	mutex_lock(&vcpu->mutex);
 	cpu = get_cpu();
 	preempt_notifier_register(&vcpu->preempt_notifier);
 	kvm_arch_vcpu_load(vcpu, cpu);
 	put_cpu();
 }
 void vcpu_put(struct kvm_vcpu *vcpu)
 {
 	preempt_disable();
 	kvm_arch_vcpu_put(vcpu);
 	preempt_notifier_unregister(&vcpu->preempt_notifier);
 	preempt_enable();
 	mutex_unlock(&vcpu->mutex);
 }
 static void ack_flush(void *_completed)
 {
 }
 static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
 {
 	int i, cpu, me;
 	cpumask_var_t cpus;
 	bool called = true;
 	struct kvm_vcpu *vcpu;
 	if (alloc_cpumask_var(&cpus, GFP_ATOMIC))
 		cpumask_clear(cpus);
 	me = get_cpu();
 	spin_lock(&kvm->requests_lock);
 	for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 		vcpu = kvm->vcpus[i];
 		if (!vcpu)
 			continue;
 		if (test_and_set_bit(req, &vcpu->requests))
 			continue;
 		cpu = vcpu->cpu;
 		if (cpus != NULL && cpu != -1 && cpu != me)
 			cpumask_set_cpu(cpu, cpus);
 	}
 	if (unlikely(cpus == NULL))
 		smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 	else if (!cpumask_empty(cpus))
 		smp_call_function_many(cpus, ack_flush, NULL, 1);
 	else
 		called = false;
 	spin_unlock(&kvm->requests_lock);
 	put_cpu();
 	free_cpumask_var(cpus);
 	return called;
 }
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
 	if (make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 		++kvm->stat.remote_tlb_flush;
 }
 void kvm_reload_remote_mmus(struct kvm *kvm)
 {
 	make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 }
 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 {
 	struct page *page;
 	int r;
 	mutex_init(&vcpu->mutex);
 	vcpu->cpu = -1;
 	vcpu->kvm = kvm;
 	vcpu->vcpu_id = id;
 	init_waitqueue_head(&vcpu->wq);
 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (!page) {
 		r = -ENOMEM;
 		goto fail;
 	}
 	vcpu->run = page_address(page);
 	r = kvm_arch_vcpu_init(vcpu);
 	if (r < 0)
 		goto fail_free_run;
 	return 0;
 fail_free_run:
 	free_page((unsigned long)vcpu->run);
 fail:
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 {
 	kvm_arch_vcpu_uninit(vcpu);
 	free_page((unsigned long)vcpu->run);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 {
 	return container_of(mn, struct kvm, mmu_notifier);
 }
 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 					     struct mm_struct *mm,
 					     unsigned long address)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush;
 	/*
 	 * When ->invalidate_page runs, the linux pte has been zapped
 	 * already but the page is still allocated until
 	 * ->invalidate_page returns. So if we increase the sequence
 	 * here the kvm page fault will notice if the spte can't be
 	 * established because the page is going to be freed. If
 	 * instead the kvm page fault establishes the spte before
 	 * ->invalidate_page runs, kvm_unmap_hva will release it
 	 * before returning.
 	 *
 	 * The sequence increase only need to be seen at spin_unlock
 	 * time, and not at spin_lock time.
 	 *
 	 * Increasing the sequence after the spin_unlock would be
 	 * unsafe because the kvm page fault could then establish the
 	 * pte after kvm_unmap_hva returned, without noticing the page
 	 * is going to be freed.
 	 */
 	spin_lock(&kvm->mmu_lock);
 	kvm->mmu_notifier_seq++;
 	need_tlb_flush = kvm_unmap_hva(kvm, address);
 	spin_unlock(&kvm->mmu_lock);
 	/* we've to flush the tlb before the pages can be freed */
 	if (need_tlb_flush)
 		kvm_flush_remote_tlbs(kvm);
 }
 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 						    struct mm_struct *mm,
 						    unsigned long start,
 						    unsigned long end)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int need_tlb_flush = 0;
 	spin_lock(&kvm->mmu_lock);
 	/*
 	 * The count increase must become visible at unlock time as no
 	 * spte can be established without taking the mmu_lock and
 	 * count is also read inside the mmu_lock critical section.
 	 */
 	kvm->mmu_notifier_count++;
 	for (; start < end; start += PAGE_SIZE)
 		need_tlb_flush |= kvm_unmap_hva(kvm, start);
 	spin_unlock(&kvm->mmu_lock);
 	/* we've to flush the tlb before the pages can be freed */
 	if (need_tlb_flush)
 		kvm_flush_remote_tlbs(kvm);
 }
 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 						  struct mm_struct *mm,
 						  unsigned long start,
 						  unsigned long end)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	spin_lock(&kvm->mmu_lock);
 	/*
 	 * This sequence increase will notify the kvm page fault that
 	 * the page that is going to be mapped in the spte could have
 	 * been freed.
 	 */
 	kvm->mmu_notifier_seq++;
 	/*
 	 * The above sequence increase must be visible before the
 	 * below count decrease but both values are read by the kvm
 	 * page fault under mmu_lock spinlock so we don't need to add
 	 * a smb_wmb() here in between the two.
 	 */
 	kvm->mmu_notifier_count--;
 	spin_unlock(&kvm->mmu_lock);
 	BUG_ON(kvm->mmu_notifier_count < 0);
 }
 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 					      struct mm_struct *mm,
 					      unsigned long address)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	int young;
 	spin_lock(&kvm->mmu_lock);
 	young = kvm_age_hva(kvm, address);
 	spin_unlock(&kvm->mmu_lock);
 	if (young)
 		kvm_flush_remote_tlbs(kvm);
 	return young;
 }
 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 				     struct mm_struct *mm)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
 	kvm_arch_flush_shadow(kvm);
 }
 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
 	.invalidate_range_start	= kvm_mmu_notifier_invalidate_range_start,
 	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
 	.clear_flush_young	= kvm_mmu_notifier_clear_flush_young,
 	.release		= kvm_mmu_notifier_release,
 };
 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 static struct kvm *kvm_create_vm(void)
 {
 	struct kvm *kvm = kvm_arch_create_vm();
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	struct page *page;
 #endif
 	if (IS_ERR(kvm))
 		goto out;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 	INIT_LIST_HEAD(&kvm->irq_routing);
 	INIT_HLIST_HEAD(&kvm->mask_notifier_list);
 #endif
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (!page) {
 		kfree(kvm);
 		return ERR_PTR(-ENOMEM);
 	}
 	kvm->coalesced_mmio_ring =
 			(struct kvm_coalesced_mmio_ring *)page_address(page);
 #endif
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	{
 		int err;
 		kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 		err = mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 		if (err) {
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 			put_page(page);
 #endif
 			kfree(kvm);
 			return ERR_PTR(err);
 		}
 	}
 #endif
 	kvm->mm = current->mm;
 	atomic_inc(&kvm->mm->mm_count);
 	spin_lock_init(&kvm->mmu_lock);
 	spin_lock_init(&kvm->requests_lock);
 	kvm_io_bus_init(&kvm->pio_bus);
 	kvm_irqfd_init(kvm);
 	mutex_init(&kvm->lock);
 	mutex_init(&kvm->irq_lock);
 	kvm_io_bus_init(&kvm->mmio_bus);
 	init_rwsem(&kvm->slots_lock);
 	atomic_set(&kvm->users_count, 1);
 	spin_lock(&kvm_lock);
 	list_add(&kvm->vm_list, &vm_list);
 	spin_unlock(&kvm_lock);
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	kvm_coalesced_mmio_init(kvm);
 #endif
 out:
 	return kvm;
 }
 /*
  * Free any memory in @free but not in @dont.
  */
 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 				  struct kvm_memory_slot *dont)
 {
 	if (!dont || free->rmap != dont->rmap)
 		vfree(free->rmap);
 	if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 		vfree(free->dirty_bitmap);
 	if (!dont || free->lpage_info != dont->lpage_info)
 		vfree(free->lpage_info);
 	free->npages = 0;
 	free->dirty_bitmap = NULL;
 	free->rmap = NULL;
 	free->lpage_info = NULL;
 }
 void kvm_free_physmem(struct kvm *kvm)
 {
 	int i;
 	for (i = 0; i < kvm->nmemslots; ++i)
 		kvm_free_physmem_slot(&kvm->memslots[i], NULL);
 }
 static void kvm_destroy_vm(struct kvm *kvm)
 {
 	struct mm_struct *mm = kvm->mm;
 	kvm_arch_sync_events(kvm);
 	spin_lock(&kvm_lock);
 	list_del(&kvm->vm_list);
 	spin_unlock(&kvm_lock);
 	kvm_free_irq_routing(kvm);
 	kvm_io_bus_destroy(&kvm->pio_bus);
 	kvm_io_bus_destroy(&kvm->mmio_bus);
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	if (kvm->coalesced_mmio_ring != NULL)
 		free_page((unsigned long)kvm->coalesced_mmio_ring);
 #endif
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 #else
 	kvm_arch_flush_shadow(kvm);
 #endif
 	kvm_arch_destroy_vm(kvm);
 	mmdrop(mm);
 }
 void kvm_get_kvm(struct kvm *kvm)
 {
 	atomic_inc(&kvm->users_count);
 }
 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 void kvm_put_kvm(struct kvm *kvm)
 {
 	if (atomic_dec_and_test(&kvm->users_count))
 		kvm_destroy_vm(kvm);
 }
 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 static int kvm_vm_release(struct inode *inode, struct file *filp)
 {
 	struct kvm *kvm = filp->private_data;
 	kvm_irqfd_release(kvm);
 	kvm_put_kvm(kvm);
 	return 0;
 }
 /*
  * Allocate some memory and give it an address in the guest physical address
  * space.
  *
  * Discontiguous memory is allowed, mostly for framebuffers.
  *
  * Must be called holding mmap_sem for write.
  */
 int __kvm_set_memory_region(struct kvm *kvm,
 			    struct kvm_userspace_memory_region *mem,
 			    int user_alloc)
 {
 	int r;
 	gfn_t base_gfn;
 	unsigned long npages, ugfn;
 	unsigned long largepages, i;
 	struct kvm_memory_slot *memslot;
 	struct kvm_memory_slot old, new;
 	r = -EINVAL;
 	/* General sanity checks */
 	if (mem->memory_size & (PAGE_SIZE - 1))
 		goto out;
 	if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 		goto out;
 	if (user_alloc && (mem->userspace_addr & (PAGE_SIZE - 1)))
 		goto out;
 	if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
 		goto out;
 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 		goto out;
 	memslot = &kvm->memslots[mem->slot];
 	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 	npages = mem->memory_size >> PAGE_SHIFT;
 	if (!npages)
 		mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
 	new = old = *memslot;
 	new.base_gfn = base_gfn;
 	new.npages = npages;
 	new.flags = mem->flags;
 	/* Disallow changing a memory slot's size. */
 	r = -EINVAL;
 	if (npages && old.npages && npages != old.npages)
 		goto out_free;
 	/* Check for overlaps */
 	r = -EEXIST;
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *s = &kvm->memslots[i];
 		if (s == memslot || !s->npages)
 			continue;
 		if (!((base_gfn + npages <= s->base_gfn) ||
 		      (base_gfn >= s->base_gfn + s->npages)))
 			goto out_free;
 	}
 	/* Free page dirty bitmap if unneeded */
 	if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 		new.dirty_bitmap = NULL;
 	r = -ENOMEM;
 	/* Allocate if a slot is being created */
 #ifndef CONFIG_S390
 	if (npages && !new.rmap) {
 		new.rmap = vmalloc(npages * sizeof(struct page *));
 		if (!new.rmap)
 			goto out_free;
 		memset(new.rmap, 0, npages * sizeof(*new.rmap));
 		new.user_alloc = user_alloc;
 		/*
 		 * hva_to_rmmap() serialzies with the mmu_lock and to be
 		 * safe it has to ignore memslots with !user_alloc &&
 		 * !userspace_addr.
 		 */
 		if (user_alloc)
 			new.userspace_addr = mem->userspace_addr;
 		else
 			new.userspace_addr = 0;
 	}
 	if (npages && !new.lpage_info) {
 		largepages = 1 + (base_gfn + npages - 1) / KVM_PAGES_PER_HPAGE;
 		largepages -= base_gfn / KVM_PAGES_PER_HPAGE;
 		new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));
 		if (!new.lpage_info)
 			goto out_free;
 		memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));
 		if (base_gfn % KVM_PAGES_PER_HPAGE)
 			new.lpage_info[0].write_count = 1;
 		if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
 			new.lpage_info[largepages-1].write_count = 1;
 		ugfn = new.userspace_addr >> PAGE_SHIFT;
 		/*
 		 * If the gfn and userspace address are not aligned wrt each
 		 * other, disable large page support for this slot
 		 */
 		if ((base_gfn ^ ugfn) & (KVM_PAGES_PER_HPAGE - 1))
 			for (i = 0; i < largepages; ++i)
 				new.lpage_info[i].write_count = 1;
 	}
 	/* Allocate page dirty bitmap if needed */
 	if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 		unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
 		new.dirty_bitmap = vmalloc(dirty_bytes);
 		if (!new.dirty_bitmap)
 			goto out_free;
 		memset(new.dirty_bitmap, 0, dirty_bytes);
 		if (old.npages)
 			kvm_arch_flush_shadow(kvm);
 	}
 #endif /* not defined CONFIG_S390 */
 	if (!npages)
 		kvm_arch_flush_shadow(kvm);
 	spin_lock(&kvm->mmu_lock);
 	if (mem->slot >= kvm->nmemslots)
 		kvm->nmemslots = mem->slot + 1;
 	*memslot = new;
 	spin_unlock(&kvm->mmu_lock);
 	r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
 	if (r) {
 		spin_lock(&kvm->mmu_lock);
 		*memslot = old;
 		spin_unlock(&kvm->mmu_lock);
 		goto out_free;
 	}
 	kvm_free_physmem_slot(&old, npages ? &new : NULL);
 	/* Slot deletion case: we have to update the current slot */
 	spin_lock(&kvm->mmu_lock);
 	if (!npages)
 		*memslot = old;
 	spin_unlock(&kvm->mmu_lock);
 #ifdef CONFIG_DMAR
 	/* map the pages in iommu page table */
 	r = kvm_iommu_map_pages(kvm, base_gfn, npages);
 	if (r)
 		goto out;
 #endif
 	return 0;
 out_free:
 	kvm_free_physmem_slot(&new, &old);
 out:
 	return r;
 }
 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 int kvm_set_memory_region(struct kvm *kvm,
 			  struct kvm_userspace_memory_region *mem,
 			  int user_alloc)
 {
 	int r;
 	down_write(&kvm->slots_lock);
 	r = __kvm_set_memory_region(kvm, mem, user_alloc);
 	up_write(&kvm->slots_lock);
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
 				   struct
 				   kvm_userspace_memory_region *mem,
 				   int user_alloc)
 {
 	if (mem->slot >= KVM_MEMORY_SLOTS)
 		return -EINVAL;
 	return kvm_set_memory_region(kvm, mem, user_alloc);
 }
 int kvm_get_dirty_log(struct kvm *kvm,
 			struct kvm_dirty_log *log, int *is_dirty)
 {
 	struct kvm_memory_slot *memslot;
 	int r, i;
 	int n;
 	unsigned long any = 0;
 	r = -EINVAL;
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 	memslot = &kvm->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
 	n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
 	for (i = 0; !any && i < n/sizeof(long); ++i)
 		any = memslot->dirty_bitmap[i];
 	r = -EFAULT;
 	if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
 		goto out;
 	if (any)
 		*is_dirty = 1;
 	r = 0;
 out:
 	return r;
 }
 int is_error_page(struct page *page)
 {
 	return page == bad_page;
 }
 EXPORT_SYMBOL_GPL(is_error_page);
 int is_error_pfn(pfn_t pfn)
 {
 	return pfn == bad_pfn;
 }
 EXPORT_SYMBOL_GPL(is_error_pfn);
 static inline unsigned long bad_hva(void)
 {
 	return PAGE_OFFSET;
 }
 int kvm_is_error_hva(unsigned long addr)
 {
 	return addr == bad_hva();
 }
 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
 struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	for (i = 0; i < kvm->nmemslots; ++i) {
 		struct kvm_memory_slot *memslot = &kvm->memslots[i];
 		if (gfn >= memslot->base_gfn
 		    && gfn < memslot->base_gfn + memslot->npages)
 			return memslot;
 	}
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(gfn_to_memslot_unaliased);
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 {
 	gfn = unalias_gfn(kvm, gfn);
 	return gfn_to_memslot_unaliased(kvm, gfn);
 }
 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	gfn = unalias_gfn(kvm, gfn);
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *memslot = &kvm->memslots[i];
 		if (gfn >= memslot->base_gfn
 		    && gfn < memslot->base_gfn + memslot->npages)
 			return 1;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
 {
 	struct kvm_memory_slot *slot;
 	gfn = unalias_gfn(kvm, gfn);
 	slot = gfn_to_memslot_unaliased(kvm, gfn);
 	if (!slot)
 		return bad_hva();
 	return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
 }
 EXPORT_SYMBOL_GPL(gfn_to_hva);
 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
 {
 	struct page *page[1];
 	unsigned long addr;
 	int npages;
 	pfn_t pfn;
 	might_sleep();
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr)) {
 		get_page(bad_page);
 		return page_to_pfn(bad_page);
 	}
 	npages = get_user_pages_fast(addr, 1, 1, page);
 	if (unlikely(npages != 1)) {
 		struct vm_area_struct *vma;
 		down_read(&current->mm->mmap_sem);
 		vma = find_vma(current->mm, addr);
 		if (vma == NULL || addr < vma->vm_start ||
 		    !(vma->vm_flags & VM_PFNMAP)) {
 			up_read(&current->mm->mmap_sem);
 			get_page(bad_page);
 			return page_to_pfn(bad_page);
 		}
 		pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
 		up_read(&current->mm->mmap_sem);
 		BUG_ON(!kvm_is_mmio_pfn(pfn));
 	} else
 		pfn = page_to_pfn(page[0]);
 	return pfn;
 }
 EXPORT_SYMBOL_GPL(gfn_to_pfn);
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
 {
 	pfn_t pfn;
 	pfn = gfn_to_pfn(kvm, gfn);
 	if (!kvm_is_mmio_pfn(pfn))
 		return pfn_to_page(pfn);
 	WARN_ON(kvm_is_mmio_pfn(pfn));
 	get_page(bad_page);
 	return bad_page;
 }
 EXPORT_SYMBOL_GPL(gfn_to_page);
 void kvm_release_page_clean(struct page *page)
 {
 	kvm_release_pfn_clean(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
 void kvm_release_pfn_clean(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		put_page(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
 void kvm_release_page_dirty(struct page *page)
 {
 	kvm_release_pfn_dirty(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
 void kvm_release_pfn_dirty(pfn_t pfn)
 {
 	kvm_set_pfn_dirty(pfn);
 	kvm_release_pfn_clean(pfn);
 }
 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
 void kvm_set_page_dirty(struct page *page)
 {
 	kvm_set_pfn_dirty(page_to_pfn(page));
 }
 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
 void kvm_set_pfn_dirty(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn)) {
 		struct page *page = pfn_to_page(pfn);
 		if (!PageReserved(page))
 			SetPageDirty(page);
 	}
 }
 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
 void kvm_set_pfn_accessed(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		mark_page_accessed(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
 void kvm_get_pfn(pfn_t pfn)
 {
 	if (!kvm_is_mmio_pfn(pfn))
 		get_page(pfn_to_page(pfn));
 }
 EXPORT_SYMBOL_GPL(kvm_get_pfn);
 static int next_segment(unsigned long len, int offset)
 {
 	if (len > PAGE_SIZE - offset)
 		return PAGE_SIZE - offset;
 	else
 		return len;
 }
 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
 			int len)
 {
 	int r;
 	unsigned long addr;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	r = copy_from_user(data, (void __user *)addr + offset, len);
 	if (r)
 		return -EFAULT;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
 	while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		data += seg;
 		++gfn;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_read_guest);
 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
 			  unsigned long len)
 {
 	int r;
 	unsigned long addr;
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int offset = offset_in_page(gpa);
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	pagefault_disable();
 	r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
 	pagefault_enable();
 	if (r)
 		return -EFAULT;
 	return 0;
 }
 EXPORT_SYMBOL(kvm_read_guest_atomic);
 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
 			 int offset, int len)
 {
 	int r;
 	unsigned long addr;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return -EFAULT;
 	r = copy_to_user((void __user *)addr + offset, data, len);
 	if (r)
 		return -EFAULT;
 	mark_page_dirty(kvm, gfn);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
 		    unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
 	while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		data += seg;
 		++gfn;
 	}
 	return 0;
 }
 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
 {
 	return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
 }
 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
 {
 	gfn_t gfn = gpa >> PAGE_SHIFT;
 	int seg;
 	int offset = offset_in_page(gpa);
 	int ret;
         while ((seg = next_segment(len, offset)) != 0) {
 		ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
 		if (ret < 0)
 			return ret;
 		offset = 0;
 		len -= seg;
 		++gfn;
 	}
 	return 0;
 }
 EXPORT_SYMBOL_GPL(kvm_clear_guest);
 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 {
 	struct kvm_memory_slot *memslot;
 	gfn = unalias_gfn(kvm, gfn);
 	memslot = gfn_to_memslot_unaliased(kvm, gfn);
 	if (memslot && memslot->dirty_bitmap) {
 		unsigned long rel_gfn = gfn - memslot->base_gfn;
 		/* avoid RMW */
 		if (!test_bit(rel_gfn, memslot->dirty_bitmap))
 			set_bit(rel_gfn, memslot->dirty_bitmap);
 	}
 }
 /*
  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  */
 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 {
 	DEFINE_WAIT(wait);
 	for (;;) {
 		prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
 		if ((kvm_arch_interrupt_allowed(vcpu) &&
 					kvm_cpu_has_interrupt(vcpu)) ||
 				kvm_arch_vcpu_runnable(vcpu)) {
 			set_bit(KVM_REQ_UNHALT, &vcpu->requests);
 			break;
 		}
 		if (kvm_cpu_has_pending_timer(vcpu))
 			break;
 		if (signal_pending(current))
 			break;
 		vcpu_put(vcpu);
 		schedule();
 		vcpu_load(vcpu);
 	}
 	finish_wait(&vcpu->wq, &wait);
 }
 void kvm_resched(struct kvm_vcpu *vcpu)
 {
 	if (!need_resched())
 		return;
 	cond_resched();
 }
 EXPORT_SYMBOL_GPL(kvm_resched);
 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
 	struct page *page;
 	if (vmf->pgoff == 0)
 		page = virt_to_page(vcpu->run);
 #ifdef CONFIG_X86
 	else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
 		page = virt_to_page(vcpu->arch.pio_data);
 #endif
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
 		page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
 #endif
 	else
 		return VM_FAULT_SIGBUS;
 	get_page(page);
 	vmf->page = page;
 	return 0;
 }
 static struct vm_operations_struct kvm_vcpu_vm_ops = {
 	.fault = kvm_vcpu_fault,
 };
 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	vma->vm_ops = &kvm_vcpu_vm_ops;
 	return 0;
 }
 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	kvm_put_kvm(vcpu->kvm);
 	return 0;
 }
 static struct file_operations kvm_vcpu_fops = {
 	.release        = kvm_vcpu_release,
 	.unlocked_ioctl = kvm_vcpu_ioctl,
 	.compat_ioctl   = kvm_vcpu_ioctl,
 	.mmap           = kvm_vcpu_mmap,
 };
 /*
  * Allocates an inode for the vcpu.
  */
 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
 {
-	int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
+	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
-	if (fd < 0)
-		kvm_put_kvm(vcpu->kvm);
-	return fd;
 }
 /*
  * Creates some virtual cpus.  Good luck creating more than one.
  */
-static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
+static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
 {
 	int r;
 	struct kvm_vcpu *vcpu;
-	if (!valid_vcpu(n))
+	vcpu = kvm_arch_vcpu_create(kvm, id);
-		return -EINVAL;
-	vcpu = kvm_arch_vcpu_create(kvm, n);
 	if (IS_ERR(vcpu))
 		return PTR_ERR(vcpu);
 	preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
 	r = kvm_arch_vcpu_setup(vcpu);
 	if (r)
 		return r;
 	mutex_lock(&kvm->lock);
-	if (kvm->vcpus[n]) {
+	if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
-		r = -EEXIST;
+		r = -EINVAL;
 		goto vcpu_destroy;
 	}
-	kvm->vcpus[n] = vcpu;
-	if (n == 0)
-		kvm->bsp_vcpu = vcpu;
-	mutex_unlock(&kvm->lock);
+	for (r = 0; r < atomic_read(&kvm->online_vcpus); r++)
+		if (kvm->vcpus[r]->vcpu_id == id) {
+			r = -EEXIST;
+			goto vcpu_destroy;
+		}
+	BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
 	/* Now it's all set up, let userspace reach it */
 	kvm_get_kvm(kvm);
 	r = create_vcpu_fd(vcpu);
-	if (r < 0)
+	if (r < 0) {
-		goto unlink;
+		kvm_put_kvm(kvm);
+		goto vcpu_destroy;
+	}
+	kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
+	smp_wmb();
+	atomic_inc(&kvm->online_vcpus);
+#ifdef CONFIG_KVM_APIC_ARCHITECTURE
+	if (kvm->bsp_vcpu_id == id)
+		kvm->bsp_vcpu = vcpu;
+#endif
+	mutex_unlock(&kvm->lock);
 	return r;
-unlink:
-	mutex_lock(&kvm->lock);
-	kvm->vcpus[n] = NULL;
 vcpu_destroy:
 	mutex_unlock(&kvm->lock);
 	kvm_arch_vcpu_destroy(vcpu);
 	return r;
 }
 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
 {
 	if (sigset) {
 		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
 		vcpu->sigset_active = 1;
 		vcpu->sigset = *sigset;
 	} else
 		vcpu->sigset_active = 0;
 	return 0;
 }
 #ifdef __KVM_HAVE_MSIX
 static int kvm_vm_ioctl_set_msix_nr(struct kvm *kvm,
 				    struct kvm_assigned_msix_nr *entry_nr)
 {
 	int r = 0;
 	struct kvm_assigned_dev_kernel *adev;
 	mutex_lock(&kvm->lock);
 	adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      entry_nr->assigned_dev_id);
 	if (!adev) {
 		r = -EINVAL;
 		goto msix_nr_out;
 	}
 	if (adev->entries_nr == 0) {
 		adev->entries_nr = entry_nr->entry_nr;
 		if (adev->entries_nr == 0 ||
 		    adev->entries_nr >= KVM_MAX_MSIX_PER_DEV) {
 			r = -EINVAL;
 			goto msix_nr_out;
 		}
 		adev->host_msix_entries = kzalloc(sizeof(struct msix_entry) *
 						entry_nr->entry_nr,
 						GFP_KERNEL);
 		if (!adev->host_msix_entries) {
 			r = -ENOMEM;
 			goto msix_nr_out;
 		}
 		adev->guest_msix_entries = kzalloc(
 				sizeof(struct kvm_guest_msix_entry) *
 				entry_nr->entry_nr, GFP_KERNEL);
 		if (!adev->guest_msix_entries) {
 			kfree(adev->host_msix_entries);
 			r = -ENOMEM;
 			goto msix_nr_out;
 		}
 	} else /* Not allowed set MSI-X number twice */
 		r = -EINVAL;
 msix_nr_out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
 				       struct kvm_assigned_msix_entry *entry)
 {
 	int r = 0, i;
 	struct kvm_assigned_dev_kernel *adev;
 	mutex_lock(&kvm->lock);
 	adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      entry->assigned_dev_id);
 	if (!adev) {
 		r = -EINVAL;
 		goto msix_entry_out;
 	}
 	for (i = 0; i < adev->entries_nr; i++)
 		if (adev->guest_msix_entries[i].vector == 0 ||
 		    adev->guest_msix_entries[i].entry == entry->entry) {
 			adev->guest_msix_entries[i].entry = entry->entry;
 			adev->guest_msix_entries[i].vector = entry->gsi;
 			adev->host_msix_entries[i].entry = entry->entry;
 			break;
 		}
 	if (i == adev->entries_nr) {
 		r = -ENOSPC;
 		goto msix_entry_out;
 	}
 msix_entry_out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 #endif
 static long kvm_vcpu_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	struct kvm_fpu *fpu = NULL;
 	struct kvm_sregs *kvm_sregs = NULL;
 	if (vcpu->kvm->mm != current->mm)
 		return -EIO;
 	switch (ioctl) {
 	case KVM_RUN:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
 		break;
 	case KVM_GET_REGS: {
 		struct kvm_regs *kvm_regs;
 		r = -ENOMEM;
 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
 		if (!kvm_regs)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
 		if (r)
 			goto out_free1;
 		r = -EFAULT;
 		if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
 			goto out_free1;
 		r = 0;
 out_free1:
 		kfree(kvm_regs);
 		break;
 	}
 	case KVM_SET_REGS: {
 		struct kvm_regs *kvm_regs;
 		r = -ENOMEM;
 		kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
 		if (!kvm_regs)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
 			goto out_free2;
 		r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
 		if (r)
 			goto out_free2;
 		r = 0;
 out_free2:
 		kfree(kvm_regs);
 		break;
 	}
 	case KVM_GET_SREGS: {
 		kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!kvm_sregs)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_SREGS: {
 		kvm_sregs = kmalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!kvm_sregs)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(kvm_sregs, argp, sizeof(struct kvm_sregs)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_GET_MP_STATE: {
 		struct kvm_mp_state mp_state;
 		r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &mp_state, sizeof mp_state))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_MP_STATE: {
 		struct kvm_mp_state mp_state;
 		r = -EFAULT;
 		if (copy_from_user(&mp_state, argp, sizeof mp_state))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_TRANSLATE: {
 		struct kvm_translation tr;
 		r = -EFAULT;
 		if (copy_from_user(&tr, argp, sizeof tr))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, &tr, sizeof tr))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_GUEST_DEBUG: {
 		struct kvm_guest_debug dbg;
 		r = -EFAULT;
 		if (copy_from_user(&dbg, argp, sizeof dbg))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_SIGNAL_MASK: {
 		struct kvm_signal_mask __user *sigmask_arg = argp;
 		struct kvm_signal_mask kvm_sigmask;
 		sigset_t sigset, *p;
 		p = NULL;
 		if (argp) {
 			r = -EFAULT;
 			if (copy_from_user(&kvm_sigmask, argp,
 					   sizeof kvm_sigmask))
 				goto out;
 			r = -EINVAL;
 			if (kvm_sigmask.len != sizeof sigset)
 				goto out;
 			r = -EFAULT;
 			if (copy_from_user(&sigset, sigmask_arg->sigset,
 					   sizeof sigset))
 				goto out;
 			p = &sigset;
 		}
 		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
 		break;
 	}
 	case KVM_GET_FPU: {
 		fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!fpu)
 			goto out;
 		r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
 		if (r)
 			goto out;
 		r = -EFAULT;
 		if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_SET_FPU: {
 		fpu = kmalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
 		r = -ENOMEM;
 		if (!fpu)
 			goto out;
 		r = -EFAULT;
 		if (copy_from_user(fpu, argp, sizeof(struct kvm_fpu)))
 			goto out;
 		r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	default:
 		r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
 	}
 out:
 	kfree(fpu);
 	kfree(kvm_sregs);
 	return r;
 }
 static long kvm_vm_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
 	struct kvm *kvm = filp->private_data;
 	void __user *argp = (void __user *)arg;
 	int r;
 	if (kvm->mm != current->mm)
 		return -EIO;
 	switch (ioctl) {
 	case KVM_CREATE_VCPU:
 		r = kvm_vm_ioctl_create_vcpu(kvm, arg);
 		if (r < 0)
 			goto out;
 		break;
 	case KVM_SET_USER_MEMORY_REGION: {
 		struct kvm_userspace_memory_region kvm_userspace_mem;
 		r = -EFAULT;
 		if (copy_from_user(&kvm_userspace_mem, argp,
 						sizeof kvm_userspace_mem))
 			goto out;
 		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_GET_DIRTY_LOG: {
 		struct kvm_dirty_log log;
 		r = -EFAULT;
 		if (copy_from_user(&log, argp, sizeof log))
 			goto out;
 		r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
 		if (r)
 			goto out;
 		break;
 	}
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	case KVM_REGISTER_COALESCED_MMIO: {
 		struct kvm_coalesced_mmio_zone zone;
 		r = -EFAULT;
 		if (copy_from_user(&zone, argp, sizeof zone))
 			goto out;
 		r = -ENXIO;
 		r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 	case KVM_UNREGISTER_COALESCED_MMIO: {
 		struct kvm_coalesced_mmio_zone zone;
 		r = -EFAULT;
 		if (copy_from_user(&zone, argp, sizeof zone))
 			goto out;
 		r = -ENXIO;
 		r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
 		if (r)
 			goto out;
 		r = 0;
 		break;
 	}
 #endif
 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
 	case KVM_ASSIGN_PCI_DEVICE: {
 		struct kvm_assigned_pci_dev assigned_dev;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
 			goto out;
 		r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_ASSIGN_IRQ: {
 		r = -EOPNOTSUPP;
 		break;
 	}
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 	case KVM_ASSIGN_DEV_IRQ: {
 		struct kvm_assigned_irq assigned_irq;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
 			goto out;
 		r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_DEASSIGN_DEV_IRQ: {
 		struct kvm_assigned_irq assigned_irq;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
 			goto out;
 		r = kvm_vm_ioctl_deassign_dev_irq(kvm, &assigned_irq);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 #endif
 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
 	case KVM_DEASSIGN_PCI_DEVICE: {
 		struct kvm_assigned_pci_dev assigned_dev;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
 			goto out;
 		r = kvm_vm_ioctl_deassign_device(kvm, &assigned_dev);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 #ifdef KVM_CAP_IRQ_ROUTING
 	case KVM_SET_GSI_ROUTING: {
 		struct kvm_irq_routing routing;
 		struct kvm_irq_routing __user *urouting;
 		struct kvm_irq_routing_entry *entries;
 		r = -EFAULT;
 		if (copy_from_user(&routing, argp, sizeof(routing)))
 			goto out;
 		r = -EINVAL;
 		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
 			goto out;
 		if (routing.flags)
 			goto out;
 		r = -ENOMEM;
 		entries = vmalloc(routing.nr * sizeof(*entries));
 		if (!entries)
 			goto out;
 		r = -EFAULT;
 		urouting = argp;
 		if (copy_from_user(entries, urouting->entries,
 				   routing.nr * sizeof(*entries)))
 			goto out_free_irq_routing;
 		r = kvm_set_irq_routing(kvm, entries, routing.nr,
 					routing.flags);
 	out_free_irq_routing:
 		vfree(entries);
 		break;
 	}
 #ifdef __KVM_HAVE_MSIX
 	case KVM_ASSIGN_SET_MSIX_NR: {
 		struct kvm_assigned_msix_nr entry_nr;
 		r = -EFAULT;
 		if (copy_from_user(&entry_nr, argp, sizeof entry_nr))
 			goto out;
 		r = kvm_vm_ioctl_set_msix_nr(kvm, &entry_nr);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_ASSIGN_SET_MSIX_ENTRY: {
 		struct kvm_assigned_msix_entry entry;
 		r = -EFAULT;
 		if (copy_from_user(&entry, argp, sizeof entry))
 			goto out;
 		r = kvm_vm_ioctl_set_msix_entry(kvm, &entry);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 #endif /* KVM_CAP_IRQ_ROUTING */
 	case KVM_IRQFD: {
 		struct kvm_irqfd data;
 		r = -EFAULT;
 		if (copy_from_user(&data, argp, sizeof data))
 			goto out;
 		r = kvm_irqfd(kvm, data.fd, data.gsi, data.flags);
 		break;
 	}
+#ifdef CONFIG_KVM_APIC_ARCHITECTURE
+	case KVM_SET_BOOT_CPU_ID:
+		r = 0;
+		if (atomic_read(&kvm->online_vcpus) != 0)
+			r = -EBUSY;
+		else
+			kvm->bsp_vcpu_id = arg;
+		break;
+#endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
 }
 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct page *page[1];
 	unsigned long addr;
 	int npages;
 	gfn_t gfn = vmf->pgoff;
 	struct kvm *kvm = vma->vm_file->private_data;
 	addr = gfn_to_hva(kvm, gfn);
 	if (kvm_is_error_hva(addr))
 		return VM_FAULT_SIGBUS;
 	npages = get_user_pages(current, current->mm, addr, 1, 1, 0, page,
 				NULL);
 	if (unlikely(npages != 1))
 		return VM_FAULT_SIGBUS;
 	vmf->page = page[0];
 	return 0;
 }
 static struct vm_operations_struct kvm_vm_vm_ops = {
 	.fault = kvm_vm_fault,
 };
 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	vma->vm_ops = &kvm_vm_vm_ops;
 	return 0;
 }
 static struct file_operations kvm_vm_fops = {
 	.release        = kvm_vm_release,
 	.unlocked_ioctl = kvm_vm_ioctl,
 	.compat_ioctl   = kvm_vm_ioctl,
 	.mmap           = kvm_vm_mmap,
 };
 static int kvm_dev_ioctl_create_vm(void)
 {
 	int fd;
 	struct kvm *kvm;
 	kvm = kvm_create_vm();
 	if (IS_ERR(kvm))
 		return PTR_ERR(kvm);
 	fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
 	if (fd < 0)
 		kvm_put_kvm(kvm);
 	return fd;
 }
 static long kvm_dev_ioctl_check_extension_generic(long arg)
 {
 	switch (arg) {
 	case KVM_CAP_USER_MEMORY:
 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
 	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
+#ifdef CONFIG_KVM_APIC_ARCHITECTURE
+	case KVM_CAP_SET_BOOT_CPU_ID:
+#endif
 		return 1;
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
 	case KVM_CAP_IRQ_ROUTING:
 		return KVM_MAX_IRQ_ROUTES;
 #endif
 	default:
 		break;
 	}
 	return kvm_dev_ioctl_check_extension(arg);
 }
 static long kvm_dev_ioctl(struct file *filp,
 			  unsigned int ioctl, unsigned long arg)
 {
 	long r = -EINVAL;
 	switch (ioctl) {
 	case KVM_GET_API_VERSION:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = KVM_API_VERSION;
 		break;
 	case KVM_CREATE_VM:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = kvm_dev_ioctl_create_vm();
 		break;
 	case KVM_CHECK_EXTENSION:
 		r = kvm_dev_ioctl_check_extension_generic(arg);
 		break;
 	case KVM_GET_VCPU_MMAP_SIZE:
 		r = -EINVAL;
 		if (arg)
 			goto out;
 		r = PAGE_SIZE;     /* struct kvm_run */
 #ifdef CONFIG_X86
 		r += PAGE_SIZE;    /* pio data page */
 #endif
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 		r += PAGE_SIZE;    /* coalesced mmio ring page */
 #endif
 		break;
 	case KVM_TRACE_ENABLE:
 	case KVM_TRACE_PAUSE:
 	case KVM_TRACE_DISABLE:
 		r = kvm_trace_ioctl(ioctl, arg);
 		break;
 	default:
 		return kvm_arch_dev_ioctl(filp, ioctl, arg);
 	}
 out:
 	return r;
 }
 static struct file_operations kvm_chardev_ops = {
 	.unlocked_ioctl = kvm_dev_ioctl,
 	.compat_ioctl   = kvm_dev_ioctl,
 };
 static struct miscdevice kvm_dev = {
 	KVM_MINOR,
 	"kvm",
 	&kvm_chardev_ops,
 };
 static void hardware_enable(void *junk)
 {
 	int cpu = raw_smp_processor_id();
 	if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
 		return;
 	cpumask_set_cpu(cpu, cpus_hardware_enabled);
 	kvm_arch_hardware_enable(NULL);
 }
 static void hardware_disable(void *junk)
 {
 	int cpu = raw_smp_processor_id();
 	if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
 		return;
 	cpumask_clear_cpu(cpu, cpus_hardware_enabled);
 	kvm_arch_hardware_disable(NULL);
 }
 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
 			   void *v)
 {
 	int cpu = (long)v;
 	val &= ~CPU_TASKS_FROZEN;
 	switch (val) {
 	case CPU_DYING:
 		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
 		       cpu);
 		hardware_disable(NULL);
 		break;
 	case CPU_UP_CANCELED:
 		printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
 		       cpu);
 		smp_call_function_single(cpu, hardware_disable, NULL, 1);
 		break;
 	case CPU_ONLINE:
 		printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
 		       cpu);
 		smp_call_function_single(cpu, hardware_enable, NULL, 1);
 		break;
 	}
 	return NOTIFY_OK;
 }
 asmlinkage void kvm_handle_fault_on_reboot(void)
 {
 	if (kvm_rebooting)
 		/* spin while reset goes on */
 		while (true)
 			;
 	/* Fault while not rebooting.  We want the trace. */
 	BUG();
 }
 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
 		      void *v)
 {
 	/*
 	 * Some (well, at least mine) BIOSes hang on reboot if
 	 * in vmx root mode.
 	 *
 	 * And Intel TXT required VMX off for all cpu when system shutdown.
 	 */
 	printk(KERN_INFO "kvm: exiting hardware virtualization\n");
 	kvm_rebooting = true;
 	on_each_cpu(hardware_disable, NULL, 1);
 	return NOTIFY_OK;
 }
 static struct notifier_block kvm_reboot_notifier = {
 	.notifier_call = kvm_reboot,
 	.priority = 0,
 };
 void kvm_io_bus_init(struct kvm_io_bus *bus)
 {
 	memset(bus, 0, sizeof(*bus));
 }
 void kvm_io_bus_destroy(struct kvm_io_bus *bus)
 {
 	int i;
 	for (i = 0; i < bus->dev_count; i++) {
 		struct kvm_io_device *pos = bus->devs[i];
 		kvm_iodevice_destructor(pos);
 	}
 }
 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
 					  gpa_t addr, int len, int is_write)
 {
 	int i;
 	for (i = 0; i < bus->dev_count; i++) {
 		struct kvm_io_device *pos = bus->devs[i];
 		if (kvm_iodevice_in_range(pos, addr, len, is_write))
 			return pos;
 	}
 	return NULL;
 }
 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
 {
 	BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
 	bus->devs[bus->dev_count++] = dev;
 }
 static struct notifier_block kvm_cpu_notifier = {
 	.notifier_call = kvm_cpu_hotplug,
 	.priority = 20, /* must be > scheduler priority */
 };
 static int vm_stat_get(void *_offset, u64 *val)
 {
 	unsigned offset = (long)_offset;
 	struct kvm *kvm;
 	*val = 0;
 	spin_lock(&kvm_lock);
 	list_for_each_entry(kvm, &vm_list, vm_list)
 		*val += *(u32 *)((void *)kvm + offset);
 	spin_unlock(&kvm_lock);
 	return 0;
 }
 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
 static int vcpu_stat_get(void *_offset, u64 *val)
 {
 	unsigned offset = (long)_offset;
 	struct kvm *kvm;
 	struct kvm_vcpu *vcpu;
 	int i;
 	*val = 0;
 	spin_lock(&kvm_lock);
 	list_for_each_entry(kvm, &vm_list, vm_list)
 		for (i = 0; i < KVM_MAX_VCPUS; ++i) {
 			vcpu = kvm->vcpus[i];
 			if (vcpu)
 				*val += *(u32 *)((void *)vcpu + offset);
 		}
 	spin_unlock(&kvm_lock);
 	return 0;
 }
 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
 static struct file_operations *stat_fops[] = {
 	[KVM_STAT_VCPU] = &vcpu_stat_fops,
 	[KVM_STAT_VM]   = &vm_stat_fops,
 };
 static void kvm_init_debug(void)
 {
 	struct kvm_stats_debugfs_item *p;
 	kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
 	for (p = debugfs_entries; p->name; ++p)
 		p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
 						(void *)(long)p->offset,
 						stat_fops[p->kind]);
 }
 static void kvm_exit_debug(void)
 {
 	struct kvm_stats_debugfs_item *p;
 	for (p = debugfs_entries; p->name; ++p)
 		debugfs_remove(p->dentry);
 	debugfs_remove(kvm_debugfs_dir);
 }
 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
 {
 	hardware_disable(NULL);
 	return 0;
 }
 static int kvm_resume(struct sys_device *dev)
 {
 	hardware_enable(NULL);
 	return 0;
 }
 static struct sysdev_class kvm_sysdev_class = {
 	.name = "kvm",
 	.suspend = kvm_suspend,
 	.resume = kvm_resume,
 };
 static struct sys_device kvm_sysdev = {
 	.id = 0,
 	.cls = &kvm_sysdev_class,
 };
 struct page *bad_page;
 pfn_t bad_pfn;
 static inline
 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
 {
 	return container_of(pn, struct kvm_vcpu, preempt_notifier);
 }
 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
 {
 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 	kvm_arch_vcpu_load(vcpu, cpu);
 }
 static void kvm_sched_out(struct preempt_notifier *pn,
 			  struct task_struct *next)
 {
 	struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
 	kvm_arch_vcpu_put(vcpu);
 }
 int kvm_init(void *opaque, unsigned int vcpu_size,
 		  struct module *module)
 {
 	int r;
 	int cpu;
 	kvm_init_debug();
 	r = kvm_arch_init(opaque);
 	if (r)
 		goto out_fail;
 	bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 	if (bad_page == NULL) {
 		r = -ENOMEM;
 		goto out;
 	}
 	bad_pfn = page_to_pfn(bad_page);
 	if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
 		r = -ENOMEM;
 		goto out_free_0;
 	}
 	r = kvm_arch_hardware_setup();
 	if (r < 0)
 		goto out_free_0a;
 	for_each_online_cpu(cpu) {
 		smp_call_function_single(cpu,
 				kvm_arch_check_processor_compat,
 				&r, 1);
 		if (r < 0)
 			goto out_free_1;
 	}
 	on_each_cpu(hardware_enable, NULL, 1);
 	r = register_cpu_notifier(&kvm_cpu_notifier);
 	if (r)
 		goto out_free_2;
 	register_reboot_notifier(&kvm_reboot_notifier);
 	r = sysdev_class_register(&kvm_sysdev_class);
 	if (r)
 		goto out_free_3;
 	r = sysdev_register(&kvm_sysdev);
 	if (r)
 		goto out_free_4;
 	/* A kmem cache lets us meet the alignment requirements of fx_save. */
 	kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
 					   __alignof__(struct kvm_vcpu),
 					   0, NULL);
 	if (!kvm_vcpu_cache) {
 		r = -ENOMEM;
 		goto out_free_5;
 	}
 	kvm_chardev_ops.owner = module;
 	kvm_vm_fops.owner = module;
 	kvm_vcpu_fops.owner = module;
 	r = misc_register(&kvm_dev);
 	if (r) {
 		printk(KERN_ERR "kvm: misc device register failed\n");
 		goto out_free;
 	}
 	kvm_preempt_ops.sched_in = kvm_sched_in;
 	kvm_preempt_ops.sched_out = kvm_sched_out;
 	return 0;
 out_free:
 	kmem_cache_destroy(kvm_vcpu_cache);
 out_free_5:
 	sysdev_unregister(&kvm_sysdev);
 out_free_4:
 	sysdev_class_unregister(&kvm_sysdev_class);
 out_free_3:
 	unregister_reboot_notifier(&kvm_reboot_notifier);
 	unregister_cpu_notifier(&kvm_cpu_notifier);
 out_free_2:
 	on_each_cpu(hardware_disable, NULL, 1);
 out_free_1:
 	kvm_arch_hardware_unsetup();
 out_free_0a:
 	free_cpumask_var(cpus_hardware_enabled);
 out_free_0:
 	__free_page(bad_page);
 out:
 	kvm_arch_exit();
 	kvm_exit_debug();
 out_fail:
 	return r;
 }
 EXPORT_SYMBOL_GPL(kvm_init);