Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit a24e809902339458416900869abdcc51a44bfd48

Authored by Andi Kleen 2010-06-10 19:10:55 +0800

Committed by Avi Kivity 2010-08-01 15:46:29 +0800

Exists in master and in 7 other branches

KVM: Fix unused but set warnings

No real bugs in this one.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

Showing 2 changed files with 1 additions and 2 deletions Inline Diff

arch/x86/kvm/paging_tmpl.h
virt/kvm/assigned-dev.c

arch/x86/kvm/paging_tmpl.h

Diff comments View file @ a24e809

 /*
  * Kernel-based Virtual Machine driver for Linux
  *
  * This module enables machines with Intel VT-x extensions to run virtual
  * machines without emulation or binary translation.
  *
  * MMU support
  *
  * Copyright (C) 2006 Qumranet, Inc.
  * Copyright 2010 Red Hat, Inc. and/or its affilates.
  *
  * Authors:
  *   Yaniv Kamay  <yaniv@qumranet.com>
  *   Avi Kivity   <avi@qumranet.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  *
  */
 /*
  * We need the mmu code to access both 32-bit and 64-bit guest ptes,
  * so the code in this file is compiled twice, once per pte size.
  */
 #if PTTYPE == 64
 	#define pt_element_t u64
 	#define guest_walker guest_walker64
 	#define FNAME(name) paging##64_##name
 	#define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
 	#define PT_LVL_ADDR_MASK(lvl) PT64_LVL_ADDR_MASK(lvl)
 	#define PT_LVL_OFFSET_MASK(lvl) PT64_LVL_OFFSET_MASK(lvl)
 	#define PT_INDEX(addr, level) PT64_INDEX(addr, level)
 	#define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level)
 	#define PT_LEVEL_BITS PT64_LEVEL_BITS
 	#ifdef CONFIG_X86_64
 	#define PT_MAX_FULL_LEVELS 4
 	#define CMPXCHG cmpxchg
 	#else
 	#define CMPXCHG cmpxchg64
 	#define PT_MAX_FULL_LEVELS 2
 	#endif
 #elif PTTYPE == 32
 	#define pt_element_t u32
 	#define guest_walker guest_walker32
 	#define FNAME(name) paging##32_##name
 	#define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
 	#define PT_LVL_ADDR_MASK(lvl) PT32_LVL_ADDR_MASK(lvl)
 	#define PT_LVL_OFFSET_MASK(lvl) PT32_LVL_OFFSET_MASK(lvl)
 	#define PT_INDEX(addr, level) PT32_INDEX(addr, level)
 	#define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level)
 	#define PT_LEVEL_BITS PT32_LEVEL_BITS
 	#define PT_MAX_FULL_LEVELS 2
 	#define CMPXCHG cmpxchg
 #else
 	#error Invalid PTTYPE value
 #endif
 #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl)
 #define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL)
 /*
  * The guest_walker structure emulates the behavior of the hardware page
  * table walker.
  */
 struct guest_walker {
 	int level;
 	gfn_t table_gfn[PT_MAX_FULL_LEVELS];
 	pt_element_t ptes[PT_MAX_FULL_LEVELS];
 	gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
 	unsigned pt_access;
 	unsigned pte_access;
 	gfn_t gfn;
 	u32 error_code;
 };
 static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
 {
 	return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
 }
 static bool FNAME(cmpxchg_gpte)(struct kvm *kvm,
 			 gfn_t table_gfn, unsigned index,
 			 pt_element_t orig_pte, pt_element_t new_pte)
 {
 	pt_element_t ret;
 	pt_element_t *table;
 	struct page *page;
 	page = gfn_to_page(kvm, table_gfn);
 	table = kmap_atomic(page, KM_USER0);
 	ret = CMPXCHG(&table[index], orig_pte, new_pte);
 	kunmap_atomic(table, KM_USER0);
 	kvm_release_page_dirty(page);
 	return (ret != orig_pte);
 }
 static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte)
 {
 	unsigned access;
 	access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
 #if PTTYPE == 64
 	if (is_nx(vcpu))
 		access &= ~(gpte >> PT64_NX_SHIFT);
 #endif
 	return access;
 }
 /*
  * Fetch a guest pte for a guest virtual address
  */
 static int FNAME(walk_addr)(struct guest_walker *walker,
 			    struct kvm_vcpu *vcpu, gva_t addr,
 			    int write_fault, int user_fault, int fetch_fault)
 {
 	pt_element_t pte;
 	gfn_t table_gfn;
 	unsigned index, pt_access, pte_access;
 	gpa_t pte_gpa;
 	int rsvd_fault = 0;
 	trace_kvm_mmu_pagetable_walk(addr, write_fault, user_fault,
 				     fetch_fault);
 walk:
 	walker->level = vcpu->arch.mmu.root_level;
 	pte = vcpu->arch.cr3;
 #if PTTYPE == 64
 	if (!is_long_mode(vcpu)) {
 		pte = kvm_pdptr_read(vcpu, (addr >> 30) & 3);
 		trace_kvm_mmu_paging_element(pte, walker->level);
 		if (!is_present_gpte(pte))
 			goto not_present;
 		--walker->level;
 	}
 #endif
 	ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
 	       (vcpu->arch.cr3 & CR3_NONPAE_RESERVED_BITS) == 0);
 	pt_access = ACC_ALL;
 	for (;;) {
 		index = PT_INDEX(addr, walker->level);
 		table_gfn = gpte_to_gfn(pte);
 		pte_gpa = gfn_to_gpa(table_gfn);
 		pte_gpa += index * sizeof(pt_element_t);
 		walker->table_gfn[walker->level - 1] = table_gfn;
 		walker->pte_gpa[walker->level - 1] = pte_gpa;
 		if (kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)))
 			goto not_present;
 		trace_kvm_mmu_paging_element(pte, walker->level);
 		if (!is_present_gpte(pte))
 			goto not_present;
 		rsvd_fault = is_rsvd_bits_set(vcpu, pte, walker->level);
 		if (rsvd_fault)
 			goto access_error;
 		if (write_fault && !is_writable_pte(pte))
 			if (user_fault || is_write_protection(vcpu))
 				goto access_error;
 		if (user_fault && !(pte & PT_USER_MASK))
 			goto access_error;
 #if PTTYPE == 64
 		if (fetch_fault && (pte & PT64_NX_MASK))
 			goto access_error;
 #endif
 		if (!(pte & PT_ACCESSED_MASK)) {
 			trace_kvm_mmu_set_accessed_bit(table_gfn, index,
 						       sizeof(pte));
 			if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn,
 			    index, pte, pte|PT_ACCESSED_MASK))
 				goto walk;
 			mark_page_dirty(vcpu->kvm, table_gfn);
 			pte |= PT_ACCESSED_MASK;
 		}
 		pte_access = pt_access & FNAME(gpte_access)(vcpu, pte);
 		walker->ptes[walker->level - 1] = pte;
 		if ((walker->level == PT_PAGE_TABLE_LEVEL) ||
 		    ((walker->level == PT_DIRECTORY_LEVEL) &&
 				is_large_pte(pte) &&
 				(PTTYPE == 64 || is_pse(vcpu))) ||
 		    ((walker->level == PT_PDPE_LEVEL) &&
 				is_large_pte(pte) &&
 				is_long_mode(vcpu))) {
 			int lvl = walker->level;
 			walker->gfn = gpte_to_gfn_lvl(pte, lvl);
 			walker->gfn += (addr & PT_LVL_OFFSET_MASK(lvl))
 					>> PAGE_SHIFT;
 			if (PTTYPE == 32 &&
 			    walker->level == PT_DIRECTORY_LEVEL &&
 			    is_cpuid_PSE36())
 				walker->gfn += pse36_gfn_delta(pte);
 			break;
 		}
 		pt_access = pte_access;
 		--walker->level;
 	}
 	if (write_fault && !is_dirty_gpte(pte)) {
 		bool ret;
 		trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
 		ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte,
 			    pte|PT_DIRTY_MASK);
 		if (ret)
 			goto walk;
 		mark_page_dirty(vcpu->kvm, table_gfn);
 		pte |= PT_DIRTY_MASK;
 		walker->ptes[walker->level - 1] = pte;
 	}
 	walker->pt_access = pt_access;
 	walker->pte_access = pte_access;
 	pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
 		 __func__, (u64)pte, pte_access, pt_access);
 	return 1;
 not_present:
 	walker->error_code = 0;
 	goto err;
 access_error:
 	walker->error_code = PFERR_PRESENT_MASK;
 err:
 	if (write_fault)
 		walker->error_code |= PFERR_WRITE_MASK;
 	if (user_fault)
 		walker->error_code |= PFERR_USER_MASK;
 	if (fetch_fault)
 		walker->error_code |= PFERR_FETCH_MASK;
 	if (rsvd_fault)
 		walker->error_code |= PFERR_RSVD_MASK;
 	trace_kvm_mmu_walker_error(walker->error_code);
 	return 0;
 }
 static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page,
 			      u64 *spte, const void *pte)
 {
 	pt_element_t gpte;
 	unsigned pte_access;
 	pfn_t pfn;
 	u64 new_spte;
 	gpte = *(const pt_element_t *)pte;
 	if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) {
 		if (!is_present_gpte(gpte)) {
 			if (page->unsync)
 				new_spte = shadow_trap_nonpresent_pte;
 			else
 				new_spte = shadow_notrap_nonpresent_pte;
 			__set_spte(spte, new_spte);
 		}
 		return;
 	}
 	pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
 	pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte);
 	if (gpte_to_gfn(gpte) != vcpu->arch.update_pte.gfn)
 		return;
 	pfn = vcpu->arch.update_pte.pfn;
 	if (is_error_pfn(pfn))
 		return;
 	if (mmu_notifier_retry(vcpu, vcpu->arch.update_pte.mmu_seq))
 		return;
 	kvm_get_pfn(pfn);
 	/*
 	 * we call mmu_set_spte() with reset_host_protection = true beacuse that
 	 * vcpu->arch.update_pte.pfn was fetched from get_user_pages(write = 1).
 	 */
 	mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0,
 		     gpte & PT_DIRTY_MASK, NULL, PT_PAGE_TABLE_LEVEL,
 		     gpte_to_gfn(gpte), pfn, true, true);
 }
 /*
  * Fetch a shadow pte for a specific level in the paging hierarchy.
  */
 static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr,
 			 struct guest_walker *gw,
 			 int user_fault, int write_fault, int hlevel,
 			 int *ptwrite, pfn_t pfn)
 {
 	unsigned access = gw->pt_access;
 	struct kvm_mmu_page *shadow_page;
 	u64 spte, *sptep = NULL;
 	int direct;
 	gfn_t table_gfn;
 	int r;
 	int level;
 	pt_element_t curr_pte;
 	struct kvm_shadow_walk_iterator iterator;
 	if (!is_present_gpte(gw->ptes[gw->level - 1]))
 		return NULL;
 	for_each_shadow_entry(vcpu, addr, iterator) {
 		level = iterator.level;
 		sptep = iterator.sptep;
 		if (iterator.level == hlevel) {
 			mmu_set_spte(vcpu, sptep, access,
 				     gw->pte_access & access,
 				     user_fault, write_fault,
 				     gw->ptes[gw->level-1] & PT_DIRTY_MASK,
 				     ptwrite, level,
 				     gw->gfn, pfn, false, true);
 			break;
 		}
 		if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep))
 			continue;
 		if (is_large_pte(*sptep)) {
 			rmap_remove(vcpu->kvm, sptep);
 			__set_spte(sptep, shadow_trap_nonpresent_pte);
 			kvm_flush_remote_tlbs(vcpu->kvm);
 		}
 		if (level <= gw->level) {
 			int delta = level - gw->level + 1;
 			direct = 1;
 			if (!is_dirty_gpte(gw->ptes[level - delta]))
 				access &= ~ACC_WRITE_MASK;
 			/*
 			 * It is a large guest pages backed by small host pages,
 			 * So we set @direct(@shadow_page->role.direct)=1, and
 			 * set @table_gfn(@shadow_page->gfn)=the base page frame
 			 * for linear translations.
 			 */
 			table_gfn = gw->gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
 			access &= gw->pte_access;
 		} else {
 			direct = 0;
 			table_gfn = gw->table_gfn[level - 2];
 		}
 		shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1,
 					       direct, access, sptep);
 		if (!direct) {
 			r = kvm_read_guest_atomic(vcpu->kvm,
 						  gw->pte_gpa[level - 2],
 						  &curr_pte, sizeof(curr_pte));
 			if (r || curr_pte != gw->ptes[level - 2]) {
 				kvm_mmu_put_page(shadow_page, sptep);
 				kvm_release_pfn_clean(pfn);
 				sptep = NULL;
 				break;
 			}
 		}
 		spte = __pa(shadow_page->spt)
 			| PT_PRESENT_MASK | PT_ACCESSED_MASK
 			| PT_WRITABLE_MASK | PT_USER_MASK;
 		*sptep = spte;
 	}
 	return sptep;
 }
 /*
  * Page fault handler.  There are several causes for a page fault:
  *   - there is no shadow pte for the guest pte
  *   - write access through a shadow pte marked read only so that we can set
  *     the dirty bit
  *   - write access to a shadow pte marked read only so we can update the page
  *     dirty bitmap, when userspace requests it
  *   - mmio access; in this case we will never install a present shadow pte
  *   - normal guest page fault due to the guest pte marked not present, not
  *     writable, or not executable
  *
  *  Returns: 1 if we need to emulate the instruction, 0 otherwise, or
  *           a negative value on error.
  */
 static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
 			       u32 error_code)
 {
 	int write_fault = error_code & PFERR_WRITE_MASK;
 	int user_fault = error_code & PFERR_USER_MASK;
 	int fetch_fault = error_code & PFERR_FETCH_MASK;
 	struct guest_walker walker;
 	u64 *sptep;
 	int write_pt = 0;
 	int r;
 	pfn_t pfn;
 	int level = PT_PAGE_TABLE_LEVEL;
 	unsigned long mmu_seq;
 	pgprintk("%s: addr %lx err %x\n", __func__, addr, error_code);
 	kvm_mmu_audit(vcpu, "pre page fault");
 	r = mmu_topup_memory_caches(vcpu);
 	if (r)
 		return r;
 	/*
 	 * Look up the guest pte for the faulting address.
 	 */
 	r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault,
 			     fetch_fault);
 	/*
 	 * The page is not mapped by the guest.  Let the guest handle it.
 	 */
 	if (!r) {
 		pgprintk("%s: guest page fault\n", __func__);
 		inject_page_fault(vcpu, addr, walker.error_code);
 		vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
 		return 0;
 	}
 	if (walker.level >= PT_DIRECTORY_LEVEL) {
 		level = min(walker.level, mapping_level(vcpu, walker.gfn));
 		walker.gfn = walker.gfn & ~(KVM_PAGES_PER_HPAGE(level) - 1);
 	}
 	mmu_seq = vcpu->kvm->mmu_notifier_seq;
 	smp_rmb();
 	pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
 	/* mmio */
 	if (is_error_pfn(pfn))
 		return kvm_handle_bad_page(vcpu->kvm, walker.gfn, pfn);
 	spin_lock(&vcpu->kvm->mmu_lock);
 	if (mmu_notifier_retry(vcpu, mmu_seq))
 		goto out_unlock;
 	kvm_mmu_free_some_pages(vcpu);
 	sptep = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault,
 			     level, &write_pt, pfn);
+	(void)sptep;
 	pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__,
 		 sptep, *sptep, write_pt);
 	if (!write_pt)
 		vcpu->arch.last_pt_write_count = 0; /* reset fork detector */
 	++vcpu->stat.pf_fixed;
 	kvm_mmu_audit(vcpu, "post page fault (fixed)");
 	spin_unlock(&vcpu->kvm->mmu_lock);
 	return write_pt;
 out_unlock:
 	spin_unlock(&vcpu->kvm->mmu_lock);
 	kvm_release_pfn_clean(pfn);
 	return 0;
 }
 static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva)
 {
 	struct kvm_shadow_walk_iterator iterator;
 	struct kvm_mmu_page *sp;
 	gpa_t pte_gpa = -1;
 	int level;
 	u64 *sptep;
 	int need_flush = 0;
 	spin_lock(&vcpu->kvm->mmu_lock);
 	for_each_shadow_entry(vcpu, gva, iterator) {
 		level = iterator.level;
 		sptep = iterator.sptep;
 		sp = page_header(__pa(sptep));
 		if (is_last_spte(*sptep, level)) {
 			int offset, shift;
 			if (!sp->unsync)
 				break;
 			shift = PAGE_SHIFT -
 				  (PT_LEVEL_BITS - PT64_LEVEL_BITS) * level;
 			offset = sp->role.quadrant << shift;
 			pte_gpa = (sp->gfn << PAGE_SHIFT) + offset;
 			pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t);
 			if (is_shadow_present_pte(*sptep)) {
 				rmap_remove(vcpu->kvm, sptep);
 				if (is_large_pte(*sptep))
 					--vcpu->kvm->stat.lpages;
 				need_flush = 1;
 			}
 			__set_spte(sptep, shadow_trap_nonpresent_pte);
 			break;
 		}
 		if (!is_shadow_present_pte(*sptep) || !sp->unsync_children)
 			break;
 	}
 	if (need_flush)
 		kvm_flush_remote_tlbs(vcpu->kvm);
 	atomic_inc(&vcpu->kvm->arch.invlpg_counter);
 	spin_unlock(&vcpu->kvm->mmu_lock);
 	if (pte_gpa == -1)
 		return;
 	if (mmu_topup_memory_caches(vcpu))
 		return;
 	kvm_mmu_pte_write(vcpu, pte_gpa, NULL, sizeof(pt_element_t), 0);
 }
 static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr, u32 access,
 			       u32 *error)
 {
 	struct guest_walker walker;
 	gpa_t gpa = UNMAPPED_GVA;
 	int r;
 	r = FNAME(walk_addr)(&walker, vcpu, vaddr,
 			     !!(access & PFERR_WRITE_MASK),
 			     !!(access & PFERR_USER_MASK),
 			     !!(access & PFERR_FETCH_MASK));
 	if (r) {
 		gpa = gfn_to_gpa(walker.gfn);
 		gpa |= vaddr & ~PAGE_MASK;
 	} else if (error)
 		*error = walker.error_code;
 	return gpa;
 }
 static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu,
 				 struct kvm_mmu_page *sp)
 {
 	int i, j, offset, r;
 	pt_element_t pt[256 / sizeof(pt_element_t)];
 	gpa_t pte_gpa;
 	if (sp->role.direct
 	    || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) {
 		nonpaging_prefetch_page(vcpu, sp);
 		return;
 	}
 	pte_gpa = gfn_to_gpa(sp->gfn);
 	if (PTTYPE == 32) {
 		offset = sp->role.quadrant << PT64_LEVEL_BITS;
 		pte_gpa += offset * sizeof(pt_element_t);
 	}
 	for (i = 0; i < PT64_ENT_PER_PAGE; i += ARRAY_SIZE(pt)) {
 		r = kvm_read_guest_atomic(vcpu->kvm, pte_gpa, pt, sizeof pt);
 		pte_gpa += ARRAY_SIZE(pt) * sizeof(pt_element_t);
 		for (j = 0; j < ARRAY_SIZE(pt); ++j)
 			if (r || is_present_gpte(pt[j]))
 				sp->spt[i+j] = shadow_trap_nonpresent_pte;
 			else
 				sp->spt[i+j] = shadow_notrap_nonpresent_pte;
 	}
 }
 /*
  * Using the cached information from sp->gfns is safe because:
  * - The spte has a reference to the struct page, so the pfn for a given gfn
  *   can't change unless all sptes pointing to it are nuked first.
  * - Alias changes zap the entire shadow cache.
  */
 static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 {
 	int i, offset, nr_present;
 	bool reset_host_protection;
 	gpa_t first_pte_gpa;
 	offset = nr_present = 0;
 	/* direct kvm_mmu_page can not be unsync. */
 	BUG_ON(sp->role.direct);
 	if (PTTYPE == 32)
 		offset = sp->role.quadrant << PT64_LEVEL_BITS;
 	first_pte_gpa = gfn_to_gpa(sp->gfn) + offset * sizeof(pt_element_t);
 	for (i = 0; i < PT64_ENT_PER_PAGE; i++) {
 		unsigned pte_access;
 		pt_element_t gpte;
 		gpa_t pte_gpa;
 		gfn_t gfn;
 		if (!is_shadow_present_pte(sp->spt[i]))
 			continue;
 		pte_gpa = first_pte_gpa + i * sizeof(pt_element_t);
 		if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte,
 					  sizeof(pt_element_t)))
 			return -EINVAL;
 		gfn = gpte_to_gfn(gpte);
 		if (unalias_gfn(vcpu->kvm, gfn) != sp->gfns[i] ||
 		      !is_present_gpte(gpte) || !(gpte & PT_ACCESSED_MASK)) {
 			u64 nonpresent;
 			rmap_remove(vcpu->kvm, &sp->spt[i]);
 			if (is_present_gpte(gpte))
 				nonpresent = shadow_trap_nonpresent_pte;
 			else
 				nonpresent = shadow_notrap_nonpresent_pte;
 			__set_spte(&sp->spt[i], nonpresent);
 			continue;
 		}
 		nr_present++;
 		pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte);
 		if (!(sp->spt[i] & SPTE_HOST_WRITEABLE)) {
 			pte_access &= ~ACC_WRITE_MASK;
 			reset_host_protection = 0;
 		} else {
 			reset_host_protection = 1;
 		}
 		set_spte(vcpu, &sp->spt[i], pte_access, 0, 0,
 			 is_dirty_gpte(gpte), PT_PAGE_TABLE_LEVEL, gfn,
 			 spte_to_pfn(sp->spt[i]), true, false,
 			 reset_host_protection);
 	}
 	return !nr_present;
 }
 #undef pt_element_t
 #undef guest_walker
 #undef FNAME
 #undef PT_BASE_ADDR_MASK
 #undef PT_INDEX
 #undef PT_LEVEL_MASK
 #undef PT_LVL_ADDR_MASK
 #undef PT_LVL_OFFSET_MASK
 #undef PT_LEVEL_BITS
 #undef PT_MAX_FULL_LEVELS
 #undef gpte_to_gfn
 #undef gpte_to_gfn_lvl
 #undef CMPXCHG

virt/kvm/assigned-dev.c

Diff comments View file @ a24e809

 /*
  * Kernel-based Virtual Machine - device assignment support
  *
  * Copyright (C) 2010 Red Hat, Inc. and/or its affiliates.
  *
  * This work is licensed under the terms of the GNU GPL, version 2.  See
  * the COPYING file in the top-level directory.
  *
  */
 #include <linux/kvm_host.h>
 #include <linux/kvm.h>
 #include <linux/uaccess.h>
 #include <linux/vmalloc.h>
 #include <linux/errno.h>
 #include <linux/spinlock.h>
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 #include <linux/slab.h>
 #include "irq.h"
 static struct kvm_assigned_dev_kernel *kvm_find_assigned_dev(struct list_head *head,
 						      int assigned_dev_id)
 {
 	struct list_head *ptr;
 	struct kvm_assigned_dev_kernel *match;
 	list_for_each(ptr, head) {
 		match = list_entry(ptr, struct kvm_assigned_dev_kernel, list);
 		if (match->assigned_dev_id == assigned_dev_id)
 			return match;
 	}
 	return NULL;
 }
 static int find_index_from_host_irq(struct kvm_assigned_dev_kernel
 				    *assigned_dev, int irq)
 {
 	int i, index;
 	struct msix_entry *host_msix_entries;
 	host_msix_entries = assigned_dev->host_msix_entries;
 	index = -1;
 	for (i = 0; i < assigned_dev->entries_nr; i++)
 		if (irq == host_msix_entries[i].vector) {
 			index = i;
 			break;
 		}
 	if (index < 0) {
 		printk(KERN_WARNING "Fail to find correlated MSI-X entry!\n");
 		return 0;
 	}
 	return index;
 }
 static void kvm_assigned_dev_interrupt_work_handler(struct work_struct *work)
 {
 	struct kvm_assigned_dev_kernel *assigned_dev;
-	struct kvm *kvm;
 	int i;
 	assigned_dev = container_of(work, struct kvm_assigned_dev_kernel,
 				    interrupt_work);
-	kvm = assigned_dev->kvm;
 	spin_lock_irq(&assigned_dev->assigned_dev_lock);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		struct kvm_guest_msix_entry *guest_entries =
 			assigned_dev->guest_msix_entries;
 		for (i = 0; i < assigned_dev->entries_nr; i++) {
 			if (!(guest_entries[i].flags &
 					KVM_ASSIGNED_MSIX_PENDING))
 				continue;
 			guest_entries[i].flags &= ~KVM_ASSIGNED_MSIX_PENDING;
 			kvm_set_irq(assigned_dev->kvm,
 				    assigned_dev->irq_source_id,
 				    guest_entries[i].vector, 1);
 		}
 	} else
 		kvm_set_irq(assigned_dev->kvm, assigned_dev->irq_source_id,
 			    assigned_dev->guest_irq, 1);
 	spin_unlock_irq(&assigned_dev->assigned_dev_lock);
 }
 static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
 {
 	unsigned long flags;
 	struct kvm_assigned_dev_kernel *assigned_dev =
 		(struct kvm_assigned_dev_kernel *) dev_id;
 	spin_lock_irqsave(&assigned_dev->assigned_dev_lock, flags);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		int index = find_index_from_host_irq(assigned_dev, irq);
 		if (index < 0)
 			goto out;
 		assigned_dev->guest_msix_entries[index].flags |=
 			KVM_ASSIGNED_MSIX_PENDING;
 	}
 	schedule_work(&assigned_dev->interrupt_work);
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_GUEST_INTX) {
 		disable_irq_nosync(irq);
 		assigned_dev->host_irq_disabled = true;
 	}
 out:
 	spin_unlock_irqrestore(&assigned_dev->assigned_dev_lock, flags);
 	return IRQ_HANDLED;
 }
 /* Ack the irq line for an assigned device */
 static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier *kian)
 {
 	struct kvm_assigned_dev_kernel *dev;
 	unsigned long flags;
 	if (kian->gsi == -1)
 		return;
 	dev = container_of(kian, struct kvm_assigned_dev_kernel,
 			   ack_notifier);
 	kvm_set_irq(dev->kvm, dev->irq_source_id, dev->guest_irq, 0);
 	/* The guest irq may be shared so this ack may be
 	 * from another device.
 	 */
 	spin_lock_irqsave(&dev->assigned_dev_lock, flags);
 	if (dev->host_irq_disabled) {
 		enable_irq(dev->host_irq);
 		dev->host_irq_disabled = false;
 	}
 	spin_unlock_irqrestore(&dev->assigned_dev_lock, flags);
 }
 static void deassign_guest_irq(struct kvm *kvm,
 			       struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	kvm_unregister_irq_ack_notifier(kvm, &assigned_dev->ack_notifier);
 	assigned_dev->ack_notifier.gsi = -1;
 	if (assigned_dev->irq_source_id != -1)
 		kvm_free_irq_source_id(kvm, assigned_dev->irq_source_id);
 	assigned_dev->irq_source_id = -1;
 	assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_GUEST_MASK);
 }
 /* The function implicit hold kvm->lock mutex due to cancel_work_sync() */
 static void deassign_host_irq(struct kvm *kvm,
 			      struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	/*
 	 * In kvm_free_device_irq, cancel_work_sync return true if:
 	 * 1. work is scheduled, and then cancelled.
 	 * 2. work callback is executed.
 	 *
 	 * The first one ensured that the irq is disabled and no more events
 	 * would happen. But for the second one, the irq may be enabled (e.g.
 	 * for MSI). So we disable irq here to prevent further events.
 	 *
 	 * Notice this maybe result in nested disable if the interrupt type is
 	 * INTx, but it's OK for we are going to free it.
 	 *
 	 * If this function is a part of VM destroy, please ensure that till
 	 * now, the kvm state is still legal for probably we also have to wait
 	 * interrupt_work done.
 	 */
 	if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX) {
 		int i;
 		for (i = 0; i < assigned_dev->entries_nr; i++)
 			disable_irq_nosync(assigned_dev->
 					   host_msix_entries[i].vector);
 		cancel_work_sync(&assigned_dev->interrupt_work);
 		for (i = 0; i < assigned_dev->entries_nr; i++)
 			free_irq(assigned_dev->host_msix_entries[i].vector,
 				 (void *)assigned_dev);
 		assigned_dev->entries_nr = 0;
 		kfree(assigned_dev->host_msix_entries);
 		kfree(assigned_dev->guest_msix_entries);
 		pci_disable_msix(assigned_dev->dev);
 	} else {
 		/* Deal with MSI and INTx */
 		disable_irq_nosync(assigned_dev->host_irq);
 		cancel_work_sync(&assigned_dev->interrupt_work);
 		free_irq(assigned_dev->host_irq, (void *)assigned_dev);
 		if (assigned_dev->irq_requested_type & KVM_DEV_IRQ_HOST_MSI)
 			pci_disable_msi(assigned_dev->dev);
 	}
 	assigned_dev->irq_requested_type &= ~(KVM_DEV_IRQ_HOST_MASK);
 }
 static int kvm_deassign_irq(struct kvm *kvm,
 			    struct kvm_assigned_dev_kernel *assigned_dev,
 			    unsigned long irq_requested_type)
 {
 	unsigned long guest_irq_type, host_irq_type;
 	if (!irqchip_in_kernel(kvm))
 		return -EINVAL;
 	/* no irq assignment to deassign */
 	if (!assigned_dev->irq_requested_type)
 		return -ENXIO;
 	host_irq_type = irq_requested_type & KVM_DEV_IRQ_HOST_MASK;
 	guest_irq_type = irq_requested_type & KVM_DEV_IRQ_GUEST_MASK;
 	if (host_irq_type)
 		deassign_host_irq(kvm, assigned_dev);
 	if (guest_irq_type)
 		deassign_guest_irq(kvm, assigned_dev);
 	return 0;
 }
 static void kvm_free_assigned_irq(struct kvm *kvm,
 				  struct kvm_assigned_dev_kernel *assigned_dev)
 {
 	kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
 }
 static void kvm_free_assigned_device(struct kvm *kvm,
 				     struct kvm_assigned_dev_kernel
 				     *assigned_dev)
 {
 	kvm_free_assigned_irq(kvm, assigned_dev);
 	pci_reset_function(assigned_dev->dev);
 	pci_release_regions(assigned_dev->dev);
 	pci_disable_device(assigned_dev->dev);
 	pci_dev_put(assigned_dev->dev);
 	list_del(&assigned_dev->list);
 	kfree(assigned_dev);
 }
 void kvm_free_all_assigned_devices(struct kvm *kvm)
 {
 	struct list_head *ptr, *ptr2;
 	struct kvm_assigned_dev_kernel *assigned_dev;
 	list_for_each_safe(ptr, ptr2, &kvm->arch.assigned_dev_head) {
 		assigned_dev = list_entry(ptr,
 					  struct kvm_assigned_dev_kernel,
 					  list);
 		kvm_free_assigned_device(kvm, assigned_dev);
 	}
 }
 static int assigned_device_enable_host_intx(struct kvm *kvm,
 					    struct kvm_assigned_dev_kernel *dev)
 {
 	dev->host_irq = dev->dev->irq;
 	/* Even though this is PCI, we don't want to use shared
 	 * interrupts. Sharing host devices with guest-assigned devices
 	 * on the same interrupt line is not a happy situation: there
 	 * are going to be long delays in accepting, acking, etc.
 	 */
 	if (request_irq(dev->host_irq, kvm_assigned_dev_intr,
 			0, "kvm_assigned_intx_device", (void *)dev))
 		return -EIO;
 	return 0;
 }
 #ifdef __KVM_HAVE_MSI
 static int assigned_device_enable_host_msi(struct kvm *kvm,
 					   struct kvm_assigned_dev_kernel *dev)
 {
 	int r;
 	if (!dev->dev->msi_enabled) {
 		r = pci_enable_msi(dev->dev);
 		if (r)
 			return r;
 	}
 	dev->host_irq = dev->dev->irq;
 	if (request_irq(dev->host_irq, kvm_assigned_dev_intr, 0,
 			"kvm_assigned_msi_device", (void *)dev)) {
 		pci_disable_msi(dev->dev);
 		return -EIO;
 	}
 	return 0;
 }
 #endif
 #ifdef __KVM_HAVE_MSIX
 static int assigned_device_enable_host_msix(struct kvm *kvm,
 					    struct kvm_assigned_dev_kernel *dev)
 {
 	int i, r = -EINVAL;
 	/* host_msix_entries and guest_msix_entries should have been
 	 * initialized */
 	if (dev->entries_nr == 0)
 		return r;
 	r = pci_enable_msix(dev->dev, dev->host_msix_entries, dev->entries_nr);
 	if (r)
 		return r;
 	for (i = 0; i < dev->entries_nr; i++) {
 		r = request_irq(dev->host_msix_entries[i].vector,
 				kvm_assigned_dev_intr, 0,
 				"kvm_assigned_msix_device",
 				(void *)dev);
 		if (r)
 			goto err;
 	}
 	return 0;
 err:
 	for (i -= 1; i >= 0; i--)
 		free_irq(dev->host_msix_entries[i].vector, (void *)dev);
 	pci_disable_msix(dev->dev);
 	return r;
 }
 #endif
 static int assigned_device_enable_guest_intx(struct kvm *kvm,
 				struct kvm_assigned_dev_kernel *dev,
 				struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = irq->guest_irq;
 	return 0;
 }
 #ifdef __KVM_HAVE_MSI
 static int assigned_device_enable_guest_msi(struct kvm *kvm,
 			struct kvm_assigned_dev_kernel *dev,
 			struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = -1;
 	dev->host_irq_disabled = false;
 	return 0;
 }
 #endif
 #ifdef __KVM_HAVE_MSIX
 static int assigned_device_enable_guest_msix(struct kvm *kvm,
 			struct kvm_assigned_dev_kernel *dev,
 			struct kvm_assigned_irq *irq)
 {
 	dev->guest_irq = irq->guest_irq;
 	dev->ack_notifier.gsi = -1;
 	dev->host_irq_disabled = false;
 	return 0;
 }
 #endif
 static int assign_host_irq(struct kvm *kvm,
 			   struct kvm_assigned_dev_kernel *dev,
 			   __u32 host_irq_type)
 {
 	int r = -EEXIST;
 	if (dev->irq_requested_type & KVM_DEV_IRQ_HOST_MASK)
 		return r;
 	switch (host_irq_type) {
 	case KVM_DEV_IRQ_HOST_INTX:
 		r = assigned_device_enable_host_intx(kvm, dev);
 		break;
 #ifdef __KVM_HAVE_MSI
 	case KVM_DEV_IRQ_HOST_MSI:
 		r = assigned_device_enable_host_msi(kvm, dev);
 		break;
 #endif
 #ifdef __KVM_HAVE_MSIX
 	case KVM_DEV_IRQ_HOST_MSIX:
 		r = assigned_device_enable_host_msix(kvm, dev);
 		break;
 #endif
 	default:
 		r = -EINVAL;
 	}
 	if (!r)
 		dev->irq_requested_type |= host_irq_type;
 	return r;
 }
 static int assign_guest_irq(struct kvm *kvm,
 			    struct kvm_assigned_dev_kernel *dev,
 			    struct kvm_assigned_irq *irq,
 			    unsigned long guest_irq_type)
 {
 	int id;
 	int r = -EEXIST;
 	if (dev->irq_requested_type & KVM_DEV_IRQ_GUEST_MASK)
 		return r;
 	id = kvm_request_irq_source_id(kvm);
 	if (id < 0)
 		return id;
 	dev->irq_source_id = id;
 	switch (guest_irq_type) {
 	case KVM_DEV_IRQ_GUEST_INTX:
 		r = assigned_device_enable_guest_intx(kvm, dev, irq);
 		break;
 #ifdef __KVM_HAVE_MSI
 	case KVM_DEV_IRQ_GUEST_MSI:
 		r = assigned_device_enable_guest_msi(kvm, dev, irq);
 		break;
 #endif
 #ifdef __KVM_HAVE_MSIX
 	case KVM_DEV_IRQ_GUEST_MSIX:
 		r = assigned_device_enable_guest_msix(kvm, dev, irq);
 		break;
 #endif
 	default:
 		r = -EINVAL;
 	}
 	if (!r) {
 		dev->irq_requested_type |= guest_irq_type;
 		kvm_register_irq_ack_notifier(kvm, &dev->ack_notifier);
 	} else
 		kvm_free_irq_source_id(kvm, dev->irq_source_id);
 	return r;
 }
 /* TODO Deal with KVM_DEV_IRQ_ASSIGNED_MASK_MSIX */
 static int kvm_vm_ioctl_assign_irq(struct kvm *kvm,
 				   struct kvm_assigned_irq *assigned_irq)
 {
 	int r = -EINVAL;
 	struct kvm_assigned_dev_kernel *match;
 	unsigned long host_irq_type, guest_irq_type;
 	if (!irqchip_in_kernel(kvm))
 		return r;
 	mutex_lock(&kvm->lock);
 	r = -ENODEV;
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_irq->assigned_dev_id);
 	if (!match)
 		goto out;
 	host_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_HOST_MASK);
 	guest_irq_type = (assigned_irq->flags & KVM_DEV_IRQ_GUEST_MASK);
 	r = -EINVAL;
 	/* can only assign one type at a time */
 	if (hweight_long(host_irq_type) > 1)
 		goto out;
 	if (hweight_long(guest_irq_type) > 1)
 		goto out;
 	if (host_irq_type == 0 && guest_irq_type == 0)
 		goto out;
 	r = 0;
 	if (host_irq_type)
 		r = assign_host_irq(kvm, match, host_irq_type);
 	if (r)
 		goto out;
 	if (guest_irq_type)
 		r = assign_guest_irq(kvm, match, assigned_irq, guest_irq_type);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_deassign_dev_irq(struct kvm *kvm,
 					 struct kvm_assigned_irq
 					 *assigned_irq)
 {
 	int r = -ENODEV;
 	struct kvm_assigned_dev_kernel *match;
 	mutex_lock(&kvm->lock);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_irq->assigned_dev_id);
 	if (!match)
 		goto out;
 	r = kvm_deassign_irq(kvm, match, assigned_irq->flags);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
 				      struct kvm_assigned_pci_dev *assigned_dev)
 {
 	int r = 0, idx;
 	struct kvm_assigned_dev_kernel *match;
 	struct pci_dev *dev;
 	mutex_lock(&kvm->lock);
 	idx = srcu_read_lock(&kvm->srcu);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_dev->assigned_dev_id);
 	if (match) {
 		/* device already assigned */
 		r = -EEXIST;
 		goto out;
 	}
 	match = kzalloc(sizeof(struct kvm_assigned_dev_kernel), GFP_KERNEL);
 	if (match == NULL) {
 		printk(KERN_INFO "%s: Couldn't allocate memory\n",
 		       __func__);
 		r = -ENOMEM;
 		goto out;
 	}
 	dev = pci_get_domain_bus_and_slot(assigned_dev->segnr,
 				   assigned_dev->busnr,
 				   assigned_dev->devfn);
 	if (!dev) {
 		printk(KERN_INFO "%s: host device not found\n", __func__);
 		r = -EINVAL;
 		goto out_free;
 	}
 	if (pci_enable_device(dev)) {
 		printk(KERN_INFO "%s: Could not enable PCI device\n", __func__);
 		r = -EBUSY;
 		goto out_put;
 	}
 	r = pci_request_regions(dev, "kvm_assigned_device");
 	if (r) {
 		printk(KERN_INFO "%s: Could not get access to device regions\n",
 		       __func__);
 		goto out_disable;
 	}
 	pci_reset_function(dev);
 	match->assigned_dev_id = assigned_dev->assigned_dev_id;
 	match->host_segnr = assigned_dev->segnr;
 	match->host_busnr = assigned_dev->busnr;
 	match->host_devfn = assigned_dev->devfn;
 	match->flags = assigned_dev->flags;
 	match->dev = dev;
 	spin_lock_init(&match->assigned_dev_lock);
 	match->irq_source_id = -1;
 	match->kvm = kvm;
 	match->ack_notifier.irq_acked = kvm_assigned_dev_ack_irq;
 	INIT_WORK(&match->interrupt_work,
 		  kvm_assigned_dev_interrupt_work_handler);
 	list_add(&match->list, &kvm->arch.assigned_dev_head);
 	if (assigned_dev->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU) {
 		if (!kvm->arch.iommu_domain) {
 			r = kvm_iommu_map_guest(kvm);
 			if (r)
 				goto out_list_del;
 		}
 		r = kvm_assign_device(kvm, match);
 		if (r)
 			goto out_list_del;
 	}
 out:
 	srcu_read_unlock(&kvm->srcu, idx);
 	mutex_unlock(&kvm->lock);
 	return r;
 out_list_del:
 	list_del(&match->list);
 	pci_release_regions(dev);
 out_disable:
 	pci_disable_device(dev);
 out_put:
 	pci_dev_put(dev);
 out_free:
 	kfree(match);
 	srcu_read_unlock(&kvm->srcu, idx);
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_deassign_device(struct kvm *kvm,
 		struct kvm_assigned_pci_dev *assigned_dev)
 {
 	int r = 0;
 	struct kvm_assigned_dev_kernel *match;
 	mutex_lock(&kvm->lock);
 	match = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      assigned_dev->assigned_dev_id);
 	if (!match) {
 		printk(KERN_INFO "%s: device hasn't been assigned before, "
 		  "so cannot be deassigned\n", __func__);
 		r = -EINVAL;
 		goto out;
 	}
 	if (match->flags & KVM_DEV_ASSIGN_ENABLE_IOMMU)
 		kvm_deassign_device(kvm, match);
 	kvm_free_assigned_device(kvm, match);
 out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 #ifdef __KVM_HAVE_MSIX
 static int kvm_vm_ioctl_set_msix_nr(struct kvm *kvm,
 				    struct kvm_assigned_msix_nr *entry_nr)
 {
 	int r = 0;
 	struct kvm_assigned_dev_kernel *adev;
 	mutex_lock(&kvm->lock);
 	adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      entry_nr->assigned_dev_id);
 	if (!adev) {
 		r = -EINVAL;
 		goto msix_nr_out;
 	}
 	if (adev->entries_nr == 0) {
 		adev->entries_nr = entry_nr->entry_nr;
 		if (adev->entries_nr == 0 ||
 		    adev->entries_nr >= KVM_MAX_MSIX_PER_DEV) {
 			r = -EINVAL;
 			goto msix_nr_out;
 		}
 		adev->host_msix_entries = kzalloc(sizeof(struct msix_entry) *
 						entry_nr->entry_nr,
 						GFP_KERNEL);
 		if (!adev->host_msix_entries) {
 			r = -ENOMEM;
 			goto msix_nr_out;
 		}
 		adev->guest_msix_entries = kzalloc(
 				sizeof(struct kvm_guest_msix_entry) *
 				entry_nr->entry_nr, GFP_KERNEL);
 		if (!adev->guest_msix_entries) {
 			kfree(adev->host_msix_entries);
 			r = -ENOMEM;
 			goto msix_nr_out;
 		}
 	} else /* Not allowed set MSI-X number twice */
 		r = -EINVAL;
 msix_nr_out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
 				       struct kvm_assigned_msix_entry *entry)
 {
 	int r = 0, i;
 	struct kvm_assigned_dev_kernel *adev;
 	mutex_lock(&kvm->lock);
 	adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
 				      entry->assigned_dev_id);
 	if (!adev) {
 		r = -EINVAL;
 		goto msix_entry_out;
 	}
 	for (i = 0; i < adev->entries_nr; i++)
 		if (adev->guest_msix_entries[i].vector == 0 ||
 		    adev->guest_msix_entries[i].entry == entry->entry) {
 			adev->guest_msix_entries[i].entry = entry->entry;
 			adev->guest_msix_entries[i].vector = entry->gsi;
 			adev->host_msix_entries[i].entry = entry->entry;
 			break;
 		}
 	if (i == adev->entries_nr) {
 		r = -ENOSPC;
 		goto msix_entry_out;
 	}
 msix_entry_out:
 	mutex_unlock(&kvm->lock);
 	return r;
 }
 #endif
 long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
 				  unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
 	int r = -ENOTTY;
 	switch (ioctl) {
 	case KVM_ASSIGN_PCI_DEVICE: {
 		struct kvm_assigned_pci_dev assigned_dev;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
 			goto out;
 		r = kvm_vm_ioctl_assign_device(kvm, &assigned_dev);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_ASSIGN_IRQ: {
 		r = -EOPNOTSUPP;
 		break;
 	}
 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
 	case KVM_ASSIGN_DEV_IRQ: {
 		struct kvm_assigned_irq assigned_irq;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
 			goto out;
 		r = kvm_vm_ioctl_assign_irq(kvm, &assigned_irq);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_DEASSIGN_DEV_IRQ: {
 		struct kvm_assigned_irq assigned_irq;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_irq, argp, sizeof assigned_irq))
 			goto out;
 		r = kvm_vm_ioctl_deassign_dev_irq(kvm, &assigned_irq);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
 	case KVM_DEASSIGN_PCI_DEVICE: {
 		struct kvm_assigned_pci_dev assigned_dev;
 		r = -EFAULT;
 		if (copy_from_user(&assigned_dev, argp, sizeof assigned_dev))
 			goto out;
 		r = kvm_vm_ioctl_deassign_device(kvm, &assigned_dev);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 #ifdef KVM_CAP_IRQ_ROUTING
 	case KVM_SET_GSI_ROUTING: {
 		struct kvm_irq_routing routing;
 		struct kvm_irq_routing __user *urouting;
 		struct kvm_irq_routing_entry *entries;
 		r = -EFAULT;
 		if (copy_from_user(&routing, argp, sizeof(routing)))
 			goto out;
 		r = -EINVAL;
 		if (routing.nr >= KVM_MAX_IRQ_ROUTES)
 			goto out;
 		if (routing.flags)
 			goto out;
 		r = -ENOMEM;
 		entries = vmalloc(routing.nr * sizeof(*entries));
 		if (!entries)
 			goto out;
 		r = -EFAULT;
 		urouting = argp;
 		if (copy_from_user(entries, urouting->entries,
 				   routing.nr * sizeof(*entries)))
 			goto out_free_irq_routing;
 		r = kvm_set_irq_routing(kvm, entries, routing.nr,
 					routing.flags);
 	out_free_irq_routing:
 		vfree(entries);
 		break;
 	}
 #endif /* KVM_CAP_IRQ_ROUTING */
 #ifdef __KVM_HAVE_MSIX
 	case KVM_ASSIGN_SET_MSIX_NR: {
 		struct kvm_assigned_msix_nr entry_nr;
 		r = -EFAULT;
 		if (copy_from_user(&entry_nr, argp, sizeof entry_nr))
 			goto out;
 		r = kvm_vm_ioctl_set_msix_nr(kvm, &entry_nr);
 		if (r)
 			goto out;
 		break;
 	}
 	case KVM_ASSIGN_SET_MSIX_ENTRY: {
 		struct kvm_assigned_msix_entry entry;
 		r = -EFAULT;
 		if (copy_from_user(&entry, argp, sizeof entry))
 			goto out;
 		r = kvm_vm_ioctl_set_msix_entry(kvm, &entry);
 		if (r)
 			goto out;
 		break;
 	}
 #endif
 	}
 out:
 	return r;
 }