KVM: x86: switch kvm_set_memory_alias to SRCU update

Using a similar two-step procedure as for memslots. Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

KVM: x86: switch kvm_set_memory_alias to SRCU update
Using a similar two-step procedure as for memslots. Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Marcelo Tosatti
1 parent b050b015ab
Showing 4 changed files with 63 additions and 11 deletions Side-by-side Diff
arch/x86/include/asm/kvm_host.h
arch/x86/kvm/x86.c
include/linux/kvm_host.h
virt/kvm/kvm_main.c
@@ -368,7 +368,11 @@
 	gfn_t base_gfn;
 	unsigned long npages;
 	gfn_t target_gfn;
+#define KVM_ALIAS_INVALID     1UL
+	unsigned long flags;
 };
+
+#define KVM_ARCH_HAS_UNALIAS_INSTANTIATION
  
 struct kvm_mem_aliases {
 	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
@@ -38,6 +38,7 @@
 #include <linux/intel-iommu.h>
 #include <linux/cpufreq.h>
 #include <linux/user-return-notifier.h>
+#include <linux/srcu.h>
 #include <trace/events/kvm.h>
 #undef TRACE_INCLUDE_FILE
 #define CREATE_TRACE_POINTS
  
  
@@ -2223,12 +2224,33 @@
 	return kvm->arch.n_alloc_mmu_pages;
 }
  
+gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
+{
+	int i;
+	struct kvm_mem_alias *alias;
+	struct kvm_mem_aliases *aliases;
+
+	aliases = rcu_dereference(kvm->arch.aliases);
+
+	for (i = 0; i < aliases->naliases; ++i) {
+		alias = &aliases->aliases[i];
+		if (alias->flags & KVM_ALIAS_INVALID)
+			continue;
+		if (gfn >= alias->base_gfn
+		    && gfn < alias->base_gfn + alias->npages)
+			return alias->target_gfn + gfn - alias->base_gfn;
+	}
+	return gfn;
+}
+
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
 	struct kvm_mem_alias *alias;
-	struct kvm_mem_aliases *aliases = kvm->arch.aliases;
+	struct kvm_mem_aliases *aliases;
  
+	aliases = rcu_dereference(kvm->arch.aliases);
+
 	for (i = 0; i < aliases->naliases; ++i) {
 		alias = &aliases->aliases[i];
 		if (gfn >= alias->base_gfn
@@ -2248,7 +2270,7 @@
 {
 	int r, n;
 	struct kvm_mem_alias *p;
-	struct kvm_mem_aliases *aliases;
+	struct kvm_mem_aliases *aliases, *old_aliases;
  
 	r = -EINVAL;
 	/* General sanity checks */
  
  
  
  
  
  
  
@@ -2265,28 +2287,48 @@
 	    < alias->target_phys_addr)
 		goto out;
  
+	r = -ENOMEM;
+	aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
+	if (!aliases)
+		goto out;
+
 	down_write(&kvm->slots_lock);
-	spin_lock(&kvm->mmu_lock);
  
-	aliases = kvm->arch.aliases;
+	/* invalidate any gfn reference in case of deletion/shrinking */
+	memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
+	aliases->aliases[alias->slot].flags |= KVM_ALIAS_INVALID;
+	old_aliases = kvm->arch.aliases;
+	rcu_assign_pointer(kvm->arch.aliases, aliases);
+	synchronize_srcu_expedited(&kvm->srcu);
+	kvm_mmu_zap_all(kvm);
+	kfree(old_aliases);
  
+	r = -ENOMEM;
+	aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
+	if (!aliases)
+		goto out_unlock;
+
+	memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
+
 	p = &aliases->aliases[alias->slot];
 	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
 	p->npages = alias->memory_size >> PAGE_SHIFT;
 	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
+	p->flags &= ~(KVM_ALIAS_INVALID);
  
 	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
 		if (aliases->aliases[n - 1].npages)
 			break;
 	aliases->naliases = n;
  
-	spin_unlock(&kvm->mmu_lock);
-	kvm_mmu_zap_all(kvm);
+	old_aliases = kvm->arch.aliases;
+	rcu_assign_pointer(kvm->arch.aliases, aliases);
+	synchronize_srcu_expedited(&kvm->srcu);
+	kfree(old_aliases);
+	r = 0;
  
+out_unlock:
 	up_write(&kvm->slots_lock);
-
-	return 0;
-
 out:
 	return r;
 }
@@ -266,6 +266,8 @@
 void kvm_disable_largepages(void);
 void kvm_arch_flush_shadow(struct kvm *kvm);
 gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
+gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn);
+
 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn);
 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
 void kvm_release_page_clean(struct page *page);
@@ -537,6 +539,10 @@
 		return 1;
 	return 0;
 }
+#endif
+
+#ifndef KVM_ARCH_HAS_UNALIAS_INSTANTIATION
+#define unalias_gfn_instantiation unalias_gfn
 #endif
  
 #ifdef CONFIG_HAVE_KVM_IRQCHIP
@@ -859,7 +859,7 @@
 	int i;
 	struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
  
-	gfn = unalias_gfn(kvm, gfn);
+	gfn = unalias_gfn_instantiation(kvm, gfn);
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *memslot = &slots->memslots[i];
  
@@ -896,7 +896,7 @@
 {
 	struct kvm_memory_slot *slot;
  
-	gfn = unalias_gfn(kvm, gfn);
+	gfn = unalias_gfn_instantiation(kvm, gfn);
 	slot = gfn_to_memslot_unaliased(kvm, gfn);
 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
 		return bad_hva();
...	...	@@ -368,7 +368,11 @@
368	368	gfn_t base_gfn;
369	369	unsigned long npages;
370	370	gfn_t target_gfn;
	371	+#define KVM_ALIAS_INVALID 1UL
	372	+ unsigned long flags;
371	373	};
	374	+
	375	+#define KVM_ARCH_HAS_UNALIAS_INSTANTIATION
372	376
373	377	struct kvm_mem_aliases {
374	378	struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
...	...	@@ -38,6 +38,7 @@
38	38	#include <linux/intel-iommu.h>
39	39	#include <linux/cpufreq.h>
40	40	#include <linux/user-return-notifier.h>
	41	+#include <linux/srcu.h>
41	42	#include <trace/events/kvm.h>
42	43	#undef TRACE_INCLUDE_FILE
43	44	#define CREATE_TRACE_POINTS
44	45
45	46
...	...	@@ -2223,12 +2224,33 @@
2223	2224	return kvm->arch.n_alloc_mmu_pages;
2224	2225	}
2225	2226
	2227	+gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn)
	2228	+{
	2229	+ int i;
	2230	+ struct kvm_mem_alias *alias;
	2231	+ struct kvm_mem_aliases *aliases;
	2232	+
	2233	+ aliases = rcu_dereference(kvm->arch.aliases);
	2234	+
	2235	+ for (i = 0; i < aliases->naliases; ++i) {
	2236	+ alias = &aliases->aliases[i];
	2237	+ if (alias->flags & KVM_ALIAS_INVALID)
	2238	+ continue;
	2239	+ if (gfn >= alias->base_gfn
	2240	+ && gfn < alias->base_gfn + alias->npages)
	2241	+ return alias->target_gfn + gfn - alias->base_gfn;
	2242	+ }
	2243	+ return gfn;
	2244	+}
	2245	+
2226	2246	gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn)
2227	2247	{
2228	2248	int i;
2229	2249	struct kvm_mem_alias *alias;
2230		- struct kvm_mem_aliases *aliases = kvm->arch.aliases;
	2250	+ struct kvm_mem_aliases *aliases;
2231	2251
	2252	+ aliases = rcu_dereference(kvm->arch.aliases);
	2253	+
2232	2254	for (i = 0; i < aliases->naliases; ++i) {
2233	2255	alias = &aliases->aliases[i];
2234	2256	if (gfn >= alias->base_gfn
...	...	@@ -2248,7 +2270,7 @@
2248	2270	{
2249	2271	int r, n;
2250	2272	struct kvm_mem_alias *p;
2251		- struct kvm_mem_aliases *aliases;
	2273	+ struct kvm_mem_aliases aliases, old_aliases;
2252	2274
2253	2275	r = -EINVAL;
2254	2276	/* General sanity checks */
2255	2277
2256	2278
2257	2279
2258	2280
2259	2281
2260	2282
2261	2283
...	...	@@ -2265,28 +2287,48 @@
2265	2287	< alias->target_phys_addr)
2266	2288	goto out;
2267	2289
	2290	+ r = -ENOMEM;
	2291	+ aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
	2292	+ if (!aliases)
	2293	+ goto out;
	2294	+
2268	2295	down_write(&kvm->slots_lock);
2269		- spin_lock(&kvm->mmu_lock);
2270	2296
2271		- aliases = kvm->arch.aliases;
	2297	+ /* invalidate any gfn reference in case of deletion/shrinking */
	2298	+ memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
	2299	+ aliases->aliases[alias->slot].flags \|= KVM_ALIAS_INVALID;
	2300	+ old_aliases = kvm->arch.aliases;
	2301	+ rcu_assign_pointer(kvm->arch.aliases, aliases);
	2302	+ synchronize_srcu_expedited(&kvm->srcu);
	2303	+ kvm_mmu_zap_all(kvm);
	2304	+ kfree(old_aliases);
2272	2305
	2306	+ r = -ENOMEM;
	2307	+ aliases = kzalloc(sizeof(struct kvm_mem_aliases), GFP_KERNEL);
	2308	+ if (!aliases)
	2309	+ goto out_unlock;
	2310	+
	2311	+ memcpy(aliases, kvm->arch.aliases, sizeof(struct kvm_mem_aliases));
	2312	+
2273	2313	p = &aliases->aliases[alias->slot];
2274	2314	p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT;
2275	2315	p->npages = alias->memory_size >> PAGE_SHIFT;
2276	2316	p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT;
	2317	+ p->flags &= ~(KVM_ALIAS_INVALID);
2277	2318
2278	2319	for (n = KVM_ALIAS_SLOTS; n > 0; --n)
2279	2320	if (aliases->aliases[n - 1].npages)
2280	2321	break;
2281	2322	aliases->naliases = n;
2282	2323
2283		- spin_unlock(&kvm->mmu_lock);
2284		- kvm_mmu_zap_all(kvm);
	2324	+ old_aliases = kvm->arch.aliases;
	2325	+ rcu_assign_pointer(kvm->arch.aliases, aliases);
	2326	+ synchronize_srcu_expedited(&kvm->srcu);
	2327	+ kfree(old_aliases);
	2328	+ r = 0;
2285	2329
	2330	+out_unlock:
2286	2331	up_write(&kvm->slots_lock);
2287		-
2288		- return 0;
2289		-
2290	2332	out:
2291	2333	return r;
2292	2334	}
...	...	@@ -266,6 +266,8 @@
266	266	void kvm_disable_largepages(void);
267	267	void kvm_arch_flush_shadow(struct kvm *kvm);
268	268	gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn);
	269	+gfn_t unalias_gfn_instantiation(struct kvm *kvm, gfn_t gfn);
	270	+
269	271	struct page gfn_to_page(struct kvm kvm, gfn_t gfn);
270	272	unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn);
271	273	void kvm_release_page_clean(struct page *page);
...	...	@@ -537,6 +539,10 @@
537	539	return 1;
538	540	return 0;
539	541	}
	542	+#endif
	543	+
	544	+#ifndef KVM_ARCH_HAS_UNALIAS_INSTANTIATION
	545	+#define unalias_gfn_instantiation unalias_gfn
540	546	#endif
541	547
542	548	#ifdef CONFIG_HAVE_KVM_IRQCHIP
...	...	@@ -859,7 +859,7 @@
859	859	int i;
860	860	struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
861	861
862		- gfn = unalias_gfn(kvm, gfn);
	862	+ gfn = unalias_gfn_instantiation(kvm, gfn);
863	863	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
864	864	struct kvm_memory_slot *memslot = &slots->memslots[i];
865	865
...	...	@@ -896,7 +896,7 @@
896	896	{
897	897	struct kvm_memory_slot *slot;
898	898
899		- gfn = unalias_gfn(kvm, gfn);
	899	+ gfn = unalias_gfn_instantiation(kvm, gfn);
900	900	slot = gfn_to_memslot_unaliased(kvm, gfn);
901	901	if (!slot \|\| slot->flags & KVM_MEMSLOT_INVALID)
902	902	return bad_hva();