KVM: convert io_bus to SRCU

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

KVM: convert io_bus to SRCU
Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
Marcelo Tosatti
1 parent a983fb2387
Showing 9 changed files with 101 additions and 75 deletions Side-by-side Diff
arch/ia64/kvm/kvm-ia64.c
arch/x86/kvm/i8254.c
arch/x86/kvm/i8259.c
arch/x86/kvm/x86.c
include/linux/kvm_host.h
virt/kvm/coalesced_mmio.c
virt/kvm/eventfd.c
virt/kvm/ioapic.c
virt/kvm/kvm_main.c
@@ -241,10 +241,10 @@
 	return 0;
 mmio:
 	if (p->dir)
-		r = kvm_io_bus_read(&vcpu->kvm->mmio_bus, p->addr,
+		r = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, p->addr,
 				    p->size, &p->data);
 	else
-		r = kvm_io_bus_write(&vcpu->kvm->mmio_bus, p->addr,
+		r = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, p->addr,
 				     p->size, &p->data);
 	if (r)
 		printk(KERN_ERR"kvm: No iodevice found! addr:%lx\n", p->addr);
@@ -645,13 +645,13 @@
 	kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier);
  
 	kvm_iodevice_init(&pit->dev, &pit_dev_ops);
-	ret = __kvm_io_bus_register_dev(&kvm->pio_bus, &pit->dev);
+	ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &pit->dev);
 	if (ret < 0)
 		goto fail;
  
 	if (flags & KVM_PIT_SPEAKER_DUMMY) {
 		kvm_iodevice_init(&pit->speaker_dev, &speaker_dev_ops);
-		ret = __kvm_io_bus_register_dev(&kvm->pio_bus,
+		ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS,
 						&pit->speaker_dev);
 		if (ret < 0)
 			goto fail_unregister;
@@ -660,7 +660,7 @@
 	return pit;
  
 fail_unregister:
-	__kvm_io_bus_unregister_dev(&kvm->pio_bus, &pit->dev);
+	kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->dev);
  
 fail:
 	if (pit->irq_source_id >= 0)
@@ -533,7 +533,9 @@
 	 * Initialize PIO device
 	 */
 	kvm_iodevice_init(&s->dev, &picdev_ops);
-	ret = kvm_io_bus_register_dev(kvm, &kvm->pio_bus, &s->dev);
+	down_write(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &s->dev);
+	up_write(&kvm->slots_lock);
 	if (ret < 0) {
 		kfree(s);
 		return NULL;
@@ -2851,7 +2851,7 @@
 	    !kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
 		return 0;
  
-	return kvm_io_bus_write(&vcpu->kvm->mmio_bus, addr, len, v);
+	return kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
 }
  
 static int vcpu_mmio_read(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *v)
@@ -2860,7 +2860,7 @@
 	    !kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
 		return 0;
  
-	return kvm_io_bus_read(&vcpu->kvm->mmio_bus, addr, len, v);
+	return kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
 }
  
 static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
  
@@ -3345,11 +3345,12 @@
 	int r;
  
 	if (vcpu->arch.pio.in)
-		r = kvm_io_bus_read(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
+		r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
 				    vcpu->arch.pio.size, pd);
 	else
-		r = kvm_io_bus_write(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
-				     vcpu->arch.pio.size, pd);
+		r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
+				     vcpu->arch.pio.port, vcpu->arch.pio.size,
+				     pd);
 	return r;
 }
  
@@ -3360,7 +3361,7 @@
 	int i, r = 0;
  
 	for (i = 0; i < io->cur_count; i++) {
-		if (kvm_io_bus_write(&vcpu->kvm->pio_bus,
+		if (kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
 				     io->port, io->size, pd)) {
 			r = -EOPNOTSUPP;
 			break;
@@ -57,20 +57,20 @@
 	struct kvm_io_device *devs[NR_IOBUS_DEVS];
 };
  
-void kvm_io_bus_init(struct kvm_io_bus *bus);
-void kvm_io_bus_destroy(struct kvm_io_bus *bus);
-int kvm_io_bus_write(struct kvm_io_bus *bus, gpa_t addr, int len,
-		     const void *val);
-int kvm_io_bus_read(struct kvm_io_bus *bus, gpa_t addr, int len,
+enum kvm_bus {
+	KVM_MMIO_BUS,
+	KVM_PIO_BUS,
+	KVM_NR_BUSES
+};
+
+int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
+		     int len, const void *val);
+int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len,
 		    void *val);
-int __kvm_io_bus_register_dev(struct kvm_io_bus *bus,
-			       struct kvm_io_device *dev);
-int kvm_io_bus_register_dev(struct kvm *kvm, struct kvm_io_bus *bus,
+int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 			    struct kvm_io_device *dev);
-void __kvm_io_bus_unregister_dev(struct kvm_io_bus *bus,
-				 struct kvm_io_device *dev);
-void kvm_io_bus_unregister_dev(struct kvm *kvm, struct kvm_io_bus *bus,
-			       struct kvm_io_device *dev);
+int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
+			      struct kvm_io_device *dev);
  
 struct kvm_vcpu {
 	struct kvm *kvm;
@@ -171,8 +171,7 @@
 	atomic_t online_vcpus;
 	struct list_head vm_list;
 	struct mutex lock;
-	struct kvm_io_bus mmio_bus;
-	struct kvm_io_bus pio_bus;
+	struct kvm_io_bus *buses[KVM_NR_BUSES];
 #ifdef CONFIG_HAVE_KVM_EVENTFD
 	struct {
 		spinlock_t        lock;
@@ -110,7 +110,9 @@
 	dev->kvm = kvm;
 	kvm->coalesced_mmio_dev = dev;
  
-	ret = kvm_io_bus_register_dev(kvm, &kvm->mmio_bus, &dev->dev);
+	down_write(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &dev->dev);
+	up_write(&kvm->slots_lock);
 	if (ret < 0)
 		goto out_free_dev;
  
@@ -463,7 +463,7 @@
 kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
 {
 	int                       pio = args->flags & KVM_IOEVENTFD_FLAG_PIO;
-	struct kvm_io_bus        *bus = pio ? &kvm->pio_bus : &kvm->mmio_bus;
+	enum kvm_bus              bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS;
 	struct _ioeventfd        *p;
 	struct eventfd_ctx       *eventfd;
 	int                       ret;
@@ -518,7 +518,7 @@
  
 	kvm_iodevice_init(&p->dev, &ioeventfd_ops);
  
-	ret = __kvm_io_bus_register_dev(bus, &p->dev);
+	ret = kvm_io_bus_register_dev(kvm, bus_idx, &p->dev);
 	if (ret < 0)
 		goto unlock_fail;
  
@@ -542,7 +542,7 @@
 kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args)
 {
 	int                       pio = args->flags & KVM_IOEVENTFD_FLAG_PIO;
-	struct kvm_io_bus        *bus = pio ? &kvm->pio_bus : &kvm->mmio_bus;
+	enum kvm_bus              bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS;
 	struct _ioeventfd        *p, *tmp;
 	struct eventfd_ctx       *eventfd;
 	int                       ret = -ENOENT;
@@ -565,7 +565,7 @@
 		if (!p->wildcard && p->datamatch != args->datamatch)
 			continue;
  
-		__kvm_io_bus_unregister_dev(bus, &p->dev);
+		kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
 		ioeventfd_release(p);
 		ret = 0;
 		break;
@@ -372,7 +372,9 @@
 	kvm_ioapic_reset(ioapic);
 	kvm_iodevice_init(&ioapic->dev, &ioapic_mmio_ops);
 	ioapic->kvm = kvm;
-	ret = kvm_io_bus_register_dev(kvm, &kvm->mmio_bus, &ioapic->dev);
+	down_write(&kvm->slots_lock);
+	ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &ioapic->dev);
+	up_write(&kvm->slots_lock);
 	if (ret < 0)
 		kfree(ioapic);
  
@@ -85,6 +85,8 @@
 static int hardware_enable_all(void);
 static void hardware_disable_all(void);
  
+static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
+
 static bool kvm_rebooting;
  
 static bool largepages_enabled = true;
@@ -367,7 +369,7 @@
  
 static struct kvm *kvm_create_vm(void)
 {
-	int r = 0;
+	int r = 0, i;
 	struct kvm *kvm = kvm_arch_create_vm();
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	struct page *page;
@@ -391,6 +393,14 @@
 		goto out_err;
 	if (init_srcu_struct(&kvm->srcu))
 		goto out_err;
+	for (i = 0; i < KVM_NR_BUSES; i++) {
+		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
+					GFP_KERNEL);
+		if (!kvm->buses[i]) {
+			cleanup_srcu_struct(&kvm->srcu);
+			goto out_err;
+		}
+	}
  
 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
 	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  
@@ -416,11 +426,9 @@
 	atomic_inc(&kvm->mm->mm_count);
 	spin_lock_init(&kvm->mmu_lock);
 	spin_lock_init(&kvm->requests_lock);
-	kvm_io_bus_init(&kvm->pio_bus);
 	kvm_eventfd_init(kvm);
 	mutex_init(&kvm->lock);
 	mutex_init(&kvm->irq_lock);
-	kvm_io_bus_init(&kvm->mmio_bus);
 	init_rwsem(&kvm->slots_lock);
 	atomic_set(&kvm->users_count, 1);
 	spin_lock(&kvm_lock);
@@ -435,6 +443,8 @@
 out_err:
 	hardware_disable_all();
 out_err_nodisable:
+	for (i = 0; i < KVM_NR_BUSES; i++)
+		kfree(kvm->buses[i]);
 	kfree(kvm->memslots);
 	kfree(kvm);
 	return ERR_PTR(r);
@@ -480,6 +490,7 @@
  
 static void kvm_destroy_vm(struct kvm *kvm)
 {
+	int i;
 	struct mm_struct *mm = kvm->mm;
  
 	kvm_arch_sync_events(kvm);
@@ -487,8 +498,8 @@
 	list_del(&kvm->vm_list);
 	spin_unlock(&kvm_lock);
 	kvm_free_irq_routing(kvm);
-	kvm_io_bus_destroy(&kvm->pio_bus);
-	kvm_io_bus_destroy(&kvm->mmio_bus);
+	for (i = 0; i < KVM_NR_BUSES; i++)
+		kvm_io_bus_destroy(kvm->buses[i]);
 	kvm_coalesced_mmio_free(kvm);
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
  
@@ -1949,13 +1960,8 @@
 	.priority = 0,
 };
  
-void kvm_io_bus_init(struct kvm_io_bus *bus)
+static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
 {
-	memset(bus, 0, sizeof(*bus));
-}
-
-void kvm_io_bus_destroy(struct kvm_io_bus *bus)
-{
 	int i;
  
 	for (i = 0; i < bus->dev_count; i++) {
  
  
@@ -1963,13 +1969,15 @@
  
 		kvm_iodevice_destructor(pos);
 	}
+	kfree(bus);
 }
  
 /* kvm_io_bus_write - called under kvm->slots_lock */
-int kvm_io_bus_write(struct kvm_io_bus *bus, gpa_t addr,
+int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
 	int i;
+	struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
 			return 0;
  
  
  
  
  
  
  
  
  
  
@@ -1977,59 +1985,71 @@
 }
  
 /* kvm_io_bus_read - called under kvm->slots_lock */
-int kvm_io_bus_read(struct kvm_io_bus *bus, gpa_t addr, int len, void *val)
+int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
+		    int len, void *val)
 {
 	int i;
+	struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
+
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
 			return 0;
 	return -EOPNOTSUPP;
 }
  
-int kvm_io_bus_register_dev(struct kvm *kvm, struct kvm_io_bus *bus,
-			     struct kvm_io_device *dev)
+/* Caller must have write lock on slots_lock. */
+int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
+			    struct kvm_io_device *dev)
 {
-	int ret;
+	struct kvm_io_bus *new_bus, *bus;
  
-	down_write(&kvm->slots_lock);
-	ret = __kvm_io_bus_register_dev(bus, dev);
-	up_write(&kvm->slots_lock);
-
-	return ret;
-}
-
-/* An unlocked version. Caller must have write lock on slots_lock. */
-int __kvm_io_bus_register_dev(struct kvm_io_bus *bus,
-			      struct kvm_io_device *dev)
-{
+	bus = kvm->buses[bus_idx];
 	if (bus->dev_count > NR_IOBUS_DEVS-1)
 		return -ENOSPC;
  
-	bus->devs[bus->dev_count++] = dev;
+	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
+	if (!new_bus)
+		return -ENOMEM;
+	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
+	new_bus->devs[new_bus->dev_count++] = dev;
+	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+	synchronize_srcu_expedited(&kvm->srcu);
+	kfree(bus);
  
 	return 0;
 }
  
-void kvm_io_bus_unregister_dev(struct kvm *kvm,
-			       struct kvm_io_bus *bus,
-			       struct kvm_io_device *dev)
+/* Caller must have write lock on slots_lock. */
+int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
+			      struct kvm_io_device *dev)
 {
-	down_write(&kvm->slots_lock);
-	__kvm_io_bus_unregister_dev(bus, dev);
-	up_write(&kvm->slots_lock);
-}
+	int i, r;
+	struct kvm_io_bus *new_bus, *bus;
  
-/* An unlocked version. Caller must have write lock on slots_lock. */
-void __kvm_io_bus_unregister_dev(struct kvm_io_bus *bus,
-				 struct kvm_io_device *dev)
-{
-	int i;
+	new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
+	if (!new_bus)
+		return -ENOMEM;
  
-	for (i = 0; i < bus->dev_count; i++)
-		if (bus->devs[i] == dev) {
-			bus->devs[i] = bus->devs[--bus->dev_count];
+	bus = kvm->buses[bus_idx];
+	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
+
+	r = -ENOENT;
+	for (i = 0; i < new_bus->dev_count; i++)
+		if (new_bus->devs[i] == dev) {
+			r = 0;
+			new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
 			break;
 		}
+
+	if (r) {
+		kfree(new_bus);
+		return r;
+	}
+
+	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+	synchronize_srcu_expedited(&kvm->srcu);
+	kfree(bus);
+	return r;
 }
  
 static struct notifier_block kvm_cpu_notifier = {
...	...	@@ -241,10 +241,10 @@
241	241	return 0;
242	242	mmio:
243	243	if (p->dir)
244		- r = kvm_io_bus_read(&vcpu->kvm->mmio_bus, p->addr,
	244	+ r = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, p->addr,
245	245	p->size, &p->data);
246	246	else
247		- r = kvm_io_bus_write(&vcpu->kvm->mmio_bus, p->addr,
	247	+ r = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, p->addr,
248	248	p->size, &p->data);
249	249	if (r)
250	250	printk(KERN_ERR"kvm: No iodevice found! addr:%lx\n", p->addr);
...	...	@@ -645,13 +645,13 @@
645	645	kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier);
646	646
647	647	kvm_iodevice_init(&pit->dev, &pit_dev_ops);
648		- ret = __kvm_io_bus_register_dev(&kvm->pio_bus, &pit->dev);
	648	+ ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &pit->dev);
649	649	if (ret < 0)
650	650	goto fail;
651	651
652	652	if (flags & KVM_PIT_SPEAKER_DUMMY) {
653	653	kvm_iodevice_init(&pit->speaker_dev, &speaker_dev_ops);
654		- ret = __kvm_io_bus_register_dev(&kvm->pio_bus,
	654	+ ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS,
655	655	&pit->speaker_dev);
656	656	if (ret < 0)
657	657	goto fail_unregister;
...	...	@@ -660,7 +660,7 @@
660	660	return pit;
661	661
662	662	fail_unregister:
663		- __kvm_io_bus_unregister_dev(&kvm->pio_bus, &pit->dev);
	663	+ kvm_io_bus_unregister_dev(kvm, KVM_PIO_BUS, &pit->dev);
664	664
665	665	fail:
666	666	if (pit->irq_source_id >= 0)
...	...	@@ -533,7 +533,9 @@
533	533	* Initialize PIO device
534	534	*/
535	535	kvm_iodevice_init(&s->dev, &picdev_ops);
536		- ret = kvm_io_bus_register_dev(kvm, &kvm->pio_bus, &s->dev);
	536	+ down_write(&kvm->slots_lock);
	537	+ ret = kvm_io_bus_register_dev(kvm, KVM_PIO_BUS, &s->dev);
	538	+ up_write(&kvm->slots_lock);
537	539	if (ret < 0) {
538	540	kfree(s);
539	541	return NULL;
...	...	@@ -2851,7 +2851,7 @@
2851	2851	!kvm_iodevice_write(&vcpu->arch.apic->dev, addr, len, v))
2852	2852	return 0;
2853	2853
2854		- return kvm_io_bus_write(&vcpu->kvm->mmio_bus, addr, len, v);
	2854	+ return kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
2855	2855	}
2856	2856
2857	2857	static int vcpu_mmio_read(struct kvm_vcpu vcpu, gpa_t addr, int len, void v)
...	...	@@ -2860,7 +2860,7 @@
2860	2860	!kvm_iodevice_read(&vcpu->arch.apic->dev, addr, len, v))
2861	2861	return 0;
2862	2862
2863		- return kvm_io_bus_read(&vcpu->kvm->mmio_bus, addr, len, v);
	2863	+ return kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, addr, len, v);
2864	2864	}
2865	2865
2866	2866	static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes,
2867	2867
...	...	@@ -3345,11 +3345,12 @@
3345	3345	int r;
3346	3346
3347	3347	if (vcpu->arch.pio.in)
3348		- r = kvm_io_bus_read(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
	3348	+ r = kvm_io_bus_read(vcpu->kvm, KVM_PIO_BUS, vcpu->arch.pio.port,
3349	3349	vcpu->arch.pio.size, pd);
3350	3350	else
3351		- r = kvm_io_bus_write(&vcpu->kvm->pio_bus, vcpu->arch.pio.port,
3352		- vcpu->arch.pio.size, pd);
	3351	+ r = kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
	3352	+ vcpu->arch.pio.port, vcpu->arch.pio.size,
	3353	+ pd);
3353	3354	return r;
3354	3355	}
3355	3356
...	...	@@ -3360,7 +3361,7 @@
3360	3361	int i, r = 0;
3361	3362
3362	3363	for (i = 0; i < io->cur_count; i++) {
3363		- if (kvm_io_bus_write(&vcpu->kvm->pio_bus,
	3364	+ if (kvm_io_bus_write(vcpu->kvm, KVM_PIO_BUS,
3364	3365	io->port, io->size, pd)) {
3365	3366	r = -EOPNOTSUPP;
3366	3367	break;
...	...	@@ -57,20 +57,20 @@
57	57	struct kvm_io_device *devs[NR_IOBUS_DEVS];
58	58	};
59	59
60		-void kvm_io_bus_init(struct kvm_io_bus *bus);
61		-void kvm_io_bus_destroy(struct kvm_io_bus *bus);
62		-int kvm_io_bus_write(struct kvm_io_bus *bus, gpa_t addr, int len,
63		- const void *val);
64		-int kvm_io_bus_read(struct kvm_io_bus *bus, gpa_t addr, int len,
	60	+enum kvm_bus {
	61	+ KVM_MMIO_BUS,
	62	+ KVM_PIO_BUS,
	63	+ KVM_NR_BUSES
	64	+};
	65	+
	66	+int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
	67	+ int len, const void *val);
	68	+int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, int len,
65	69	void *val);
66		-int __kvm_io_bus_register_dev(struct kvm_io_bus *bus,
67		- struct kvm_io_device *dev);
68		-int kvm_io_bus_register_dev(struct kvm kvm, struct kvm_io_bus bus,
	70	+int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
69	71	struct kvm_io_device *dev);
70		-void __kvm_io_bus_unregister_dev(struct kvm_io_bus *bus,
71		- struct kvm_io_device *dev);
72		-void kvm_io_bus_unregister_dev(struct kvm kvm, struct kvm_io_bus bus,
73		- struct kvm_io_device *dev);
	72	+int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
	73	+ struct kvm_io_device *dev);
74	74
75	75	struct kvm_vcpu {
76	76	struct kvm *kvm;
...	...	@@ -171,8 +171,7 @@
171	171	atomic_t online_vcpus;
172	172	struct list_head vm_list;
173	173	struct mutex lock;
174		- struct kvm_io_bus mmio_bus;
175		- struct kvm_io_bus pio_bus;
	174	+ struct kvm_io_bus *buses[KVM_NR_BUSES];
176	175	#ifdef CONFIG_HAVE_KVM_EVENTFD
177	176	struct {
178	177	spinlock_t lock;
...	...	@@ -110,7 +110,9 @@
110	110	dev->kvm = kvm;
111	111	kvm->coalesced_mmio_dev = dev;
112	112
113		- ret = kvm_io_bus_register_dev(kvm, &kvm->mmio_bus, &dev->dev);
	113	+ down_write(&kvm->slots_lock);
	114	+ ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &dev->dev);
	115	+ up_write(&kvm->slots_lock);
114	116	if (ret < 0)
115	117	goto out_free_dev;
116	118
...	...	@@ -463,7 +463,7 @@
463	463	kvm_assign_ioeventfd(struct kvm kvm, struct kvm_ioeventfd args)
464	464	{
465	465	int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO;
466		- struct kvm_io_bus *bus = pio ? &kvm->pio_bus : &kvm->mmio_bus;
	466	+ enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS;
467	467	struct _ioeventfd *p;
468	468	struct eventfd_ctx *eventfd;
469	469	int ret;
...	...	@@ -518,7 +518,7 @@
518	518
519	519	kvm_iodevice_init(&p->dev, &ioeventfd_ops);
520	520
521		- ret = __kvm_io_bus_register_dev(bus, &p->dev);
	521	+ ret = kvm_io_bus_register_dev(kvm, bus_idx, &p->dev);
522	522	if (ret < 0)
523	523	goto unlock_fail;
524	524
...	...	@@ -542,7 +542,7 @@
542	542	kvm_deassign_ioeventfd(struct kvm kvm, struct kvm_ioeventfd args)
543	543	{
544	544	int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO;
545		- struct kvm_io_bus *bus = pio ? &kvm->pio_bus : &kvm->mmio_bus;
	545	+ enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS;
546	546	struct _ioeventfd p, tmp;
547	547	struct eventfd_ctx *eventfd;
548	548	int ret = -ENOENT;
...	...	@@ -565,7 +565,7 @@
565	565	if (!p->wildcard && p->datamatch != args->datamatch)
566	566	continue;
567	567
568		- __kvm_io_bus_unregister_dev(bus, &p->dev);
	568	+ kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev);
569	569	ioeventfd_release(p);
570	570	ret = 0;
571	571	break;
...	...	@@ -372,7 +372,9 @@
372	372	kvm_ioapic_reset(ioapic);
373	373	kvm_iodevice_init(&ioapic->dev, &ioapic_mmio_ops);
374	374	ioapic->kvm = kvm;
375		- ret = kvm_io_bus_register_dev(kvm, &kvm->mmio_bus, &ioapic->dev);
	375	+ down_write(&kvm->slots_lock);
	376	+ ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, &ioapic->dev);
	377	+ up_write(&kvm->slots_lock);
376	378	if (ret < 0)
377	379	kfree(ioapic);
378	380
...	...	@@ -85,6 +85,8 @@
85	85	static int hardware_enable_all(void);
86	86	static void hardware_disable_all(void);
87	87
	88	+static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
	89	+
88	90	static bool kvm_rebooting;
89	91
90	92	static bool largepages_enabled = true;
...	...	@@ -367,7 +369,7 @@
367	369
368	370	static struct kvm *kvm_create_vm(void)
369	371	{
370		- int r = 0;
	372	+ int r = 0, i;
371	373	struct kvm *kvm = kvm_arch_create_vm();
372	374	#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
373	375	struct page *page;
...	...	@@ -391,6 +393,14 @@
391	393	goto out_err;
392	394	if (init_srcu_struct(&kvm->srcu))
393	395	goto out_err;
	396	+ for (i = 0; i < KVM_NR_BUSES; i++) {
	397	+ kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
	398	+ GFP_KERNEL);
	399	+ if (!kvm->buses[i]) {
	400	+ cleanup_srcu_struct(&kvm->srcu);
	401	+ goto out_err;
	402	+ }
	403	+ }
394	404
395	405	#ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
396	406	page = alloc_page(GFP_KERNEL \| __GFP_ZERO);
397	407
...	...	@@ -416,11 +426,9 @@
416	426	atomic_inc(&kvm->mm->mm_count);
417	427	spin_lock_init(&kvm->mmu_lock);
418	428	spin_lock_init(&kvm->requests_lock);
419		- kvm_io_bus_init(&kvm->pio_bus);
420	429	kvm_eventfd_init(kvm);
421	430	mutex_init(&kvm->lock);
422	431	mutex_init(&kvm->irq_lock);
423		- kvm_io_bus_init(&kvm->mmio_bus);
424	432	init_rwsem(&kvm->slots_lock);
425	433	atomic_set(&kvm->users_count, 1);
426	434	spin_lock(&kvm_lock);
...	...	@@ -435,6 +443,8 @@
435	443	out_err:
436	444	hardware_disable_all();
437	445	out_err_nodisable:
	446	+ for (i = 0; i < KVM_NR_BUSES; i++)
	447	+ kfree(kvm->buses[i]);
438	448	kfree(kvm->memslots);
439	449	kfree(kvm);
440	450	return ERR_PTR(r);
...	...	@@ -480,6 +490,7 @@
480	490
481	491	static void kvm_destroy_vm(struct kvm *kvm)
482	492	{
	493	+ int i;
483	494	struct mm_struct *mm = kvm->mm;
484	495
485	496	kvm_arch_sync_events(kvm);
...	...	@@ -487,8 +498,8 @@
487	498	list_del(&kvm->vm_list);
488	499	spin_unlock(&kvm_lock);
489	500	kvm_free_irq_routing(kvm);
490		- kvm_io_bus_destroy(&kvm->pio_bus);
491		- kvm_io_bus_destroy(&kvm->mmio_bus);
	501	+ for (i = 0; i < KVM_NR_BUSES; i++)
	502	+ kvm_io_bus_destroy(kvm->buses[i]);
492	503	kvm_coalesced_mmio_free(kvm);
493	504	#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
494	505	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
495	506
...	...	@@ -1949,13 +1960,8 @@
1949	1960	.priority = 0,
1950	1961	};
1951	1962
1952		-void kvm_io_bus_init(struct kvm_io_bus *bus)
	1963	+static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1953	1964	{
1954		- memset(bus, 0, sizeof(*bus));
1955		-}
1956		-
1957		-void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1958		-{
1959	1965	int i;
1960	1966
1961	1967	for (i = 0; i < bus->dev_count; i++) {
1962	1968
1963	1969
...	...	@@ -1963,13 +1969,15 @@
1963	1969
1964	1970	kvm_iodevice_destructor(pos);
1965	1971	}
	1972	+ kfree(bus);
1966	1973	}
1967	1974
1968	1975	/* kvm_io_bus_write - called under kvm->slots_lock */
1969		-int kvm_io_bus_write(struct kvm_io_bus *bus, gpa_t addr,
	1976	+int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
1970	1977	int len, const void *val)
1971	1978	{
1972	1979	int i;
	1980	+ struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
1973	1981	for (i = 0; i < bus->dev_count; i++)
1974	1982	if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
1975	1983	return 0;
1976	1984
1977	1985
1978	1986
1979	1987
1980	1988
1981	1989
1982	1990
1983	1991
1984	1992
1985	1993
...	...	@@ -1977,59 +1985,71 @@
1977	1985	}
1978	1986
1979	1987	/* kvm_io_bus_read - called under kvm->slots_lock */
1980		-int kvm_io_bus_read(struct kvm_io_bus bus, gpa_t addr, int len, void val)
	1988	+int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
	1989	+ int len, void *val)
1981	1990	{
1982	1991	int i;
	1992	+ struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
	1993	+
1983	1994	for (i = 0; i < bus->dev_count; i++)
1984	1995	if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
1985	1996	return 0;
1986	1997	return -EOPNOTSUPP;
1987	1998	}
1988	1999
1989		-int kvm_io_bus_register_dev(struct kvm kvm, struct kvm_io_bus bus,
1990		- struct kvm_io_device *dev)
	2000	+/* Caller must have write lock on slots_lock. */
	2001	+int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
	2002	+ struct kvm_io_device *dev)
1991	2003	{
1992		- int ret;
	2004	+ struct kvm_io_bus new_bus, bus;
1993	2005
1994		- down_write(&kvm->slots_lock);
1995		- ret = __kvm_io_bus_register_dev(bus, dev);
1996		- up_write(&kvm->slots_lock);
1997		-
1998		- return ret;
1999		-}
2000		-
2001		-/* An unlocked version. Caller must have write lock on slots_lock. */
2002		-int __kvm_io_bus_register_dev(struct kvm_io_bus *bus,
2003		- struct kvm_io_device *dev)
2004		-{
	2006	+ bus = kvm->buses[bus_idx];
2005	2007	if (bus->dev_count > NR_IOBUS_DEVS-1)
2006	2008	return -ENOSPC;
2007	2009
2008		- bus->devs[bus->dev_count++] = dev;
	2010	+ new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	2011	+ if (!new_bus)
	2012	+ return -ENOMEM;
	2013	+ memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
	2014	+ new_bus->devs[new_bus->dev_count++] = dev;
	2015	+ rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	2016	+ synchronize_srcu_expedited(&kvm->srcu);
	2017	+ kfree(bus);
2009	2018
2010	2019	return 0;
2011	2020	}
2012	2021
2013		-void kvm_io_bus_unregister_dev(struct kvm *kvm,
2014		- struct kvm_io_bus *bus,
2015		- struct kvm_io_device *dev)
	2022	+/* Caller must have write lock on slots_lock. */
	2023	+int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
	2024	+ struct kvm_io_device *dev)
2016	2025	{
2017		- down_write(&kvm->slots_lock);
2018		- __kvm_io_bus_unregister_dev(bus, dev);
2019		- up_write(&kvm->slots_lock);
2020		-}
	2026	+ int i, r;
	2027	+ struct kvm_io_bus new_bus, bus;
2021	2028
2022		-/* An unlocked version. Caller must have write lock on slots_lock. */
2023		-void __kvm_io_bus_unregister_dev(struct kvm_io_bus *bus,
2024		- struct kvm_io_device *dev)
2025		-{
2026		- int i;
	2029	+ new_bus = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
	2030	+ if (!new_bus)
	2031	+ return -ENOMEM;
2027	2032
2028		- for (i = 0; i < bus->dev_count; i++)
2029		- if (bus->devs[i] == dev) {
2030		- bus->devs[i] = bus->devs[--bus->dev_count];
	2033	+ bus = kvm->buses[bus_idx];
	2034	+ memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
	2035	+
	2036	+ r = -ENOENT;
	2037	+ for (i = 0; i < new_bus->dev_count; i++)
	2038	+ if (new_bus->devs[i] == dev) {
	2039	+ r = 0;
	2040	+ new_bus->devs[i] = new_bus->devs[--new_bus->dev_count];
2031	2041	break;
2032	2042	}
	2043	+
	2044	+ if (r) {
	2045	+ kfree(new_bus);
	2046	+ return r;
	2047	+ }
	2048	+
	2049	+ rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
	2050	+ synchronize_srcu_expedited(&kvm->srcu);
	2051	+ kfree(bus);
	2052	+ return r;
2033	2053	}
2034	2054
2035	2055	static struct notifier_block kvm_cpu_notifier = {