Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 0b22bd0ba0d500ab40e7673c591c594ec5447342

Authored by Michael S. Tsirkin
Committed by Rusty Russell
1 parent 964fe080d9
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

virtio-pci: fix per-vq MSI-X request logic 

Commit f68d24082e22ccee3077d11aeb6dc5354f0ca7f1
in 2.6.32-rc1 broke requesting IRQs for per-VQ MSI-X vectors:
- vector number was used instead of the vector itself
- we try to request an IRQ for VQ which does not
  have a callback handler

This is a regression that causes warnings in kernel log,
potentially lower performance as we need to scan vq list,
and might cause system failure if the interrupt
requested is in fact needed by another system.

This was not noticed earlier because in most cases
we were falling back on shared interrupt for all vqs.

The warnings often look like this:

virtio-pci 0000:00:03.0: irq 26 for MSI/MSI-X
virtio-pci 0000:00:03.0: irq 27 for MSI/MSI-X
virtio-pci 0000:00:03.0: irq 28 for MSI/MSI-X
IRQ handler type mismatch for IRQ 1
current handler: i8042
Pid: 2400, comm: modprobe Tainted: G        W
2.6.32-rc3-11952-gf3ed8d8-dirty #1
Call Trace:
 [<ffffffff81072aed>] ? __setup_irq+0x299/0x304
 [<ffffffff81072ff3>] ? request_threaded_irq+0x144/0x1c1
 [<ffffffff813455af>] ? vring_interrupt+0x0/0x30
 [<ffffffff81346598>] ? vp_try_to_find_vqs+0x583/0x5c7
 [<ffffffffa0015188>] ? skb_recv_done+0x0/0x34 [virtio_net]
 [<ffffffff81346609>] ? vp_find_vqs+0x2d/0x83
 [<ffffffff81345d00>] ? vp_get+0x3c/0x4e
 [<ffffffffa0016373>] ? virtnet_probe+0x2f1/0x428 [virtio_net]
 [<ffffffffa0015188>] ? skb_recv_done+0x0/0x34 [virtio_net]
 [<ffffffffa00150d8>] ? skb_xmit_done+0x0/0x39 [virtio_net]
 [<ffffffff8110ab92>] ? sysfs_do_create_link+0xcb/0x116
 [<ffffffff81345cc2>] ? vp_get_status+0x14/0x16
 [<ffffffff81345464>] ? virtio_dev_probe+0xa9/0xc8
 [<ffffffff8122b11c>] ? driver_probe_device+0x8d/0x128
 [<ffffffff8122b206>] ? __driver_attach+0x4f/0x6f
 [<ffffffff8122b1b7>] ? __driver_attach+0x0/0x6f
 [<ffffffff8122a9f9>] ? bus_for_each_dev+0x43/0x74
 [<ffffffff8122a374>] ? bus_add_driver+0xea/0x22d
 [<ffffffff8122b4a3>] ? driver_register+0xa7/0x111
 [<ffffffffa001a000>] ? init+0x0/0xc [virtio_net]
 [<ffffffff81009051>] ? do_one_initcall+0x50/0x148
 [<ffffffff8106e117>] ? sys_init_module+0xc5/0x21a
 [<ffffffff8100af02>] ? system_call_fastpath+0x16/0x1b
virtio-pci 0000:00:03.0: irq 26 for MSI/MSI-X
virtio-pci 0000:00:03.0: irq 27 for MSI/MSI-X

Reported-by: Marcelo Tosatti <mtosatti@redhat.com>
Reported-by: Shirley Ma <xma@us.ibm.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Showing 1 changed file with 15 additions and 12 deletions Inline Diff

drivers/virtio/virtio_pci.c
Diff comments   View file @ 0b22bd0
1 /* 1 /*
2 * Virtio PCI driver 2 * Virtio PCI driver
3 * 3 *
4 * This module allows virtio devices to be used over a virtual PCI device. 4 * This module allows virtio devices to be used over a virtual PCI device.
5 * This can be used with QEMU based VMMs like KVM or Xen. 5 * This can be used with QEMU based VMMs like KVM or Xen.
6 * 6 *
7 * Copyright IBM Corp. 2007 7 * Copyright IBM Corp. 2007
8 * 8 *
9 * Authors: 9 * Authors:
10 * Anthony Liguori <aliguori@us.ibm.com> 10 * Anthony Liguori <aliguori@us.ibm.com>
11 * 11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or later. 12 * This work is licensed under the terms of the GNU GPL, version 2 or later.
13 * See the COPYING file in the top-level directory. 13 * See the COPYING file in the top-level directory.
14 * 14 *
15 */ 15 */
16 16
17 #include <linux/module.h> 17 #include <linux/module.h>
18 #include <linux/list.h> 18 #include <linux/list.h>
19 #include <linux/pci.h> 19 #include <linux/pci.h>
20 #include <linux/interrupt.h> 20 #include <linux/interrupt.h>
21 #include <linux/virtio.h> 21 #include <linux/virtio.h>
22 #include <linux/virtio_config.h> 22 #include <linux/virtio_config.h>
23 #include <linux/virtio_ring.h> 23 #include <linux/virtio_ring.h>
24 #include <linux/virtio_pci.h> 24 #include <linux/virtio_pci.h>
25 #include <linux/highmem.h> 25 #include <linux/highmem.h>
26 #include <linux/spinlock.h> 26 #include <linux/spinlock.h>
27 27
28 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>"); 28 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
29 MODULE_DESCRIPTION("virtio-pci"); 29 MODULE_DESCRIPTION("virtio-pci");
30 MODULE_LICENSE("GPL"); 30 MODULE_LICENSE("GPL");
31 MODULE_VERSION("1"); 31 MODULE_VERSION("1");
32 32
33 /* Our device structure */ 33 /* Our device structure */
34 struct virtio_pci_device 34 struct virtio_pci_device
35 { 35 {
36 struct virtio_device vdev; 36 struct virtio_device vdev;
37 struct pci_dev *pci_dev; 37 struct pci_dev *pci_dev;
38 38
39 /* the IO mapping for the PCI config space */ 39 /* the IO mapping for the PCI config space */
40 void __iomem *ioaddr; 40 void __iomem *ioaddr;
41 41
42 /* a list of queues so we can dispatch IRQs */ 42 /* a list of queues so we can dispatch IRQs */
43 spinlock_t lock; 43 spinlock_t lock;
44 struct list_head virtqueues; 44 struct list_head virtqueues;
45 45
46 /* MSI-X support */ 46 /* MSI-X support */
47 int msix_enabled; 47 int msix_enabled;
48 int intx_enabled; 48 int intx_enabled;
49 struct msix_entry *msix_entries; 49 struct msix_entry *msix_entries;
50 /* Name strings for interrupts. This size should be enough, 50 /* Name strings for interrupts. This size should be enough,
51 * and I'm too lazy to allocate each name separately. */ 51 * and I'm too lazy to allocate each name separately. */
52 char (*msix_names)[256]; 52 char (*msix_names)[256];
53 /* Number of available vectors */ 53 /* Number of available vectors */
54 unsigned msix_vectors; 54 unsigned msix_vectors;
55 /* Vectors allocated, excluding per-vq vectors if any */ 55 /* Vectors allocated, excluding per-vq vectors if any */
56 unsigned msix_used_vectors; 56 unsigned msix_used_vectors;
57 /* Whether we have vector per vq */ 57 /* Whether we have vector per vq */
58 bool per_vq_vectors; 58 bool per_vq_vectors;
59 }; 59 };
60 60
61 /* Constants for MSI-X */ 61 /* Constants for MSI-X */
62 /* Use first vector for configuration changes, second and the rest for 62 /* Use first vector for configuration changes, second and the rest for
63 * virtqueues Thus, we need at least 2 vectors for MSI. */ 63 * virtqueues Thus, we need at least 2 vectors for MSI. */
64 enum { 64 enum {
65 VP_MSIX_CONFIG_VECTOR = 0, 65 VP_MSIX_CONFIG_VECTOR = 0,
66 VP_MSIX_VQ_VECTOR = 1, 66 VP_MSIX_VQ_VECTOR = 1,
67 }; 67 };
68 68
69 struct virtio_pci_vq_info 69 struct virtio_pci_vq_info
70 { 70 {
71 /* the actual virtqueue */ 71 /* the actual virtqueue */
72 struct virtqueue *vq; 72 struct virtqueue *vq;
73 73
74 /* the number of entries in the queue */ 74 /* the number of entries in the queue */
75 int num; 75 int num;
76 76
77 /* the index of the queue */ 77 /* the index of the queue */
78 int queue_index; 78 int queue_index;
79 79
80 /* the virtual address of the ring queue */ 80 /* the virtual address of the ring queue */
81 void *queue; 81 void *queue;
82 82
83 /* the list node for the virtqueues list */ 83 /* the list node for the virtqueues list */
84 struct list_head node; 84 struct list_head node;
85 85
86 /* MSI-X vector (or none) */ 86 /* MSI-X vector (or none) */
87 unsigned msix_vector; 87 unsigned msix_vector;
88 }; 88 };
89 89
90 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */ 90 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
91 static struct pci_device_id virtio_pci_id_table[] = { 91 static struct pci_device_id virtio_pci_id_table[] = {
92 { 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, 92 { 0x1af4, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
93 { 0 }, 93 { 0 },
94 }; 94 };
95 95
96 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table); 96 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
97 97
98 /* A PCI device has it's own struct device and so does a virtio device so 98 /* A PCI device has it's own struct device and so does a virtio device so
99 * we create a place for the virtio devices to show up in sysfs. I think it 99 * we create a place for the virtio devices to show up in sysfs. I think it
100 * would make more sense for virtio to not insist on having it's own device. */ 100 * would make more sense for virtio to not insist on having it's own device. */
101 static struct device *virtio_pci_root; 101 static struct device *virtio_pci_root;
102 102
103 /* Convert a generic virtio device to our structure */ 103 /* Convert a generic virtio device to our structure */
104 static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev) 104 static struct virtio_pci_device *to_vp_device(struct virtio_device *vdev)
105 { 105 {
106 return container_of(vdev, struct virtio_pci_device, vdev); 106 return container_of(vdev, struct virtio_pci_device, vdev);
107 } 107 }
108 108
109 /* virtio config->get_features() implementation */ 109 /* virtio config->get_features() implementation */
110 static u32 vp_get_features(struct virtio_device *vdev) 110 static u32 vp_get_features(struct virtio_device *vdev)
111 { 111 {
112 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 112 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
113 113
114 /* When someone needs more than 32 feature bits, we'll need to 114 /* When someone needs more than 32 feature bits, we'll need to
115 * steal a bit to indicate that the rest are somewhere else. */ 115 * steal a bit to indicate that the rest are somewhere else. */
116 return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES); 116 return ioread32(vp_dev->ioaddr + VIRTIO_PCI_HOST_FEATURES);
117 } 117 }
118 118
119 /* virtio config->finalize_features() implementation */ 119 /* virtio config->finalize_features() implementation */
120 static void vp_finalize_features(struct virtio_device *vdev) 120 static void vp_finalize_features(struct virtio_device *vdev)
121 { 121 {
122 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 122 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
123 123
124 /* Give virtio_ring a chance to accept features. */ 124 /* Give virtio_ring a chance to accept features. */
125 vring_transport_features(vdev); 125 vring_transport_features(vdev);
126 126
127 /* We only support 32 feature bits. */ 127 /* We only support 32 feature bits. */
128 BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1); 128 BUILD_BUG_ON(ARRAY_SIZE(vdev->features) != 1);
129 iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES); 129 iowrite32(vdev->features[0], vp_dev->ioaddr+VIRTIO_PCI_GUEST_FEATURES);
130 } 130 }
131 131
132 /* virtio config->get() implementation */ 132 /* virtio config->get() implementation */
133 static void vp_get(struct virtio_device *vdev, unsigned offset, 133 static void vp_get(struct virtio_device *vdev, unsigned offset,
134 void *buf, unsigned len) 134 void *buf, unsigned len)
135 { 135 {
136 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 136 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
137 void __iomem *ioaddr = vp_dev->ioaddr + 137 void __iomem *ioaddr = vp_dev->ioaddr +
138 VIRTIO_PCI_CONFIG(vp_dev) + offset; 138 VIRTIO_PCI_CONFIG(vp_dev) + offset;
139 u8 *ptr = buf; 139 u8 *ptr = buf;
140 int i; 140 int i;
141 141
142 for (i = 0; i < len; i++) 142 for (i = 0; i < len; i++)
143 ptr[i] = ioread8(ioaddr + i); 143 ptr[i] = ioread8(ioaddr + i);
144 } 144 }
145 145
146 /* the config->set() implementation. it's symmetric to the config->get() 146 /* the config->set() implementation. it's symmetric to the config->get()
147 * implementation */ 147 * implementation */
148 static void vp_set(struct virtio_device *vdev, unsigned offset, 148 static void vp_set(struct virtio_device *vdev, unsigned offset,
149 const void *buf, unsigned len) 149 const void *buf, unsigned len)
150 { 150 {
151 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 151 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
152 void __iomem *ioaddr = vp_dev->ioaddr + 152 void __iomem *ioaddr = vp_dev->ioaddr +
153 VIRTIO_PCI_CONFIG(vp_dev) + offset; 153 VIRTIO_PCI_CONFIG(vp_dev) + offset;
154 const u8 *ptr = buf; 154 const u8 *ptr = buf;
155 int i; 155 int i;
156 156
157 for (i = 0; i < len; i++) 157 for (i = 0; i < len; i++)
158 iowrite8(ptr[i], ioaddr + i); 158 iowrite8(ptr[i], ioaddr + i);
159 } 159 }
160 160
161 /* config->{get,set}_status() implementations */ 161 /* config->{get,set}_status() implementations */
162 static u8 vp_get_status(struct virtio_device *vdev) 162 static u8 vp_get_status(struct virtio_device *vdev)
163 { 163 {
164 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 164 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
165 return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS); 165 return ioread8(vp_dev->ioaddr + VIRTIO_PCI_STATUS);
166 } 166 }
167 167
168 static void vp_set_status(struct virtio_device *vdev, u8 status) 168 static void vp_set_status(struct virtio_device *vdev, u8 status)
169 { 169 {
170 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 170 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
171 /* We should never be setting status to 0. */ 171 /* We should never be setting status to 0. */
172 BUG_ON(status == 0); 172 BUG_ON(status == 0);
173 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 173 iowrite8(status, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
174 } 174 }
175 175
176 static void vp_reset(struct virtio_device *vdev) 176 static void vp_reset(struct virtio_device *vdev)
177 { 177 {
178 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 178 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
179 /* 0 status means a reset. */ 179 /* 0 status means a reset. */
180 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS); 180 iowrite8(0, vp_dev->ioaddr + VIRTIO_PCI_STATUS);
181 } 181 }
182 182
183 /* the notify function used when creating a virt queue */ 183 /* the notify function used when creating a virt queue */
184 static void vp_notify(struct virtqueue *vq) 184 static void vp_notify(struct virtqueue *vq)
185 { 185 {
186 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 186 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
187 struct virtio_pci_vq_info *info = vq->priv; 187 struct virtio_pci_vq_info *info = vq->priv;
188 188
189 /* we write the queue's selector into the notification register to 189 /* we write the queue's selector into the notification register to
190 * signal the other end */ 190 * signal the other end */
191 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY); 191 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NOTIFY);
192 } 192 }
193 193
194 /* Handle a configuration change: Tell driver if it wants to know. */ 194 /* Handle a configuration change: Tell driver if it wants to know. */
195 static irqreturn_t vp_config_changed(int irq, void *opaque) 195 static irqreturn_t vp_config_changed(int irq, void *opaque)
196 { 196 {
197 struct virtio_pci_device *vp_dev = opaque; 197 struct virtio_pci_device *vp_dev = opaque;
198 struct virtio_driver *drv; 198 struct virtio_driver *drv;
199 drv = container_of(vp_dev->vdev.dev.driver, 199 drv = container_of(vp_dev->vdev.dev.driver,
200 struct virtio_driver, driver); 200 struct virtio_driver, driver);
201 201
202 if (drv && drv->config_changed) 202 if (drv && drv->config_changed)
203 drv->config_changed(&vp_dev->vdev); 203 drv->config_changed(&vp_dev->vdev);
204 return IRQ_HANDLED; 204 return IRQ_HANDLED;
205 } 205 }
206 206
207 /* Notify all virtqueues on an interrupt. */ 207 /* Notify all virtqueues on an interrupt. */
208 static irqreturn_t vp_vring_interrupt(int irq, void *opaque) 208 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
209 { 209 {
210 struct virtio_pci_device *vp_dev = opaque; 210 struct virtio_pci_device *vp_dev = opaque;
211 struct virtio_pci_vq_info *info; 211 struct virtio_pci_vq_info *info;
212 irqreturn_t ret = IRQ_NONE; 212 irqreturn_t ret = IRQ_NONE;
213 unsigned long flags; 213 unsigned long flags;
214 214
215 spin_lock_irqsave(&vp_dev->lock, flags); 215 spin_lock_irqsave(&vp_dev->lock, flags);
216 list_for_each_entry(info, &vp_dev->virtqueues, node) { 216 list_for_each_entry(info, &vp_dev->virtqueues, node) {
217 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED) 217 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
218 ret = IRQ_HANDLED; 218 ret = IRQ_HANDLED;
219 } 219 }
220 spin_unlock_irqrestore(&vp_dev->lock, flags); 220 spin_unlock_irqrestore(&vp_dev->lock, flags);
221 221
222 return ret; 222 return ret;
223 } 223 }
224 224
225 /* A small wrapper to also acknowledge the interrupt when it's handled. 225 /* A small wrapper to also acknowledge the interrupt when it's handled.
226 * I really need an EIO hook for the vring so I can ack the interrupt once we 226 * I really need an EIO hook for the vring so I can ack the interrupt once we
227 * know that we'll be handling the IRQ but before we invoke the callback since 227 * know that we'll be handling the IRQ but before we invoke the callback since
228 * the callback may notify the host which results in the host attempting to 228 * the callback may notify the host which results in the host attempting to
229 * raise an interrupt that we would then mask once we acknowledged the 229 * raise an interrupt that we would then mask once we acknowledged the
230 * interrupt. */ 230 * interrupt. */
231 static irqreturn_t vp_interrupt(int irq, void *opaque) 231 static irqreturn_t vp_interrupt(int irq, void *opaque)
232 { 232 {
233 struct virtio_pci_device *vp_dev = opaque; 233 struct virtio_pci_device *vp_dev = opaque;
234 u8 isr; 234 u8 isr;
235 235
236 /* reading the ISR has the effect of also clearing it so it's very 236 /* reading the ISR has the effect of also clearing it so it's very
237 * important to save off the value. */ 237 * important to save off the value. */
238 isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR); 238 isr = ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
239 239
240 /* It's definitely not us if the ISR was not high */ 240 /* It's definitely not us if the ISR was not high */
241 if (!isr) 241 if (!isr)
242 return IRQ_NONE; 242 return IRQ_NONE;
243 243
244 /* Configuration change? Tell driver if it wants to know. */ 244 /* Configuration change? Tell driver if it wants to know. */
245 if (isr & VIRTIO_PCI_ISR_CONFIG) 245 if (isr & VIRTIO_PCI_ISR_CONFIG)
246 vp_config_changed(irq, opaque); 246 vp_config_changed(irq, opaque);
247 247
248 return vp_vring_interrupt(irq, opaque); 248 return vp_vring_interrupt(irq, opaque);
249 } 249 }
250 250
251 static void vp_free_vectors(struct virtio_device *vdev) 251 static void vp_free_vectors(struct virtio_device *vdev)
252 { 252 {
253 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 253 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
254 int i; 254 int i;
255 255
256 if (vp_dev->intx_enabled) { 256 if (vp_dev->intx_enabled) {
257 free_irq(vp_dev->pci_dev->irq, vp_dev); 257 free_irq(vp_dev->pci_dev->irq, vp_dev);
258 vp_dev->intx_enabled = 0; 258 vp_dev->intx_enabled = 0;
259 } 259 }
260 260
261 for (i = 0; i < vp_dev->msix_used_vectors; ++i) 261 for (i = 0; i < vp_dev->msix_used_vectors; ++i)
262 free_irq(vp_dev->msix_entries[i].vector, vp_dev); 262 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
263 263
264 if (vp_dev->msix_enabled) { 264 if (vp_dev->msix_enabled) {
265 /* Disable the vector used for configuration */ 265 /* Disable the vector used for configuration */
266 iowrite16(VIRTIO_MSI_NO_VECTOR, 266 iowrite16(VIRTIO_MSI_NO_VECTOR,
267 vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 267 vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
268 /* Flush the write out to device */ 268 /* Flush the write out to device */
269 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 269 ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
270 270
271 pci_disable_msix(vp_dev->pci_dev); 271 pci_disable_msix(vp_dev->pci_dev);
272 vp_dev->msix_enabled = 0; 272 vp_dev->msix_enabled = 0;
273 vp_dev->msix_vectors = 0; 273 vp_dev->msix_vectors = 0;
274 } 274 }
275 275
276 vp_dev->msix_used_vectors = 0; 276 vp_dev->msix_used_vectors = 0;
277 kfree(vp_dev->msix_names); 277 kfree(vp_dev->msix_names);
278 vp_dev->msix_names = NULL; 278 vp_dev->msix_names = NULL;
279 kfree(vp_dev->msix_entries); 279 kfree(vp_dev->msix_entries);
280 vp_dev->msix_entries = NULL; 280 vp_dev->msix_entries = NULL;
281 } 281 }
282 282
283 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, 283 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
284 bool per_vq_vectors) 284 bool per_vq_vectors)
285 { 285 {
286 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 286 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
287 const char *name = dev_name(&vp_dev->vdev.dev); 287 const char *name = dev_name(&vp_dev->vdev.dev);
288 unsigned i, v; 288 unsigned i, v;
289 int err = -ENOMEM; 289 int err = -ENOMEM;
290 290
291 vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries, 291 vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
292 GFP_KERNEL); 292 GFP_KERNEL);
293 if (!vp_dev->msix_entries) 293 if (!vp_dev->msix_entries)
294 goto error; 294 goto error;
295 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names, 295 vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
296 GFP_KERNEL); 296 GFP_KERNEL);
297 if (!vp_dev->msix_names) 297 if (!vp_dev->msix_names)
298 goto error; 298 goto error;
299 299
300 for (i = 0; i < nvectors; ++i) 300 for (i = 0; i < nvectors; ++i)
301 vp_dev->msix_entries[i].entry = i; 301 vp_dev->msix_entries[i].entry = i;
302 302
303 /* pci_enable_msix returns positive if we can't get this many. */ 303 /* pci_enable_msix returns positive if we can't get this many. */
304 err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors); 304 err = pci_enable_msix(vp_dev->pci_dev, vp_dev->msix_entries, nvectors);
305 if (err > 0) 305 if (err > 0)
306 err = -ENOSPC; 306 err = -ENOSPC;
307 if (err) 307 if (err)
308 goto error; 308 goto error;
309 vp_dev->msix_vectors = nvectors; 309 vp_dev->msix_vectors = nvectors;
310 vp_dev->msix_enabled = 1; 310 vp_dev->msix_enabled = 1;
311 311
312 /* Set the vector used for configuration */ 312 /* Set the vector used for configuration */
313 v = vp_dev->msix_used_vectors; 313 v = vp_dev->msix_used_vectors;
314 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, 314 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
315 "%s-config", name); 315 "%s-config", name);
316 err = request_irq(vp_dev->msix_entries[v].vector, 316 err = request_irq(vp_dev->msix_entries[v].vector,
317 vp_config_changed, 0, vp_dev->msix_names[v], 317 vp_config_changed, 0, vp_dev->msix_names[v],
318 vp_dev); 318 vp_dev);
319 if (err) 319 if (err)
320 goto error; 320 goto error;
321 ++vp_dev->msix_used_vectors; 321 ++vp_dev->msix_used_vectors;
322 322
323 iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 323 iowrite16(v, vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
324 /* Verify we had enough resources to assign the vector */ 324 /* Verify we had enough resources to assign the vector */
325 v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR); 325 v = ioread16(vp_dev->ioaddr + VIRTIO_MSI_CONFIG_VECTOR);
326 if (v == VIRTIO_MSI_NO_VECTOR) { 326 if (v == VIRTIO_MSI_NO_VECTOR) {
327 err = -EBUSY; 327 err = -EBUSY;
328 goto error; 328 goto error;
329 } 329 }
330 330
331 if (!per_vq_vectors) { 331 if (!per_vq_vectors) {
332 /* Shared vector for all VQs */ 332 /* Shared vector for all VQs */
333 v = vp_dev->msix_used_vectors; 333 v = vp_dev->msix_used_vectors;
334 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names, 334 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
335 "%s-virtqueues", name); 335 "%s-virtqueues", name);
336 err = request_irq(vp_dev->msix_entries[v].vector, 336 err = request_irq(vp_dev->msix_entries[v].vector,
337 vp_vring_interrupt, 0, vp_dev->msix_names[v], 337 vp_vring_interrupt, 0, vp_dev->msix_names[v],
338 vp_dev); 338 vp_dev);
339 if (err) 339 if (err)
340 goto error; 340 goto error;
341 ++vp_dev->msix_used_vectors; 341 ++vp_dev->msix_used_vectors;
342 } 342 }
343 return 0; 343 return 0;
344 error: 344 error:
345 vp_free_vectors(vdev); 345 vp_free_vectors(vdev);
346 return err; 346 return err;
347 } 347 }
348 348
349 static int vp_request_intx(struct virtio_device *vdev) 349 static int vp_request_intx(struct virtio_device *vdev)
350 { 350 {
351 int err; 351 int err;
352 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 352 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
353 353
354 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, 354 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
355 IRQF_SHARED, dev_name(&vdev->dev), vp_dev); 355 IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
356 if (!err) 356 if (!err)
357 vp_dev->intx_enabled = 1; 357 vp_dev->intx_enabled = 1;
358 return err; 358 return err;
359 } 359 }
360 360
361 static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index, 361 static struct virtqueue *setup_vq(struct virtio_device *vdev, unsigned index,
362 void (*callback)(struct virtqueue *vq), 362 void (*callback)(struct virtqueue *vq),
363 const char *name, 363 const char *name,
364 u16 msix_vec) 364 u16 msix_vec)
365 { 365 {
366 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 366 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
367 struct virtio_pci_vq_info *info; 367 struct virtio_pci_vq_info *info;
368 struct virtqueue *vq; 368 struct virtqueue *vq;
369 unsigned long flags, size; 369 unsigned long flags, size;
370 u16 num; 370 u16 num;
371 int err; 371 int err;
372 372
373 /* Select the queue we're interested in */ 373 /* Select the queue we're interested in */
374 iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 374 iowrite16(index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
375 375
376 /* Check if queue is either not available or already active. */ 376 /* Check if queue is either not available or already active. */
377 num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM); 377 num = ioread16(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_NUM);
378 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN)) 378 if (!num || ioread32(vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN))
379 return ERR_PTR(-ENOENT); 379 return ERR_PTR(-ENOENT);
380 380
381 /* allocate and fill out our structure the represents an active 381 /* allocate and fill out our structure the represents an active
382 * queue */ 382 * queue */
383 info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL); 383 info = kmalloc(sizeof(struct virtio_pci_vq_info), GFP_KERNEL);
384 if (!info) 384 if (!info)
385 return ERR_PTR(-ENOMEM); 385 return ERR_PTR(-ENOMEM);
386 386
387 info->queue_index = index; 387 info->queue_index = index;
388 info->num = num; 388 info->num = num;
389 info->msix_vector = msix_vec; 389 info->msix_vector = msix_vec;
390 390
391 size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN)); 391 size = PAGE_ALIGN(vring_size(num, VIRTIO_PCI_VRING_ALIGN));
392 info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO); 392 info->queue = alloc_pages_exact(size, GFP_KERNEL|__GFP_ZERO);
393 if (info->queue == NULL) { 393 if (info->queue == NULL) {
394 err = -ENOMEM; 394 err = -ENOMEM;
395 goto out_info; 395 goto out_info;
396 } 396 }
397 397
398 /* activate the queue */ 398 /* activate the queue */
399 iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT, 399 iowrite32(virt_to_phys(info->queue) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT,
400 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 400 vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
401 401
402 /* create the vring */ 402 /* create the vring */
403 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN, 403 vq = vring_new_virtqueue(info->num, VIRTIO_PCI_VRING_ALIGN,
404 vdev, info->queue, vp_notify, callback, name); 404 vdev, info->queue, vp_notify, callback, name);
405 if (!vq) { 405 if (!vq) {
406 err = -ENOMEM; 406 err = -ENOMEM;
407 goto out_activate_queue; 407 goto out_activate_queue;
408 } 408 }
409 409
410 vq->priv = info; 410 vq->priv = info;
411 info->vq = vq; 411 info->vq = vq;
412 412
413 if (msix_vec != VIRTIO_MSI_NO_VECTOR) { 413 if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
414 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 414 iowrite16(msix_vec, vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
415 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 415 msix_vec = ioread16(vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
416 if (msix_vec == VIRTIO_MSI_NO_VECTOR) { 416 if (msix_vec == VIRTIO_MSI_NO_VECTOR) {
417 err = -EBUSY; 417 err = -EBUSY;
418 goto out_assign; 418 goto out_assign;
419 } 419 }
420 } 420 }
421 421
422 spin_lock_irqsave(&vp_dev->lock, flags); 422 spin_lock_irqsave(&vp_dev->lock, flags);
423 list_add(&info->node, &vp_dev->virtqueues); 423 list_add(&info->node, &vp_dev->virtqueues);
424 spin_unlock_irqrestore(&vp_dev->lock, flags); 424 spin_unlock_irqrestore(&vp_dev->lock, flags);
425 425
426 return vq; 426 return vq;
427 427
428 out_assign: 428 out_assign:
429 vring_del_virtqueue(vq); 429 vring_del_virtqueue(vq);
430 out_activate_queue: 430 out_activate_queue:
431 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 431 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
432 free_pages_exact(info->queue, size); 432 free_pages_exact(info->queue, size);
433 out_info: 433 out_info:
434 kfree(info); 434 kfree(info);
435 return ERR_PTR(err); 435 return ERR_PTR(err);
436 } 436 }
437 437
438 static void vp_del_vq(struct virtqueue *vq) 438 static void vp_del_vq(struct virtqueue *vq)
439 { 439 {
440 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 440 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
441 struct virtio_pci_vq_info *info = vq->priv; 441 struct virtio_pci_vq_info *info = vq->priv;
442 unsigned long flags, size; 442 unsigned long flags, size;
443 443
444 spin_lock_irqsave(&vp_dev->lock, flags); 444 spin_lock_irqsave(&vp_dev->lock, flags);
445 list_del(&info->node); 445 list_del(&info->node);
446 spin_unlock_irqrestore(&vp_dev->lock, flags); 446 spin_unlock_irqrestore(&vp_dev->lock, flags);
447 447
448 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL); 448 iowrite16(info->queue_index, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_SEL);
449 449
450 if (vp_dev->msix_enabled) { 450 if (vp_dev->msix_enabled) {
451 iowrite16(VIRTIO_MSI_NO_VECTOR, 451 iowrite16(VIRTIO_MSI_NO_VECTOR,
452 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR); 452 vp_dev->ioaddr + VIRTIO_MSI_QUEUE_VECTOR);
453 /* Flush the write out to device */ 453 /* Flush the write out to device */
454 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR); 454 ioread8(vp_dev->ioaddr + VIRTIO_PCI_ISR);
455 } 455 }
456 456
457 vring_del_virtqueue(vq); 457 vring_del_virtqueue(vq);
458 458
459 /* Select and deactivate the queue */ 459 /* Select and deactivate the queue */
460 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN); 460 iowrite32(0, vp_dev->ioaddr + VIRTIO_PCI_QUEUE_PFN);
461 461
462 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN)); 462 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_PCI_VRING_ALIGN));
463 free_pages_exact(info->queue, size); 463 free_pages_exact(info->queue, size);
464 kfree(info); 464 kfree(info);
465 } 465 }
466 466
467 /* the config->del_vqs() implementation */ 467 /* the config->del_vqs() implementation */
468 static void vp_del_vqs(struct virtio_device *vdev) 468 static void vp_del_vqs(struct virtio_device *vdev)
469 { 469 {
470 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 470 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
471 struct virtqueue *vq, *n; 471 struct virtqueue *vq, *n;
472 struct virtio_pci_vq_info *info; 472 struct virtio_pci_vq_info *info;
473 473
474 list_for_each_entry_safe(vq, n, &vdev->vqs, list) { 474 list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
475 info = vq->priv; 475 info = vq->priv;
476 if (vp_dev->per_vq_vectors) 476 if (vp_dev->per_vq_vectors)
477 free_irq(vp_dev->msix_entries[info->msix_vector].vector, 477 free_irq(vp_dev->msix_entries[info->msix_vector].vector,
478 vq); 478 vq);
479 vp_del_vq(vq); 479 vp_del_vq(vq);
480 } 480 }
481 vp_dev->per_vq_vectors = false; 481 vp_dev->per_vq_vectors = false;
482 482
483 vp_free_vectors(vdev); 483 vp_free_vectors(vdev);
484 } 484 }
485 485
486 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs, 486 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
487 struct virtqueue *vqs[], 487 struct virtqueue *vqs[],
488 vq_callback_t *callbacks[], 488 vq_callback_t *callbacks[],
489 const char *names[], 489 const char *names[],
490 bool use_msix, 490 bool use_msix,
491 bool per_vq_vectors) 491 bool per_vq_vectors)
492 { 492 {
493 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 493 struct virtio_pci_device *vp_dev = to_vp_device(vdev);
494 u16 msix_vec; 494 u16 msix_vec;
495 int i, err, nvectors, allocated_vectors; 495 int i, err, nvectors, allocated_vectors;
496 496
497 if (!use_msix) { 497 if (!use_msix) {
498 /* Old style: one normal interrupt for change and all vqs. */ 498 /* Old style: one normal interrupt for change and all vqs. */
499 err = vp_request_intx(vdev); 499 err = vp_request_intx(vdev);
500 if (err) 500 if (err)
501 goto error_request; 501 goto error_request;
502 } else { 502 } else {
503 if (per_vq_vectors) { 503 if (per_vq_vectors) {
504 /* Best option: one for change interrupt, one per vq. */ 504 /* Best option: one for change interrupt, one per vq. */
505 nvectors = 1; 505 nvectors = 1;
506 for (i = 0; i < nvqs; ++i) 506 for (i = 0; i < nvqs; ++i)
507 if (callbacks[i]) 507 if (callbacks[i])
508 ++nvectors; 508 ++nvectors;
509 } else { 509 } else {
510 /* Second best: one for change, shared for all vqs. */ 510 /* Second best: one for change, shared for all vqs. */
511 nvectors = 2; 511 nvectors = 2;
512 } 512 }
513 513
514 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors); 514 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
515 if (err) 515 if (err)
516 goto error_request; 516 goto error_request;
517 } 517 }
518 518
519 vp_dev->per_vq_vectors = per_vq_vectors; 519 vp_dev->per_vq_vectors = per_vq_vectors;
520 allocated_vectors = vp_dev->msix_used_vectors; 520 allocated_vectors = vp_dev->msix_used_vectors;
521 for (i = 0; i < nvqs; ++i) { 521 for (i = 0; i < nvqs; ++i) {
522 if (!callbacks[i] || !vp_dev->msix_enabled) 522 if (!callbacks[i] || !vp_dev->msix_enabled)
523 msix_vec = VIRTIO_MSI_NO_VECTOR; 523 msix_vec = VIRTIO_MSI_NO_VECTOR;
524 else if (vp_dev->per_vq_vectors) 524 else if (vp_dev->per_vq_vectors)
525 msix_vec = allocated_vectors++; 525 msix_vec = allocated_vectors++;
526 else 526 else
527 msix_vec = VP_MSIX_VQ_VECTOR; 527 msix_vec = VP_MSIX_VQ_VECTOR;
528 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec); 528 vqs[i] = setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
529 if (IS_ERR(vqs[i])) { 529 if (IS_ERR(vqs[i])) {
530 err = PTR_ERR(vqs[i]); 530 err = PTR_ERR(vqs[i]);
531 goto error_find; 531 goto error_find;
532 } 532 }
533
534 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
535 continue;
536
533 /* allocate per-vq irq if available and necessary */ 537 /* allocate per-vq irq if available and necessary */
534 if (vp_dev->per_vq_vectors) { 538 snprintf(vp_dev->msix_names[msix_vec],
535 snprintf(vp_dev->msix_names[msix_vec], 539 sizeof *vp_dev->msix_names,
536 sizeof *vp_dev->msix_names, 540 "%s-%s",
537 "%s-%s", 541 dev_name(&vp_dev->vdev.dev), names[i]);
538 dev_name(&vp_dev->vdev.dev), names[i]); 542 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
539 err = request_irq(msix_vec, vring_interrupt, 0, 543 vring_interrupt, 0,
540 vp_dev->msix_names[msix_vec], 544 vp_dev->msix_names[msix_vec],
541 vqs[i]); 545 vqs[i]);
542 if (err) { 546 if (err) {
543 vp_del_vq(vqs[i]); 547 vp_del_vq(vqs[i]);
544 goto error_find; 548 goto error_find;
545 }
546 } 549 }
547 } 550 }
548 return 0; 551 return 0;
549 552
550 error_find: 553 error_find:
551 vp_del_vqs(vdev); 554 vp_del_vqs(vdev);
552 555
553 error_request: 556 error_request:
554 return err; 557 return err;
555 } 558 }
556 559
557 /* the config->find_vqs() implementation */ 560 /* the config->find_vqs() implementation */
558 static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, 561 static int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
559 struct virtqueue *vqs[], 562 struct virtqueue *vqs[],
560 vq_callback_t *callbacks[], 563 vq_callback_t *callbacks[],
561 const char *names[]) 564 const char *names[])
562 { 565 {
563 int err; 566 int err;
564 567
565 /* Try MSI-X with one vector per queue. */ 568 /* Try MSI-X with one vector per queue. */
566 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true); 569 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
567 if (!err) 570 if (!err)
568 return 0; 571 return 0;
569 /* Fallback: MSI-X with one vector for config, one shared for queues. */ 572 /* Fallback: MSI-X with one vector for config, one shared for queues. */
570 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, 573 err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
571 true, false); 574 true, false);
572 if (!err) 575 if (!err)
573 return 0; 576 return 0;
574 /* Finally fall back to regular interrupts. */ 577 /* Finally fall back to regular interrupts. */
575 return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, 578 return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
576 false, false); 579 false, false);
577 } 580 }
578 581
579 static struct virtio_config_ops virtio_pci_config_ops = { 582 static struct virtio_config_ops virtio_pci_config_ops = {
580 .get = vp_get, 583 .get = vp_get,
581 .set = vp_set, 584 .set = vp_set,
582 .get_status = vp_get_status, 585 .get_status = vp_get_status,
583 .set_status = vp_set_status, 586 .set_status = vp_set_status,
584 .reset = vp_reset, 587 .reset = vp_reset,
585 .find_vqs = vp_find_vqs, 588 .find_vqs = vp_find_vqs,
586 .del_vqs = vp_del_vqs, 589 .del_vqs = vp_del_vqs,
587 .get_features = vp_get_features, 590 .get_features = vp_get_features,
588 .finalize_features = vp_finalize_features, 591 .finalize_features = vp_finalize_features,
589 }; 592 };
590 593
591 static void virtio_pci_release_dev(struct device *_d) 594 static void virtio_pci_release_dev(struct device *_d)
592 { 595 {
593 struct virtio_device *dev = container_of(_d, struct virtio_device, dev); 596 struct virtio_device *dev = container_of(_d, struct virtio_device, dev);
594 struct virtio_pci_device *vp_dev = to_vp_device(dev); 597 struct virtio_pci_device *vp_dev = to_vp_device(dev);
595 struct pci_dev *pci_dev = vp_dev->pci_dev; 598 struct pci_dev *pci_dev = vp_dev->pci_dev;
596 599
597 vp_del_vqs(dev); 600 vp_del_vqs(dev);
598 pci_set_drvdata(pci_dev, NULL); 601 pci_set_drvdata(pci_dev, NULL);
599 pci_iounmap(pci_dev, vp_dev->ioaddr); 602 pci_iounmap(pci_dev, vp_dev->ioaddr);
600 pci_release_regions(pci_dev); 603 pci_release_regions(pci_dev);
601 pci_disable_device(pci_dev); 604 pci_disable_device(pci_dev);
602 kfree(vp_dev); 605 kfree(vp_dev);
603 } 606 }
604 607
605 /* the PCI probing function */ 608 /* the PCI probing function */
606 static int __devinit virtio_pci_probe(struct pci_dev *pci_dev, 609 static int __devinit virtio_pci_probe(struct pci_dev *pci_dev,
607 const struct pci_device_id *id) 610 const struct pci_device_id *id)
608 { 611 {
609 struct virtio_pci_device *vp_dev; 612 struct virtio_pci_device *vp_dev;
610 int err; 613 int err;
611 614
612 /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */ 615 /* We only own devices >= 0x1000 and <= 0x103f: leave the rest. */
613 if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f) 616 if (pci_dev->device < 0x1000 || pci_dev->device > 0x103f)
614 return -ENODEV; 617 return -ENODEV;
615 618
616 if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) { 619 if (pci_dev->revision != VIRTIO_PCI_ABI_VERSION) {
617 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n", 620 printk(KERN_ERR "virtio_pci: expected ABI version %d, got %d\n",
618 VIRTIO_PCI_ABI_VERSION, pci_dev->revision); 621 VIRTIO_PCI_ABI_VERSION, pci_dev->revision);
619 return -ENODEV; 622 return -ENODEV;
620 } 623 }
621 624
622 /* allocate our structure and fill it out */ 625 /* allocate our structure and fill it out */
623 vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL); 626 vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
624 if (vp_dev == NULL) 627 if (vp_dev == NULL)
625 return -ENOMEM; 628 return -ENOMEM;
626 629
627 vp_dev->vdev.dev.parent = virtio_pci_root; 630 vp_dev->vdev.dev.parent = virtio_pci_root;
628 vp_dev->vdev.dev.release = virtio_pci_release_dev; 631 vp_dev->vdev.dev.release = virtio_pci_release_dev;
629 vp_dev->vdev.config = &virtio_pci_config_ops; 632 vp_dev->vdev.config = &virtio_pci_config_ops;
630 vp_dev->pci_dev = pci_dev; 633 vp_dev->pci_dev = pci_dev;
631 INIT_LIST_HEAD(&vp_dev->virtqueues); 634 INIT_LIST_HEAD(&vp_dev->virtqueues);
632 spin_lock_init(&vp_dev->lock); 635 spin_lock_init(&vp_dev->lock);
633 636
634 /* enable the device */ 637 /* enable the device */
635 err = pci_enable_device(pci_dev); 638 err = pci_enable_device(pci_dev);
636 if (err) 639 if (err)
637 goto out; 640 goto out;
638 641
639 err = pci_request_regions(pci_dev, "virtio-pci"); 642 err = pci_request_regions(pci_dev, "virtio-pci");
640 if (err) 643 if (err)
641 goto out_enable_device; 644 goto out_enable_device;
642 645
643 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0); 646 vp_dev->ioaddr = pci_iomap(pci_dev, 0, 0);
644 if (vp_dev->ioaddr == NULL) 647 if (vp_dev->ioaddr == NULL)
645 goto out_req_regions; 648 goto out_req_regions;
646 649
647 pci_set_drvdata(pci_dev, vp_dev); 650 pci_set_drvdata(pci_dev, vp_dev);
648 651
649 /* we use the subsystem vendor/device id as the virtio vendor/device 652 /* we use the subsystem vendor/device id as the virtio vendor/device
650 * id. this allows us to use the same PCI vendor/device id for all 653 * id. this allows us to use the same PCI vendor/device id for all
651 * virtio devices and to identify the particular virtio driver by 654 * virtio devices and to identify the particular virtio driver by
652 * the subsytem ids */ 655 * the subsytem ids */
653 vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor; 656 vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
654 vp_dev->vdev.id.device = pci_dev->subsystem_device; 657 vp_dev->vdev.id.device = pci_dev->subsystem_device;
655 658
656 /* finally register the virtio device */ 659 /* finally register the virtio device */
657 err = register_virtio_device(&vp_dev->vdev); 660 err = register_virtio_device(&vp_dev->vdev);
658 if (err) 661 if (err)
659 goto out_set_drvdata; 662 goto out_set_drvdata;
660 663
661 return 0; 664 return 0;
662 665
663 out_set_drvdata: 666 out_set_drvdata:
664 pci_set_drvdata(pci_dev, NULL); 667 pci_set_drvdata(pci_dev, NULL);
665 pci_iounmap(pci_dev, vp_dev->ioaddr); 668 pci_iounmap(pci_dev, vp_dev->ioaddr);
666 out_req_regions: 669 out_req_regions:
667 pci_release_regions(pci_dev); 670 pci_release_regions(pci_dev);
668 out_enable_device: 671 out_enable_device:
669 pci_disable_device(pci_dev); 672 pci_disable_device(pci_dev);
670 out: 673 out:
671 kfree(vp_dev); 674 kfree(vp_dev);
672 return err; 675 return err;
673 } 676 }
674 677
675 static void __devexit virtio_pci_remove(struct pci_dev *pci_dev) 678 static void __devexit virtio_pci_remove(struct pci_dev *pci_dev)
676 { 679 {
677 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev); 680 struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
678 681
679 unregister_virtio_device(&vp_dev->vdev); 682 unregister_virtio_device(&vp_dev->vdev);
680 } 683 }
681 684
682 #ifdef CONFIG_PM 685 #ifdef CONFIG_PM
683 static int virtio_pci_suspend(struct pci_dev *pci_dev, pm_message_t state) 686 static int virtio_pci_suspend(struct pci_dev *pci_dev, pm_message_t state)
684 { 687 {
685 pci_save_state(pci_dev); 688 pci_save_state(pci_dev);
686 pci_set_power_state(pci_dev, PCI_D3hot); 689 pci_set_power_state(pci_dev, PCI_D3hot);
687 return 0; 690 return 0;
688 } 691 }
689 692
690 static int virtio_pci_resume(struct pci_dev *pci_dev) 693 static int virtio_pci_resume(struct pci_dev *pci_dev)
691 { 694 {
692 pci_restore_state(pci_dev); 695 pci_restore_state(pci_dev);
693 pci_set_power_state(pci_dev, PCI_D0); 696 pci_set_power_state(pci_dev, PCI_D0);
694 return 0; 697 return 0;
695 } 698 }
696 #endif 699 #endif
697 700
698 static struct pci_driver virtio_pci_driver = { 701 static struct pci_driver virtio_pci_driver = {
699 .name = "virtio-pci", 702 .name = "virtio-pci",
700 .id_table = virtio_pci_id_table, 703 .id_table = virtio_pci_id_table,
701 .probe = virtio_pci_probe, 704 .probe = virtio_pci_probe,
702 .remove = virtio_pci_remove, 705 .remove = virtio_pci_remove,
703 #ifdef CONFIG_PM 706 #ifdef CONFIG_PM
704 .suspend = virtio_pci_suspend, 707 .suspend = virtio_pci_suspend,
705 .resume = virtio_pci_resume, 708 .resume = virtio_pci_resume,
706 #endif 709 #endif
707 }; 710 };
708 711
709 static int __init virtio_pci_init(void) 712 static int __init virtio_pci_init(void)
710 { 713 {
711 int err; 714 int err;
712 715
713 virtio_pci_root = root_device_register("virtio-pci"); 716 virtio_pci_root = root_device_register("virtio-pci");
714 if (IS_ERR(virtio_pci_root)) 717 if (IS_ERR(virtio_pci_root))
715 return PTR_ERR(virtio_pci_root); 718 return PTR_ERR(virtio_pci_root);
716 719
717 err = pci_register_driver(&virtio_pci_driver); 720 err = pci_register_driver(&virtio_pci_driver);
718 if (err) 721 if (err)
719 root_device_unregister(virtio_pci_root); 722 root_device_unregister(virtio_pci_root);
720 723
721 return err; 724 return err;
722 } 725 }
723 726
724 module_init(virtio_pci_init); 727 module_init(virtio_pci_init);
725 728
726 static void __exit virtio_pci_exit(void) 729 static void __exit virtio_pci_exit(void)
727 { 730 {
728 pci_unregister_driver(&virtio_pci_driver); 731 pci_unregister_driver(&virtio_pci_driver);
729 root_device_unregister(virtio_pci_root); 732 root_device_unregister(virtio_pci_root);
730 } 733 }
731 734
732 module_exit(virtio_pci_exit); 735 module_exit(virtio_pci_exit);