Blame view
drivers/vhost/vdpa.c
25.3 KB
4c8cf3188 vhost: introduce ... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 |
// SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2018-2020 Intel Corporation. * Copyright (C) 2020 Red Hat, Inc. * * Author: Tiwei Bie <tiwei.bie@intel.com> * Jason Wang <jasowang@redhat.com> * * Thanks Michael S. Tsirkin for the valuable comments and * suggestions. And thanks to Cunming Liang and Zhihong Wang for all * their supports. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/cdev.h> #include <linux/device.h> |
ddd89d0a0 vhost_vdpa: suppo... |
18 |
#include <linux/mm.h> |
4c8cf3188 vhost: introduce ... |
19 20 21 22 23 24 25 26 |
#include <linux/iommu.h> #include <linux/uuid.h> #include <linux/vdpa.h> #include <linux/nospec.h> #include <linux/vhost.h> #include <linux/virtio_net.h> #include "vhost.h" |
653055b9a vhost-vdpa: suppo... |
27 |
enum { |
25abc060d vhost-vdpa: suppo... |
28 29 30 |
VHOST_VDPA_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) | (1ULL << VHOST_BACKEND_F_IOTLB_BATCH), |
653055b9a vhost-vdpa: suppo... |
31 |
}; |
4c8cf3188 vhost: introduce ... |
32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
#define VHOST_VDPA_DEV_MAX (1U << MINORBITS) struct vhost_vdpa { struct vhost_dev vdev; struct iommu_domain *domain; struct vhost_virtqueue *vqs; struct completion completion; struct vdpa_device *vdpa; struct device dev; struct cdev cdev; atomic_t opened; int nvqs; int virtio_id; int minor; |
776f39500 vhost_vdpa: Suppo... |
46 |
struct eventfd_ctx *config_ctx; |
25abc060d vhost-vdpa: suppo... |
47 |
int in_batch; |
1b48dc03e vhost: vdpa: repo... |
48 |
struct vdpa_iova_range range; |
4c8cf3188 vhost: introduce ... |
49 50 51 52 53 |
}; static DEFINE_IDA(vhost_vdpa_ida); static dev_t vhost_vdpa_major; |
4c8cf3188 vhost: introduce ... |
54 55 56 57 58 59 60 61 62 63 64 65 66 |
static void handle_vq_kick(struct vhost_work *work) { struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, poll.work); struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); const struct vdpa_config_ops *ops = v->vdpa->config; ops->kick_vq(v->vdpa, vq - v->vqs); } static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) { struct vhost_virtqueue *vq = private; |
265a0ad87 vhost: introduce ... |
67 |
struct eventfd_ctx *call_ctx = vq->call_ctx.ctx; |
4c8cf3188 vhost: introduce ... |
68 69 70 71 72 73 |
if (call_ctx) eventfd_signal(call_ctx, 1); return IRQ_HANDLED; } |
776f39500 vhost_vdpa: Suppo... |
74 75 76 77 78 79 80 81 82 83 |
static irqreturn_t vhost_vdpa_config_cb(void *private) { struct vhost_vdpa *v = private; struct eventfd_ctx *config_ctx = v->config_ctx; if (config_ctx) eventfd_signal(config_ctx, 1); return IRQ_HANDLED; } |
2cf1ba9a4 vhost_vdpa: imple... |
84 85 86 87 88 89 90 91 92 93 94 |
static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid) { struct vhost_virtqueue *vq = &v->vqs[qid]; const struct vdpa_config_ops *ops = v->vdpa->config; struct vdpa_device *vdpa = v->vdpa; int ret, irq; if (!ops->get_vq_irq) return; irq = ops->get_vq_irq(vdpa, qid); |
2cf1ba9a4 vhost_vdpa: imple... |
95 |
irq_bypass_unregister_producer(&vq->call_ctx.producer); |
86e182fe1 vhost_vdpa: remov... |
96 |
if (!vq->call_ctx.ctx || irq < 0) |
2cf1ba9a4 vhost_vdpa: imple... |
97 |
return; |
2cf1ba9a4 vhost_vdpa: imple... |
98 99 100 101 |
vq->call_ctx.producer.token = vq->call_ctx.ctx; vq->call_ctx.producer.irq = irq; ret = irq_bypass_register_producer(&vq->call_ctx.producer); |
e01afe36d vdpa: handle irq ... |
102 103 104 105 |
if (unlikely(ret)) dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret = %d ", qid, vq->call_ctx.producer.token, ret); |
2cf1ba9a4 vhost_vdpa: imple... |
106 107 108 109 110 |
} static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) { struct vhost_virtqueue *vq = &v->vqs[qid]; |
2cf1ba9a4 vhost_vdpa: imple... |
111 |
irq_bypass_unregister_producer(&vq->call_ctx.producer); |
2cf1ba9a4 vhost_vdpa: imple... |
112 |
} |
4c8cf3188 vhost: introduce ... |
113 114 115 |
static void vhost_vdpa_reset(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; |
4c8cf3188 vhost: introduce ... |
116 |
|
0d234007a vhost/vdpa: switc... |
117 |
vdpa_reset(vdpa); |
25abc060d vhost-vdpa: suppo... |
118 |
v->in_batch = 0; |
4c8cf3188 vhost: introduce ... |
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 |
} static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u32 device_id; device_id = ops->get_device_id(vdpa); if (copy_to_user(argp, &device_id, sizeof(device_id))) return -EFAULT; return 0; } static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u8 status; status = ops->get_status(vdpa); if (copy_to_user(statusp, &status, sizeof(status))) return -EFAULT; return 0; } static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; |
2cf1ba9a4 vhost_vdpa: imple... |
153 154 155 |
u8 status, status_old; int nvqs = v->nvqs; u16 i; |
4c8cf3188 vhost: introduce ... |
156 157 158 |
if (copy_from_user(&status, statusp, sizeof(status))) return -EFAULT; |
2cf1ba9a4 vhost_vdpa: imple... |
159 |
status_old = ops->get_status(vdpa); |
4c8cf3188 vhost: introduce ... |
160 161 162 163 164 165 166 167 |
/* * Userspace shouldn't remove status bits unless reset the * status to 0. */ if (status != 0 && (ops->get_status(vdpa) & ~status) != 0) return -EINVAL; ops->set_status(vdpa, status); |
2cf1ba9a4 vhost_vdpa: imple... |
168 169 170 171 172 173 174 |
if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) for (i = 0; i < nvqs; i++) vhost_vdpa_setup_vq_irq(v, i); if ((status_old & VIRTIO_CONFIG_S_DRIVER_OK) && !(status & VIRTIO_CONFIG_S_DRIVER_OK)) for (i = 0; i < nvqs; i++) vhost_vdpa_unsetup_vq_irq(v, i); |
4c8cf3188 vhost: introduce ... |
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
return 0; } static int vhost_vdpa_config_validate(struct vhost_vdpa *v, struct vhost_vdpa_config *c) { long size = 0; switch (v->virtio_id) { case VIRTIO_ID_NET: size = sizeof(struct virtio_net_config); break; } if (c->len == 0) return -EINVAL; if (c->len > size - c->off) return -E2BIG; return 0; } static long vhost_vdpa_get_config(struct vhost_vdpa *v, struct vhost_vdpa_config __user *c) { struct vdpa_device *vdpa = v->vdpa; |
4c8cf3188 vhost: introduce ... |
202 203 204 205 206 207 208 209 210 211 212 |
struct vhost_vdpa_config config; unsigned long size = offsetof(struct vhost_vdpa_config, buf); u8 *buf; if (copy_from_user(&config, c, size)) return -EFAULT; if (vhost_vdpa_config_validate(v, &config)) return -EINVAL; buf = kvzalloc(config.len, GFP_KERNEL); if (!buf) return -ENOMEM; |
0d234007a vhost/vdpa: switc... |
213 |
vdpa_get_config(vdpa, config.off, buf, config.len); |
4c8cf3188 vhost: introduce ... |
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 |
if (copy_to_user(c->buf, buf, config.len)) { kvfree(buf); return -EFAULT; } kvfree(buf); return 0; } static long vhost_vdpa_set_config(struct vhost_vdpa *v, struct vhost_vdpa_config __user *c) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; struct vhost_vdpa_config config; unsigned long size = offsetof(struct vhost_vdpa_config, buf); u8 *buf; if (copy_from_user(&config, c, size)) return -EFAULT; if (vhost_vdpa_config_validate(v, &config)) return -EINVAL; buf = kvzalloc(config.len, GFP_KERNEL); if (!buf) return -ENOMEM; if (copy_from_user(buf, c->buf, config.len)) { kvfree(buf); return -EFAULT; } ops->set_config(vdpa, config.off, buf, config.len); kvfree(buf); return 0; } static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u64 features; features = ops->get_features(vdpa); |
4c8cf3188 vhost: introduce ... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 |
if (copy_to_user(featurep, &features, sizeof(features))) return -EFAULT; return 0; } static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u64 features; /* * It's not allowed to change the features after they have * been negotiated. */ if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) return -EBUSY; if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; |
0d234007a vhost/vdpa: switc... |
281 |
if (vdpa_set_features(vdpa, features)) |
4c8cf3188 vhost: introduce ... |
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 |
return -EINVAL; return 0; } static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; u16 num; num = ops->get_vq_num_max(vdpa); if (copy_to_user(argp, &num, sizeof(num))) return -EFAULT; return 0; } |
776f39500 vhost_vdpa: Suppo... |
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 |
static void vhost_vdpa_config_put(struct vhost_vdpa *v) { if (v->config_ctx) eventfd_ctx_put(v->config_ctx); } static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) { struct vdpa_callback cb; int fd; struct eventfd_ctx *ctx; cb.callback = vhost_vdpa_config_cb; cb.private = v->vdpa; if (copy_from_user(&fd, argp, sizeof(fd))) return -EFAULT; ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); swap(ctx, v->config_ctx); if (!IS_ERR_OR_NULL(ctx)) eventfd_ctx_put(ctx); if (IS_ERR(v->config_ctx)) return PTR_ERR(v->config_ctx); v->vdpa->config->set_config_cb(v->vdpa, &cb); return 0; } |
2cf1ba9a4 vhost_vdpa: imple... |
330 |
|
1b48dc03e vhost: vdpa: repo... |
331 332 333 334 335 336 |
static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp) { struct vhost_vdpa_iova_range range = { .first = v->range.first, .last = v->range.last, }; |
2c602741b vhost_vdpa: retur... |
337 338 339 |
if (copy_to_user(argp, &range, sizeof(range))) return -EFAULT; return 0; |
1b48dc03e vhost: vdpa: repo... |
340 |
} |
4c8cf3188 vhost: introduce ... |
341 342 343 344 345 |
static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; |
aac50c0bd net/vdpa: Use str... |
346 |
struct vdpa_vq_state vq_state; |
4c8cf3188 vhost: introduce ... |
347 348 349 |
struct vdpa_callback cb; struct vhost_virtqueue *vq; struct vhost_vring_state s; |
4c8cf3188 vhost: introduce ... |
350 351 352 353 354 355 356 357 358 359 360 361 |
u32 idx; long r; r = get_user(idx, (u32 __user *)argp); if (r < 0) return r; if (idx >= v->nvqs) return -ENOBUFS; idx = array_index_nospec(idx, v->nvqs); vq = &v->vqs[idx]; |
b0bd82bf7 vhost-vdpa: refin... |
362 363 |
switch (cmd) { case VHOST_VDPA_SET_VRING_ENABLE: |
4c8cf3188 vhost: introduce ... |
364 365 366 367 |
if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; ops->set_vq_ready(vdpa, idx, s.num); return 0; |
b0bd82bf7 vhost-vdpa: refin... |
368 |
case VHOST_GET_VRING_BASE: |
23750e39d vdpa: Modify get_... |
369 370 371 |
r = ops->get_vq_state(v->vdpa, idx, &vq_state); if (r) return r; |
aac50c0bd net/vdpa: Use str... |
372 |
vq->last_avail_idx = vq_state.avail_index; |
b0bd82bf7 vhost-vdpa: refin... |
373 374 |
break; } |
4c8cf3188 vhost: introduce ... |
375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 |
r = vhost_vring_ioctl(&v->vdev, cmd, argp); if (r) return r; switch (cmd) { case VHOST_SET_VRING_ADDR: if (ops->set_vq_address(vdpa, idx, (u64)(uintptr_t)vq->desc, (u64)(uintptr_t)vq->avail, (u64)(uintptr_t)vq->used)) r = -EINVAL; break; case VHOST_SET_VRING_BASE: |
aac50c0bd net/vdpa: Use str... |
390 391 |
vq_state.avail_index = vq->last_avail_idx; if (ops->set_vq_state(vdpa, idx, &vq_state)) |
4c8cf3188 vhost: introduce ... |
392 393 394 395 |
r = -EINVAL; break; case VHOST_SET_VRING_CALL: |
265a0ad87 vhost: introduce ... |
396 |
if (vq->call_ctx.ctx) { |
4c8cf3188 vhost: introduce ... |
397 398 399 400 401 402 403 |
cb.callback = vhost_vdpa_virtqueue_cb; cb.private = vq; } else { cb.callback = NULL; cb.private = NULL; } ops->set_vq_cb(vdpa, idx, &cb); |
2cf1ba9a4 vhost_vdpa: imple... |
404 |
vhost_vdpa_setup_vq_irq(v, idx); |
4c8cf3188 vhost: introduce ... |
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 |
break; case VHOST_SET_VRING_NUM: ops->set_vq_num(vdpa, idx, vq->num); break; } return r; } static long vhost_vdpa_unlocked_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct vhost_vdpa *v = filep->private_data; struct vhost_dev *d = &v->vdev; void __user *argp = (void __user *)arg; |
a127c5bbb vhost-vdpa: fix b... |
421 422 |
u64 __user *featurep = argp; u64 features; |
7922460e3 vhost_vdpa: Retur... |
423 |
long r = 0; |
4c8cf3188 vhost: introduce ... |
424 |
|
a127c5bbb vhost-vdpa: fix b... |
425 |
if (cmd == VHOST_SET_BACKEND_FEATURES) { |
7922460e3 vhost_vdpa: Retur... |
426 427 |
if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; |
a127c5bbb vhost-vdpa: fix b... |
428 429 430 431 432 |
if (features & ~VHOST_VDPA_BACKEND_FEATURES) return -EOPNOTSUPP; vhost_set_backend_features(&v->vdev, features); return 0; } |
4c8cf3188 vhost: introduce ... |
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 |
mutex_lock(&d->mutex); switch (cmd) { case VHOST_VDPA_GET_DEVICE_ID: r = vhost_vdpa_get_device_id(v, argp); break; case VHOST_VDPA_GET_STATUS: r = vhost_vdpa_get_status(v, argp); break; case VHOST_VDPA_SET_STATUS: r = vhost_vdpa_set_status(v, argp); break; case VHOST_VDPA_GET_CONFIG: r = vhost_vdpa_get_config(v, argp); break; case VHOST_VDPA_SET_CONFIG: r = vhost_vdpa_set_config(v, argp); break; case VHOST_GET_FEATURES: r = vhost_vdpa_get_features(v, argp); break; case VHOST_SET_FEATURES: r = vhost_vdpa_set_features(v, argp); break; case VHOST_VDPA_GET_VRING_NUM: r = vhost_vdpa_get_vring_num(v, argp); break; case VHOST_SET_LOG_BASE: case VHOST_SET_LOG_FD: r = -ENOIOCTLCMD; break; |
776f39500 vhost_vdpa: Suppo... |
464 465 466 |
case VHOST_VDPA_SET_CONFIG_CALL: r = vhost_vdpa_set_config_call(v, argp); break; |
a127c5bbb vhost-vdpa: fix b... |
467 468 |
case VHOST_GET_BACKEND_FEATURES: features = VHOST_VDPA_BACKEND_FEATURES; |
7922460e3 vhost_vdpa: Retur... |
469 470 |
if (copy_to_user(featurep, &features, sizeof(features))) r = -EFAULT; |
a127c5bbb vhost-vdpa: fix b... |
471 |
break; |
1b48dc03e vhost: vdpa: repo... |
472 473 474 |
case VHOST_VDPA_GET_IOVA_RANGE: r = vhost_vdpa_get_iova_range(v, argp); break; |
4c8cf3188 vhost: introduce ... |
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 |
default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) r = vhost_vdpa_vring_ioctl(v, cmd, argp); break; } mutex_unlock(&d->mutex); return r; } static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last) { struct vhost_dev *dev = &v->vdev; struct vhost_iotlb *iotlb = dev->iotlb; struct vhost_iotlb_map *map; struct page *page; unsigned long pfn, pinned; while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { pinned = map->size >> PAGE_SHIFT; for (pfn = map->addr >> PAGE_SHIFT; pinned > 0; pfn++, pinned--) { page = pfn_to_page(pfn); if (map->perm & VHOST_ACCESS_WO) set_page_dirty_lock(page); unpin_user_page(page); } atomic64_sub(map->size >> PAGE_SHIFT, &dev->mm->pinned_vm); vhost_iotlb_map_free(iotlb, map); } } static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v) { struct vhost_dev *dev = &v->vdev; vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1); kfree(dev->iotlb); dev->iotlb = NULL; } static int perm_to_iommu_flags(u32 perm) { int flags = 0; switch (perm) { case VHOST_ACCESS_WO: flags |= IOMMU_WRITE; break; case VHOST_ACCESS_RO: flags |= IOMMU_READ; break; case VHOST_ACCESS_RW: flags |= (IOMMU_WRITE | IOMMU_READ); break; default: WARN(1, "invalidate vhost IOTLB permission "); break; } return flags | IOMMU_CACHE; } static int vhost_vdpa_map(struct vhost_vdpa *v, u64 iova, u64 size, u64 pa, u32 perm) { struct vhost_dev *dev = &v->vdev; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; int r = 0; r = vhost_iotlb_add_range(dev->iotlb, iova, iova + size - 1, pa, perm); if (r) return r; |
25abc060d vhost-vdpa: suppo... |
552 |
if (ops->dma_map) { |
4c8cf3188 vhost: introduce ... |
553 |
r = ops->dma_map(vdpa, iova, size, pa, perm); |
25abc060d vhost-vdpa: suppo... |
554 555 556 557 |
} else if (ops->set_map) { if (!v->in_batch) r = ops->set_map(vdpa, dev->iotlb); } else { |
4c8cf3188 vhost: introduce ... |
558 559 |
r = iommu_map(v->domain, iova, pa, size, perm_to_iommu_flags(perm)); |
25abc060d vhost-vdpa: suppo... |
560 |
} |
4c8cf3188 vhost: introduce ... |
561 |
|
1477c8aeb vhost-vdpa: fix v... |
562 563 |
if (r) vhost_iotlb_del_range(dev->iotlb, iova, iova + size - 1); |
ad89653f7 vhost-vdpa: fix p... |
564 565 |
else atomic64_add(size >> PAGE_SHIFT, &dev->mm->pinned_vm); |
1477c8aeb vhost-vdpa: fix v... |
566 |
|
4c8cf3188 vhost: introduce ... |
567 568 569 570 571 572 573 574 575 576 |
return r; } static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size) { struct vhost_dev *dev = &v->vdev; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1); |
25abc060d vhost-vdpa: suppo... |
577 |
if (ops->dma_map) { |
4c8cf3188 vhost: introduce ... |
578 |
ops->dma_unmap(vdpa, iova, size); |
25abc060d vhost-vdpa: suppo... |
579 580 581 582 |
} else if (ops->set_map) { if (!v->in_batch) ops->set_map(vdpa, dev->iotlb); } else { |
4c8cf3188 vhost: introduce ... |
583 |
iommu_unmap(v->domain, iova, size); |
25abc060d vhost-vdpa: suppo... |
584 |
} |
4c8cf3188 vhost: introduce ... |
585 586 587 588 589 590 591 592 |
} static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, struct vhost_iotlb_msg *msg) { struct vhost_dev *dev = &v->vdev; struct vhost_iotlb *iotlb = dev->iotlb; struct page **page_list; |
5e1a3149e Revert "vhost-vdp... |
593 |
unsigned long list_size = PAGE_SIZE / sizeof(struct page *); |
4c8cf3188 vhost: introduce ... |
594 |
unsigned int gup_flags = FOLL_LONGTERM; |
5e1a3149e Revert "vhost-vdp... |
595 |
unsigned long npages, cur_base, map_pfn, last_pfn = 0; |
ad89653f7 vhost-vdpa: fix p... |
596 |
unsigned long lock_limit, sz2pin, nchunks, i; |
4c8cf3188 vhost: introduce ... |
597 |
u64 iova = msg->iova; |
ad89653f7 vhost-vdpa: fix p... |
598 |
long pinned; |
4c8cf3188 vhost: introduce ... |
599 |
int ret = 0; |
1b48dc03e vhost: vdpa: repo... |
600 601 602 |
if (msg->iova < v->range.first || msg->iova + msg->size - 1 > v->range.last) return -EINVAL; |
4c8cf3188 vhost: introduce ... |
603 604 605 |
if (vhost_iotlb_itree_first(iotlb, msg->iova, msg->iova + msg->size - 1)) return -EEXIST; |
ad89653f7 vhost-vdpa: fix p... |
606 |
/* Limit the use of memory for bookkeeping */ |
5e1a3149e Revert "vhost-vdp... |
607 608 609 |
page_list = (struct page **) __get_free_page(GFP_KERNEL); if (!page_list) return -ENOMEM; |
4c8cf3188 vhost: introduce ... |
610 611 612 613 |
if (msg->perm & VHOST_ACCESS_WO) gup_flags |= FOLL_WRITE; npages = PAGE_ALIGN(msg->size + (iova & ~PAGE_MASK)) >> PAGE_SHIFT; |
ad89653f7 vhost-vdpa: fix p... |
614 615 616 617 |
if (!npages) { ret = -EINVAL; goto free; } |
4c8cf3188 vhost: introduce ... |
618 |
|
d8ed45c5d mmap locking API:... |
619 |
mmap_read_lock(dev->mm); |
4c8cf3188 vhost: introduce ... |
620 |
|
4c8cf3188 vhost: introduce ... |
621 |
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; |
ad89653f7 vhost-vdpa: fix p... |
622 |
if (npages + atomic64_read(&dev->mm->pinned_vm) > lock_limit) { |
5e1a3149e Revert "vhost-vdp... |
623 |
ret = -ENOMEM; |
ad89653f7 vhost-vdpa: fix p... |
624 |
goto unlock; |
7ed9e3d97 vhost-vdpa: fix p... |
625 |
} |
4c8cf3188 vhost: introduce ... |
626 |
|
5e1a3149e Revert "vhost-vdp... |
627 |
cur_base = msg->uaddr & PAGE_MASK; |
7ed9e3d97 vhost-vdpa: fix p... |
628 |
iova &= PAGE_MASK; |
ad89653f7 vhost-vdpa: fix p... |
629 |
nchunks = 0; |
5e1a3149e Revert "vhost-vdp... |
630 631 |
while (npages) { |
ad89653f7 vhost-vdpa: fix p... |
632 633 634 635 636 637 638 639 640 641 |
sz2pin = min_t(unsigned long, npages, list_size); pinned = pin_user_pages(cur_base, sz2pin, gup_flags, page_list, NULL); if (sz2pin != pinned) { if (pinned < 0) { ret = pinned; } else { unpin_user_pages(page_list, pinned); ret = -ENOMEM; } |
5e1a3149e Revert "vhost-vdp... |
642 |
goto out; |
ad89653f7 vhost-vdpa: fix p... |
643 644 |
} nchunks++; |
5e1a3149e Revert "vhost-vdp... |
645 646 647 |
if (!last_pfn) map_pfn = page_to_pfn(page_list[0]); |
ad89653f7 vhost-vdpa: fix p... |
648 |
for (i = 0; i < pinned; i++) { |
5e1a3149e Revert "vhost-vdp... |
649 650 651 652 653 654 |
unsigned long this_pfn = page_to_pfn(page_list[i]); u64 csize; if (last_pfn && (this_pfn != last_pfn + 1)) { /* Pin a contiguous chunk of memory */ csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT; |
ad89653f7 vhost-vdpa: fix p... |
655 656 657 658 659 660 661 662 663 664 665 666 667 668 |
ret = vhost_vdpa_map(v, iova, csize, map_pfn << PAGE_SHIFT, msg->perm); if (ret) { /* * Unpin the pages that are left unmapped * from this point on in the current * page_list. The remaining outstanding * ones which may stride across several * chunks will be covered in the common * error path subsequently. */ unpin_user_pages(&page_list[i], pinned - i); |
5e1a3149e Revert "vhost-vdp... |
669 |
goto out; |
ad89653f7 vhost-vdpa: fix p... |
670 |
} |
5e1a3149e Revert "vhost-vdp... |
671 672 |
map_pfn = this_pfn; iova += csize; |
ad89653f7 vhost-vdpa: fix p... |
673 |
nchunks = 0; |
4c8cf3188 vhost: introduce ... |
674 |
} |
5e1a3149e Revert "vhost-vdp... |
675 676 |
last_pfn = this_pfn; |
4c8cf3188 vhost: introduce ... |
677 |
} |
5e1a3149e Revert "vhost-vdp... |
678 |
|
ad89653f7 vhost-vdpa: fix p... |
679 680 |
cur_base += pinned << PAGE_SHIFT; npages -= pinned; |
4c8cf3188 vhost: introduce ... |
681 |
} |
5e1a3149e Revert "vhost-vdp... |
682 683 684 |
/* Pin the rest chunk */ ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT, map_pfn << PAGE_SHIFT, msg->perm); |
4c8cf3188 vhost: introduce ... |
685 |
out: |
5e1a3149e Revert "vhost-vdp... |
686 |
if (ret) { |
ad89653f7 vhost-vdpa: fix p... |
687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 |
if (nchunks) { unsigned long pfn; /* * Unpin the outstanding pages which are yet to be * mapped but haven't due to vdpa_map() or * pin_user_pages() failure. * * Mapped pages are accounted in vdpa_map(), hence * the corresponding unpinning will be handled by * vdpa_unmap(). */ WARN_ON(!last_pfn); for (pfn = map_pfn; pfn <= last_pfn; pfn++) unpin_user_page(pfn_to_page(pfn)); } |
4c8cf3188 vhost: introduce ... |
703 |
vhost_vdpa_unmap(v, msg->iova, msg->size); |
5e1a3149e Revert "vhost-vdp... |
704 |
} |
ad89653f7 vhost-vdpa: fix p... |
705 |
unlock: |
d8ed45c5d mmap locking API:... |
706 |
mmap_read_unlock(dev->mm); |
ad89653f7 vhost-vdpa: fix p... |
707 |
free: |
5e1a3149e Revert "vhost-vdp... |
708 |
free_page((unsigned long)page_list); |
4c8cf3188 vhost: introduce ... |
709 710 711 712 713 714 715 |
return ret; } static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, struct vhost_iotlb_msg *msg) { struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev); |
25abc060d vhost-vdpa: suppo... |
716 717 |
struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; |
4c8cf3188 vhost: introduce ... |
718 719 720 721 722 723 724 725 726 727 728 729 730 |
int r = 0; r = vhost_dev_check_owner(dev); if (r) return r; switch (msg->type) { case VHOST_IOTLB_UPDATE: r = vhost_vdpa_process_iotlb_update(v, msg); break; case VHOST_IOTLB_INVALIDATE: vhost_vdpa_unmap(v, msg->iova, msg->size); break; |
25abc060d vhost-vdpa: suppo... |
731 732 733 734 735 736 737 738 |
case VHOST_IOTLB_BATCH_BEGIN: v->in_batch = true; break; case VHOST_IOTLB_BATCH_END: if (v->in_batch && ops->set_map) ops->set_map(vdpa, dev->iotlb); v->in_batch = false; break; |
4c8cf3188 vhost: introduce ... |
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 |
default: r = -EINVAL; break; } return r; } static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct vhost_vdpa *v = file->private_data; struct vhost_dev *dev = &v->vdev; return vhost_chr_write_iter(dev, from); } static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; struct device *dma_dev = vdpa_get_dma_dev(vdpa); struct bus_type *bus; int ret; /* Device want to do DMA by itself */ if (ops->set_map || ops->dma_map) return 0; bus = dma_dev->bus; if (!bus) return -EFAULT; if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) return -ENOTSUPP; v->domain = iommu_domain_alloc(bus); if (!v->domain) return -EIO; ret = iommu_attach_device(v->domain, dma_dev); if (ret) goto err_attach; return 0; err_attach: iommu_domain_free(v->domain); return ret; } static void vhost_vdpa_free_domain(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; struct device *dma_dev = vdpa_get_dma_dev(vdpa); if (v->domain) { iommu_detach_device(v->domain, dma_dev); iommu_domain_free(v->domain); } v->domain = NULL; } |
1b48dc03e vhost: vdpa: repo... |
803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 |
static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v) { struct vdpa_iova_range *range = &v->range; struct iommu_domain_geometry geo; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; if (ops->get_iova_range) { *range = ops->get_iova_range(vdpa); } else if (v->domain && !iommu_domain_get_attr(v->domain, DOMAIN_ATTR_GEOMETRY, &geo) && geo.force_aperture) { range->first = geo.aperture_start; range->last = geo.aperture_end; } else { range->first = 0; range->last = ULLONG_MAX; } } |
4c8cf3188 vhost: introduce ... |
823 824 825 826 827 828 829 830 |
static int vhost_vdpa_open(struct inode *inode, struct file *filep) { struct vhost_vdpa *v; struct vhost_dev *dev; struct vhost_virtqueue **vqs; int nvqs, i, r, opened; v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); |
4c8cf3188 vhost: introduce ... |
831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 |
opened = atomic_cmpxchg(&v->opened, 0, 1); if (opened) return -EBUSY; nvqs = v->nvqs; vhost_vdpa_reset(v); vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL); if (!vqs) { r = -ENOMEM; goto err; } dev = &v->vdev; for (i = 0; i < nvqs; i++) { vqs[i] = &v->vqs[i]; vqs[i]->handle_kick = handle_vq_kick; } |
01fcb1cbc vhost: allow devi... |
850 |
vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false, |
4c8cf3188 vhost: introduce ... |
851 852 853 854 855 856 857 858 859 860 861 |
vhost_vdpa_process_iotlb_msg); dev->iotlb = vhost_iotlb_alloc(0, 0); if (!dev->iotlb) { r = -ENOMEM; goto err_init_iotlb; } r = vhost_vdpa_alloc_domain(v); if (r) goto err_init_iotlb; |
1b48dc03e vhost: vdpa: repo... |
862 |
vhost_vdpa_set_iova_range(v); |
4c8cf3188 vhost: introduce ... |
863 864 865 866 867 868 |
filep->private_data = v; return 0; err_init_iotlb: vhost_dev_cleanup(&v->vdev); |
37787e9f8 vhost vdpa: fix v... |
869 |
kfree(vqs); |
4c8cf3188 vhost: introduce ... |
870 871 872 873 |
err: atomic_dec(&v->opened); return r; } |
2cf1ba9a4 vhost_vdpa: imple... |
874 875 876 877 878 879 880 881 882 883 884 |
static void vhost_vdpa_clean_irq(struct vhost_vdpa *v) { struct vhost_virtqueue *vq; int i; for (i = 0; i < v->nvqs; i++) { vq = &v->vqs[i]; if (vq->call_ctx.producer.irq) irq_bypass_unregister_producer(&vq->call_ctx.producer); } } |
4c8cf3188 vhost: introduce ... |
885 886 887 888 889 890 891 892 893 894 895 |
static int vhost_vdpa_release(struct inode *inode, struct file *filep) { struct vhost_vdpa *v = filep->private_data; struct vhost_dev *d = &v->vdev; mutex_lock(&d->mutex); filep->private_data = NULL; vhost_vdpa_reset(v); vhost_dev_stop(&v->vdev); vhost_vdpa_iotlb_free(v); vhost_vdpa_free_domain(v); |
776f39500 vhost_vdpa: Suppo... |
896 |
vhost_vdpa_config_put(v); |
2cf1ba9a4 vhost_vdpa: imple... |
897 |
vhost_vdpa_clean_irq(v); |
4c8cf3188 vhost: introduce ... |
898 899 900 901 902 903 904 905 906 |
vhost_dev_cleanup(&v->vdev); kfree(v->vdev.vqs); mutex_unlock(&d->mutex); atomic_dec(&v->opened); complete(&v->completion); return 0; } |
4b4e4867d vhost_vdpa: disab... |
907 |
#ifdef CONFIG_MMU |
ddd89d0a0 vhost_vdpa: suppo... |
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 |
static vm_fault_t vhost_vdpa_fault(struct vm_fault *vmf) { struct vhost_vdpa *v = vmf->vma->vm_file->private_data; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; struct vdpa_notification_area notify; struct vm_area_struct *vma = vmf->vma; u16 index = vma->vm_pgoff; notify = ops->get_vq_notification(vdpa, index); vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); if (remap_pfn_range(vma, vmf->address & PAGE_MASK, notify.addr >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot)) return VM_FAULT_SIGBUS; return VM_FAULT_NOPAGE; } static const struct vm_operations_struct vhost_vdpa_vm_ops = { .fault = vhost_vdpa_fault, }; static int vhost_vdpa_mmap(struct file *file, struct vm_area_struct *vma) { struct vhost_vdpa *v = vma->vm_file->private_data; struct vdpa_device *vdpa = v->vdpa; const struct vdpa_config_ops *ops = vdpa->config; struct vdpa_notification_area notify; |
c09cc2c31 vhost_vdpa: Fix p... |
938 |
unsigned long index = vma->vm_pgoff; |
ddd89d0a0 vhost_vdpa: suppo... |
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 |
if (vma->vm_end - vma->vm_start != PAGE_SIZE) return -EINVAL; if ((vma->vm_flags & VM_SHARED) == 0) return -EINVAL; if (vma->vm_flags & VM_READ) return -EINVAL; if (index > 65535) return -EINVAL; if (!ops->get_vq_notification) return -ENOTSUPP; /* To be safe and easily modelled by userspace, We only * support the doorbell which sits on the page boundary and * does not share the page with other registers. */ notify = ops->get_vq_notification(vdpa, index); if (notify.addr & (PAGE_SIZE - 1)) return -EINVAL; if (vma->vm_end - vma->vm_start != notify.size) return -ENOTSUPP; vma->vm_ops = &vhost_vdpa_vm_ops; return 0; } |
4b4e4867d vhost_vdpa: disab... |
964 |
#endif /* CONFIG_MMU */ |
ddd89d0a0 vhost_vdpa: suppo... |
965 |
|
4c8cf3188 vhost: introduce ... |
966 967 968 969 970 971 |
static const struct file_operations vhost_vdpa_fops = { .owner = THIS_MODULE, .open = vhost_vdpa_open, .release = vhost_vdpa_release, .write_iter = vhost_vdpa_chr_write_iter, .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, |
4b4e4867d vhost_vdpa: disab... |
972 |
#ifdef CONFIG_MMU |
ddd89d0a0 vhost_vdpa: suppo... |
973 |
.mmap = vhost_vdpa_mmap, |
4b4e4867d vhost_vdpa: disab... |
974 |
#endif /* CONFIG_MMU */ |
4c8cf3188 vhost: introduce ... |
975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 |
.compat_ioctl = compat_ptr_ioctl, }; static void vhost_vdpa_release_dev(struct device *device) { struct vhost_vdpa *v = container_of(device, struct vhost_vdpa, dev); ida_simple_remove(&vhost_vdpa_ida, v->minor); kfree(v->vqs); kfree(v); } static int vhost_vdpa_probe(struct vdpa_device *vdpa) { const struct vdpa_config_ops *ops = vdpa->config; struct vhost_vdpa *v; |
a9974489b vdpa: remove hard... |
992 |
int minor; |
4c8cf3188 vhost: introduce ... |
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 |
int r; /* Currently, we only accept the network devices. */ if (ops->get_device_id(vdpa) != VIRTIO_ID_NET) return -ENOTSUPP; v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL); if (!v) return -ENOMEM; minor = ida_simple_get(&vhost_vdpa_ida, 0, VHOST_VDPA_DEV_MAX, GFP_KERNEL); if (minor < 0) { kfree(v); return minor; } atomic_set(&v->opened, 0); v->minor = minor; v->vdpa = vdpa; |
a9974489b vdpa: remove hard... |
1013 |
v->nvqs = vdpa->nvqs; |
4c8cf3188 vhost: introduce ... |
1014 1015 1016 1017 1018 1019 |
v->virtio_id = ops->get_device_id(vdpa); device_initialize(&v->dev); v->dev.release = vhost_vdpa_release_dev; v->dev.parent = &vdpa->dev; v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); |
a9974489b vdpa: remove hard... |
1020 |
v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue), |
4c8cf3188 vhost: introduce ... |
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 |
GFP_KERNEL); if (!v->vqs) { r = -ENOMEM; goto err; } r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); if (r) goto err; cdev_init(&v->cdev, &vhost_vdpa_fops); v->cdev.owner = THIS_MODULE; r = cdev_device_add(&v->cdev, &v->dev); if (r) goto err; init_completion(&v->completion); vdpa_set_drvdata(vdpa, v); return 0; err: put_device(&v->dev); return r; } static void vhost_vdpa_remove(struct vdpa_device *vdpa) { struct vhost_vdpa *v = vdpa_get_drvdata(vdpa); int opened; cdev_device_del(&v->cdev, &v->dev); do { opened = atomic_cmpxchg(&v->opened, 0, 1); if (!opened) break; wait_for_completion(&v->completion); } while (1); put_device(&v->dev); } static struct vdpa_driver vhost_vdpa_driver = { .driver = { .name = "vhost_vdpa", }, .probe = vhost_vdpa_probe, .remove = vhost_vdpa_remove, }; static int __init vhost_vdpa_init(void) { int r; r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX, "vhost-vdpa"); if (r) goto err_alloc_chrdev; r = vdpa_register_driver(&vhost_vdpa_driver); if (r) goto err_vdpa_register_driver; return 0; err_vdpa_register_driver: unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); err_alloc_chrdev: return r; } module_init(vhost_vdpa_init); static void __exit vhost_vdpa_exit(void) { vdpa_unregister_driver(&vhost_vdpa_driver); unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); } module_exit(vhost_vdpa_exit); MODULE_VERSION("0.0.1"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("vDPA-based vhost backend for virtio"); |