// SPDX-License-Identifier: GPL-2.0-only
/*
 * Virtio over ivshmem front-end device driver
 *
 * Copyright (c) Siemens AG, 2019
 */

#include <linux/delay.h>
#include <linux/ivshmem.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/pci.h>
#include <linux/virtio.h>
#include <linux/virtio_config.h>
#include <linux/virtio_ring.h>

#define DRV_NAME "virtio-ivshmem"

#define VIRTIO_IVSHMEM_PREFERRED_ALLOC_CHUNKS	4096

#define VIRTIO_STATE_READY	cpu_to_le32(1)

struct virtio_ivshmem_header {
	__le32 revision;
	__le32 size;

	__le32 write_transaction;

	__le32 device_features;
	__le32 device_features_sel;
	__le32 driver_features;
	__le32 driver_features_sel;

	__le32 queue_sel;

	__le16 queue_size;
	__le16 queue_device_vector;
	__le16 queue_driver_vector;
	__le16 queue_enable;
	__le64 queue_desc;
	__le64 queue_driver;
	__le64 queue_device;

	__u8 config_event;
	__u8 queue_event;
	__u8 __reserved[2];
	__le32 device_status;

	__le32 config_generation;
	__u8 config[];
};

#define VI_REG_OFFSET(reg)	offsetof(struct virtio_ivshmem_header, reg)

struct virtio_ivshmem_device {
	struct virtio_device vdev;
	struct pci_dev *pci_dev;

	struct ivshm_regs __iomem *ivshm_regs;

	unsigned int num_vectors;
	bool per_vq_vector;
	char *config_irq_name;
	char *queues_irq_name;

	u32 peer_id;
	u32 *peer_state;

	void *shmem;
	resource_size_t shmem_sz;
	struct virtio_ivshmem_header *virtio_header;

	spinlock_t alloc_lock;
	unsigned long *alloc_bitmap;
	unsigned int alloc_shift;
	void **map_src_addr;

	/* a list of queues so we can dispatch IRQs */
	spinlock_t virtqueues_lock;
	struct list_head virtqueues;
};

struct virtio_ivshmem_vq_info {
	/* the actual virtqueue */
	struct virtqueue *vq;

	/* vector to use for signaling the device */
	unsigned int device_vector;
	/* vector used by the device for signaling the driver */
	unsigned int driver_vector;

	char *irq_name;

	/* the list node for the virtqueues list */
	struct list_head node;
};

static inline unsigned int get_custom_order(unsigned long size,
					    unsigned int shift)
{
	size--;
	size >>= shift;
#if BITS_PER_LONG == 32
	return fls(size);
#else
	return fls64(size);
#endif
}

static inline struct virtio_ivshmem_device *
to_virtio_ivshmem_device(struct virtio_device *vdev)
{
	return container_of(vdev, struct virtio_ivshmem_device, vdev);
}

static bool vi_synchronize_reg_write(struct virtio_ivshmem_device *vi_dev)
{
	while (READ_ONCE(vi_dev->virtio_header->write_transaction)) {
		if (READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY) {
			dev_err_ratelimited(&vi_dev->pci_dev->dev,
					    "backend failed!");
			return false;
		}
		cpu_relax();
	}
	return true;
}

static bool vi_reg_write(struct virtio_ivshmem_device *vi_dev, unsigned int reg,
			 u64 value, unsigned int size)
{
	u8 *reg_area = (u8 *)vi_dev->virtio_header;

	if (!vi_synchronize_reg_write(vi_dev))
		return false;

	if (size == 1)
		*(u8 *)(reg_area + reg) = (u8)value;
	else if (size == 2)
		*(u16 *)(reg_area + reg) = cpu_to_le16((u16)value);
	else if (size == 4)
		*(u32 *)(reg_area + reg) = cpu_to_le32((u32)value);
	else if (size == 8)
		*(u64 *)(reg_area + reg) = cpu_to_le64(value);
	else
		BUG();
	virt_wmb();

	vi_dev->virtio_header->write_transaction = cpu_to_le32(reg);
	virt_wmb();

	writel((vi_dev->peer_id << 16), &vi_dev->ivshm_regs->doorbell);

	return true;
}

static bool vi_reg_write16(struct virtio_ivshmem_device *vi_dev,
			   unsigned int reg, u32 value)
{
	return vi_reg_write(vi_dev, reg, value, 2);
}

static bool vi_reg_write32(struct virtio_ivshmem_device *vi_dev,
			   unsigned int reg, u32 value)
{
	return vi_reg_write(vi_dev, reg, value, 4);
}

static bool vi_reg_write64(struct virtio_ivshmem_device *vi_dev,
			   unsigned int reg, u64 value)
{
	return vi_reg_write(vi_dev, reg, value, 8);
}

static void vi_get(struct virtio_device *vdev, unsigned offset,
		   void *buf, unsigned len)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	__le16 w;
	__le32 l;
	__le64 q;

	switch (len) {
	case 1:
		*(u8 *)buf = *(u8 *)(vi_dev->virtio_header->config + offset);
		break;
	case 2:
		w = *(u16 *)(vi_dev->virtio_header->config + offset);
		*(u16 *)buf = le16_to_cpu(w);
		break;
	case 4:
		l = *(u32 *)(vi_dev->virtio_header->config + offset);
		*(u32 *)buf = le32_to_cpu(l);
		break;
	case 8:
		q = *(u64 *)(vi_dev->virtio_header->config + offset);
		*(u64 *)buf = le64_to_cpu(q);
		break;
	default:
		BUG();
	}
}

static void vi_set(struct virtio_device *vdev, unsigned offset,
		   const void *buf, unsigned len)
{
	u64 value;

	switch (len) {
	case 1:
		value = *(u8 *)buf;
		break;
	case 2:
		value = *(u16 *)buf;
		break;
	case 4:
		value = *(u32 *)buf;
		break;
	case 8:
		value = *(u64 *)buf;
		break;
	default:
		BUG();
	}
	vi_reg_write(to_virtio_ivshmem_device(vdev),
		     offsetof(struct virtio_ivshmem_header, config) + offset,
		     value, len);
}

static u32 vi_generation(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	u32 gen  = READ_ONCE(vi_dev->virtio_header->config_generation);

	while (gen & 1) {
		if (READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY) {
			dev_err_ratelimited(&vi_dev->pci_dev->dev,
					    "backend failed!");
			return 0;
		}
		cpu_relax();

		gen = READ_ONCE(vi_dev->virtio_header->config_generation);
	}
	return gen;
}

static u8 vi_get_status(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	return le32_to_cpu(vi_dev->virtio_header->device_status) & 0xff;
}

static void vi_set_status(struct virtio_device *vdev, u8 status)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	/* We should never be setting status to 0. */
	BUG_ON(status == 0);

	vi_reg_write32(vi_dev, VI_REG_OFFSET(device_status), status);
}

static void vi_reset(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	/* 0 status means a reset. */
	vi_reg_write32(vi_dev, VI_REG_OFFSET(device_status), 0);
}

static u64 vi_get_features(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	u64 features;

	if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(device_features_sel), 1) ||
	    !vi_synchronize_reg_write(vi_dev))
		return 0;
	features = le32_to_cpu(vi_dev->virtio_header->device_features);
	features <<= 32;

	if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(device_features_sel), 0) ||
	    !vi_synchronize_reg_write(vi_dev))
		return 0;
	features |= le32_to_cpu(vi_dev->virtio_header->device_features);

	return features;
}

static int vi_finalize_features(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	/* Give virtio_ring a chance to accept features. */
	vring_transport_features(vdev);

	if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
		dev_err(&vdev->dev,
			"virtio: device does not have VIRTIO_F_VERSION_1\n");
		return -EINVAL;
	}

	if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features_sel), 1) ||
	    !vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features),
			    (u32)(vdev->features >> 32)))
		return -ENODEV;

	if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features_sel), 0) ||
	    !vi_reg_write32(vi_dev, VI_REG_OFFSET(driver_features),
			    (u32)vdev->features))
		return -ENODEV;

	return 0;
}

/* the notify function used when creating a virt queue */
static bool vi_notify(struct virtqueue *vq)
{
	struct virtio_ivshmem_vq_info *info = vq->priv;
	struct virtio_ivshmem_device *vi_dev =
		to_virtio_ivshmem_device(vq->vdev);

	virt_wmb();
	writel((vi_dev->peer_id << 16) | info->device_vector,
	       &vi_dev->ivshm_regs->doorbell);

	return true;
}

static irqreturn_t vi_config_interrupt(int irq, void *opaque)
{
	struct virtio_ivshmem_device *vi_dev = opaque;

	if (unlikely(READ_ONCE(*vi_dev->peer_state) != VIRTIO_STATE_READY)) {
		virtio_break_device(&vi_dev->vdev);
		vi_dev->virtio_header->config_event = 0;
		vi_dev->virtio_header->queue_event = 0;
		dev_err(&vi_dev->pci_dev->dev, "backend failed!");
		return IRQ_HANDLED;
	}

	if (unlikely(READ_ONCE(vi_dev->virtio_header->config_event) & 1)) {
		vi_dev->virtio_header->config_event = 0;
		virt_wmb();
		virtio_config_changed(&vi_dev->vdev);
		return IRQ_HANDLED;
	}

	return IRQ_NONE;
}

static irqreturn_t vi_queues_interrupt(int irq, void *opaque)
{
	struct virtio_ivshmem_device *vi_dev = opaque;
	struct virtio_ivshmem_vq_info *info;
	irqreturn_t ret = IRQ_NONE;

	if (likely(READ_ONCE(vi_dev->virtio_header->queue_event) & 1)) {
		vi_dev->virtio_header->queue_event = 0;
		virt_wmb();
		spin_lock(&vi_dev->virtqueues_lock);
		list_for_each_entry(info, &vi_dev->virtqueues, node)
			ret |= vring_interrupt(irq, info->vq);
		spin_unlock(&vi_dev->virtqueues_lock);
	}

	return ret;
}

static irqreturn_t vi_interrupt(int irq, void *opaque)
{
	return vi_config_interrupt(irq, opaque) |
		vi_queues_interrupt(irq, opaque);
}

static struct virtqueue *vi_setup_vq(struct virtio_device *vdev,
				     unsigned int index,
				     void (*callback)(struct virtqueue *vq),
				     const char *name, bool ctx,
				     unsigned int irq_vector)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	struct virtio_ivshmem_vq_info *info;
	struct virtqueue *vq;
	unsigned long flags;
	unsigned int size;
	int irq, err;

	/* Select the queue we're interested in */
	if (!vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_sel), index) ||
	    !vi_synchronize_reg_write(vi_dev))
		return ERR_PTR(-ENODEV);

	/* Queue shouldn't already be set up. */
	if (vi_dev->virtio_header->queue_enable)
		return ERR_PTR(-ENOENT);

	/* Allocate and fill out our active queue description */
	info = kzalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		return ERR_PTR(-ENOMEM);

	size = vi_dev->virtio_header->queue_size;
	if (size == 0) {
		err = -ENOENT;
		goto error_new_virtqueue;
	}

	info->device_vector = vi_dev->virtio_header->queue_device_vector;
	info->driver_vector = irq_vector;

	/* Create the vring */
	vq = vring_create_virtqueue(index, size, SMP_CACHE_BYTES, vdev, true,
				    true, ctx, vi_notify, callback, name);
	if (!vq) {
		err = -ENOMEM;
		goto error_new_virtqueue;
	}

	if (callback && vi_dev->per_vq_vector) {
		irq = pci_irq_vector(vi_dev->pci_dev, info->driver_vector);
		info->irq_name = kasprintf(GFP_KERNEL, "%s-%s",
					   dev_name(&vdev->dev), name);
		if (!info->irq_name) {
			err = -ENOMEM;
			goto error_setup_virtqueue;
		}

		err = request_irq(irq, vring_interrupt, 0, info->irq_name, vq);
		if (err)
			goto error_setup_virtqueue;
	}

	/* Activate the queue */
	if (!vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_size),
			    virtqueue_get_vring_size(vq)) ||
	    !vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_driver_vector),
			    info->driver_vector) ||
	    !vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_desc),
			    virtqueue_get_desc_addr(vq)) ||
	    !vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_driver),
			    virtqueue_get_avail_addr(vq)) ||
	    !vi_reg_write64(vi_dev, VI_REG_OFFSET(queue_device),
			    virtqueue_get_used_addr(vq)) ||
	    !vi_reg_write16(vi_dev, VI_REG_OFFSET(queue_enable), 1)) {
		err = -ENODEV;
		goto error_setup_virtqueue;
	}

	vq->priv = info;
	info->vq = vq;

	spin_lock_irqsave(&vi_dev->virtqueues_lock, flags);
	list_add(&info->node, &vi_dev->virtqueues);
	spin_unlock_irqrestore(&vi_dev->virtqueues_lock, flags);

	return vq;

error_setup_virtqueue:
	vring_del_virtqueue(vq);

error_new_virtqueue:
	vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_enable), 0);
	kfree(info);
	return ERR_PTR(err);
}

static void vi_del_vq(struct virtqueue *vq)
{
	struct virtio_ivshmem_device *vi_dev =
		to_virtio_ivshmem_device(vq->vdev);
	struct virtio_ivshmem_vq_info *info = vq->priv;
	unsigned long flags;

	spin_lock_irqsave(&vi_dev->virtqueues_lock, flags);
	list_del(&info->node);
	spin_unlock_irqrestore(&vi_dev->virtqueues_lock, flags);

	/* Select and deactivate the queue */
	vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_sel), vq->index);
	vi_reg_write32(vi_dev, VI_REG_OFFSET(queue_enable), 0);

	vring_del_virtqueue(vq);

	if (info->driver_vector) {
		free_irq(pci_irq_vector(vi_dev->pci_dev, info->driver_vector),
			 vq);
		kfree(info->irq_name);
	}

	kfree(info);
}

static void vi_del_vqs(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	struct virtqueue *vq, *n;

	list_for_each_entry_safe(vq, n, &vdev->vqs, list)
		vi_del_vq(vq);

	free_irq(pci_irq_vector(vi_dev->pci_dev, 0), vi_dev);
	if (!vi_dev->per_vq_vector && vi_dev->num_vectors > 1)
		free_irq(pci_irq_vector(vi_dev->pci_dev, 1), vi_dev);
	pci_free_irq_vectors(vi_dev->pci_dev);

	kfree(vi_dev->config_irq_name);
	vi_dev->config_irq_name = NULL;
	kfree(vi_dev->queues_irq_name);
	vi_dev->queues_irq_name = NULL;
}

static int vi_find_vqs(struct virtio_device *vdev, unsigned nvqs,
		       struct virtqueue *vqs[],
		       vq_callback_t *callbacks[],
		       const char * const names[],
		       const bool *ctx,
		       struct irq_affinity *desc)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);
	unsigned int vq_vector, desired_vectors;
	int err, vectors, i, queue_idx = 0;

	desired_vectors = 1; /* one for config events */
	for (i = 0; i < nvqs; i++)
		if (callbacks[i])
			desired_vectors++;

	vectors = pci_alloc_irq_vectors(vi_dev->pci_dev, desired_vectors,
					desired_vectors, PCI_IRQ_MSIX);
	if (vectors != desired_vectors) {
		vectors = pci_alloc_irq_vectors(vi_dev->pci_dev, 1, 2,
						PCI_IRQ_LEGACY | PCI_IRQ_MSIX);
		if (vectors < 0)
			return vectors;
	}

	vi_dev->num_vectors = vectors;
	vi_dev->per_vq_vector = vectors == desired_vectors;

	if (vectors == 1) {
		vq_vector = 0;
		err = request_irq(pci_irq_vector(vi_dev->pci_dev, 0),
				  vi_interrupt, IRQF_SHARED,
				  dev_name(&vdev->dev), vi_dev);
		if (err)
			goto error_common_irq;
	} else {
		vq_vector = 1;
		vi_dev->config_irq_name = kasprintf(GFP_KERNEL, "%s-config",
						    dev_name(&vdev->dev));
		if (!vi_dev->config_irq_name) {
			err = -ENOMEM;
			goto error_common_irq;
		}

		err = request_irq(pci_irq_vector(vi_dev->pci_dev, 0),
				  vi_config_interrupt, 0,
				  vi_dev->config_irq_name, vi_dev);
		if (err)
			goto error_common_irq;
	}

	if (!vi_dev->per_vq_vector && vectors > 1) {
		vi_dev->queues_irq_name = kasprintf(GFP_KERNEL, "%s-virtqueues",
						    dev_name(&vdev->dev));
		if (!vi_dev->queues_irq_name) {
			err = -ENOMEM;
			goto error_queues_irq;
		}

		err = request_irq(pci_irq_vector(vi_dev->pci_dev, 1),
				  vi_queues_interrupt, 0,
				  vi_dev->queues_irq_name, vi_dev);
		if (err)
			goto error_queues_irq;
	}

	for (i = 0; i < nvqs; ++i) {
		if (!names[i]) {
			vqs[i] = NULL;
			continue;
		}

		vqs[i] = vi_setup_vq(vdev, queue_idx++, callbacks[i], names[i],
				     ctx ? ctx[i] : false, vq_vector);
		if (IS_ERR(vqs[i])) {
			vi_del_vqs(vdev);
			return PTR_ERR(vqs[i]);
		}

		if (vi_dev->per_vq_vector)
			vq_vector++;
	}

	writel(IVSHM_INT_ENABLE, &vi_dev->ivshm_regs->int_control);

	return 0;

error_queues_irq:
	free_irq(pci_irq_vector(vi_dev->pci_dev, 0), vi_dev);
	kfree(vi_dev->config_irq_name);
	vi_dev->config_irq_name = NULL;

error_common_irq:
	kfree(vi_dev->queues_irq_name);
	vi_dev->queues_irq_name = NULL;
	pci_free_irq_vectors(vi_dev->pci_dev);
	return err;
}

static const char *vi_bus_name(struct virtio_device *vdev)
{
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	return pci_name(vi_dev->pci_dev);
}

static const struct virtio_config_ops virtio_ivshmem_config_ops = {
	.get			= vi_get,
	.set			= vi_set,
	.generation		= vi_generation,
	.get_status		= vi_get_status,
	.set_status		= vi_set_status,
	.reset			= vi_reset,
	.find_vqs		= vi_find_vqs,
	.del_vqs		= vi_del_vqs,
	.get_features		= vi_get_features,
	.finalize_features	= vi_finalize_features,
	.bus_name		= vi_bus_name,
};

static void virtio_ivshmem_release_dev(struct device *_d)
{
	struct virtio_device *vdev = dev_to_virtio(_d);
	struct virtio_ivshmem_device *vi_dev = to_virtio_ivshmem_device(vdev);

	devm_kfree(&vi_dev->pci_dev->dev, vi_dev);
}

static u64 get_config_qword(struct pci_dev *pci_dev, unsigned int pos)
{
	u32 lo, hi;

	pci_read_config_dword(pci_dev, pos, &lo);
	pci_read_config_dword(pci_dev, pos + 4, &hi);
	return lo | ((u64)hi << 32);
}

static void *vi_dma_alloc(struct device *dev, size_t size,
			  dma_addr_t *dma_handle, gfp_t flag,
			  unsigned long attrs)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	int order = get_custom_order(size, vi_dev->alloc_shift);
	int chunk = -ENOMEM;
	unsigned long flags;
	void *addr;

	spin_lock_irqsave(&vi_dev->alloc_lock, flags);
	chunk = bitmap_find_free_region(vi_dev->alloc_bitmap,
					vi_dev->shmem_sz >> vi_dev->alloc_shift,
					order);
	spin_unlock_irqrestore(&vi_dev->alloc_lock, flags);

	if (chunk < 0) {
		if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
			dev_warn(dev,
				 "shared memory is full (size: %zd bytes)\n",
				 size);
		return NULL;
	}

	*dma_handle = chunk << vi_dev->alloc_shift;
	addr = vi_dev->shmem + *dma_handle;
	memset(addr, 0, size);

	return addr;
}

static void vi_dma_free(struct device *dev, size_t size, void *vaddr,
			dma_addr_t dma_handle, unsigned long attrs)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	int order = get_custom_order(size, vi_dev->alloc_shift);
	int chunk = (int)(dma_handle >> vi_dev->alloc_shift);
	unsigned long flags;

	spin_lock_irqsave(&vi_dev->alloc_lock, flags);
	bitmap_release_region(vi_dev->alloc_bitmap, chunk, order);
	spin_unlock_irqrestore(&vi_dev->alloc_lock, flags);
}

static dma_addr_t vi_dma_map_page(struct device *dev, struct page *page,
				  unsigned long offset, size_t size,
				  enum dma_data_direction dir,
				  unsigned long attrs)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	void *buffer, *orig_addr;
	dma_addr_t dma_addr;

	buffer = vi_dma_alloc(dev, size, &dma_addr, 0, attrs);
	if (!buffer)
		return DMA_MAPPING_ERROR;

	orig_addr = page_address(page) + offset;
	vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift] = orig_addr;

	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
	    (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
		memcpy(buffer, orig_addr, size);

	return dma_addr;
}

static void vi_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
			      size_t size, enum dma_data_direction dir,
			      unsigned long attrs)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
	void *buffer = vi_dev->shmem + dma_addr;

	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
            ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
		memcpy(orig_addr, buffer, size);

	vi_dma_free(dev, size, buffer, dma_addr, attrs);
}

static void
vi_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr,
			   size_t size, enum dma_data_direction dir)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
	void *buffer = vi_dev->shmem + dma_addr;

	memcpy(orig_addr, buffer, size);
}

static void
vi_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_addr,
			      size_t size, enum dma_data_direction dir)
{
	struct pci_dev *pci_dev = to_pci_dev(dev);
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);
	void *orig_addr = vi_dev->map_src_addr[dma_addr >> vi_dev->alloc_shift];
	void *buffer = vi_dev->shmem + dma_addr;

	memcpy(buffer, orig_addr, size);
}

static const struct dma_map_ops virtio_ivshmem_dma_ops = {
	.alloc = vi_dma_alloc,
	.free = vi_dma_free,
	.map_page = vi_dma_map_page,
	.unmap_page = vi_dma_unmap_page,
	.sync_single_for_cpu = vi_dma_sync_single_for_cpu,
	.sync_single_for_device = vi_dma_sync_single_for_device,
};

static int virtio_ivshmem_probe(struct pci_dev *pci_dev,
				const struct pci_device_id *pci_id)
{
	unsigned int chunks, chunk_size, bitmap_size;
	struct virtio_ivshmem_device *vi_dev;
	resource_size_t section_sz;
	phys_addr_t section_addr;
	unsigned int cap_pos;
	u32 *state_table;
	int vendor_cap;
	u32 id, dword;
	int ret;

	vi_dev = devm_kzalloc(&pci_dev->dev, sizeof(*vi_dev), GFP_KERNEL);
	if (!vi_dev)
		return -ENOMEM;

	pci_set_drvdata(pci_dev, vi_dev);
	vi_dev->vdev.dev.parent = &pci_dev->dev;
	vi_dev->vdev.dev.release = virtio_ivshmem_release_dev;
	vi_dev->vdev.config = &virtio_ivshmem_config_ops;
	vi_dev->vdev.id.device = pci_dev->class & IVSHM_PROTO_VIRTIO_DEVID_MASK;
	vi_dev->vdev.id.vendor = pci_dev->subsystem_vendor;
	vi_dev->pci_dev = pci_dev;

	spin_lock_init(&vi_dev->virtqueues_lock);
	INIT_LIST_HEAD(&vi_dev->virtqueues);

	ret = pcim_enable_device(pci_dev);
	if (ret)
		return ret;

	ret = pcim_iomap_regions(pci_dev, BIT(0), DRV_NAME);
	if (ret)
		return ret;

	vi_dev->ivshm_regs = pcim_iomap_table(pci_dev)[0];

	id = readl(&vi_dev->ivshm_regs->id);
	if (id > 1) {
		dev_err(&pci_dev->dev, "invalid ID %d\n", id);
		return -EINVAL;
	}
	if (readl(&vi_dev->ivshm_regs->max_peers) != 2) {
		dev_err(&pci_dev->dev, "number of peers must be 2\n");
		return -EINVAL;
	}

	vi_dev->peer_id = !id;

	vendor_cap = pci_find_capability(pci_dev, PCI_CAP_ID_VNDR);
	if (vendor_cap < 0) {
		dev_err(&pci_dev->dev, "missing vendor capability\n");
		return -EINVAL;
	}

	if (pci_resource_len(pci_dev, 2) > 0) {
		section_addr = pci_resource_start(pci_dev, 2);
	} else {
		cap_pos = vendor_cap + IVSHM_CFG_ADDRESS;
		section_addr = get_config_qword(pci_dev, cap_pos);
	}

	cap_pos = vendor_cap + IVSHM_CFG_STATE_TAB_SZ;
	pci_read_config_dword(pci_dev, cap_pos, &dword);
	section_sz = dword;

	if (!devm_request_mem_region(&pci_dev->dev, section_addr, section_sz,
				     DRV_NAME))
		return -EBUSY;

	state_table = devm_memremap(&pci_dev->dev, section_addr, section_sz,
				    MEMREMAP_WB);
	if (!state_table)
		return -ENOMEM;

	vi_dev->peer_state = &state_table[vi_dev->peer_id];

	section_addr += section_sz;

	cap_pos = vendor_cap + IVSHM_CFG_RW_SECTION_SZ;
	section_sz = get_config_qword(pci_dev, cap_pos);
	if (section_sz < 2 * PAGE_SIZE) {
		dev_err(&pci_dev->dev, "R/W section too small\n");
		return -EINVAL;
	}

	vi_dev->shmem_sz = section_sz;
	vi_dev->shmem = devm_memremap(&pci_dev->dev, section_addr, section_sz,
				      MEMREMAP_WB);
	if (!vi_dev->shmem)
		return -ENOMEM;

	vi_dev->virtio_header = vi_dev->shmem;
	if (vi_dev->virtio_header->revision < 1) {
		dev_err(&pci_dev->dev, "invalid virtio-ivshmem revision\n");
		return -EINVAL;
	}

	spin_lock_init(&vi_dev->alloc_lock);

	chunk_size = vi_dev->shmem_sz / VIRTIO_IVSHMEM_PREFERRED_ALLOC_CHUNKS;
	if (chunk_size < SMP_CACHE_BYTES)
		chunk_size = SMP_CACHE_BYTES;
	if (chunk_size > PAGE_SIZE)
		chunk_size = PAGE_SIZE;
	vi_dev->alloc_shift = get_custom_order(chunk_size, 0);

	chunks = vi_dev->shmem_sz >> vi_dev->alloc_shift;
	bitmap_size = BITS_TO_LONGS(chunks) * sizeof(long);
	vi_dev->alloc_bitmap = devm_kzalloc(&pci_dev->dev,
					    bitmap_size,
					    GFP_KERNEL);
	if (!vi_dev->alloc_bitmap)
		return -ENOMEM;

	/* mark the header chunks used */
	bitmap_set(vi_dev->alloc_bitmap, 0,
		   1 << get_custom_order(vi_dev->virtio_header->size,
					 vi_dev->alloc_shift));

	vi_dev->map_src_addr = devm_kzalloc(&pci_dev->dev,
					    chunks * sizeof(void *),
					    GFP_KERNEL);
	if (!vi_dev->map_src_addr)
		return -ENOMEM;

	set_dma_ops(&pci_dev->dev, &virtio_ivshmem_dma_ops);

	if (*vi_dev->peer_state != VIRTIO_STATE_READY) {
		dev_err(&pci_dev->dev, "backend not ready\n");
		return -ENODEV;
	}

	pci_set_master(pci_dev);
	pci_write_config_byte(pci_dev, vendor_cap + IVSHM_CFG_PRIV_CNTL, 0);

	writel(VIRTIO_STATE_READY, &vi_dev->ivshm_regs->state);

	ret = register_virtio_device(&vi_dev->vdev);
	if (ret) {
		dev_err(&pci_dev->dev, "failed to register device\n");
		writel(0, &vi_dev->ivshm_regs->state);
		put_device(&vi_dev->vdev.dev);
	}

	return ret;
}

static void virtio_ivshmem_remove(struct pci_dev *pci_dev)
{
	struct virtio_ivshmem_device *vi_dev = pci_get_drvdata(pci_dev);

	writel(0, &vi_dev->ivshm_regs->state);
	writel(0, &vi_dev->ivshm_regs->int_control);

	unregister_virtio_device(&vi_dev->vdev);
}

static const struct pci_device_id virtio_ivshmem_id_table[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_SIEMENS, PCI_DEVICE_ID_IVSHMEM),
	  (PCI_CLASS_OTHERS << 16) | IVSHM_PROTO_VIRTIO_FRONT, 0xffff00 },
	{ 0 }
};

MODULE_DEVICE_TABLE(pci, virtio_ivshmem_id_table);

static struct pci_driver virtio_ivshmem_driver = {
	.name		= DRV_NAME,
	.id_table	= virtio_ivshmem_id_table,
	.probe		= virtio_ivshmem_probe,
	.remove		= virtio_ivshmem_remove,
};

module_pci_driver(virtio_ivshmem_driver);

MODULE_AUTHOR("Jan Kiszka <jan.kiszka@siemens.com>");
MODULE_DESCRIPTION("Driver for ivshmem-based virtio front-end devices");
MODULE_LICENSE("GPL v2");