/*

   * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.

   * Author: Joerg Roedel <joerg.roedel@amd.com>
   *         Leo Duran <leo.duran@amd.com>
   *
   * This program is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 as published
   * by the Free Software Foundation.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU General Public License for more details.
   *
   * You should have received a copy of the GNU General Public License
   * along with this program; if not, write to the Free Software
   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
   */

  #include <linux/ratelimit.h>

  #include <linux/pci.h>

  #include <linux/pci-ats.h>

  #include <linux/bitmap.h>

  #include <linux/slab.h>

  #include <linux/debugfs.h>

  #include <linux/scatterlist.h>

  #include <linux/dma-mapping.h>

  #include <linux/iommu-helper.h>

  #include <linux/iommu.h>

  #include <linux/delay.h>

  #include <linux/amd-iommu.h>

  #include <linux/notifier.h>
  #include <linux/export.h>

  #include <asm/msidef.h>

  #include <asm/proto.h>

  #include <asm/iommu.h>

  #include <asm/gart.h>

  #include <asm/dma.h>

  
  #include "amd_iommu_proto.h"
  #include "amd_iommu_types.h"

  
  #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))

  #define LOOP_TIMEOUT	100000

  /*
   * This bitmap is used to advertise the page sizes our hardware support
   * to the IOMMU core, which will then use this information to split
   * physically contiguous memory regions it is mapping into page sizes
   * that we support.
   *
   * Traditionally the IOMMU core just handed us the mappings directly,
   * after making sure the size is an order of a 4KiB page and that the
   * mapping has natural alignment.
   *
   * To retain this behavior, we currently advertise that we support
   * all page sizes that are an order of 4KiB.
   *
   * If at some point we'd like to utilize the IOMMU core's new behavior,
   * we could change this to advertise the real page sizes we support.
   */
  #define AMD_IOMMU_PGSIZES	(~0xFFFUL)

  static DEFINE_RWLOCK(amd_iommu_devtable_lock);

  /* A list of preallocated protection domains */
  static LIST_HEAD(iommu_pd_list);
  static DEFINE_SPINLOCK(iommu_pd_list_lock);

  /* List of all available dev_data structures */
  static LIST_HEAD(dev_data_list);
  static DEFINE_SPINLOCK(dev_data_list_lock);

  /*
   * Domain for untranslated devices - only allocated
   * if iommu=pt passed on kernel cmd line.
   */
  static struct protection_domain *pt_domain;

  static struct iommu_ops amd_iommu_ops;

  static ATOMIC_NOTIFIER_HEAD(ppr_notifier);

  int amd_iommu_max_glx_val = -1;

  /*
   * general struct to manage commands send to an IOMMU
   */

  struct iommu_cmd {

  	u32 data[4];
  };

  static void update_domain(struct protection_domain *domain);

  static int __init alloc_passthrough_domain(void);

  /****************************************************************************
   *
   * Helper functions
   *
   ****************************************************************************/

  static struct iommu_dev_data *alloc_dev_data(u16 devid)

  {
  	struct iommu_dev_data *dev_data;
  	unsigned long flags;
  
  	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
  	if (!dev_data)
  		return NULL;

  	dev_data->devid = devid;

  	atomic_set(&dev_data->bind, 0);
  
  	spin_lock_irqsave(&dev_data_list_lock, flags);
  	list_add_tail(&dev_data->dev_data_list, &dev_data_list);
  	spin_unlock_irqrestore(&dev_data_list_lock, flags);
  
  	return dev_data;
  }
  
  static void free_dev_data(struct iommu_dev_data *dev_data)
  {
  	unsigned long flags;
  
  	spin_lock_irqsave(&dev_data_list_lock, flags);
  	list_del(&dev_data->dev_data_list);
  	spin_unlock_irqrestore(&dev_data_list_lock, flags);
  
  	kfree(dev_data);
  }

  static struct iommu_dev_data *search_dev_data(u16 devid)
  {
  	struct iommu_dev_data *dev_data;
  	unsigned long flags;
  
  	spin_lock_irqsave(&dev_data_list_lock, flags);
  	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
  		if (dev_data->devid == devid)
  			goto out_unlock;
  	}
  
  	dev_data = NULL;
  
  out_unlock:
  	spin_unlock_irqrestore(&dev_data_list_lock, flags);
  
  	return dev_data;
  }
  
  static struct iommu_dev_data *find_dev_data(u16 devid)
  {
  	struct iommu_dev_data *dev_data;
  
  	dev_data = search_dev_data(devid);
  
  	if (dev_data == NULL)
  		dev_data = alloc_dev_data(devid);
  
  	return dev_data;
  }

  static inline u16 get_device_id(struct device *dev)
  {
  	struct pci_dev *pdev = to_pci_dev(dev);
  
  	return calc_devid(pdev->bus->number, pdev->devfn);
  }

  static struct iommu_dev_data *get_dev_data(struct device *dev)
  {
  	return dev->archdata.iommu;
  }

  static bool pci_iommuv2_capable(struct pci_dev *pdev)
  {
  	static const int caps[] = {
  		PCI_EXT_CAP_ID_ATS,

  		PCI_EXT_CAP_ID_PRI,
  		PCI_EXT_CAP_ID_PASID,

  	};
  	int i, pos;
  
  	for (i = 0; i < 3; ++i) {
  		pos = pci_find_ext_capability(pdev, caps[i]);
  		if (pos == 0)
  			return false;
  	}
  
  	return true;
  }

  static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
  {
  	struct iommu_dev_data *dev_data;
  
  	dev_data = get_dev_data(&pdev->dev);
  
  	return dev_data->errata & (1 << erratum) ? true : false;
  }

  /*
   * In this function the list of preallocated protection domains is traversed to
   * find the domain for a specific device
   */
  static struct dma_ops_domain *find_protection_domain(u16 devid)
  {
  	struct dma_ops_domain *entry, *ret = NULL;
  	unsigned long flags;
  	u16 alias = amd_iommu_alias_table[devid];
  
  	if (list_empty(&iommu_pd_list))
  		return NULL;
  
  	spin_lock_irqsave(&iommu_pd_list_lock, flags);
  
  	list_for_each_entry(entry, &iommu_pd_list, list) {
  		if (entry->target_dev == devid ||
  		    entry->target_dev == alias) {
  			ret = entry;
  			break;
  		}
  	}
  
  	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
  
  	return ret;
  }

  /*
   * This function checks if the driver got a valid device from the caller to
   * avoid dereferencing invalid pointers.
   */
  static bool check_device(struct device *dev)
  {
  	u16 devid;
  
  	if (!dev || !dev->dma_mask)
  		return false;
  
  	/* No device or no PCI device */

  	if (dev->bus != &pci_bus_type)

  		return false;
  
  	devid = get_device_id(dev);
  
  	/* Out of our scope? */
  	if (devid > amd_iommu_last_bdf)
  		return false;
  
  	if (amd_iommu_rlookup_table[devid] == NULL)
  		return false;
  
  	return true;
  }

  static int iommu_init_device(struct device *dev)
  {

  	struct pci_dev *pdev = to_pci_dev(dev);

  	struct iommu_dev_data *dev_data;

  	u16 alias;

  
  	if (dev->archdata.iommu)
  		return 0;

  	dev_data = find_dev_data(get_device_id(dev));

  	if (!dev_data)
  		return -ENOMEM;

  	alias = amd_iommu_alias_table[dev_data->devid];

  	if (alias != dev_data->devid) {

  		struct iommu_dev_data *alias_data;

  		alias_data = find_dev_data(alias);
  		if (alias_data == NULL) {
  			pr_err("AMD-Vi: Warning: Unhandled device %s
  ",
  					dev_name(dev));

  			free_dev_data(dev_data);
  			return -ENOTSUPP;
  		}

  		dev_data->alias_data = alias_data;

  	}

  	if (pci_iommuv2_capable(pdev)) {
  		struct amd_iommu *iommu;
  
  		iommu              = amd_iommu_rlookup_table[dev_data->devid];
  		dev_data->iommu_v2 = iommu->is_iommu_v2;
  	}

  	dev->archdata.iommu = dev_data;

  	return 0;
  }

  static void iommu_ignore_device(struct device *dev)
  {
  	u16 devid, alias;
  
  	devid = get_device_id(dev);
  	alias = amd_iommu_alias_table[devid];
  
  	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
  	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
  
  	amd_iommu_rlookup_table[devid] = NULL;
  	amd_iommu_rlookup_table[alias] = NULL;
  }

  static void iommu_uninit_device(struct device *dev)
  {

  	/*
  	 * Nothing to do here - we keep dev_data around for unplugged devices
  	 * and reuse it when the device is re-plugged - not doing so would
  	 * introduce a ton of races.
  	 */

  }

  
  void __init amd_iommu_uninit_devices(void)
  {

  	struct iommu_dev_data *dev_data, *n;

  	struct pci_dev *pdev = NULL;
  
  	for_each_pci_dev(pdev) {
  
  		if (!check_device(&pdev->dev))
  			continue;
  
  		iommu_uninit_device(&pdev->dev);
  	}

  
  	/* Free all of our dev_data structures */
  	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
  		free_dev_data(dev_data);

  }
  
  int __init amd_iommu_init_devices(void)
  {
  	struct pci_dev *pdev = NULL;
  	int ret = 0;
  
  	for_each_pci_dev(pdev) {
  
  		if (!check_device(&pdev->dev))
  			continue;
  
  		ret = iommu_init_device(&pdev->dev);

  		if (ret == -ENOTSUPP)
  			iommu_ignore_device(&pdev->dev);
  		else if (ret)

  			goto out_free;
  	}
  
  	return 0;
  
  out_free:
  
  	amd_iommu_uninit_devices();
  
  	return ret;
  }

  #ifdef CONFIG_AMD_IOMMU_STATS
  
  /*
   * Initialization code for statistics collection
   */

  DECLARE_STATS_COUNTER(compl_wait);

  DECLARE_STATS_COUNTER(cnt_map_single);

  DECLARE_STATS_COUNTER(cnt_unmap_single);

  DECLARE_STATS_COUNTER(cnt_map_sg);

  DECLARE_STATS_COUNTER(cnt_unmap_sg);

  DECLARE_STATS_COUNTER(cnt_alloc_coherent);

  DECLARE_STATS_COUNTER(cnt_free_coherent);

  DECLARE_STATS_COUNTER(cross_page);

  DECLARE_STATS_COUNTER(domain_flush_single);

  DECLARE_STATS_COUNTER(domain_flush_all);

  DECLARE_STATS_COUNTER(alloced_io_mem);

  DECLARE_STATS_COUNTER(total_map_requests);

  DECLARE_STATS_COUNTER(complete_ppr);
  DECLARE_STATS_COUNTER(invalidate_iotlb);
  DECLARE_STATS_COUNTER(invalidate_iotlb_all);
  DECLARE_STATS_COUNTER(pri_requests);

  static struct dentry *stats_dir;

  static struct dentry *de_fflush;
  
  static void amd_iommu_stats_add(struct __iommu_counter *cnt)
  {
  	if (stats_dir == NULL)
  		return;
  
  	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
  				       &cnt->value);
  }
  
  static void amd_iommu_stats_init(void)
  {
  	stats_dir = debugfs_create_dir("amd-iommu", NULL);
  	if (stats_dir == NULL)
  		return;

  	de_fflush  = debugfs_create_bool("fullflush", 0444, stats_dir,
  					 (u32 *)&amd_iommu_unmap_flush);

  
  	amd_iommu_stats_add(&compl_wait);

  	amd_iommu_stats_add(&cnt_map_single);

  	amd_iommu_stats_add(&cnt_unmap_single);

  	amd_iommu_stats_add(&cnt_map_sg);

  	amd_iommu_stats_add(&cnt_unmap_sg);

  	amd_iommu_stats_add(&cnt_alloc_coherent);

  	amd_iommu_stats_add(&cnt_free_coherent);

  	amd_iommu_stats_add(&cross_page);

  	amd_iommu_stats_add(&domain_flush_single);

  	amd_iommu_stats_add(&domain_flush_all);

  	amd_iommu_stats_add(&alloced_io_mem);

  	amd_iommu_stats_add(&total_map_requests);

  	amd_iommu_stats_add(&complete_ppr);
  	amd_iommu_stats_add(&invalidate_iotlb);
  	amd_iommu_stats_add(&invalidate_iotlb_all);
  	amd_iommu_stats_add(&pri_requests);

  }
  
  #endif

  /****************************************************************************
   *

   * Interrupt handling functions
   *
   ****************************************************************************/

  static void dump_dte_entry(u16 devid)
  {
  	int i;

  	for (i = 0; i < 4; ++i)
  		pr_err("AMD-Vi: DTE[%d]: %016llx
  ", i,

  			amd_iommu_dev_table[devid].data[i]);
  }

  static void dump_command(unsigned long phys_addr)
  {
  	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
  	int i;
  
  	for (i = 0; i < 4; ++i)
  		pr_err("AMD-Vi: CMD[%d]: %08x
  ", i, cmd->data[i]);
  }

  static void iommu_print_event(struct amd_iommu *iommu, void *__evt)

  {
  	u32 *event = __evt;
  	int type  = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
  	int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
  	int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
  	int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
  	u64 address = (u64)(((u64)event[3]) << 32) | event[2];

  	printk(KERN_ERR "AMD-Vi: Event logged [");

  
  	switch (type) {
  	case EVENT_TYPE_ILL_DEV:
  		printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
  		       "address=0x%016llx flags=0x%04x]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       address, flags);

  		dump_dte_entry(devid);

  		break;
  	case EVENT_TYPE_IO_FAULT:
  		printk("IO_PAGE_FAULT device=%02x:%02x.%x "
  		       "domain=0x%04x address=0x%016llx flags=0x%04x]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       domid, address, flags);
  		break;
  	case EVENT_TYPE_DEV_TAB_ERR:
  		printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  		       "address=0x%016llx flags=0x%04x]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       address, flags);
  		break;
  	case EVENT_TYPE_PAGE_TAB_ERR:
  		printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
  		       "domain=0x%04x address=0x%016llx flags=0x%04x]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       domid, address, flags);
  		break;
  	case EVENT_TYPE_ILL_CMD:
  		printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]
  ", address);

  		dump_command(address);

  		break;
  	case EVENT_TYPE_CMD_HARD_ERR:
  		printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
  		       "flags=0x%04x]
  ", address, flags);
  		break;
  	case EVENT_TYPE_IOTLB_INV_TO:
  		printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
  		       "address=0x%016llx]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       address);
  		break;
  	case EVENT_TYPE_INV_DEV_REQ:
  		printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
  		       "address=0x%016llx flags=0x%04x]
  ",
  		       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
  		       address, flags);
  		break;
  	default:
  		printk(KERN_ERR "UNKNOWN type=0x%02x]
  ", type);
  	}
  }
  
  static void iommu_poll_events(struct amd_iommu *iommu)
  {
  	u32 head, tail;
  	unsigned long flags;
  
  	spin_lock_irqsave(&iommu->lock, flags);
  
  	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
  
  	while (head != tail) {

  		iommu_print_event(iommu, iommu->evt_buf + head);

  		head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
  	}
  
  	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
  
  	spin_unlock_irqrestore(&iommu->lock, flags);
  }

  static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u32 head)
  {
  	struct amd_iommu_fault fault;
  	volatile u64 *raw;
  	int i;

  	INC_STATS_COUNTER(pri_requests);

  	raw = (u64 *)(iommu->ppr_log + head);
  
  	/*
  	 * Hardware bug: Interrupt may arrive before the entry is written to
  	 * memory. If this happens we need to wait for the entry to arrive.
  	 */
  	for (i = 0; i < LOOP_TIMEOUT; ++i) {
  		if (PPR_REQ_TYPE(raw[0]) != 0)
  			break;
  		udelay(1);
  	}
  
  	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
  		pr_err_ratelimited("AMD-Vi: Unknown PPR request received
  ");
  		return;
  	}
  
  	fault.address   = raw[1];
  	fault.pasid     = PPR_PASID(raw[0]);
  	fault.device_id = PPR_DEVID(raw[0]);
  	fault.tag       = PPR_TAG(raw[0]);
  	fault.flags     = PPR_FLAGS(raw[0]);
  
  	/*
  	 * To detect the hardware bug we need to clear the entry
  	 * to back to zero.
  	 */
  	raw[0] = raw[1] = 0;
  
  	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
  }
  
  static void iommu_poll_ppr_log(struct amd_iommu *iommu)
  {
  	unsigned long flags;
  	u32 head, tail;
  
  	if (iommu->ppr_log == NULL)
  		return;
  
  	spin_lock_irqsave(&iommu->lock, flags);
  
  	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
  
  	while (head != tail) {
  
  		/* Handle PPR entry */
  		iommu_handle_ppr_entry(iommu, head);
  
  		/* Update and refresh ring-buffer state*/
  		head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
  		writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
  		tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
  	}
  
  	/* enable ppr interrupts again */
  	writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
  
  	spin_unlock_irqrestore(&iommu->lock, flags);
  }

  irqreturn_t amd_iommu_int_thread(int irq, void *data)

  {

  	struct amd_iommu *iommu;

  	for_each_iommu(iommu) {

  		iommu_poll_events(iommu);

  		iommu_poll_ppr_log(iommu);
  	}

  
  	return IRQ_HANDLED;

  }

  irqreturn_t amd_iommu_int_handler(int irq, void *data)
  {
  	return IRQ_WAKE_THREAD;
  }

  /****************************************************************************
   *

   * IOMMU command queuing functions
   *
   ****************************************************************************/

  static int wait_on_sem(volatile u64 *sem)
  {
  	int i = 0;
  
  	while (*sem == 0 && i < LOOP_TIMEOUT) {
  		udelay(1);
  		i += 1;
  	}
  
  	if (i == LOOP_TIMEOUT) {
  		pr_alert("AMD-Vi: Completion-Wait loop timed out
  ");
  		return -EIO;
  	}
  
  	return 0;
  }
  
  static void copy_cmd_to_buffer(struct amd_iommu *iommu,
  			       struct iommu_cmd *cmd,
  			       u32 tail)

  {

  	u8 *target;

  	target = iommu->cmd_buf + tail;

  	tail   = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  
  	/* Copy command to buffer */
  	memcpy(target, cmd, sizeof(*cmd));
  
  	/* Tell the IOMMU about it */

  	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

  }

  static void build_completion_wait(struct iommu_cmd *cmd, u64 address)

  {

  	WARN_ON(address & 0x7ULL);

  	memset(cmd, 0, sizeof(*cmd));

  	cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
  	cmd->data[1] = upper_32_bits(__pa(address));
  	cmd->data[2] = 1;

  	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
  }

  static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
  {
  	memset(cmd, 0, sizeof(*cmd));
  	cmd->data[0] = devid;
  	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
  }

  static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
  				  size_t size, u16 domid, int pde)
  {
  	u64 pages;
  	int s;
  
  	pages = iommu_num_pages(address, size, PAGE_SIZE);
  	s     = 0;
  
  	if (pages > 1) {
  		/*
  		 * If we have to flush more than one page, flush all
  		 * TLB entries for this domain
  		 */
  		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  		s = 1;
  	}
  
  	address &= PAGE_MASK;
  
  	memset(cmd, 0, sizeof(*cmd));
  	cmd->data[1] |= domid;
  	cmd->data[2]  = lower_32_bits(address);
  	cmd->data[3]  = upper_32_bits(address);
  	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  	if (s) /* size bit - we flush more than one 4kb page */
  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  	if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  }

  static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
  				  u64 address, size_t size)
  {
  	u64 pages;
  	int s;
  
  	pages = iommu_num_pages(address, size, PAGE_SIZE);
  	s     = 0;
  
  	if (pages > 1) {
  		/*
  		 * If we have to flush more than one page, flush all
  		 * TLB entries for this domain
  		 */
  		address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
  		s = 1;
  	}
  
  	address &= PAGE_MASK;
  
  	memset(cmd, 0, sizeof(*cmd));
  	cmd->data[0]  = devid;
  	cmd->data[0] |= (qdep & 0xff) << 24;
  	cmd->data[1]  = devid;
  	cmd->data[2]  = lower_32_bits(address);
  	cmd->data[3]  = upper_32_bits(address);
  	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
  	if (s)
  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  }

  static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
  				  u64 address, bool size)
  {
  	memset(cmd, 0, sizeof(*cmd));
  
  	address &= ~(0xfffULL);
  
  	cmd->data[0]  = pasid & PASID_MASK;
  	cmd->data[1]  = domid;
  	cmd->data[2]  = lower_32_bits(address);
  	cmd->data[3]  = upper_32_bits(address);
  	cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
  	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
  	if (size)
  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
  }
  
  static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
  				  int qdep, u64 address, bool size)
  {
  	memset(cmd, 0, sizeof(*cmd));
  
  	address &= ~(0xfffULL);
  
  	cmd->data[0]  = devid;
  	cmd->data[0] |= (pasid & 0xff) << 16;
  	cmd->data[0] |= (qdep  & 0xff) << 24;
  	cmd->data[1]  = devid;
  	cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
  	cmd->data[2]  = lower_32_bits(address);
  	cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
  	cmd->data[3]  = upper_32_bits(address);
  	if (size)
  		cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
  	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
  }

  static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
  			       int status, int tag, bool gn)
  {
  	memset(cmd, 0, sizeof(*cmd));
  
  	cmd->data[0]  = devid;
  	if (gn) {
  		cmd->data[1]  = pasid & PASID_MASK;
  		cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
  	}
  	cmd->data[3]  = tag & 0x1ff;
  	cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
  
  	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
  }

  static void build_inv_all(struct iommu_cmd *cmd)
  {
  	memset(cmd, 0, sizeof(*cmd));
  	CMD_SET_TYPE(cmd, CMD_INV_ALL);

  }

  /*

   * Writes the command to the IOMMUs command buffer and informs the

   * hardware about the new command.

   */

  static int iommu_queue_command_sync(struct amd_iommu *iommu,
  				    struct iommu_cmd *cmd,
  				    bool sync)

  {

  	u32 left, tail, head, next_tail;

  	unsigned long flags;

  	WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);

  
  again:

  	spin_lock_irqsave(&iommu->lock, flags);

  	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
  	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
  	next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
  	left      = (head - next_tail) % iommu->cmd_buf_size;

  	if (left <= 2) {
  		struct iommu_cmd sync_cmd;
  		volatile u64 sem = 0;
  		int ret;

  		build_completion_wait(&sync_cmd, (u64)&sem);
  		copy_cmd_to_buffer(iommu, &sync_cmd, tail);

  		spin_unlock_irqrestore(&iommu->lock, flags);
  
  		if ((ret = wait_on_sem(&sem)) != 0)
  			return ret;
  
  		goto again;

  	}

  	copy_cmd_to_buffer(iommu, cmd, tail);
  
  	/* We need to sync now to make sure all commands are processed */

  	iommu->need_sync = sync;

  	spin_unlock_irqrestore(&iommu->lock, flags);

  	return 0;

  }

  static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
  {
  	return iommu_queue_command_sync(iommu, cmd, true);
  }

  /*
   * This function queues a completion wait command into the command
   * buffer of an IOMMU
   */

  static int iommu_completion_wait(struct amd_iommu *iommu)

  {
  	struct iommu_cmd cmd;

  	volatile u64 sem = 0;

  	int ret;

  	if (!iommu->need_sync)

  		return 0;

  	build_completion_wait(&cmd, (u64)&sem);

  	ret = iommu_queue_command_sync(iommu, &cmd, false);

  	if (ret)

  		return ret;

  	return wait_on_sem(&sem);

  }

  static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)

  {

  	struct iommu_cmd cmd;

  	build_inv_dte(&cmd, devid);

  	return iommu_queue_command(iommu, &cmd);
  }

  static void iommu_flush_dte_all(struct amd_iommu *iommu)
  {
  	u32 devid;

  	for (devid = 0; devid <= 0xffff; ++devid)
  		iommu_flush_dte(iommu, devid);

  	iommu_completion_wait(iommu);
  }

  /*
   * This function uses heavy locking and may disable irqs for some time. But
   * this is no issue because it is only called during resume.
   */
  static void iommu_flush_tlb_all(struct amd_iommu *iommu)
  {
  	u32 dom_id;

  	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
  		struct iommu_cmd cmd;
  		build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
  				      dom_id, 1);
  		iommu_queue_command(iommu, &cmd);
  	}

  	iommu_completion_wait(iommu);

  }

  static void iommu_flush_all(struct amd_iommu *iommu)

  {

  	struct iommu_cmd cmd;

  	build_inv_all(&cmd);

  	iommu_queue_command(iommu, &cmd);
  	iommu_completion_wait(iommu);
  }

  void iommu_flush_all_caches(struct amd_iommu *iommu)
  {

  	if (iommu_feature(iommu, FEATURE_IA)) {
  		iommu_flush_all(iommu);
  	} else {
  		iommu_flush_dte_all(iommu);
  		iommu_flush_tlb_all(iommu);

  	}
  }

  /*

   * Command send function for flushing on-device TLB

   */

  static int device_flush_iotlb(struct iommu_dev_data *dev_data,
  			      u64 address, size_t size)

  {
  	struct amd_iommu *iommu;

  	struct iommu_cmd cmd;

  	int qdep;

  	qdep     = dev_data->ats.qdep;
  	iommu    = amd_iommu_rlookup_table[dev_data->devid];

  	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);

  
  	return iommu_queue_command(iommu, &cmd);

  }

  /*

   * Command send function for invalidating a device table entry

   */

  static int device_flush_dte(struct iommu_dev_data *dev_data)

  {

  	struct amd_iommu *iommu;

  	int ret;

  	iommu = amd_iommu_rlookup_table[dev_data->devid];

  	ret = iommu_flush_dte(iommu, dev_data->devid);

  	if (ret)
  		return ret;

  	if (dev_data->ats.enabled)

  		ret = device_flush_iotlb(dev_data, 0, ~0UL);

  	return ret;

  }

  /*
   * TLB invalidation function which is called from the mapping functions.
   * It invalidates a single PTE if the range to flush is within a single
   * page. Otherwise it flushes the whole TLB of the IOMMU.
   */

  static void __domain_flush_pages(struct protection_domain *domain,
  				 u64 address, size_t size, int pde)

  {

  	struct iommu_dev_data *dev_data;

  	struct iommu_cmd cmd;
  	int ret = 0, i;

  	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);

  	for (i = 0; i < amd_iommus_present; ++i) {
  		if (!domain->dev_iommu[i])
  			continue;
  
  		/*
  		 * Devices of this domain are behind this IOMMU
  		 * We need a TLB flush
  		 */

  		ret |= iommu_queue_command(amd_iommus[i], &cmd);

  	}

  	list_for_each_entry(dev_data, &domain->dev_list, list) {

  		if (!dev_data->ats.enabled)

  			continue;

  		ret |= device_flush_iotlb(dev_data, address, size);

  	}

  	WARN_ON(ret);

  }

  static void domain_flush_pages(struct protection_domain *domain,
  			       u64 address, size_t size)

  {

  	__domain_flush_pages(domain, address, size, 0);

  }

  /* Flush the whole IO/TLB for a given protection domain */

  static void domain_flush_tlb(struct protection_domain *domain)

  {

  	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);

  }

  /* Flush the whole IO/TLB for a given protection domain - including PDE */

  static void domain_flush_tlb_pde(struct protection_domain *domain)

  {

  	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);

  }

  static void domain_flush_complete(struct protection_domain *domain)

  {

  	int i;

  	for (i = 0; i < amd_iommus_present; ++i) {
  		if (!domain->dev_iommu[i])
  			continue;

  		/*
  		 * Devices of this domain are behind this IOMMU
  		 * We need to wait for completion of all commands.
  		 */
  		iommu_completion_wait(amd_iommus[i]);

  	}

  }

  /*

   * This function flushes the DTEs for all devices in domain

   */

  static void domain_flush_devices(struct protection_domain *domain)

  {

  	struct iommu_dev_data *dev_data;

  	list_for_each_entry(dev_data, &domain->dev_list, list)

  		device_flush_dte(dev_data);

  }

  /****************************************************************************
   *
   * The functions below are used the create the page table mappings for
   * unity mapped regions.
   *
   ****************************************************************************/
  
  /*

   * This function is used to add another level to an IO page table. Adding
   * another level increases the size of the address space by 9 bits to a size up
   * to 64 bits.
   */
  static bool increase_address_space(struct protection_domain *domain,
  				   gfp_t gfp)
  {
  	u64 *pte;
  
  	if (domain->mode == PAGE_MODE_6_LEVEL)
  		/* address space already 64 bit large */
  		return false;
  
  	pte = (void *)get_zeroed_page(gfp);
  	if (!pte)
  		return false;
  
  	*pte             = PM_LEVEL_PDE(domain->mode,
  					virt_to_phys(domain->pt_root));
  	domain->pt_root  = pte;
  	domain->mode    += 1;
  	domain->updated  = true;
  
  	return true;
  }
  
  static u64 *alloc_pte(struct protection_domain *domain,
  		      unsigned long address,

  		      unsigned long page_size,

  		      u64 **pte_page,
  		      gfp_t gfp)
  {

  	int level, end_lvl;

  	u64 *pte, *page;

  
  	BUG_ON(!is_power_of_2(page_size));

  
  	while (address > PM_LEVEL_SIZE(domain->mode))
  		increase_address_space(domain, gfp);

  	level   = domain->mode - 1;
  	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
  	address = PAGE_SIZE_ALIGN(address, page_size);
  	end_lvl = PAGE_SIZE_LEVEL(page_size);

  
  	while (level > end_lvl) {
  		if (!IOMMU_PTE_PRESENT(*pte)) {
  			page = (u64 *)get_zeroed_page(gfp);
  			if (!page)
  				return NULL;
  			*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
  		}

  		/* No level skipping support yet */
  		if (PM_PTE_LEVEL(*pte) != level)
  			return NULL;

  		level -= 1;
  
  		pte = IOMMU_PTE_PAGE(*pte);
  
  		if (pte_page && level == end_lvl)
  			*pte_page = pte;
  
  		pte = &pte[PM_LEVEL_INDEX(level, address)];
  	}
  
  	return pte;
  }
  
  /*
   * This function checks if there is a PTE for a given dma address. If
   * there is one, it returns the pointer to it.
   */

  static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)

  {
  	int level;
  	u64 *pte;

  	if (address > PM_LEVEL_SIZE(domain->mode))
  		return NULL;
  
  	level   =  domain->mode - 1;
  	pte     = &domain->pt_root[PM_LEVEL_INDEX(level, address)];

  	while (level > 0) {
  
  		/* Not Present */

  		if (!IOMMU_PTE_PRESENT(*pte))
  			return NULL;

  		/* Large PTE */
  		if (PM_PTE_LEVEL(*pte) == 0x07) {
  			unsigned long pte_mask, __pte;
  
  			/*
  			 * If we have a series of large PTEs, make
  			 * sure to return a pointer to the first one.
  			 */
  			pte_mask = PTE_PAGE_SIZE(*pte);
  			pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
  			__pte    = ((unsigned long)pte) & pte_mask;
  
  			return (u64 *)__pte;
  		}
  
  		/* No level skipping support yet */
  		if (PM_PTE_LEVEL(*pte) != level)
  			return NULL;

  		level -= 1;

  		/* Walk to the next level */

  		pte = IOMMU_PTE_PAGE(*pte);
  		pte = &pte[PM_LEVEL_INDEX(level, address)];

  	}
  
  	return pte;
  }
  
  /*

   * Generic mapping functions. It maps a physical address into a DMA
   * address space. It allocates the page table pages if necessary.
   * In the future it can be extended to a generic mapping function
   * supporting all features of AMD IOMMU page tables like level skipping
   * and full 64 bit address spaces.
   */

  static int iommu_map_page(struct protection_domain *dom,
  			  unsigned long bus_addr,
  			  unsigned long phys_addr,

  			  int prot,

  			  unsigned long page_size)

  {

  	u64 __pte, *pte;

  	int i, count;

  	if (!(prot & IOMMU_PROT_MASK))

  		return -EINVAL;

  	bus_addr  = PAGE_ALIGN(bus_addr);
  	phys_addr = PAGE_ALIGN(phys_addr);
  	count     = PAGE_SIZE_PTE_COUNT(page_size);
  	pte       = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
  
  	for (i = 0; i < count; ++i)
  		if (IOMMU_PTE_PRESENT(pte[i]))
  			return -EBUSY;

  	if (page_size > PAGE_SIZE) {
  		__pte = PAGE_SIZE_PTE(phys_addr, page_size);
  		__pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
  	} else
  		__pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;

  	if (prot & IOMMU_PROT_IR)
  		__pte |= IOMMU_PTE_IR;
  	if (prot & IOMMU_PROT_IW)
  		__pte |= IOMMU_PTE_IW;

  	for (i = 0; i < count; ++i)
  		pte[i] = __pte;

  	update_domain(dom);

  	return 0;
  }

  static unsigned long iommu_unmap_page(struct protection_domain *dom,
  				      unsigned long bus_addr,
  				      unsigned long page_size)

  {

  	unsigned long long unmap_size, unmapped;
  	u64 *pte;
  
  	BUG_ON(!is_power_of_2(page_size));
  
  	unmapped = 0;

  	while (unmapped < page_size) {
  
  		pte = fetch_pte(dom, bus_addr);
  
  		if (!pte) {
  			/*
  			 * No PTE for this address
  			 * move forward in 4kb steps
  			 */
  			unmap_size = PAGE_SIZE;
  		} else if (PM_PTE_LEVEL(*pte) == 0) {
  			/* 4kb PTE found for this address */
  			unmap_size = PAGE_SIZE;
  			*pte       = 0ULL;
  		} else {
  			int count, i;
  
  			/* Large PTE found which maps this address */
  			unmap_size = PTE_PAGE_SIZE(*pte);
  			count      = PAGE_SIZE_PTE_COUNT(unmap_size);
  			for (i = 0; i < count; i++)
  				pte[i] = 0ULL;
  		}
  
  		bus_addr  = (bus_addr & ~(unmap_size - 1)) + unmap_size;
  		unmapped += unmap_size;
  	}
  
  	BUG_ON(!is_power_of_2(unmapped));

  	return unmapped;

  }

  /*
   * This function checks if a specific unity mapping entry is needed for
   * this specific IOMMU.
   */

  static int iommu_for_unity_map(struct amd_iommu *iommu,
  			       struct unity_map_entry *entry)
  {
  	u16 bdf, i;
  
  	for (i = entry->devid_start; i <= entry->devid_end; ++i) {
  		bdf = amd_iommu_alias_table[i];
  		if (amd_iommu_rlookup_table[bdf] == iommu)
  			return 1;
  	}
  
  	return 0;
  }

  /*

   * This function actually applies the mapping to the page table of the
   * dma_ops domain.
   */

  static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
  			     struct unity_map_entry *e)
  {
  	u64 addr;
  	int ret;
  
  	for (addr = e->address_start; addr < e->address_end;
  	     addr += PAGE_SIZE) {

  		ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,

  				     PAGE_SIZE);

  		if (ret)
  			return ret;
  		/*
  		 * if unity mapping is in aperture range mark the page
  		 * as allocated in the aperture
  		 */
  		if (addr < dma_dom->aperture_size)

  			__set_bit(addr >> PAGE_SHIFT,

  				  dma_dom->aperture[0]->bitmap);

  	}
  
  	return 0;
  }

  /*

   * Init the unity mappings for a specific IOMMU in the system
   *
   * Basically iterates over all unity mapping entries and applies them to
   * the default domain DMA of that IOMMU if necessary.
   */
  static int iommu_init_unity_mappings(struct amd_iommu *iommu)
  {
  	struct unity_map_entry *entry;
  	int ret;
  
  	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
  		if (!iommu_for_unity_map(iommu, entry))
  			continue;
  		ret = dma_ops_unity_map(iommu->default_dom, entry);
  		if (ret)
  			return ret;
  	}
  
  	return 0;
  }
  
  /*

   * Inits the unity mappings required for a specific device
   */

  static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  					  u16 devid)
  {
  	struct unity_map_entry *e;
  	int ret;
  
  	list_for_each_entry(e, &amd_iommu_unity_map, list) {
  		if (!(devid >= e->devid_start && devid <= e->devid_end))
  			continue;
  		ret = dma_ops_unity_map(dma_dom, e);
  		if (ret)
  			return ret;
  	}
  
  	return 0;
  }

  /****************************************************************************
   *
   * The next functions belong to the address allocator for the dma_ops
   * interface functions. They work like the allocators in the other IOMMU
   * drivers. Its basically a bitmap which marks the allocated pages in
   * the aperture. Maybe it could be enhanced in the future to a more
   * efficient allocator.
   *
   ****************************************************************************/

  /*

   * The address allocator core functions.

   *
   * called with domain->lock held
   */

  /*

   * Used to reserve address ranges in the aperture (e.g. for exclusion
   * ranges.
   */
  static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
  				      unsigned long start_page,
  				      unsigned int pages)
  {
  	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
  
  	if (start_page + pages > last_page)
  		pages = last_page - start_page;
  
  	for (i = start_page; i < start_page + pages; ++i) {
  		int index = i / APERTURE_RANGE_PAGES;
  		int page  = i % APERTURE_RANGE_PAGES;
  		__set_bit(page, dom->aperture[index]->bitmap);
  	}
  }
  
  /*

   * This function is used to add a new aperture range to an existing
   * aperture in case of dma_ops domain allocation or address allocation
   * failure.
   */

  static int alloc_new_range(struct dma_ops_domain *dma_dom,

  			   bool populate, gfp_t gfp)
  {
  	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;

  	struct amd_iommu *iommu;

  	unsigned long i, old_size;

  #ifdef CONFIG_IOMMU_STRESS
  	populate = false;
  #endif

  	if (index >= APERTURE_MAX_RANGES)
  		return -ENOMEM;
  
  	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
  	if (!dma_dom->aperture[index])
  		return -ENOMEM;
  
  	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
  	if (!dma_dom->aperture[index]->bitmap)
  		goto out_free;
  
  	dma_dom->aperture[index]->offset = dma_dom->aperture_size;
  
  	if (populate) {
  		unsigned long address = dma_dom->aperture_size;
  		int i, num_ptes = APERTURE_RANGE_PAGES / 512;
  		u64 *pte, *pte_page;
  
  		for (i = 0; i < num_ptes; ++i) {

  			pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,

  					&pte_page, gfp);
  			if (!pte)
  				goto out_free;
  
  			dma_dom->aperture[index]->pte_pages[i] = pte_page;
  
  			address += APERTURE_RANGE_SIZE / 64;
  		}
  	}

  	old_size                = dma_dom->aperture_size;

  	dma_dom->aperture_size += APERTURE_RANGE_SIZE;

  	/* Reserve address range used for MSI messages */
  	if (old_size < MSI_ADDR_BASE_LO &&
  	    dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
  		unsigned long spage;
  		int pages;
  
  		pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
  		spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
  
  		dma_ops_reserve_addresses(dma_dom, spage, pages);
  	}

  	/* Initialize the exclusion range if necessary */

  	for_each_iommu(iommu) {
  		if (iommu->exclusion_start &&
  		    iommu->exclusion_start >= dma_dom->aperture[index]->offset
  		    && iommu->exclusion_start < dma_dom->aperture_size) {
  			unsigned long startpage;
  			int pages = iommu_num_pages(iommu->exclusion_start,
  						    iommu->exclusion_length,
  						    PAGE_SIZE);
  			startpage = iommu->exclusion_start >> PAGE_SHIFT;
  			dma_ops_reserve_addresses(dma_dom, startpage, pages);
  		}

  	}
  
  	/*
  	 * Check for areas already mapped as present in the new aperture
  	 * range and mark those pages as reserved in the allocator. Such
  	 * mappings may already exist as a result of requested unity
  	 * mappings for devices.
  	 */
  	for (i = dma_dom->aperture[index]->offset;
  	     i < dma_dom->aperture_size;
  	     i += PAGE_SIZE) {

  		u64 *pte = fetch_pte(&dma_dom->domain, i);

  		if (!pte || !IOMMU_PTE_PRESENT(*pte))
  			continue;

  		dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);

  	}

  	update_domain(&dma_dom->domain);

  	return 0;
  
  out_free:

  	update_domain(&dma_dom->domain);

  	free_page((unsigned long)dma_dom->aperture[index]->bitmap);
  
  	kfree(dma_dom->aperture[index]);
  	dma_dom->aperture[index] = NULL;
  
  	return -ENOMEM;
  }

  static unsigned long dma_ops_area_alloc(struct device *dev,
  					struct dma_ops_domain *dom,
  					unsigned int pages,
  					unsigned long align_mask,
  					u64 dma_mask,
  					unsigned long start)
  {

  	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;

  	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
  	int i = start >> APERTURE_RANGE_SHIFT;
  	unsigned long boundary_size;
  	unsigned long address = -1;
  	unsigned long limit;

  	next_bit >>= PAGE_SHIFT;

  	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
  			PAGE_SIZE) >> PAGE_SHIFT;
  
  	for (;i < max_index; ++i) {
  		unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
  
  		if (dom->aperture[i]->offset >= dma_mask)
  			break;
  
  		limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
  					       dma_mask >> PAGE_SHIFT);
  
  		address = iommu_area_alloc(dom->aperture[i]->bitmap,
  					   limit, next_bit, pages, 0,
  					    boundary_size, align_mask);
  		if (address != -1) {
  			address = dom->aperture[i]->offset +
  				  (address << PAGE_SHIFT);

  			dom->next_address = address + (pages << PAGE_SHIFT);

  			break;
  		}
  
  		next_bit = 0;
  	}
  
  	return address;
  }

  static unsigned long dma_ops_alloc_addresses(struct device *dev,
  					     struct dma_ops_domain *dom,

  					     unsigned int pages,

  					     unsigned long align_mask,
  					     u64 dma_mask)

  {

  	unsigned long address;

  #ifdef CONFIG_IOMMU_STRESS
  	dom->next_address = 0;
  	dom->need_flush = true;
  #endif

  	address = dma_ops_area_alloc(dev, dom, pages, align_mask,

  				     dma_mask, dom->next_address);

  	if (address == -1) {

  		dom->next_address = 0;

  		address = dma_ops_area_alloc(dev, dom, pages, align_mask,
  					     dma_mask, 0);

  		dom->need_flush = true;
  	}

  	if (unlikely(address == -1))

  		address = DMA_ERROR_CODE;

  
  	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
  
  	return address;
  }

  /*
   * The address free function.
   *
   * called with domain->lock held
   */

  static void dma_ops_free_addresses(struct dma_ops_domain *dom,
  				   unsigned long address,
  				   unsigned int pages)
  {

  	unsigned i = address >> APERTURE_RANGE_SHIFT;
  	struct aperture_range *range = dom->aperture[i];

  	BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);

  #ifdef CONFIG_IOMMU_STRESS
  	if (i < 4)
  		return;
  #endif

  	if (address >= dom->next_address)

  		dom->need_flush = true;

  
  	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;

  	bitmap_clear(range->bitmap, address, pages);

  }

  /****************************************************************************
   *
   * The next functions belong to the domain allocation. A domain is
   * allocated for every IOMMU as the default domain. If device isolation
   * is enabled, every device get its own domain. The most important thing
   * about domains is the page table mapping the DMA address space they
   * contain.
   *
   ****************************************************************************/

  /*
   * This function adds a protection domain to the global protection domain list
   */
  static void add_domain_to_list(struct protection_domain *domain)
  {
  	unsigned long flags;
  
  	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  	list_add(&domain->list, &amd_iommu_pd_list);
  	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  }
  
  /*
   * This function removes a protection domain to the global
   * protection domain list
   */
  static void del_domain_from_list(struct protection_domain *domain)
  {
  	unsigned long flags;
  
  	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
  	list_del(&domain->list);
  	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
  }

  static u16 domain_id_alloc(void)
  {
  	unsigned long flags;
  	int id;
  
  	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
  	BUG_ON(id == 0);
  	if (id > 0 && id < MAX_DOMAIN_ID)
  		__set_bit(id, amd_iommu_pd_alloc_bitmap);
  	else
  		id = 0;
  	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  
  	return id;
  }

  static void domain_id_free(int id)
  {
  	unsigned long flags;
  
  	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
  	if (id > 0 && id < MAX_DOMAIN_ID)
  		__clear_bit(id, amd_iommu_pd_alloc_bitmap);
  	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
  }

  static void free_pagetable(struct protection_domain *domain)

  {
  	int i, j;
  	u64 *p1, *p2, *p3;

  	p1 = domain->pt_root;

  
  	if (!p1)
  		return;
  
  	for (i = 0; i < 512; ++i) {
  		if (!IOMMU_PTE_PRESENT(p1[i]))
  			continue;
  
  		p2 = IOMMU_PTE_PAGE(p1[i]);

  		for (j = 0; j < 512; ++j) {

  			if (!IOMMU_PTE_PRESENT(p2[j]))
  				continue;
  			p3 = IOMMU_PTE_PAGE(p2[j]);
  			free_page((unsigned long)p3);
  		}
  
  		free_page((unsigned long)p2);
  	}
  
  	free_page((unsigned long)p1);

  
  	domain->pt_root = NULL;

  }

  static void free_gcr3_tbl_level1(u64 *tbl)
  {
  	u64 *ptr;
  	int i;
  
  	for (i = 0; i < 512; ++i) {
  		if (!(tbl[i] & GCR3_VALID))
  			continue;
  
  		ptr = __va(tbl[i] & PAGE_MASK);
  
  		free_page((unsigned long)ptr);
  	}
  }
  
  static void free_gcr3_tbl_level2(u64 *tbl)
  {
  	u64 *ptr;
  	int i;
  
  	for (i = 0; i < 512; ++i) {
  		if (!(tbl[i] & GCR3_VALID))
  			continue;
  
  		ptr = __va(tbl[i] & PAGE_MASK);
  
  		free_gcr3_tbl_level1(ptr);
  	}
  }

  static void free_gcr3_table(struct protection_domain *domain)
  {

  	if (domain->glx == 2)
  		free_gcr3_tbl_level2(domain->gcr3_tbl);
  	else if (domain->glx == 1)
  		free_gcr3_tbl_level1(domain->gcr3_tbl);
  	else if (domain->glx != 0)
  		BUG();

  	free_page((unsigned long)domain->gcr3_tbl);
  }

  /*
   * Free a domain, only used if something went wrong in the
   * allocation path and we need to free an already allocated page table
   */

  static void dma_ops_domain_free(struct dma_ops_domain *dom)
  {

  	int i;

  	if (!dom)
  		return;

  	del_domain_from_list(&dom->domain);

  	free_pagetable(&dom->domain);

  	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
  		if (!dom->aperture[i])
  			continue;
  		free_page((unsigned long)dom->aperture[i]->bitmap);
  		kfree(dom->aperture[i]);
  	}

  
  	kfree(dom);
  }

  /*
   * Allocates a new protection domain usable for the dma_ops functions.

   * It also initializes the page table and the address allocator data

   * structures required for the dma_ops interface
   */

  static struct dma_ops_domain *dma_ops_domain_alloc(void)

  {
  	struct dma_ops_domain *dma_dom;

  
  	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
  	if (!dma_dom)
  		return NULL;
  
  	spin_lock_init(&dma_dom->domain.lock);
  
  	dma_dom->domain.id = domain_id_alloc();
  	if (dma_dom->domain.id == 0)
  		goto free_dma_dom;

  	INIT_LIST_HEAD(&dma_dom->domain.dev_list);

  	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;

  	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);

  	dma_dom->domain.flags = PD_DMA_OPS_MASK;

  	dma_dom->domain.priv = dma_dom;
  	if (!dma_dom->domain.pt_root)
  		goto free_dma_dom;

  	dma_dom->need_flush = false;

  	dma_dom->target_dev = 0xffff;

  	add_domain_to_list(&dma_dom->domain);

  	if (alloc_new_range(dma_dom, true, GFP_KERNEL))

  		goto free_dma_dom;

  	/*

  	 * mark the first page as allocated so we never return 0 as
  	 * a valid dma-address. So we can use 0 as error value

  	 */

  	dma_dom->aperture[0]->bitmap[0] = 1;

  	dma_dom->next_address = 0;

  
  	return dma_dom;
  
  free_dma_dom:
  	dma_ops_domain_free(dma_dom);
  
  	return NULL;
  }

  /*

   * little helper function to check whether a given protection domain is a
   * dma_ops domain
   */
  static bool dma_ops_domain(struct protection_domain *domain)
  {
  	return domain->flags & PD_DMA_OPS_MASK;
  }

  static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)

  {

  	u64 pte_root = 0;

  	u64 flags = 0;

  	if (domain->mode != PAGE_MODE_NONE)
  		pte_root = virt_to_phys(domain->pt_root);

  	pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
  		    << DEV_ENTRY_MODE_SHIFT;
  	pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;

  	flags = amd_iommu_dev_table[devid].data[1];

  	if (ats)
  		flags |= DTE_FLAG_IOTLB;

  	if (domain->flags & PD_IOMMUV2_MASK) {
  		u64 gcr3 = __pa(domain->gcr3_tbl);
  		u64 glx  = domain->glx;
  		u64 tmp;
  
  		pte_root |= DTE_FLAG_GV;
  		pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
  
  		/* First mask out possible old values for GCR3 table */
  		tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
  		flags    &= ~tmp;
  
  		tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
  		flags    &= ~tmp;
  
  		/* Encode GCR3 table into DTE */
  		tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
  		pte_root |= tmp;
  
  		tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
  		flags    |= tmp;
  
  		tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
  		flags    |= tmp;
  	}

  	flags &= ~(0xffffUL);
  	flags |= domain->id;
  
  	amd_iommu_dev_table[devid].data[1]  = flags;
  	amd_iommu_dev_table[devid].data[0]  = pte_root;

  }
  
  static void clear_dte_entry(u16 devid)
  {

  	/* remove entry from the device table seen by the hardware */
  	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
  	amd_iommu_dev_table[devid].data[1] = 0;

  
  	amd_iommu_apply_erratum_63(devid);

  }

  static void do_attach(struct iommu_dev_data *dev_data,
  		      struct protection_domain *domain)

  {

  	struct amd_iommu *iommu;

  	bool ats;

  	iommu = amd_iommu_rlookup_table[dev_data->devid];
  	ats   = dev_data->ats.enabled;

  
  	/* Update data structures */
  	dev_data->domain = domain;
  	list_add(&dev_data->list, &domain->dev_list);

  	set_dte_entry(dev_data->devid, domain, ats);

  
  	/* Do reference counting */
  	domain->dev_iommu[iommu->index] += 1;
  	domain->dev_cnt                 += 1;
  
  	/* Flush the DTE entry */

  	device_flush_dte(dev_data);

  }

  static void do_detach(struct iommu_dev_data *dev_data)

  {

  	struct amd_iommu *iommu;

  	iommu = amd_iommu_rlookup_table[dev_data->devid];

  
  	/* decrease reference counters */

  	dev_data->domain->dev_iommu[iommu->index] -= 1;
  	dev_data->domain->dev_cnt                 -= 1;
  
  	/* Update data structures */
  	dev_data->domain = NULL;
  	list_del(&dev_data->list);

  	clear_dte_entry(dev_data->devid);

  	/* Flush the DTE entry */

  	device_flush_dte(dev_data);

  }
  
  /*
   * If a device is not yet associated with a domain, this function does
   * assigns it visible for the hardware
   */

  static int __attach_device(struct iommu_dev_data *dev_data,

  			   struct protection_domain *domain)

  {

  	int ret;

  	/* lock domain */
  	spin_lock(&domain->lock);

  	if (dev_data->alias_data != NULL) {
  		struct iommu_dev_data *alias_data = dev_data->alias_data;

  		/* Some sanity checks */
  		ret = -EBUSY;
  		if (alias_data->domain != NULL &&
  				alias_data->domain != domain)
  			goto out_unlock;

  		if (dev_data->domain != NULL &&
  				dev_data->domain != domain)
  			goto out_unlock;

  		/* Do real assignment */

  		if (alias_data->domain == NULL)

  			do_attach(alias_data, domain);

  
  		atomic_inc(&alias_data->bind);

  	}

  	if (dev_data->domain == NULL)

  		do_attach(dev_data, domain);

  	atomic_inc(&dev_data->bind);

  	ret = 0;
  
  out_unlock:

  	/* ready */
  	spin_unlock(&domain->lock);

  	return ret;

  }

  
  static void pdev_iommuv2_disable(struct pci_dev *pdev)
  {
  	pci_disable_ats(pdev);
  	pci_disable_pri(pdev);
  	pci_disable_pasid(pdev);
  }

  /* FIXME: Change generic reset-function to do the same */
  static int pri_reset_while_enabled(struct pci_dev *pdev)
  {
  	u16 control;
  	int pos;

  	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);

  	if (!pos)
  		return -EINVAL;

  	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
  	control |= PCI_PRI_CTRL_RESET;
  	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);

  
  	return 0;
  }

  static int pdev_iommuv2_enable(struct pci_dev *pdev)
  {

  	bool reset_enable;
  	int reqs, ret;
  
  	/* FIXME: Hardcode number of outstanding requests for now */
  	reqs = 32;
  	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
  		reqs = 1;
  	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);

  
  	/* Only allow access to user-accessible pages */
  	ret = pci_enable_pasid(pdev, 0);
  	if (ret)
  		goto out_err;
  
  	/* First reset the PRI state of the device */
  	ret = pci_reset_pri(pdev);
  	if (ret)
  		goto out_err;

  	/* Enable PRI */
  	ret = pci_enable_pri(pdev, reqs);

  	if (ret)
  		goto out_err;

  	if (reset_enable) {
  		ret = pri_reset_while_enabled(pdev);
  		if (ret)
  			goto out_err;
  	}

  	ret = pci_enable_ats(pdev, PAGE_SHIFT);
  	if (ret)
  		goto out_err;
  
  	return 0;
  
  out_err:
  	pci_disable_pri(pdev);
  	pci_disable_pasid(pdev);
  
  	return ret;
  }

  /* FIXME: Move this to PCI code */
  #define PCI_PRI_TLP_OFF		(1 << 2)
  
  bool pci_pri_tlp_required(struct pci_dev *pdev)
  {
  	u16 control;
  	int pos;

  	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);

  	if (!pos)
  		return false;

  	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);

  
  	return (control & PCI_PRI_TLP_OFF) ? true : false;
  }

  /*
   * If a device is not yet associated with a domain, this function does
   * assigns it visible for the hardware
   */

  static int attach_device(struct device *dev,
  			 struct protection_domain *domain)

  {

  	struct pci_dev *pdev = to_pci_dev(dev);

  	struct iommu_dev_data *dev_data;

  	unsigned long flags;

  	int ret;

  	dev_data = get_dev_data(dev);

  	if (domain->flags & PD_IOMMUV2_MASK) {
  		if (!dev_data->iommu_v2 || !dev_data->passthrough)
  			return -EINVAL;
  
  		if (pdev_iommuv2_enable(pdev) != 0)
  			return -EINVAL;
  
  		dev_data->ats.enabled = true;
  		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);

  		dev_data->pri_tlp     = pci_pri_tlp_required(pdev);

  	} else if (amd_iommu_iotlb_sup &&
  		   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {

  		dev_data->ats.enabled = true;
  		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
  	}

  	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

  	ret = __attach_device(dev_data, domain);

  	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);

  	/*
  	 * We might boot into a crash-kernel here. The crashed kernel
  	 * left the caches in the IOMMU dirty. So we have to flush
  	 * here to evict all dirty stuff.
  	 */

  	domain_flush_tlb_pde(domain);

  
  	return ret;

  }

  /*
   * Removes a device from a protection domain (unlocked)
   */

  static void __detach_device(struct iommu_dev_data *dev_data)

  {

  	struct protection_domain *domain;

  	unsigned long flags;

  	BUG_ON(!dev_data->domain);

  	domain = dev_data->domain;
  
  	spin_lock_irqsave(&domain->lock, flags);

  	if (dev_data->alias_data != NULL) {
  		struct iommu_dev_data *alias_data = dev_data->alias_data;

  		if (atomic_dec_and_test(&alias_data->bind))

  			do_detach(alias_data);

  	}

  	if (atomic_dec_and_test(&dev_data->bind))

  		do_detach(dev_data);

  	spin_unlock_irqrestore(&domain->lock, flags);

  
  	/*
  	 * If we run in passthrough mode the device must be assigned to the

  	 * passthrough domain if it is detached from any other domain.
  	 * Make sure we can deassign from the pt_domain itself.

  	 */

  	if (dev_data->passthrough &&

  	    (dev_data->domain == NULL && domain != pt_domain))

  		__attach_device(dev_data, pt_domain);

  }
  
  /*
   * Removes a device from a protection domain (with devtable_lock held)
   */

  static void detach_device(struct device *dev)

  {

  	struct protection_domain *domain;

  	struct iommu_dev_data *dev_data;

  	unsigned long flags;

  	dev_data = get_dev_data(dev);

  	domain   = dev_data->domain;

  	/* lock device table */
  	write_lock_irqsave(&amd_iommu_devtable_lock, flags);

  	__detach_device(dev_data);