Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit 989d216f86bbef47059c10aca77be9f56305a7ec

Authored by Linus Torvalds 2014-05-22 03:29:39 +0800

Exists in ti-lsk-linux-4.1.y and in 12 other branches

Merge tag 'iommu-fixes-v3.15-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu

Pull iommu fixes from Joerg Roedel:
 "Three fixes for the AMD IOMMU driver:
   - fix a locking issue around get_user_pages()
   - fix two issues with device aliasing and exclusion range handling"

* tag 'iommu-fixes-v3.15-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu:
  iommu/amd: fix enabling exclusion range for an exact device
  iommu/amd: Take mmap_sem when calling get_user_pages
  iommu/amd: Fix interrupt remapping for aliased devices

Showing 3 changed files Inline Diff

drivers/iommu/amd_iommu.c
drivers/iommu/amd_iommu_init.c
drivers/iommu/amd_iommu_v2.c

drivers/iommu/amd_iommu.c

Diff comments View file @ 989d216

1	/*	1	/*
2	* Copyright (C) 2007-2010 Advanced Micro Devices, Inc.	2	* Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3	* Author: Joerg Roedel <joerg.roedel@amd.com>	3	* Author: Joerg Roedel <joerg.roedel@amd.com>
4	* Leo Duran <leo.duran@amd.com>	4	* Leo Duran <leo.duran@amd.com>
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify it	6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms of the GNU General Public License version 2 as published	7	* under the terms of the GNU General Public License version 2 as published
8	* by the Free Software Foundation.	8	* by the Free Software Foundation.
9	*	9	*
10	* This program is distributed in the hope that it will be useful,	10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.	13	* GNU General Public License for more details.
14	*	14	*
15	* You should have received a copy of the GNU General Public License	15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software	16	* along with this program; if not, write to the Free Software
17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA	17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18	*/	18	*/
19		19
20	#include <linux/ratelimit.h>	20	#include <linux/ratelimit.h>
21	#include <linux/pci.h>	21	#include <linux/pci.h>
22	#include <linux/pci-ats.h>	22	#include <linux/pci-ats.h>
23	#include <linux/bitmap.h>	23	#include <linux/bitmap.h>
24	#include <linux/slab.h>	24	#include <linux/slab.h>
25	#include <linux/debugfs.h>	25	#include <linux/debugfs.h>
26	#include <linux/scatterlist.h>	26	#include <linux/scatterlist.h>
27	#include <linux/dma-mapping.h>	27	#include <linux/dma-mapping.h>
28	#include <linux/iommu-helper.h>	28	#include <linux/iommu-helper.h>
29	#include <linux/iommu.h>	29	#include <linux/iommu.h>
30	#include <linux/delay.h>	30	#include <linux/delay.h>
31	#include <linux/amd-iommu.h>	31	#include <linux/amd-iommu.h>
32	#include <linux/notifier.h>	32	#include <linux/notifier.h>
33	#include <linux/export.h>	33	#include <linux/export.h>
34	#include <linux/irq.h>	34	#include <linux/irq.h>
35	#include <linux/msi.h>	35	#include <linux/msi.h>
36	#include <asm/irq_remapping.h>	36	#include <asm/irq_remapping.h>
37	#include <asm/io_apic.h>	37	#include <asm/io_apic.h>
38	#include <asm/apic.h>	38	#include <asm/apic.h>
39	#include <asm/hw_irq.h>	39	#include <asm/hw_irq.h>
40	#include <asm/msidef.h>	40	#include <asm/msidef.h>
41	#include <asm/proto.h>	41	#include <asm/proto.h>
42	#include <asm/iommu.h>	42	#include <asm/iommu.h>
43	#include <asm/gart.h>	43	#include <asm/gart.h>
44	#include <asm/dma.h>	44	#include <asm/dma.h>
45		45
46	#include "amd_iommu_proto.h"	46	#include "amd_iommu_proto.h"
47	#include "amd_iommu_types.h"	47	#include "amd_iommu_types.h"
48	#include "irq_remapping.h"	48	#include "irq_remapping.h"
49	#include "pci.h"	49	#include "pci.h"
50		50
51	#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] \|= ((t) << 28))	51	#define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] \|= ((t) << 28))
52		52
53	#define LOOP_TIMEOUT 100000	53	#define LOOP_TIMEOUT 100000
54		54
55	/*	55	/*
56	* This bitmap is used to advertise the page sizes our hardware support	56	* This bitmap is used to advertise the page sizes our hardware support
57	* to the IOMMU core, which will then use this information to split	57	* to the IOMMU core, which will then use this information to split
58	* physically contiguous memory regions it is mapping into page sizes	58	* physically contiguous memory regions it is mapping into page sizes
59	* that we support.	59	* that we support.
60	*	60	*
61	* 512GB Pages are not supported due to a hardware bug	61	* 512GB Pages are not supported due to a hardware bug
62	*/	62	*/
63	#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))	63	#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))
64		64
65	static DEFINE_RWLOCK(amd_iommu_devtable_lock);	65	static DEFINE_RWLOCK(amd_iommu_devtable_lock);
66		66
67	/* A list of preallocated protection domains */	67	/* A list of preallocated protection domains */
68	static LIST_HEAD(iommu_pd_list);	68	static LIST_HEAD(iommu_pd_list);
69	static DEFINE_SPINLOCK(iommu_pd_list_lock);	69	static DEFINE_SPINLOCK(iommu_pd_list_lock);
70		70
71	/* List of all available dev_data structures */	71	/* List of all available dev_data structures */
72	static LIST_HEAD(dev_data_list);	72	static LIST_HEAD(dev_data_list);
73	static DEFINE_SPINLOCK(dev_data_list_lock);	73	static DEFINE_SPINLOCK(dev_data_list_lock);
74		74
75	LIST_HEAD(ioapic_map);	75	LIST_HEAD(ioapic_map);
76	LIST_HEAD(hpet_map);	76	LIST_HEAD(hpet_map);
77		77
78	/*	78	/*
79	* Domain for untranslated devices - only allocated	79	* Domain for untranslated devices - only allocated
80	* if iommu=pt passed on kernel cmd line.	80	* if iommu=pt passed on kernel cmd line.
81	*/	81	*/
82	static struct protection_domain *pt_domain;	82	static struct protection_domain *pt_domain;
83		83
84	static struct iommu_ops amd_iommu_ops;	84	static struct iommu_ops amd_iommu_ops;
85		85
86	static ATOMIC_NOTIFIER_HEAD(ppr_notifier);	86	static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
87	int amd_iommu_max_glx_val = -1;	87	int amd_iommu_max_glx_val = -1;
88		88
89	static struct dma_map_ops amd_iommu_dma_ops;	89	static struct dma_map_ops amd_iommu_dma_ops;
90		90
91	/*	91	/*
92	* general struct to manage commands send to an IOMMU	92	* general struct to manage commands send to an IOMMU
93	*/	93	*/
94	struct iommu_cmd {	94	struct iommu_cmd {
95	u32 data[4];	95	u32 data[4];
96	};	96	};
97		97
98	struct kmem_cache *amd_iommu_irq_cache;	98	struct kmem_cache *amd_iommu_irq_cache;
99		99
100	static void update_domain(struct protection_domain *domain);	100	static void update_domain(struct protection_domain *domain);
101	static int __init alloc_passthrough_domain(void);	101	static int __init alloc_passthrough_domain(void);
102		102
103	/****************************************************************************	103	/****************************************************************************
104	*	104	*
105	* Helper functions	105	* Helper functions
106	*	106	*
107	****************************************************************************/	107	****************************************************************************/
108		108
109	static struct iommu_dev_data *alloc_dev_data(u16 devid)	109	static struct iommu_dev_data *alloc_dev_data(u16 devid)
110	{	110	{
111	struct iommu_dev_data *dev_data;	111	struct iommu_dev_data *dev_data;
112	unsigned long flags;	112	unsigned long flags;
113		113
114	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);	114	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
115	if (!dev_data)	115	if (!dev_data)
116	return NULL;	116	return NULL;
117		117
118	dev_data->devid = devid;	118	dev_data->devid = devid;
119	atomic_set(&dev_data->bind, 0);	119	atomic_set(&dev_data->bind, 0);
120		120
121	spin_lock_irqsave(&dev_data_list_lock, flags);	121	spin_lock_irqsave(&dev_data_list_lock, flags);
122	list_add_tail(&dev_data->dev_data_list, &dev_data_list);	122	list_add_tail(&dev_data->dev_data_list, &dev_data_list);
123	spin_unlock_irqrestore(&dev_data_list_lock, flags);	123	spin_unlock_irqrestore(&dev_data_list_lock, flags);
124		124
125	return dev_data;	125	return dev_data;
126	}	126	}
127		127
128	static void free_dev_data(struct iommu_dev_data *dev_data)	128	static void free_dev_data(struct iommu_dev_data *dev_data)
129	{	129	{
130	unsigned long flags;	130	unsigned long flags;
131		131
132	spin_lock_irqsave(&dev_data_list_lock, flags);	132	spin_lock_irqsave(&dev_data_list_lock, flags);
133	list_del(&dev_data->dev_data_list);	133	list_del(&dev_data->dev_data_list);
134	spin_unlock_irqrestore(&dev_data_list_lock, flags);	134	spin_unlock_irqrestore(&dev_data_list_lock, flags);
135		135
136	if (dev_data->group)	136	if (dev_data->group)
137	iommu_group_put(dev_data->group);	137	iommu_group_put(dev_data->group);
138		138
139	kfree(dev_data);	139	kfree(dev_data);
140	}	140	}
141		141
142	static struct iommu_dev_data *search_dev_data(u16 devid)	142	static struct iommu_dev_data *search_dev_data(u16 devid)
143	{	143	{
144	struct iommu_dev_data *dev_data;	144	struct iommu_dev_data *dev_data;
145	unsigned long flags;	145	unsigned long flags;
146		146
147	spin_lock_irqsave(&dev_data_list_lock, flags);	147	spin_lock_irqsave(&dev_data_list_lock, flags);
148	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {	148	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
149	if (dev_data->devid == devid)	149	if (dev_data->devid == devid)
150	goto out_unlock;	150	goto out_unlock;
151	}	151	}
152		152
153	dev_data = NULL;	153	dev_data = NULL;
154		154
155	out_unlock:	155	out_unlock:
156	spin_unlock_irqrestore(&dev_data_list_lock, flags);	156	spin_unlock_irqrestore(&dev_data_list_lock, flags);
157		157
158	return dev_data;	158	return dev_data;
159	}	159	}
160		160
161	static struct iommu_dev_data *find_dev_data(u16 devid)	161	static struct iommu_dev_data *find_dev_data(u16 devid)
162	{	162	{
163	struct iommu_dev_data *dev_data;	163	struct iommu_dev_data *dev_data;
164		164
165	dev_data = search_dev_data(devid);	165	dev_data = search_dev_data(devid);
166		166
167	if (dev_data == NULL)	167	if (dev_data == NULL)
168	dev_data = alloc_dev_data(devid);	168	dev_data = alloc_dev_data(devid);
169		169
170	return dev_data;	170	return dev_data;
171	}	171	}
172		172
173	static inline u16 get_device_id(struct device *dev)	173	static inline u16 get_device_id(struct device *dev)
174	{	174	{
175	struct pci_dev *pdev = to_pci_dev(dev);	175	struct pci_dev *pdev = to_pci_dev(dev);
176		176
177	return PCI_DEVID(pdev->bus->number, pdev->devfn);	177	return PCI_DEVID(pdev->bus->number, pdev->devfn);
178	}	178	}
179		179
180	static struct iommu_dev_data get_dev_data(struct device dev)	180	static struct iommu_dev_data get_dev_data(struct device dev)
181	{	181	{
182	return dev->archdata.iommu;	182	return dev->archdata.iommu;
183	}	183	}
184		184
185	static bool pci_iommuv2_capable(struct pci_dev *pdev)	185	static bool pci_iommuv2_capable(struct pci_dev *pdev)
186	{	186	{
187	static const int caps[] = {	187	static const int caps[] = {
188	PCI_EXT_CAP_ID_ATS,	188	PCI_EXT_CAP_ID_ATS,
189	PCI_EXT_CAP_ID_PRI,	189	PCI_EXT_CAP_ID_PRI,
190	PCI_EXT_CAP_ID_PASID,	190	PCI_EXT_CAP_ID_PASID,
191	};	191	};
192	int i, pos;	192	int i, pos;
193		193
194	for (i = 0; i < 3; ++i) {	194	for (i = 0; i < 3; ++i) {
195	pos = pci_find_ext_capability(pdev, caps[i]);	195	pos = pci_find_ext_capability(pdev, caps[i]);
196	if (pos == 0)	196	if (pos == 0)
197	return false;	197	return false;
198	}	198	}
199		199
200	return true;	200	return true;
201	}	201	}
202		202
203	static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)	203	static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
204	{	204	{
205	struct iommu_dev_data *dev_data;	205	struct iommu_dev_data *dev_data;
206		206
207	dev_data = get_dev_data(&pdev->dev);	207	dev_data = get_dev_data(&pdev->dev);
208		208
209	return dev_data->errata & (1 << erratum) ? true : false;	209	return dev_data->errata & (1 << erratum) ? true : false;
210	}	210	}
211		211
212	/*	212	/*
213	* In this function the list of preallocated protection domains is traversed to	213	* In this function the list of preallocated protection domains is traversed to
214	* find the domain for a specific device	214	* find the domain for a specific device
215	*/	215	*/
216	static struct dma_ops_domain *find_protection_domain(u16 devid)	216	static struct dma_ops_domain *find_protection_domain(u16 devid)
217	{	217	{
218	struct dma_ops_domain entry, ret = NULL;	218	struct dma_ops_domain entry, ret = NULL;
219	unsigned long flags;	219	unsigned long flags;
220	u16 alias = amd_iommu_alias_table[devid];	220	u16 alias = amd_iommu_alias_table[devid];
221		221
222	if (list_empty(&iommu_pd_list))	222	if (list_empty(&iommu_pd_list))
223	return NULL;	223	return NULL;
224		224
225	spin_lock_irqsave(&iommu_pd_list_lock, flags);	225	spin_lock_irqsave(&iommu_pd_list_lock, flags);
226		226
227	list_for_each_entry(entry, &iommu_pd_list, list) {	227	list_for_each_entry(entry, &iommu_pd_list, list) {
228	if (entry->target_dev == devid \|\|	228	if (entry->target_dev == devid \|\|
229	entry->target_dev == alias) {	229	entry->target_dev == alias) {
230	ret = entry;	230	ret = entry;
231	break;	231	break;
232	}	232	}
233	}	233	}
234		234
235	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);	235	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
236		236
237	return ret;	237	return ret;
238	}	238	}
239		239
240	/*	240	/*
241	* This function checks if the driver got a valid device from the caller to	241	* This function checks if the driver got a valid device from the caller to
242	* avoid dereferencing invalid pointers.	242	* avoid dereferencing invalid pointers.
243	*/	243	*/
244	static bool check_device(struct device *dev)	244	static bool check_device(struct device *dev)
245	{	245	{
246	u16 devid;	246	u16 devid;
247		247
248	if (!dev \|\| !dev->dma_mask)	248	if (!dev \|\| !dev->dma_mask)
249	return false;	249	return false;
250		250
251	/* No PCI device */	251	/* No PCI device */
252	if (!dev_is_pci(dev))	252	if (!dev_is_pci(dev))
253	return false;	253	return false;
254		254
255	devid = get_device_id(dev);	255	devid = get_device_id(dev);
256		256
257	/* Out of our scope? */	257	/* Out of our scope? */
258	if (devid > amd_iommu_last_bdf)	258	if (devid > amd_iommu_last_bdf)
259	return false;	259	return false;
260		260
261	if (amd_iommu_rlookup_table[devid] == NULL)	261	if (amd_iommu_rlookup_table[devid] == NULL)
262	return false;	262	return false;
263		263
264	return true;	264	return true;
265	}	265	}
266		266
267	static struct pci_bus find_hosted_bus(struct pci_bus bus)	267	static struct pci_bus find_hosted_bus(struct pci_bus bus)
268	{	268	{
269	while (!bus->self) {	269	while (!bus->self) {
270	if (!pci_is_root_bus(bus))	270	if (!pci_is_root_bus(bus))
271	bus = bus->parent;	271	bus = bus->parent;
272	else	272	else
273	return ERR_PTR(-ENODEV);	273	return ERR_PTR(-ENODEV);
274	}	274	}
275		275
276	return bus;	276	return bus;
277	}	277	}
278		278
279	#define REQ_ACS_FLAGS (PCI_ACS_SV \| PCI_ACS_RR \| PCI_ACS_CR \| PCI_ACS_UF)	279	#define REQ_ACS_FLAGS (PCI_ACS_SV \| PCI_ACS_RR \| PCI_ACS_CR \| PCI_ACS_UF)
280		280
281	static struct pci_dev get_isolation_root(struct pci_dev pdev)	281	static struct pci_dev get_isolation_root(struct pci_dev pdev)
282	{	282	{
283	struct pci_dev *dma_pdev = pdev;	283	struct pci_dev *dma_pdev = pdev;
284		284
285	/* Account for quirked devices */	285	/* Account for quirked devices */
286	swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));	286	swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
287		287
288	/*	288	/*
289	* If it's a multifunction device that does not support our	289	* If it's a multifunction device that does not support our
290	* required ACS flags, add to the same group as lowest numbered	290	* required ACS flags, add to the same group as lowest numbered
291	* function that also does not suport the required ACS flags.	291	* function that also does not suport the required ACS flags.
292	*/	292	*/
293	if (dma_pdev->multifunction &&	293	if (dma_pdev->multifunction &&
294	!pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {	294	!pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
295	u8 i, slot = PCI_SLOT(dma_pdev->devfn);	295	u8 i, slot = PCI_SLOT(dma_pdev->devfn);
296		296
297	for (i = 0; i < 8; i++) {	297	for (i = 0; i < 8; i++) {
298	struct pci_dev *tmp;	298	struct pci_dev *tmp;
299		299
300	tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));	300	tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
301	if (!tmp)	301	if (!tmp)
302	continue;	302	continue;
303		303
304	if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {	304	if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
305	swap_pci_ref(&dma_pdev, tmp);	305	swap_pci_ref(&dma_pdev, tmp);
306	break;	306	break;
307	}	307	}
308	pci_dev_put(tmp);	308	pci_dev_put(tmp);
309	}	309	}
310	}	310	}
311		311
312	/*	312	/*
313	* Devices on the root bus go through the iommu. If that's not us,	313	* Devices on the root bus go through the iommu. If that's not us,
314	* find the next upstream device and test ACS up to the root bus.	314	* find the next upstream device and test ACS up to the root bus.
315	* Finding the next device may require skipping virtual buses.	315	* Finding the next device may require skipping virtual buses.
316	*/	316	*/
317	while (!pci_is_root_bus(dma_pdev->bus)) {	317	while (!pci_is_root_bus(dma_pdev->bus)) {
318	struct pci_bus *bus = find_hosted_bus(dma_pdev->bus);	318	struct pci_bus *bus = find_hosted_bus(dma_pdev->bus);
319	if (IS_ERR(bus))	319	if (IS_ERR(bus))
320	break;	320	break;
321		321
322	if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))	322	if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
323	break;	323	break;
324		324
325	swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));	325	swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
326	}	326	}
327		327
328	return dma_pdev;	328	return dma_pdev;
329	}	329	}
330		330
331	static int use_pdev_iommu_group(struct pci_dev pdev, struct device dev)	331	static int use_pdev_iommu_group(struct pci_dev pdev, struct device dev)
332	{	332	{
333	struct iommu_group *group = iommu_group_get(&pdev->dev);	333	struct iommu_group *group = iommu_group_get(&pdev->dev);
334	int ret;	334	int ret;
335		335
336	if (!group) {	336	if (!group) {
337	group = iommu_group_alloc();	337	group = iommu_group_alloc();
338	if (IS_ERR(group))	338	if (IS_ERR(group))
339	return PTR_ERR(group);	339	return PTR_ERR(group);
340		340
341	WARN_ON(&pdev->dev != dev);	341	WARN_ON(&pdev->dev != dev);
342	}	342	}
343		343
344	ret = iommu_group_add_device(group, dev);	344	ret = iommu_group_add_device(group, dev);
345	iommu_group_put(group);	345	iommu_group_put(group);
346	return ret;	346	return ret;
347	}	347	}
348		348
349	static int use_dev_data_iommu_group(struct iommu_dev_data *dev_data,	349	static int use_dev_data_iommu_group(struct iommu_dev_data *dev_data,
350	struct device *dev)	350	struct device *dev)
351	{	351	{
352	if (!dev_data->group) {	352	if (!dev_data->group) {
353	struct iommu_group *group = iommu_group_alloc();	353	struct iommu_group *group = iommu_group_alloc();
354	if (IS_ERR(group))	354	if (IS_ERR(group))
355	return PTR_ERR(group);	355	return PTR_ERR(group);
356		356
357	dev_data->group = group;	357	dev_data->group = group;
358	}	358	}
359		359
360	return iommu_group_add_device(dev_data->group, dev);	360	return iommu_group_add_device(dev_data->group, dev);
361	}	361	}
362		362
363	static int init_iommu_group(struct device *dev)	363	static int init_iommu_group(struct device *dev)
364	{	364	{
365	struct iommu_dev_data *dev_data;	365	struct iommu_dev_data *dev_data;
366	struct iommu_group *group;	366	struct iommu_group *group;
367	struct pci_dev *dma_pdev;	367	struct pci_dev *dma_pdev;
368	int ret;	368	int ret;
369		369
370	group = iommu_group_get(dev);	370	group = iommu_group_get(dev);
371	if (group) {	371	if (group) {
372	iommu_group_put(group);	372	iommu_group_put(group);
373	return 0;	373	return 0;
374	}	374	}
375		375
376	dev_data = find_dev_data(get_device_id(dev));	376	dev_data = find_dev_data(get_device_id(dev));
377	if (!dev_data)	377	if (!dev_data)
378	return -ENOMEM;	378	return -ENOMEM;
379		379
380	if (dev_data->alias_data) {	380	if (dev_data->alias_data) {
381	u16 alias;	381	u16 alias;
382	struct pci_bus *bus;	382	struct pci_bus *bus;
383		383
384	if (dev_data->alias_data->group)	384	if (dev_data->alias_data->group)
385	goto use_group;	385	goto use_group;
386		386
387	/*	387	/*
388	* If the alias device exists, it's effectively just a first	388	* If the alias device exists, it's effectively just a first
389	* level quirk for finding the DMA source.	389	* level quirk for finding the DMA source.
390	*/	390	*/
391	alias = amd_iommu_alias_table[dev_data->devid];	391	alias = amd_iommu_alias_table[dev_data->devid];
392	dma_pdev = pci_get_bus_and_slot(alias >> 8, alias & 0xff);	392	dma_pdev = pci_get_bus_and_slot(alias >> 8, alias & 0xff);
393	if (dma_pdev) {	393	if (dma_pdev) {
394	dma_pdev = get_isolation_root(dma_pdev);	394	dma_pdev = get_isolation_root(dma_pdev);
395	goto use_pdev;	395	goto use_pdev;
396	}	396	}
397		397
398	/*	398	/*
399	* If the alias is virtual, try to find a parent device	399	* If the alias is virtual, try to find a parent device
400	* and test whether the IOMMU group is actualy rooted above	400	* and test whether the IOMMU group is actualy rooted above
401	* the alias. Be careful to also test the parent device if	401	* the alias. Be careful to also test the parent device if
402	* we think the alias is the root of the group.	402	* we think the alias is the root of the group.
403	*/	403	*/
404	bus = pci_find_bus(0, alias >> 8);	404	bus = pci_find_bus(0, alias >> 8);
405	if (!bus)	405	if (!bus)
406	goto use_group;	406	goto use_group;
407		407
408	bus = find_hosted_bus(bus);	408	bus = find_hosted_bus(bus);
409	if (IS_ERR(bus) \|\| !bus->self)	409	if (IS_ERR(bus) \|\| !bus->self)
410	goto use_group;	410	goto use_group;
411		411
412	dma_pdev = get_isolation_root(pci_dev_get(bus->self));	412	dma_pdev = get_isolation_root(pci_dev_get(bus->self));
413	if (dma_pdev != bus->self \|\| (dma_pdev->multifunction &&	413	if (dma_pdev != bus->self \|\| (dma_pdev->multifunction &&
414	!pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)))	414	!pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)))
415	goto use_pdev;	415	goto use_pdev;
416		416
417	pci_dev_put(dma_pdev);	417	pci_dev_put(dma_pdev);
418	goto use_group;	418	goto use_group;
419	}	419	}
420		420
421	dma_pdev = get_isolation_root(pci_dev_get(to_pci_dev(dev)));	421	dma_pdev = get_isolation_root(pci_dev_get(to_pci_dev(dev)));
422	use_pdev:	422	use_pdev:
423	ret = use_pdev_iommu_group(dma_pdev, dev);	423	ret = use_pdev_iommu_group(dma_pdev, dev);
424	pci_dev_put(dma_pdev);	424	pci_dev_put(dma_pdev);
425	return ret;	425	return ret;
426	use_group:	426	use_group:
427	return use_dev_data_iommu_group(dev_data->alias_data, dev);	427	return use_dev_data_iommu_group(dev_data->alias_data, dev);
428	}	428	}
429		429
430	static int iommu_init_device(struct device *dev)	430	static int iommu_init_device(struct device *dev)
431	{	431	{
432	struct pci_dev *pdev = to_pci_dev(dev);	432	struct pci_dev *pdev = to_pci_dev(dev);
433	struct iommu_dev_data *dev_data;	433	struct iommu_dev_data *dev_data;
434	u16 alias;	434	u16 alias;
435	int ret;	435	int ret;
436		436
437	if (dev->archdata.iommu)	437	if (dev->archdata.iommu)
438	return 0;	438	return 0;
439		439
440	dev_data = find_dev_data(get_device_id(dev));	440	dev_data = find_dev_data(get_device_id(dev));
441	if (!dev_data)	441	if (!dev_data)
442	return -ENOMEM;	442	return -ENOMEM;
443		443
444	alias = amd_iommu_alias_table[dev_data->devid];	444	alias = amd_iommu_alias_table[dev_data->devid];
445	if (alias != dev_data->devid) {	445	if (alias != dev_data->devid) {
446	struct iommu_dev_data *alias_data;	446	struct iommu_dev_data *alias_data;
447		447
448	alias_data = find_dev_data(alias);	448	alias_data = find_dev_data(alias);
449	if (alias_data == NULL) {	449	if (alias_data == NULL) {
450	pr_err("AMD-Vi: Warning: Unhandled device %s\n",	450	pr_err("AMD-Vi: Warning: Unhandled device %s\n",
451	dev_name(dev));	451	dev_name(dev));
452	free_dev_data(dev_data);	452	free_dev_data(dev_data);
453	return -ENOTSUPP;	453	return -ENOTSUPP;
454	}	454	}
455	dev_data->alias_data = alias_data;	455	dev_data->alias_data = alias_data;
456	}	456	}
457		457
458	ret = init_iommu_group(dev);	458	ret = init_iommu_group(dev);
459	if (ret) {	459	if (ret) {
460	free_dev_data(dev_data);	460	free_dev_data(dev_data);
461	return ret;	461	return ret;
462	}	462	}
463		463
464	if (pci_iommuv2_capable(pdev)) {	464	if (pci_iommuv2_capable(pdev)) {
465	struct amd_iommu *iommu;	465	struct amd_iommu *iommu;
466		466
467	iommu = amd_iommu_rlookup_table[dev_data->devid];	467	iommu = amd_iommu_rlookup_table[dev_data->devid];
468	dev_data->iommu_v2 = iommu->is_iommu_v2;	468	dev_data->iommu_v2 = iommu->is_iommu_v2;
469	}	469	}
470		470
471	dev->archdata.iommu = dev_data;	471	dev->archdata.iommu = dev_data;
472		472
473	return 0;	473	return 0;
474	}	474	}
475		475
476	static void iommu_ignore_device(struct device *dev)	476	static void iommu_ignore_device(struct device *dev)
477	{	477	{
478	u16 devid, alias;	478	u16 devid, alias;
479		479
480	devid = get_device_id(dev);	480	devid = get_device_id(dev);
481	alias = amd_iommu_alias_table[devid];	481	alias = amd_iommu_alias_table[devid];
482		482
483	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));	483	memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
484	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));	484	memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
485		485
486	amd_iommu_rlookup_table[devid] = NULL;	486	amd_iommu_rlookup_table[devid] = NULL;
487	amd_iommu_rlookup_table[alias] = NULL;	487	amd_iommu_rlookup_table[alias] = NULL;
488	}	488	}
489		489
490	static void iommu_uninit_device(struct device *dev)	490	static void iommu_uninit_device(struct device *dev)
491	{	491	{
492	iommu_group_remove_device(dev);	492	iommu_group_remove_device(dev);
493		493
494	/*	494	/*
495	* Nothing to do here - we keep dev_data around for unplugged devices	495	* Nothing to do here - we keep dev_data around for unplugged devices
496	* and reuse it when the device is re-plugged - not doing so would	496	* and reuse it when the device is re-plugged - not doing so would
497	* introduce a ton of races.	497	* introduce a ton of races.
498	*/	498	*/
499	}	499	}
500		500
501	void __init amd_iommu_uninit_devices(void)	501	void __init amd_iommu_uninit_devices(void)
502	{	502	{
503	struct iommu_dev_data dev_data, n;	503	struct iommu_dev_data dev_data, n;
504	struct pci_dev *pdev = NULL;	504	struct pci_dev *pdev = NULL;
505		505
506	for_each_pci_dev(pdev) {	506	for_each_pci_dev(pdev) {
507		507
508	if (!check_device(&pdev->dev))	508	if (!check_device(&pdev->dev))
509	continue;	509	continue;
510		510
511	iommu_uninit_device(&pdev->dev);	511	iommu_uninit_device(&pdev->dev);
512	}	512	}
513		513
514	/* Free all of our dev_data structures */	514	/* Free all of our dev_data structures */
515	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)	515	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
516	free_dev_data(dev_data);	516	free_dev_data(dev_data);
517	}	517	}
518		518
519	int __init amd_iommu_init_devices(void)	519	int __init amd_iommu_init_devices(void)
520	{	520	{
521	struct pci_dev *pdev = NULL;	521	struct pci_dev *pdev = NULL;
522	int ret = 0;	522	int ret = 0;
523		523
524	for_each_pci_dev(pdev) {	524	for_each_pci_dev(pdev) {
525		525
526	if (!check_device(&pdev->dev))	526	if (!check_device(&pdev->dev))
527	continue;	527	continue;
528		528
529	ret = iommu_init_device(&pdev->dev);	529	ret = iommu_init_device(&pdev->dev);
530	if (ret == -ENOTSUPP)	530	if (ret == -ENOTSUPP)
531	iommu_ignore_device(&pdev->dev);	531	iommu_ignore_device(&pdev->dev);
532	else if (ret)	532	else if (ret)
533	goto out_free;	533	goto out_free;
534	}	534	}
535		535
536	return 0;	536	return 0;
537		537
538	out_free:	538	out_free:
539		539
540	amd_iommu_uninit_devices();	540	amd_iommu_uninit_devices();
541		541
542	return ret;	542	return ret;
543	}	543	}
544	#ifdef CONFIG_AMD_IOMMU_STATS	544	#ifdef CONFIG_AMD_IOMMU_STATS
545		545
546	/*	546	/*
547	* Initialization code for statistics collection	547	* Initialization code for statistics collection
548	*/	548	*/
549		549
550	DECLARE_STATS_COUNTER(compl_wait);	550	DECLARE_STATS_COUNTER(compl_wait);
551	DECLARE_STATS_COUNTER(cnt_map_single);	551	DECLARE_STATS_COUNTER(cnt_map_single);
552	DECLARE_STATS_COUNTER(cnt_unmap_single);	552	DECLARE_STATS_COUNTER(cnt_unmap_single);
553	DECLARE_STATS_COUNTER(cnt_map_sg);	553	DECLARE_STATS_COUNTER(cnt_map_sg);
554	DECLARE_STATS_COUNTER(cnt_unmap_sg);	554	DECLARE_STATS_COUNTER(cnt_unmap_sg);
555	DECLARE_STATS_COUNTER(cnt_alloc_coherent);	555	DECLARE_STATS_COUNTER(cnt_alloc_coherent);
556	DECLARE_STATS_COUNTER(cnt_free_coherent);	556	DECLARE_STATS_COUNTER(cnt_free_coherent);
557	DECLARE_STATS_COUNTER(cross_page);	557	DECLARE_STATS_COUNTER(cross_page);
558	DECLARE_STATS_COUNTER(domain_flush_single);	558	DECLARE_STATS_COUNTER(domain_flush_single);
559	DECLARE_STATS_COUNTER(domain_flush_all);	559	DECLARE_STATS_COUNTER(domain_flush_all);
560	DECLARE_STATS_COUNTER(alloced_io_mem);	560	DECLARE_STATS_COUNTER(alloced_io_mem);
561	DECLARE_STATS_COUNTER(total_map_requests);	561	DECLARE_STATS_COUNTER(total_map_requests);
562	DECLARE_STATS_COUNTER(complete_ppr);	562	DECLARE_STATS_COUNTER(complete_ppr);
563	DECLARE_STATS_COUNTER(invalidate_iotlb);	563	DECLARE_STATS_COUNTER(invalidate_iotlb);
564	DECLARE_STATS_COUNTER(invalidate_iotlb_all);	564	DECLARE_STATS_COUNTER(invalidate_iotlb_all);
565	DECLARE_STATS_COUNTER(pri_requests);	565	DECLARE_STATS_COUNTER(pri_requests);
566		566
567	static struct dentry *stats_dir;	567	static struct dentry *stats_dir;
568	static struct dentry *de_fflush;	568	static struct dentry *de_fflush;
569		569
570	static void amd_iommu_stats_add(struct __iommu_counter *cnt)	570	static void amd_iommu_stats_add(struct __iommu_counter *cnt)
571	{	571	{
572	if (stats_dir == NULL)	572	if (stats_dir == NULL)
573	return;	573	return;
574		574
575	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,	575	cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
576	&cnt->value);	576	&cnt->value);
577	}	577	}
578		578
579	static void amd_iommu_stats_init(void)	579	static void amd_iommu_stats_init(void)
580	{	580	{
581	stats_dir = debugfs_create_dir("amd-iommu", NULL);	581	stats_dir = debugfs_create_dir("amd-iommu", NULL);
582	if (stats_dir == NULL)	582	if (stats_dir == NULL)
583	return;	583	return;
584		584
585	de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,	585	de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
586	&amd_iommu_unmap_flush);	586	&amd_iommu_unmap_flush);
587		587
588	amd_iommu_stats_add(&compl_wait);	588	amd_iommu_stats_add(&compl_wait);
589	amd_iommu_stats_add(&cnt_map_single);	589	amd_iommu_stats_add(&cnt_map_single);
590	amd_iommu_stats_add(&cnt_unmap_single);	590	amd_iommu_stats_add(&cnt_unmap_single);
591	amd_iommu_stats_add(&cnt_map_sg);	591	amd_iommu_stats_add(&cnt_map_sg);
592	amd_iommu_stats_add(&cnt_unmap_sg);	592	amd_iommu_stats_add(&cnt_unmap_sg);
593	amd_iommu_stats_add(&cnt_alloc_coherent);	593	amd_iommu_stats_add(&cnt_alloc_coherent);
594	amd_iommu_stats_add(&cnt_free_coherent);	594	amd_iommu_stats_add(&cnt_free_coherent);
595	amd_iommu_stats_add(&cross_page);	595	amd_iommu_stats_add(&cross_page);
596	amd_iommu_stats_add(&domain_flush_single);	596	amd_iommu_stats_add(&domain_flush_single);
597	amd_iommu_stats_add(&domain_flush_all);	597	amd_iommu_stats_add(&domain_flush_all);
598	amd_iommu_stats_add(&alloced_io_mem);	598	amd_iommu_stats_add(&alloced_io_mem);
599	amd_iommu_stats_add(&total_map_requests);	599	amd_iommu_stats_add(&total_map_requests);
600	amd_iommu_stats_add(&complete_ppr);	600	amd_iommu_stats_add(&complete_ppr);
601	amd_iommu_stats_add(&invalidate_iotlb);	601	amd_iommu_stats_add(&invalidate_iotlb);
602	amd_iommu_stats_add(&invalidate_iotlb_all);	602	amd_iommu_stats_add(&invalidate_iotlb_all);
603	amd_iommu_stats_add(&pri_requests);	603	amd_iommu_stats_add(&pri_requests);
604	}	604	}
605		605
606	#endif	606	#endif
607		607
608	/****************************************************************************	608	/****************************************************************************
609	*	609	*
610	* Interrupt handling functions	610	* Interrupt handling functions
611	*	611	*
612	****************************************************************************/	612	****************************************************************************/
613		613
614	static void dump_dte_entry(u16 devid)	614	static void dump_dte_entry(u16 devid)
615	{	615	{
616	int i;	616	int i;
617		617
618	for (i = 0; i < 4; ++i)	618	for (i = 0; i < 4; ++i)
619	pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,	619	pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
620	amd_iommu_dev_table[devid].data[i]);	620	amd_iommu_dev_table[devid].data[i]);
621	}	621	}
622		622
623	static void dump_command(unsigned long phys_addr)	623	static void dump_command(unsigned long phys_addr)
624	{	624	{
625	struct iommu_cmd *cmd = phys_to_virt(phys_addr);	625	struct iommu_cmd *cmd = phys_to_virt(phys_addr);
626	int i;	626	int i;
627		627
628	for (i = 0; i < 4; ++i)	628	for (i = 0; i < 4; ++i)
629	pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);	629	pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
630	}	630	}
631		631
632	static void iommu_print_event(struct amd_iommu iommu, void __evt)	632	static void iommu_print_event(struct amd_iommu iommu, void __evt)
633	{	633	{
634	int type, devid, domid, flags;	634	int type, devid, domid, flags;
635	volatile u32 *event = __evt;	635	volatile u32 *event = __evt;
636	int count = 0;	636	int count = 0;
637	u64 address;	637	u64 address;
638		638
639	retry:	639	retry:
640	type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;	640	type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
641	devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;	641	devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
642	domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;	642	domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
643	flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;	643	flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
644	address = (u64)(((u64)event[3]) << 32) \| event[2];	644	address = (u64)(((u64)event[3]) << 32) \| event[2];
645		645
646	if (type == 0) {	646	if (type == 0) {
647	/* Did we hit the erratum? */	647	/* Did we hit the erratum? */
648	if (++count == LOOP_TIMEOUT) {	648	if (++count == LOOP_TIMEOUT) {
649	pr_err("AMD-Vi: No event written to event log\n");	649	pr_err("AMD-Vi: No event written to event log\n");
650	return;	650	return;
651	}	651	}
652	udelay(1);	652	udelay(1);
653	goto retry;	653	goto retry;
654	}	654	}
655		655
656	printk(KERN_ERR "AMD-Vi: Event logged [");	656	printk(KERN_ERR "AMD-Vi: Event logged [");
657		657
658	switch (type) {	658	switch (type) {
659	case EVENT_TYPE_ILL_DEV:	659	case EVENT_TYPE_ILL_DEV:
660	printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "	660	printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
661	"address=0x%016llx flags=0x%04x]\n",	661	"address=0x%016llx flags=0x%04x]\n",
662	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	662	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
663	address, flags);	663	address, flags);
664	dump_dte_entry(devid);	664	dump_dte_entry(devid);
665	break;	665	break;
666	case EVENT_TYPE_IO_FAULT:	666	case EVENT_TYPE_IO_FAULT:
667	printk("IO_PAGE_FAULT device=%02x:%02x.%x "	667	printk("IO_PAGE_FAULT device=%02x:%02x.%x "
668	"domain=0x%04x address=0x%016llx flags=0x%04x]\n",	668	"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
669	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	669	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
670	domid, address, flags);	670	domid, address, flags);
671	break;	671	break;
672	case EVENT_TYPE_DEV_TAB_ERR:	672	case EVENT_TYPE_DEV_TAB_ERR:
673	printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "	673	printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
674	"address=0x%016llx flags=0x%04x]\n",	674	"address=0x%016llx flags=0x%04x]\n",
675	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	675	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
676	address, flags);	676	address, flags);
677	break;	677	break;
678	case EVENT_TYPE_PAGE_TAB_ERR:	678	case EVENT_TYPE_PAGE_TAB_ERR:
679	printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "	679	printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
680	"domain=0x%04x address=0x%016llx flags=0x%04x]\n",	680	"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
681	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	681	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
682	domid, address, flags);	682	domid, address, flags);
683	break;	683	break;
684	case EVENT_TYPE_ILL_CMD:	684	case EVENT_TYPE_ILL_CMD:
685	printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);	685	printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
686	dump_command(address);	686	dump_command(address);
687	break;	687	break;
688	case EVENT_TYPE_CMD_HARD_ERR:	688	case EVENT_TYPE_CMD_HARD_ERR:
689	printk("COMMAND_HARDWARE_ERROR address=0x%016llx "	689	printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
690	"flags=0x%04x]\n", address, flags);	690	"flags=0x%04x]\n", address, flags);
691	break;	691	break;
692	case EVENT_TYPE_IOTLB_INV_TO:	692	case EVENT_TYPE_IOTLB_INV_TO:
693	printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "	693	printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
694	"address=0x%016llx]\n",	694	"address=0x%016llx]\n",
695	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	695	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
696	address);	696	address);
697	break;	697	break;
698	case EVENT_TYPE_INV_DEV_REQ:	698	case EVENT_TYPE_INV_DEV_REQ:
699	printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "	699	printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
700	"address=0x%016llx flags=0x%04x]\n",	700	"address=0x%016llx flags=0x%04x]\n",
701	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),	701	PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),
702	address, flags);	702	address, flags);
703	break;	703	break;
704	default:	704	default:
705	printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);	705	printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
706	}	706	}
707		707
708	memset(__evt, 0, 4 * sizeof(u32));	708	memset(__evt, 0, 4 * sizeof(u32));
709	}	709	}
710		710
711	static void iommu_poll_events(struct amd_iommu *iommu)	711	static void iommu_poll_events(struct amd_iommu *iommu)
712	{	712	{
713	u32 head, tail;	713	u32 head, tail;
714		714
715	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);	715	head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
716	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);	716	tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
717		717
718	while (head != tail) {	718	while (head != tail) {
719	iommu_print_event(iommu, iommu->evt_buf + head);	719	iommu_print_event(iommu, iommu->evt_buf + head);
720	head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;	720	head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
721	}	721	}
722		722
723	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);	723	writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
724	}	724	}
725		725
726	static void iommu_handle_ppr_entry(struct amd_iommu iommu, u64 raw)	726	static void iommu_handle_ppr_entry(struct amd_iommu iommu, u64 raw)
727	{	727	{
728	struct amd_iommu_fault fault;	728	struct amd_iommu_fault fault;
729		729
730	INC_STATS_COUNTER(pri_requests);	730	INC_STATS_COUNTER(pri_requests);
731		731
732	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {	732	if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
733	pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");	733	pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
734	return;	734	return;
735	}	735	}
736		736
737	fault.address = raw[1];	737	fault.address = raw[1];
738	fault.pasid = PPR_PASID(raw[0]);	738	fault.pasid = PPR_PASID(raw[0]);
739	fault.device_id = PPR_DEVID(raw[0]);	739	fault.device_id = PPR_DEVID(raw[0]);
740	fault.tag = PPR_TAG(raw[0]);	740	fault.tag = PPR_TAG(raw[0]);
741	fault.flags = PPR_FLAGS(raw[0]);	741	fault.flags = PPR_FLAGS(raw[0]);
742		742
743	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);	743	atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
744	}	744	}
745		745
746	static void iommu_poll_ppr_log(struct amd_iommu *iommu)	746	static void iommu_poll_ppr_log(struct amd_iommu *iommu)
747	{	747	{
748	u32 head, tail;	748	u32 head, tail;
749		749
750	if (iommu->ppr_log == NULL)	750	if (iommu->ppr_log == NULL)
751	return;	751	return;
752		752
753	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);	753	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
754	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);	754	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
755		755
756	while (head != tail) {	756	while (head != tail) {
757	volatile u64 *raw;	757	volatile u64 *raw;
758	u64 entry[2];	758	u64 entry[2];
759	int i;	759	int i;
760		760
761	raw = (u64 *)(iommu->ppr_log + head);	761	raw = (u64 *)(iommu->ppr_log + head);
762		762
763	/*	763	/*
764	* Hardware bug: Interrupt may arrive before the entry is	764	* Hardware bug: Interrupt may arrive before the entry is
765	* written to memory. If this happens we need to wait for the	765	* written to memory. If this happens we need to wait for the
766	* entry to arrive.	766	* entry to arrive.
767	*/	767	*/
768	for (i = 0; i < LOOP_TIMEOUT; ++i) {	768	for (i = 0; i < LOOP_TIMEOUT; ++i) {
769	if (PPR_REQ_TYPE(raw[0]) != 0)	769	if (PPR_REQ_TYPE(raw[0]) != 0)
770	break;	770	break;
771	udelay(1);	771	udelay(1);
772	}	772	}
773		773
774	/* Avoid memcpy function-call overhead */	774	/* Avoid memcpy function-call overhead */
775	entry[0] = raw[0];	775	entry[0] = raw[0];
776	entry[1] = raw[1];	776	entry[1] = raw[1];
777		777
778	/*	778	/*
779	* To detect the hardware bug we need to clear the entry	779	* To detect the hardware bug we need to clear the entry
780	* back to zero.	780	* back to zero.
781	*/	781	*/
782	raw[0] = raw[1] = 0UL;	782	raw[0] = raw[1] = 0UL;
783		783
784	/* Update head pointer of hardware ring-buffer */	784	/* Update head pointer of hardware ring-buffer */
785	head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;	785	head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
786	writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);	786	writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
787		787
788	/* Handle PPR entry */	788	/* Handle PPR entry */
789	iommu_handle_ppr_entry(iommu, entry);	789	iommu_handle_ppr_entry(iommu, entry);
790		790
791	/* Refresh ring-buffer information */	791	/* Refresh ring-buffer information */
792	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);	792	head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
793	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);	793	tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
794	}	794	}
795	}	795	}
796		796
797	irqreturn_t amd_iommu_int_thread(int irq, void *data)	797	irqreturn_t amd_iommu_int_thread(int irq, void *data)
798	{	798	{
799	struct amd_iommu iommu = (struct amd_iommu ) data;	799	struct amd_iommu iommu = (struct amd_iommu ) data;
800	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);	800	u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
801		801
802	while (status & (MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK)) {	802	while (status & (MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK)) {
803	/* Enable EVT and PPR interrupts again */	803	/* Enable EVT and PPR interrupts again */
804	writel((MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK),	804	writel((MMIO_STATUS_EVT_INT_MASK \| MMIO_STATUS_PPR_INT_MASK),
805	iommu->mmio_base + MMIO_STATUS_OFFSET);	805	iommu->mmio_base + MMIO_STATUS_OFFSET);
806		806
807	if (status & MMIO_STATUS_EVT_INT_MASK) {	807	if (status & MMIO_STATUS_EVT_INT_MASK) {
808	pr_devel("AMD-Vi: Processing IOMMU Event Log\n");	808	pr_devel("AMD-Vi: Processing IOMMU Event Log\n");
809	iommu_poll_events(iommu);	809	iommu_poll_events(iommu);
810	}	810	}
811		811
812	if (status & MMIO_STATUS_PPR_INT_MASK) {	812	if (status & MMIO_STATUS_PPR_INT_MASK) {
813	pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");	813	pr_devel("AMD-Vi: Processing IOMMU PPR Log\n");
814	iommu_poll_ppr_log(iommu);	814	iommu_poll_ppr_log(iommu);
815	}	815	}
816		816
817	/*	817	/*
818	* Hardware bug: ERBT1312	818	* Hardware bug: ERBT1312
819	* When re-enabling interrupt (by writing 1	819	* When re-enabling interrupt (by writing 1
820	* to clear the bit), the hardware might also try to set	820	* to clear the bit), the hardware might also try to set
821	* the interrupt bit in the event status register.	821	* the interrupt bit in the event status register.
822	* In this scenario, the bit will be set, and disable	822	* In this scenario, the bit will be set, and disable
823	* subsequent interrupts.	823	* subsequent interrupts.
824	*	824	*
825	* Workaround: The IOMMU driver should read back the	825	* Workaround: The IOMMU driver should read back the
826	* status register and check if the interrupt bits are cleared.	826	* status register and check if the interrupt bits are cleared.
827	* If not, driver will need to go through the interrupt handler	827	* If not, driver will need to go through the interrupt handler
828	* again and re-clear the bits	828	* again and re-clear the bits
829	*/	829	*/
830	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);	830	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
831	}	831	}
832	return IRQ_HANDLED;	832	return IRQ_HANDLED;
833	}	833	}
834		834
835	irqreturn_t amd_iommu_int_handler(int irq, void *data)	835	irqreturn_t amd_iommu_int_handler(int irq, void *data)
836	{	836	{
837	return IRQ_WAKE_THREAD;	837	return IRQ_WAKE_THREAD;
838	}	838	}
839		839
840	/****************************************************************************	840	/****************************************************************************
841	*	841	*
842	* IOMMU command queuing functions	842	* IOMMU command queuing functions
843	*	843	*
844	****************************************************************************/	844	****************************************************************************/
845		845
846	static int wait_on_sem(volatile u64 *sem)	846	static int wait_on_sem(volatile u64 *sem)
847	{	847	{
848	int i = 0;	848	int i = 0;
849		849
850	while (*sem == 0 && i < LOOP_TIMEOUT) {	850	while (*sem == 0 && i < LOOP_TIMEOUT) {
851	udelay(1);	851	udelay(1);
852	i += 1;	852	i += 1;
853	}	853	}
854		854
855	if (i == LOOP_TIMEOUT) {	855	if (i == LOOP_TIMEOUT) {
856	pr_alert("AMD-Vi: Completion-Wait loop timed out\n");	856	pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
857	return -EIO;	857	return -EIO;
858	}	858	}
859		859
860	return 0;	860	return 0;
861	}	861	}
862		862
863	static void copy_cmd_to_buffer(struct amd_iommu *iommu,	863	static void copy_cmd_to_buffer(struct amd_iommu *iommu,
864	struct iommu_cmd *cmd,	864	struct iommu_cmd *cmd,
865	u32 tail)	865	u32 tail)
866	{	866	{
867	u8 *target;	867	u8 *target;
868		868
869	target = iommu->cmd_buf + tail;	869	target = iommu->cmd_buf + tail;
870	tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;	870	tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
871		871
872	/* Copy command to buffer */	872	/* Copy command to buffer */
873	memcpy(target, cmd, sizeof(*cmd));	873	memcpy(target, cmd, sizeof(*cmd));
874		874
875	/* Tell the IOMMU about it */	875	/* Tell the IOMMU about it */
876	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);	876	writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
877	}	877	}
878		878
879	static void build_completion_wait(struct iommu_cmd *cmd, u64 address)	879	static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
880	{	880	{
881	WARN_ON(address & 0x7ULL);	881	WARN_ON(address & 0x7ULL);
882		882
883	memset(cmd, 0, sizeof(*cmd));	883	memset(cmd, 0, sizeof(*cmd));
884	cmd->data[0] = lower_32_bits(__pa(address)) \| CMD_COMPL_WAIT_STORE_MASK;	884	cmd->data[0] = lower_32_bits(__pa(address)) \| CMD_COMPL_WAIT_STORE_MASK;
885	cmd->data[1] = upper_32_bits(__pa(address));	885	cmd->data[1] = upper_32_bits(__pa(address));
886	cmd->data[2] = 1;	886	cmd->data[2] = 1;
887	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);	887	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
888	}	888	}
889		889
890	static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)	890	static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
891	{	891	{
892	memset(cmd, 0, sizeof(*cmd));	892	memset(cmd, 0, sizeof(*cmd));
893	cmd->data[0] = devid;	893	cmd->data[0] = devid;
894	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);	894	CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
895	}	895	}
896		896
897	static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,	897	static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
898	size_t size, u16 domid, int pde)	898	size_t size, u16 domid, int pde)
899	{	899	{
900	u64 pages;	900	u64 pages;
901	int s;	901	int s;
902		902
903	pages = iommu_num_pages(address, size, PAGE_SIZE);	903	pages = iommu_num_pages(address, size, PAGE_SIZE);
904	s = 0;	904	s = 0;
905		905
906	if (pages > 1) {	906	if (pages > 1) {
907	/*	907	/*
908	* If we have to flush more than one page, flush all	908	* If we have to flush more than one page, flush all
909	* TLB entries for this domain	909	* TLB entries for this domain
910	*/	910	*/
911	address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;	911	address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
912	s = 1;	912	s = 1;
913	}	913	}
914		914
915	address &= PAGE_MASK;	915	address &= PAGE_MASK;
916		916
917	memset(cmd, 0, sizeof(*cmd));	917	memset(cmd, 0, sizeof(*cmd));
918	cmd->data[1] \|= domid;	918	cmd->data[1] \|= domid;
919	cmd->data[2] = lower_32_bits(address);	919	cmd->data[2] = lower_32_bits(address);
920	cmd->data[3] = upper_32_bits(address);	920	cmd->data[3] = upper_32_bits(address);
921	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);	921	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
922	if (s) /* size bit - we flush more than one 4kb page */	922	if (s) /* size bit - we flush more than one 4kb page */
923	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;	923	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;
924	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */	924	if (pde) /* PDE bit - we want to flush everything, not only the PTEs */
925	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_PDE_MASK;	925	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_PDE_MASK;
926	}	926	}
927		927
928	static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,	928	static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
929	u64 address, size_t size)	929	u64 address, size_t size)
930	{	930	{
931	u64 pages;	931	u64 pages;
932	int s;	932	int s;
933		933
934	pages = iommu_num_pages(address, size, PAGE_SIZE);	934	pages = iommu_num_pages(address, size, PAGE_SIZE);
935	s = 0;	935	s = 0;
936		936
937	if (pages > 1) {	937	if (pages > 1) {
938	/*	938	/*
939	* If we have to flush more than one page, flush all	939	* If we have to flush more than one page, flush all
940	* TLB entries for this domain	940	* TLB entries for this domain
941	*/	941	*/
942	address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;	942	address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
943	s = 1;	943	s = 1;
944	}	944	}
945		945
946	address &= PAGE_MASK;	946	address &= PAGE_MASK;
947		947
948	memset(cmd, 0, sizeof(*cmd));	948	memset(cmd, 0, sizeof(*cmd));
949	cmd->data[0] = devid;	949	cmd->data[0] = devid;
950	cmd->data[0] \|= (qdep & 0xff) << 24;	950	cmd->data[0] \|= (qdep & 0xff) << 24;
951	cmd->data[1] = devid;	951	cmd->data[1] = devid;
952	cmd->data[2] = lower_32_bits(address);	952	cmd->data[2] = lower_32_bits(address);
953	cmd->data[3] = upper_32_bits(address);	953	cmd->data[3] = upper_32_bits(address);
954	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);	954	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
955	if (s)	955	if (s)
956	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;	956	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;
957	}	957	}
958		958
959	static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,	959	static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
960	u64 address, bool size)	960	u64 address, bool size)
961	{	961	{
962	memset(cmd, 0, sizeof(*cmd));	962	memset(cmd, 0, sizeof(*cmd));
963		963
964	address &= ~(0xfffULL);	964	address &= ~(0xfffULL);
965		965
966	cmd->data[0] = pasid;	966	cmd->data[0] = pasid;
967	cmd->data[1] = domid;	967	cmd->data[1] = domid;
968	cmd->data[2] = lower_32_bits(address);	968	cmd->data[2] = lower_32_bits(address);
969	cmd->data[3] = upper_32_bits(address);	969	cmd->data[3] = upper_32_bits(address);
970	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_PDE_MASK;	970	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_PDE_MASK;
971	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_GN_MASK;	971	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_GN_MASK;
972	if (size)	972	if (size)
973	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;	973	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;
974	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);	974	CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
975	}	975	}
976		976
977	static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,	977	static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
978	int qdep, u64 address, bool size)	978	int qdep, u64 address, bool size)
979	{	979	{
980	memset(cmd, 0, sizeof(*cmd));	980	memset(cmd, 0, sizeof(*cmd));
981		981
982	address &= ~(0xfffULL);	982	address &= ~(0xfffULL);
983		983
984	cmd->data[0] = devid;	984	cmd->data[0] = devid;
985	cmd->data[0] \|= ((pasid >> 8) & 0xff) << 16;	985	cmd->data[0] \|= ((pasid >> 8) & 0xff) << 16;
986	cmd->data[0] \|= (qdep & 0xff) << 24;	986	cmd->data[0] \|= (qdep & 0xff) << 24;
987	cmd->data[1] = devid;	987	cmd->data[1] = devid;
988	cmd->data[1] \|= (pasid & 0xff) << 16;	988	cmd->data[1] \|= (pasid & 0xff) << 16;
989	cmd->data[2] = lower_32_bits(address);	989	cmd->data[2] = lower_32_bits(address);
990	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_GN_MASK;	990	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_GN_MASK;
991	cmd->data[3] = upper_32_bits(address);	991	cmd->data[3] = upper_32_bits(address);
992	if (size)	992	if (size)
993	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;	993	cmd->data[2] \|= CMD_INV_IOMMU_PAGES_SIZE_MASK;
994	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);	994	CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
995	}	995	}
996		996
997	static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,	997	static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
998	int status, int tag, bool gn)	998	int status, int tag, bool gn)
999	{	999	{
1000	memset(cmd, 0, sizeof(*cmd));	1000	memset(cmd, 0, sizeof(*cmd));
1001		1001
1002	cmd->data[0] = devid;	1002	cmd->data[0] = devid;
1003	if (gn) {	1003	if (gn) {
1004	cmd->data[1] = pasid;	1004	cmd->data[1] = pasid;
1005	cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;	1005	cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
1006	}	1006	}
1007	cmd->data[3] = tag & 0x1ff;	1007	cmd->data[3] = tag & 0x1ff;
1008	cmd->data[3] \|= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;	1008	cmd->data[3] \|= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
1009		1009
1010	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);	1010	CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
1011	}	1011	}
1012		1012
1013	static void build_inv_all(struct iommu_cmd *cmd)	1013	static void build_inv_all(struct iommu_cmd *cmd)
1014	{	1014	{
1015	memset(cmd, 0, sizeof(*cmd));	1015	memset(cmd, 0, sizeof(*cmd));
1016	CMD_SET_TYPE(cmd, CMD_INV_ALL);	1016	CMD_SET_TYPE(cmd, CMD_INV_ALL);
1017	}	1017	}
1018		1018
1019	static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)	1019	static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
1020	{	1020	{
1021	memset(cmd, 0, sizeof(*cmd));	1021	memset(cmd, 0, sizeof(*cmd));
1022	cmd->data[0] = devid;	1022	cmd->data[0] = devid;
1023	CMD_SET_TYPE(cmd, CMD_INV_IRT);	1023	CMD_SET_TYPE(cmd, CMD_INV_IRT);
1024	}	1024	}
1025		1025
1026	/*	1026	/*
1027	* Writes the command to the IOMMUs command buffer and informs the	1027	* Writes the command to the IOMMUs command buffer and informs the
1028	* hardware about the new command.	1028	* hardware about the new command.
1029	*/	1029	*/
1030	static int iommu_queue_command_sync(struct amd_iommu *iommu,	1030	static int iommu_queue_command_sync(struct amd_iommu *iommu,
1031	struct iommu_cmd *cmd,	1031	struct iommu_cmd *cmd,
1032	bool sync)	1032	bool sync)
1033	{	1033	{
1034	u32 left, tail, head, next_tail;	1034	u32 left, tail, head, next_tail;
1035	unsigned long flags;	1035	unsigned long flags;
1036		1036
1037	WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);	1037	WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
1038		1038
1039	again:	1039	again:
1040	spin_lock_irqsave(&iommu->lock, flags);	1040	spin_lock_irqsave(&iommu->lock, flags);
1041		1041
1042	head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);	1042	head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
1043	tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);	1043	tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
1044	next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;	1044	next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
1045	left = (head - next_tail) % iommu->cmd_buf_size;	1045	left = (head - next_tail) % iommu->cmd_buf_size;
1046		1046
1047	if (left <= 2) {	1047	if (left <= 2) {
1048	struct iommu_cmd sync_cmd;	1048	struct iommu_cmd sync_cmd;
1049	volatile u64 sem = 0;	1049	volatile u64 sem = 0;
1050	int ret;	1050	int ret;
1051		1051
1052	build_completion_wait(&sync_cmd, (u64)&sem);	1052	build_completion_wait(&sync_cmd, (u64)&sem);
1053	copy_cmd_to_buffer(iommu, &sync_cmd, tail);	1053	copy_cmd_to_buffer(iommu, &sync_cmd, tail);
1054		1054
1055	spin_unlock_irqrestore(&iommu->lock, flags);	1055	spin_unlock_irqrestore(&iommu->lock, flags);
1056		1056
1057	if ((ret = wait_on_sem(&sem)) != 0)	1057	if ((ret = wait_on_sem(&sem)) != 0)
1058	return ret;	1058	return ret;
1059		1059
1060	goto again;	1060	goto again;
1061	}	1061	}
1062		1062
1063	copy_cmd_to_buffer(iommu, cmd, tail);	1063	copy_cmd_to_buffer(iommu, cmd, tail);
1064		1064
1065	/* We need to sync now to make sure all commands are processed */	1065	/* We need to sync now to make sure all commands are processed */
1066	iommu->need_sync = sync;	1066	iommu->need_sync = sync;
1067		1067
1068	spin_unlock_irqrestore(&iommu->lock, flags);	1068	spin_unlock_irqrestore(&iommu->lock, flags);
1069		1069
1070	return 0;	1070	return 0;
1071	}	1071	}
1072		1072
1073	static int iommu_queue_command(struct amd_iommu iommu, struct iommu_cmd cmd)	1073	static int iommu_queue_command(struct amd_iommu iommu, struct iommu_cmd cmd)
1074	{	1074	{
1075	return iommu_queue_command_sync(iommu, cmd, true);	1075	return iommu_queue_command_sync(iommu, cmd, true);
1076	}	1076	}
1077		1077
1078	/*	1078	/*
1079	* This function queues a completion wait command into the command	1079	* This function queues a completion wait command into the command
1080	* buffer of an IOMMU	1080	* buffer of an IOMMU
1081	*/	1081	*/
1082	static int iommu_completion_wait(struct amd_iommu *iommu)	1082	static int iommu_completion_wait(struct amd_iommu *iommu)
1083	{	1083	{
1084	struct iommu_cmd cmd;	1084	struct iommu_cmd cmd;
1085	volatile u64 sem = 0;	1085	volatile u64 sem = 0;
1086	int ret;	1086	int ret;
1087		1087
1088	if (!iommu->need_sync)	1088	if (!iommu->need_sync)
1089	return 0;	1089	return 0;
1090		1090
1091	build_completion_wait(&cmd, (u64)&sem);	1091	build_completion_wait(&cmd, (u64)&sem);
1092		1092
1093	ret = iommu_queue_command_sync(iommu, &cmd, false);	1093	ret = iommu_queue_command_sync(iommu, &cmd, false);
1094	if (ret)	1094	if (ret)
1095	return ret;	1095	return ret;
1096		1096
1097	return wait_on_sem(&sem);	1097	return wait_on_sem(&sem);
1098	}	1098	}
1099		1099
1100	static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)	1100	static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
1101	{	1101	{
1102	struct iommu_cmd cmd;	1102	struct iommu_cmd cmd;
1103		1103
1104	build_inv_dte(&cmd, devid);	1104	build_inv_dte(&cmd, devid);
1105		1105
1106	return iommu_queue_command(iommu, &cmd);	1106	return iommu_queue_command(iommu, &cmd);
1107	}	1107	}
1108		1108
1109	static void iommu_flush_dte_all(struct amd_iommu *iommu)	1109	static void iommu_flush_dte_all(struct amd_iommu *iommu)
1110	{	1110	{
1111	u32 devid;	1111	u32 devid;
1112		1112
1113	for (devid = 0; devid <= 0xffff; ++devid)	1113	for (devid = 0; devid <= 0xffff; ++devid)
1114	iommu_flush_dte(iommu, devid);	1114	iommu_flush_dte(iommu, devid);
1115		1115
1116	iommu_completion_wait(iommu);	1116	iommu_completion_wait(iommu);
1117	}	1117	}
1118		1118
1119	/*	1119	/*
1120	* This function uses heavy locking and may disable irqs for some time. But	1120	* This function uses heavy locking and may disable irqs for some time. But
1121	* this is no issue because it is only called during resume.	1121	* this is no issue because it is only called during resume.
1122	*/	1122	*/
1123	static void iommu_flush_tlb_all(struct amd_iommu *iommu)	1123	static void iommu_flush_tlb_all(struct amd_iommu *iommu)
1124	{	1124	{
1125	u32 dom_id;	1125	u32 dom_id;
1126		1126
1127	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {	1127	for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
1128	struct iommu_cmd cmd;	1128	struct iommu_cmd cmd;
1129	build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,	1129	build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
1130	dom_id, 1);	1130	dom_id, 1);
1131	iommu_queue_command(iommu, &cmd);	1131	iommu_queue_command(iommu, &cmd);
1132	}	1132	}
1133		1133
1134	iommu_completion_wait(iommu);	1134	iommu_completion_wait(iommu);
1135	}	1135	}
1136		1136
1137	static void iommu_flush_all(struct amd_iommu *iommu)	1137	static void iommu_flush_all(struct amd_iommu *iommu)
1138	{	1138	{
1139	struct iommu_cmd cmd;	1139	struct iommu_cmd cmd;
1140		1140
1141	build_inv_all(&cmd);	1141	build_inv_all(&cmd);
1142		1142
1143	iommu_queue_command(iommu, &cmd);	1143	iommu_queue_command(iommu, &cmd);
1144	iommu_completion_wait(iommu);	1144	iommu_completion_wait(iommu);
1145	}	1145	}
1146		1146
1147	static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)	1147	static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)
1148	{	1148	{
1149	struct iommu_cmd cmd;	1149	struct iommu_cmd cmd;
1150		1150
1151	build_inv_irt(&cmd, devid);	1151	build_inv_irt(&cmd, devid);
1152		1152
1153	iommu_queue_command(iommu, &cmd);	1153	iommu_queue_command(iommu, &cmd);
1154	}	1154	}
1155		1155
1156	static void iommu_flush_irt_all(struct amd_iommu *iommu)	1156	static void iommu_flush_irt_all(struct amd_iommu *iommu)
1157	{	1157	{
1158	u32 devid;	1158	u32 devid;
1159		1159
1160	for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)	1160	for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++)
1161	iommu_flush_irt(iommu, devid);	1161	iommu_flush_irt(iommu, devid);
1162		1162
1163	iommu_completion_wait(iommu);	1163	iommu_completion_wait(iommu);
1164	}	1164	}
1165		1165
1166	void iommu_flush_all_caches(struct amd_iommu *iommu)	1166	void iommu_flush_all_caches(struct amd_iommu *iommu)
1167	{	1167	{
1168	if (iommu_feature(iommu, FEATURE_IA)) {	1168	if (iommu_feature(iommu, FEATURE_IA)) {
1169	iommu_flush_all(iommu);	1169	iommu_flush_all(iommu);
1170	} else {	1170	} else {
1171	iommu_flush_dte_all(iommu);	1171	iommu_flush_dte_all(iommu);
1172	iommu_flush_irt_all(iommu);	1172	iommu_flush_irt_all(iommu);
1173	iommu_flush_tlb_all(iommu);	1173	iommu_flush_tlb_all(iommu);
1174	}	1174	}
1175	}	1175	}
1176		1176
1177	/*	1177	/*
1178	* Command send function for flushing on-device TLB	1178	* Command send function for flushing on-device TLB
1179	*/	1179	*/
1180	static int device_flush_iotlb(struct iommu_dev_data *dev_data,	1180	static int device_flush_iotlb(struct iommu_dev_data *dev_data,
1181	u64 address, size_t size)	1181	u64 address, size_t size)
1182	{	1182	{
1183	struct amd_iommu *iommu;	1183	struct amd_iommu *iommu;
1184	struct iommu_cmd cmd;	1184	struct iommu_cmd cmd;
1185	int qdep;	1185	int qdep;
1186		1186
1187	qdep = dev_data->ats.qdep;	1187	qdep = dev_data->ats.qdep;
1188	iommu = amd_iommu_rlookup_table[dev_data->devid];	1188	iommu = amd_iommu_rlookup_table[dev_data->devid];
1189		1189
1190	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);	1190	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
1191		1191
1192	return iommu_queue_command(iommu, &cmd);	1192	return iommu_queue_command(iommu, &cmd);
1193	}	1193	}
1194		1194
1195	/*	1195	/*
1196	* Command send function for invalidating a device table entry	1196	* Command send function for invalidating a device table entry
1197	*/	1197	*/
1198	static int device_flush_dte(struct iommu_dev_data *dev_data)	1198	static int device_flush_dte(struct iommu_dev_data *dev_data)
1199	{	1199	{
1200	struct amd_iommu *iommu;	1200	struct amd_iommu *iommu;
1201	int ret;	1201	int ret;
1202		1202
1203	iommu = amd_iommu_rlookup_table[dev_data->devid];	1203	iommu = amd_iommu_rlookup_table[dev_data->devid];
1204		1204
1205	ret = iommu_flush_dte(iommu, dev_data->devid);	1205	ret = iommu_flush_dte(iommu, dev_data->devid);
1206	if (ret)	1206	if (ret)
1207	return ret;	1207	return ret;
1208		1208
1209	if (dev_data->ats.enabled)	1209	if (dev_data->ats.enabled)
1210	ret = device_flush_iotlb(dev_data, 0, ~0UL);	1210	ret = device_flush_iotlb(dev_data, 0, ~0UL);
1211		1211
1212	return ret;	1212	return ret;
1213	}	1213	}
1214		1214
1215	/*	1215	/*
1216	* TLB invalidation function which is called from the mapping functions.	1216	* TLB invalidation function which is called from the mapping functions.
1217	* It invalidates a single PTE if the range to flush is within a single	1217	* It invalidates a single PTE if the range to flush is within a single
1218	* page. Otherwise it flushes the whole TLB of the IOMMU.	1218	* page. Otherwise it flushes the whole TLB of the IOMMU.
1219	*/	1219	*/
1220	static void __domain_flush_pages(struct protection_domain *domain,	1220	static void __domain_flush_pages(struct protection_domain *domain,
1221	u64 address, size_t size, int pde)	1221	u64 address, size_t size, int pde)
1222	{	1222	{
1223	struct iommu_dev_data *dev_data;	1223	struct iommu_dev_data *dev_data;
1224	struct iommu_cmd cmd;	1224	struct iommu_cmd cmd;
1225	int ret = 0, i;	1225	int ret = 0, i;
1226		1226
1227	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);	1227	build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
1228		1228
1229	for (i = 0; i < amd_iommus_present; ++i) {	1229	for (i = 0; i < amd_iommus_present; ++i) {
1230	if (!domain->dev_iommu[i])	1230	if (!domain->dev_iommu[i])
1231	continue;	1231	continue;
1232		1232
1233	/*	1233	/*
1234	* Devices of this domain are behind this IOMMU	1234	* Devices of this domain are behind this IOMMU
1235	* We need a TLB flush	1235	* We need a TLB flush
1236	*/	1236	*/
1237	ret \|= iommu_queue_command(amd_iommus[i], &cmd);	1237	ret \|= iommu_queue_command(amd_iommus[i], &cmd);
1238	}	1238	}
1239		1239
1240	list_for_each_entry(dev_data, &domain->dev_list, list) {	1240	list_for_each_entry(dev_data, &domain->dev_list, list) {
1241		1241
1242	if (!dev_data->ats.enabled)	1242	if (!dev_data->ats.enabled)
1243	continue;	1243	continue;
1244		1244
1245	ret \|= device_flush_iotlb(dev_data, address, size);	1245	ret \|= device_flush_iotlb(dev_data, address, size);
1246	}	1246	}
1247		1247
1248	WARN_ON(ret);	1248	WARN_ON(ret);
1249	}	1249	}
1250		1250
1251	static void domain_flush_pages(struct protection_domain *domain,	1251	static void domain_flush_pages(struct protection_domain *domain,
1252	u64 address, size_t size)	1252	u64 address, size_t size)
1253	{	1253	{
1254	__domain_flush_pages(domain, address, size, 0);	1254	__domain_flush_pages(domain, address, size, 0);
1255	}	1255	}
1256		1256
1257	/* Flush the whole IO/TLB for a given protection domain */	1257	/* Flush the whole IO/TLB for a given protection domain */
1258	static void domain_flush_tlb(struct protection_domain *domain)	1258	static void domain_flush_tlb(struct protection_domain *domain)
1259	{	1259	{
1260	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);	1260	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
1261	}	1261	}
1262		1262
1263	/* Flush the whole IO/TLB for a given protection domain - including PDE */	1263	/* Flush the whole IO/TLB for a given protection domain - including PDE */
1264	static void domain_flush_tlb_pde(struct protection_domain *domain)	1264	static void domain_flush_tlb_pde(struct protection_domain *domain)
1265	{	1265	{
1266	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);	1266	__domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
1267	}	1267	}
1268		1268
1269	static void domain_flush_complete(struct protection_domain *domain)	1269	static void domain_flush_complete(struct protection_domain *domain)
1270	{	1270	{
1271	int i;	1271	int i;
1272		1272
1273	for (i = 0; i < amd_iommus_present; ++i) {	1273	for (i = 0; i < amd_iommus_present; ++i) {
1274	if (!domain->dev_iommu[i])	1274	if (!domain->dev_iommu[i])
1275	continue;	1275	continue;
1276		1276
1277	/*	1277	/*
1278	* Devices of this domain are behind this IOMMU	1278	* Devices of this domain are behind this IOMMU
1279	* We need to wait for completion of all commands.	1279	* We need to wait for completion of all commands.
1280	*/	1280	*/
1281	iommu_completion_wait(amd_iommus[i]);	1281	iommu_completion_wait(amd_iommus[i]);
1282	}	1282	}
1283	}	1283	}
1284		1284
1285		1285
1286	/*	1286	/*
1287	* This function flushes the DTEs for all devices in domain	1287	* This function flushes the DTEs for all devices in domain
1288	*/	1288	*/
1289	static void domain_flush_devices(struct protection_domain *domain)	1289	static void domain_flush_devices(struct protection_domain *domain)
1290	{	1290	{
1291	struct iommu_dev_data *dev_data;	1291	struct iommu_dev_data *dev_data;
1292		1292
1293	list_for_each_entry(dev_data, &domain->dev_list, list)	1293	list_for_each_entry(dev_data, &domain->dev_list, list)
1294	device_flush_dte(dev_data);	1294	device_flush_dte(dev_data);
1295	}	1295	}
1296		1296
1297	/****************************************************************************	1297	/****************************************************************************
1298	*	1298	*
1299	* The functions below are used the create the page table mappings for	1299	* The functions below are used the create the page table mappings for
1300	* unity mapped regions.	1300	* unity mapped regions.
1301	*	1301	*
1302	****************************************************************************/	1302	****************************************************************************/
1303		1303
1304	/*	1304	/*
1305	* This function is used to add another level to an IO page table. Adding	1305	* This function is used to add another level to an IO page table. Adding
1306	* another level increases the size of the address space by 9 bits to a size up	1306	* another level increases the size of the address space by 9 bits to a size up
1307	* to 64 bits.	1307	* to 64 bits.
1308	*/	1308	*/
1309	static bool increase_address_space(struct protection_domain *domain,	1309	static bool increase_address_space(struct protection_domain *domain,
1310	gfp_t gfp)	1310	gfp_t gfp)
1311	{	1311	{
1312	u64 *pte;	1312	u64 *pte;
1313		1313
1314	if (domain->mode == PAGE_MODE_6_LEVEL)	1314	if (domain->mode == PAGE_MODE_6_LEVEL)
1315	/* address space already 64 bit large */	1315	/* address space already 64 bit large */
1316	return false;	1316	return false;
1317		1317
1318	pte = (void *)get_zeroed_page(gfp);	1318	pte = (void *)get_zeroed_page(gfp);
1319	if (!pte)	1319	if (!pte)
1320	return false;	1320	return false;
1321		1321
1322	*pte = PM_LEVEL_PDE(domain->mode,	1322	*pte = PM_LEVEL_PDE(domain->mode,
1323	virt_to_phys(domain->pt_root));	1323	virt_to_phys(domain->pt_root));
1324	domain->pt_root = pte;	1324	domain->pt_root = pte;
1325	domain->mode += 1;	1325	domain->mode += 1;
1326	domain->updated = true;	1326	domain->updated = true;
1327		1327
1328	return true;	1328	return true;
1329	}	1329	}
1330		1330
1331	static u64 alloc_pte(struct protection_domain domain,	1331	static u64 alloc_pte(struct protection_domain domain,
1332	unsigned long address,	1332	unsigned long address,
1333	unsigned long page_size,	1333	unsigned long page_size,
1334	u64 **pte_page,	1334	u64 **pte_page,
1335	gfp_t gfp)	1335	gfp_t gfp)
1336	{	1336	{
1337	int level, end_lvl;	1337	int level, end_lvl;
1338	u64 pte, page;	1338	u64 pte, page;
1339		1339
1340	BUG_ON(!is_power_of_2(page_size));	1340	BUG_ON(!is_power_of_2(page_size));
1341		1341
1342	while (address > PM_LEVEL_SIZE(domain->mode))	1342	while (address > PM_LEVEL_SIZE(domain->mode))
1343	increase_address_space(domain, gfp);	1343	increase_address_space(domain, gfp);
1344		1344
1345	level = domain->mode - 1;	1345	level = domain->mode - 1;
1346	pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];	1346	pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1347	address = PAGE_SIZE_ALIGN(address, page_size);	1347	address = PAGE_SIZE_ALIGN(address, page_size);
1348	end_lvl = PAGE_SIZE_LEVEL(page_size);	1348	end_lvl = PAGE_SIZE_LEVEL(page_size);
1349		1349
1350	while (level > end_lvl) {	1350	while (level > end_lvl) {
1351	if (!IOMMU_PTE_PRESENT(*pte)) {	1351	if (!IOMMU_PTE_PRESENT(*pte)) {
1352	page = (u64 *)get_zeroed_page(gfp);	1352	page = (u64 *)get_zeroed_page(gfp);
1353	if (!page)	1353	if (!page)
1354	return NULL;	1354	return NULL;
1355	*pte = PM_LEVEL_PDE(level, virt_to_phys(page));	1355	*pte = PM_LEVEL_PDE(level, virt_to_phys(page));
1356	}	1356	}
1357		1357
1358	/* No level skipping support yet */	1358	/* No level skipping support yet */
1359	if (PM_PTE_LEVEL(*pte) != level)	1359	if (PM_PTE_LEVEL(*pte) != level)
1360	return NULL;	1360	return NULL;
1361		1361
1362	level -= 1;	1362	level -= 1;
1363		1363
1364	pte = IOMMU_PTE_PAGE(*pte);	1364	pte = IOMMU_PTE_PAGE(*pte);
1365		1365
1366	if (pte_page && level == end_lvl)	1366	if (pte_page && level == end_lvl)
1367	*pte_page = pte;	1367	*pte_page = pte;
1368		1368
1369	pte = &pte[PM_LEVEL_INDEX(level, address)];	1369	pte = &pte[PM_LEVEL_INDEX(level, address)];
1370	}	1370	}
1371		1371
1372	return pte;	1372	return pte;
1373	}	1373	}
1374		1374
1375	/*	1375	/*
1376	* This function checks if there is a PTE for a given dma address. If	1376	* This function checks if there is a PTE for a given dma address. If
1377	* there is one, it returns the pointer to it.	1377	* there is one, it returns the pointer to it.
1378	*/	1378	*/
1379	static u64 fetch_pte(struct protection_domain domain, unsigned long address)	1379	static u64 fetch_pte(struct protection_domain domain, unsigned long address)
1380	{	1380	{
1381	int level;	1381	int level;
1382	u64 *pte;	1382	u64 *pte;
1383		1383
1384	if (address > PM_LEVEL_SIZE(domain->mode))	1384	if (address > PM_LEVEL_SIZE(domain->mode))
1385	return NULL;	1385	return NULL;
1386		1386
1387	level = domain->mode - 1;	1387	level = domain->mode - 1;
1388	pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];	1388	pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1389		1389
1390	while (level > 0) {	1390	while (level > 0) {
1391		1391
1392	/* Not Present */	1392	/* Not Present */
1393	if (!IOMMU_PTE_PRESENT(*pte))	1393	if (!IOMMU_PTE_PRESENT(*pte))
1394	return NULL;	1394	return NULL;
1395		1395
1396	/* Large PTE */	1396	/* Large PTE */
1397	if (PM_PTE_LEVEL(*pte) == 0x07) {	1397	if (PM_PTE_LEVEL(*pte) == 0x07) {
1398	unsigned long pte_mask, __pte;	1398	unsigned long pte_mask, __pte;
1399		1399
1400	/*	1400	/*
1401	* If we have a series of large PTEs, make	1401	* If we have a series of large PTEs, make
1402	* sure to return a pointer to the first one.	1402	* sure to return a pointer to the first one.
1403	*/	1403	*/
1404	pte_mask = PTE_PAGE_SIZE(*pte);	1404	pte_mask = PTE_PAGE_SIZE(*pte);
1405	pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);	1405	pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
1406	__pte = ((unsigned long)pte) & pte_mask;	1406	__pte = ((unsigned long)pte) & pte_mask;
1407		1407
1408	return (u64 *)__pte;	1408	return (u64 *)__pte;
1409	}	1409	}
1410		1410
1411	/* No level skipping support yet */	1411	/* No level skipping support yet */
1412	if (PM_PTE_LEVEL(*pte) != level)	1412	if (PM_PTE_LEVEL(*pte) != level)
1413	return NULL;	1413	return NULL;
1414		1414
1415	level -= 1;	1415	level -= 1;
1416		1416
1417	/* Walk to the next level */	1417	/* Walk to the next level */
1418	pte = IOMMU_PTE_PAGE(*pte);	1418	pte = IOMMU_PTE_PAGE(*pte);
1419	pte = &pte[PM_LEVEL_INDEX(level, address)];	1419	pte = &pte[PM_LEVEL_INDEX(level, address)];
1420	}	1420	}
1421		1421
1422	return pte;	1422	return pte;
1423	}	1423	}
1424		1424
1425	/*	1425	/*
1426	* Generic mapping functions. It maps a physical address into a DMA	1426	* Generic mapping functions. It maps a physical address into a DMA
1427	* address space. It allocates the page table pages if necessary.	1427	* address space. It allocates the page table pages if necessary.
1428	* In the future it can be extended to a generic mapping function	1428	* In the future it can be extended to a generic mapping function
1429	* supporting all features of AMD IOMMU page tables like level skipping	1429	* supporting all features of AMD IOMMU page tables like level skipping
1430	* and full 64 bit address spaces.	1430	* and full 64 bit address spaces.
1431	*/	1431	*/
1432	static int iommu_map_page(struct protection_domain *dom,	1432	static int iommu_map_page(struct protection_domain *dom,
1433	unsigned long bus_addr,	1433	unsigned long bus_addr,
1434	unsigned long phys_addr,	1434	unsigned long phys_addr,
1435	int prot,	1435	int prot,
1436	unsigned long page_size)	1436	unsigned long page_size)
1437	{	1437	{
1438	u64 __pte, *pte;	1438	u64 __pte, *pte;
1439	int i, count;	1439	int i, count;
1440		1440
1441	if (!(prot & IOMMU_PROT_MASK))	1441	if (!(prot & IOMMU_PROT_MASK))
1442	return -EINVAL;	1442	return -EINVAL;
1443		1443
1444	bus_addr = PAGE_ALIGN(bus_addr);	1444	bus_addr = PAGE_ALIGN(bus_addr);
1445	phys_addr = PAGE_ALIGN(phys_addr);	1445	phys_addr = PAGE_ALIGN(phys_addr);
1446	count = PAGE_SIZE_PTE_COUNT(page_size);	1446	count = PAGE_SIZE_PTE_COUNT(page_size);
1447	pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);	1447	pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1448		1448
1449	for (i = 0; i < count; ++i)	1449	for (i = 0; i < count; ++i)
1450	if (IOMMU_PTE_PRESENT(pte[i]))	1450	if (IOMMU_PTE_PRESENT(pte[i]))
1451	return -EBUSY;	1451	return -EBUSY;
1452		1452
1453	if (page_size > PAGE_SIZE) {	1453	if (page_size > PAGE_SIZE) {
1454	__pte = PAGE_SIZE_PTE(phys_addr, page_size);	1454	__pte = PAGE_SIZE_PTE(phys_addr, page_size);
1455	__pte \|= PM_LEVEL_ENC(7) \| IOMMU_PTE_P \| IOMMU_PTE_FC;	1455	__pte \|= PM_LEVEL_ENC(7) \| IOMMU_PTE_P \| IOMMU_PTE_FC;
1456	} else	1456	} else
1457	__pte = phys_addr \| IOMMU_PTE_P \| IOMMU_PTE_FC;	1457	__pte = phys_addr \| IOMMU_PTE_P \| IOMMU_PTE_FC;
1458		1458
1459	if (prot & IOMMU_PROT_IR)	1459	if (prot & IOMMU_PROT_IR)
1460	__pte \|= IOMMU_PTE_IR;	1460	__pte \|= IOMMU_PTE_IR;
1461	if (prot & IOMMU_PROT_IW)	1461	if (prot & IOMMU_PROT_IW)
1462	__pte \|= IOMMU_PTE_IW;	1462	__pte \|= IOMMU_PTE_IW;
1463		1463
1464	for (i = 0; i < count; ++i)	1464	for (i = 0; i < count; ++i)
1465	pte[i] = __pte;	1465	pte[i] = __pte;
1466		1466
1467	update_domain(dom);	1467	update_domain(dom);
1468		1468
1469	return 0;	1469	return 0;
1470	}	1470	}
1471		1471
1472	static unsigned long iommu_unmap_page(struct protection_domain *dom,	1472	static unsigned long iommu_unmap_page(struct protection_domain *dom,
1473	unsigned long bus_addr,	1473	unsigned long bus_addr,
1474	unsigned long page_size)	1474	unsigned long page_size)
1475	{	1475	{
1476	unsigned long long unmap_size, unmapped;	1476	unsigned long long unmap_size, unmapped;
1477	u64 *pte;	1477	u64 *pte;
1478		1478
1479	BUG_ON(!is_power_of_2(page_size));	1479	BUG_ON(!is_power_of_2(page_size));
1480		1480
1481	unmapped = 0;	1481	unmapped = 0;
1482		1482
1483	while (unmapped < page_size) {	1483	while (unmapped < page_size) {
1484		1484
1485	pte = fetch_pte(dom, bus_addr);	1485	pte = fetch_pte(dom, bus_addr);
1486		1486
1487	if (!pte) {	1487	if (!pte) {
1488	/*	1488	/*
1489	* No PTE for this address	1489	* No PTE for this address
1490	* move forward in 4kb steps	1490	* move forward in 4kb steps
1491	*/	1491	*/
1492	unmap_size = PAGE_SIZE;	1492	unmap_size = PAGE_SIZE;
1493	} else if (PM_PTE_LEVEL(*pte) == 0) {	1493	} else if (PM_PTE_LEVEL(*pte) == 0) {
1494	/* 4kb PTE found for this address */	1494	/* 4kb PTE found for this address */
1495	unmap_size = PAGE_SIZE;	1495	unmap_size = PAGE_SIZE;
1496	*pte = 0ULL;	1496	*pte = 0ULL;
1497	} else {	1497	} else {
1498	int count, i;	1498	int count, i;
1499		1499
1500	/* Large PTE found which maps this address */	1500	/* Large PTE found which maps this address */
1501	unmap_size = PTE_PAGE_SIZE(*pte);	1501	unmap_size = PTE_PAGE_SIZE(*pte);
1502		1502
1503	/* Only unmap from the first pte in the page */	1503	/* Only unmap from the first pte in the page */
1504	if ((unmap_size - 1) & bus_addr)	1504	if ((unmap_size - 1) & bus_addr)
1505	break;	1505	break;
1506	count = PAGE_SIZE_PTE_COUNT(unmap_size);	1506	count = PAGE_SIZE_PTE_COUNT(unmap_size);
1507	for (i = 0; i < count; i++)	1507	for (i = 0; i < count; i++)
1508	pte[i] = 0ULL;	1508	pte[i] = 0ULL;
1509	}	1509	}
1510		1510
1511	bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;	1511	bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
1512	unmapped += unmap_size;	1512	unmapped += unmap_size;
1513	}	1513	}
1514		1514
1515	BUG_ON(unmapped && !is_power_of_2(unmapped));	1515	BUG_ON(unmapped && !is_power_of_2(unmapped));
1516		1516
1517	return unmapped;	1517	return unmapped;
1518	}	1518	}
1519		1519
1520	/*	1520	/*
1521	* This function checks if a specific unity mapping entry is needed for	1521	* This function checks if a specific unity mapping entry is needed for
1522	* this specific IOMMU.	1522	* this specific IOMMU.
1523	*/	1523	*/
1524	static int iommu_for_unity_map(struct amd_iommu *iommu,	1524	static int iommu_for_unity_map(struct amd_iommu *iommu,
1525	struct unity_map_entry *entry)	1525	struct unity_map_entry *entry)
1526	{	1526	{
1527	u16 bdf, i;	1527	u16 bdf, i;
1528		1528
1529	for (i = entry->devid_start; i <= entry->devid_end; ++i) {	1529	for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1530	bdf = amd_iommu_alias_table[i];	1530	bdf = amd_iommu_alias_table[i];
1531	if (amd_iommu_rlookup_table[bdf] == iommu)	1531	if (amd_iommu_rlookup_table[bdf] == iommu)
1532	return 1;	1532	return 1;
1533	}	1533	}
1534		1534
1535	return 0;	1535	return 0;
1536	}	1536	}
1537		1537
1538	/*	1538	/*
1539	* This function actually applies the mapping to the page table of the	1539	* This function actually applies the mapping to the page table of the
1540	* dma_ops domain.	1540	* dma_ops domain.
1541	*/	1541	*/
1542	static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,	1542	static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1543	struct unity_map_entry *e)	1543	struct unity_map_entry *e)
1544	{	1544	{
1545	u64 addr;	1545	u64 addr;
1546	int ret;	1546	int ret;
1547		1547
1548	for (addr = e->address_start; addr < e->address_end;	1548	for (addr = e->address_start; addr < e->address_end;
1549	addr += PAGE_SIZE) {	1549	addr += PAGE_SIZE) {
1550	ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,	1550	ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1551	PAGE_SIZE);	1551	PAGE_SIZE);
1552	if (ret)	1552	if (ret)
1553	return ret;	1553	return ret;
1554	/*	1554	/*
1555	* if unity mapping is in aperture range mark the page	1555	* if unity mapping is in aperture range mark the page
1556	* as allocated in the aperture	1556	* as allocated in the aperture
1557	*/	1557	*/
1558	if (addr < dma_dom->aperture_size)	1558	if (addr < dma_dom->aperture_size)
1559	__set_bit(addr >> PAGE_SHIFT,	1559	__set_bit(addr >> PAGE_SHIFT,
1560	dma_dom->aperture[0]->bitmap);	1560	dma_dom->aperture[0]->bitmap);
1561	}	1561	}
1562		1562
1563	return 0;	1563	return 0;
1564	}	1564	}
1565		1565
1566	/*	1566	/*
1567	* Init the unity mappings for a specific IOMMU in the system	1567	* Init the unity mappings for a specific IOMMU in the system
1568	*	1568	*
1569	* Basically iterates over all unity mapping entries and applies them to	1569	* Basically iterates over all unity mapping entries and applies them to
1570	* the default domain DMA of that IOMMU if necessary.	1570	* the default domain DMA of that IOMMU if necessary.
1571	*/	1571	*/
1572	static int iommu_init_unity_mappings(struct amd_iommu *iommu)	1572	static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1573	{	1573	{
1574	struct unity_map_entry *entry;	1574	struct unity_map_entry *entry;
1575	int ret;	1575	int ret;
1576		1576
1577	list_for_each_entry(entry, &amd_iommu_unity_map, list) {	1577	list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1578	if (!iommu_for_unity_map(iommu, entry))	1578	if (!iommu_for_unity_map(iommu, entry))
1579	continue;	1579	continue;
1580	ret = dma_ops_unity_map(iommu->default_dom, entry);	1580	ret = dma_ops_unity_map(iommu->default_dom, entry);
1581	if (ret)	1581	if (ret)
1582	return ret;	1582	return ret;
1583	}	1583	}
1584		1584
1585	return 0;	1585	return 0;
1586	}	1586	}
1587		1587
1588	/*	1588	/*
1589	* Inits the unity mappings required for a specific device	1589	* Inits the unity mappings required for a specific device
1590	*/	1590	*/
1591	static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,	1591	static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1592	u16 devid)	1592	u16 devid)
1593	{	1593	{
1594	struct unity_map_entry *e;	1594	struct unity_map_entry *e;
1595	int ret;	1595	int ret;
1596		1596
1597	list_for_each_entry(e, &amd_iommu_unity_map, list) {	1597	list_for_each_entry(e, &amd_iommu_unity_map, list) {
1598	if (!(devid >= e->devid_start && devid <= e->devid_end))	1598	if (!(devid >= e->devid_start && devid <= e->devid_end))
1599	continue;	1599	continue;
1600	ret = dma_ops_unity_map(dma_dom, e);	1600	ret = dma_ops_unity_map(dma_dom, e);
1601	if (ret)	1601	if (ret)
1602	return ret;	1602	return ret;
1603	}	1603	}
1604		1604
1605	return 0;	1605	return 0;
1606	}	1606	}
1607		1607
1608	/****************************************************************************	1608	/****************************************************************************
1609	*	1609	*
1610	* The next functions belong to the address allocator for the dma_ops	1610	* The next functions belong to the address allocator for the dma_ops
1611	* interface functions. They work like the allocators in the other IOMMU	1611	* interface functions. They work like the allocators in the other IOMMU
1612	* drivers. Its basically a bitmap which marks the allocated pages in	1612	* drivers. Its basically a bitmap which marks the allocated pages in
1613	* the aperture. Maybe it could be enhanced in the future to a more	1613	* the aperture. Maybe it could be enhanced in the future to a more
1614	* efficient allocator.	1614	* efficient allocator.
1615	*	1615	*
1616	****************************************************************************/	1616	****************************************************************************/
1617		1617
1618	/*	1618	/*
1619	* The address allocator core functions.	1619	* The address allocator core functions.
1620	*	1620	*
1621	* called with domain->lock held	1621	* called with domain->lock held
1622	*/	1622	*/
1623		1623
1624	/*	1624	/*
1625	* Used to reserve address ranges in the aperture (e.g. for exclusion	1625	* Used to reserve address ranges in the aperture (e.g. for exclusion
1626	* ranges.	1626	* ranges.
1627	*/	1627	*/
1628	static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,	1628	static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1629	unsigned long start_page,	1629	unsigned long start_page,
1630	unsigned int pages)	1630	unsigned int pages)
1631	{	1631	{
1632	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;	1632	unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1633		1633
1634	if (start_page + pages > last_page)	1634	if (start_page + pages > last_page)
1635	pages = last_page - start_page;	1635	pages = last_page - start_page;
1636		1636
1637	for (i = start_page; i < start_page + pages; ++i) {	1637	for (i = start_page; i < start_page + pages; ++i) {
1638	int index = i / APERTURE_RANGE_PAGES;	1638	int index = i / APERTURE_RANGE_PAGES;
1639	int page = i % APERTURE_RANGE_PAGES;	1639	int page = i % APERTURE_RANGE_PAGES;
1640	__set_bit(page, dom->aperture[index]->bitmap);	1640	__set_bit(page, dom->aperture[index]->bitmap);
1641	}	1641	}
1642	}	1642	}
1643		1643
1644	/*	1644	/*
1645	* This function is used to add a new aperture range to an existing	1645	* This function is used to add a new aperture range to an existing
1646	* aperture in case of dma_ops domain allocation or address allocation	1646	* aperture in case of dma_ops domain allocation or address allocation
1647	* failure.	1647	* failure.
1648	*/	1648	*/
1649	static int alloc_new_range(struct dma_ops_domain *dma_dom,	1649	static int alloc_new_range(struct dma_ops_domain *dma_dom,
1650	bool populate, gfp_t gfp)	1650	bool populate, gfp_t gfp)
1651	{	1651	{
1652	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;	1652	int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1653	struct amd_iommu *iommu;	1653	struct amd_iommu *iommu;
1654	unsigned long i, old_size;	1654	unsigned long i, old_size;
1655		1655
1656	#ifdef CONFIG_IOMMU_STRESS	1656	#ifdef CONFIG_IOMMU_STRESS
1657	populate = false;	1657	populate = false;
1658	#endif	1658	#endif
1659		1659
1660	if (index >= APERTURE_MAX_RANGES)	1660	if (index >= APERTURE_MAX_RANGES)
1661	return -ENOMEM;	1661	return -ENOMEM;
1662		1662
1663	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);	1663	dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1664	if (!dma_dom->aperture[index])	1664	if (!dma_dom->aperture[index])
1665	return -ENOMEM;	1665	return -ENOMEM;
1666		1666
1667	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);	1667	dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1668	if (!dma_dom->aperture[index]->bitmap)	1668	if (!dma_dom->aperture[index]->bitmap)
1669	goto out_free;	1669	goto out_free;
1670		1670
1671	dma_dom->aperture[index]->offset = dma_dom->aperture_size;	1671	dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1672		1672
1673	if (populate) {	1673	if (populate) {
1674	unsigned long address = dma_dom->aperture_size;	1674	unsigned long address = dma_dom->aperture_size;
1675	int i, num_ptes = APERTURE_RANGE_PAGES / 512;	1675	int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1676	u64 pte, pte_page;	1676	u64 pte, pte_page;
1677		1677
1678	for (i = 0; i < num_ptes; ++i) {	1678	for (i = 0; i < num_ptes; ++i) {
1679	pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,	1679	pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1680	&pte_page, gfp);	1680	&pte_page, gfp);
1681	if (!pte)	1681	if (!pte)
1682	goto out_free;	1682	goto out_free;
1683		1683
1684	dma_dom->aperture[index]->pte_pages[i] = pte_page;	1684	dma_dom->aperture[index]->pte_pages[i] = pte_page;
1685		1685
1686	address += APERTURE_RANGE_SIZE / 64;	1686	address += APERTURE_RANGE_SIZE / 64;
1687	}	1687	}
1688	}	1688	}
1689		1689
1690	old_size = dma_dom->aperture_size;	1690	old_size = dma_dom->aperture_size;
1691	dma_dom->aperture_size += APERTURE_RANGE_SIZE;	1691	dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1692		1692
1693	/* Reserve address range used for MSI messages */	1693	/* Reserve address range used for MSI messages */
1694	if (old_size < MSI_ADDR_BASE_LO &&	1694	if (old_size < MSI_ADDR_BASE_LO &&
1695	dma_dom->aperture_size > MSI_ADDR_BASE_LO) {	1695	dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1696	unsigned long spage;	1696	unsigned long spage;
1697	int pages;	1697	int pages;
1698		1698
1699	pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);	1699	pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1700	spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;	1700	spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1701		1701
1702	dma_ops_reserve_addresses(dma_dom, spage, pages);	1702	dma_ops_reserve_addresses(dma_dom, spage, pages);
1703	}	1703	}
1704		1704
1705	/* Initialize the exclusion range if necessary */	1705	/* Initialize the exclusion range if necessary */
1706	for_each_iommu(iommu) {	1706	for_each_iommu(iommu) {
1707	if (iommu->exclusion_start &&	1707	if (iommu->exclusion_start &&
1708	iommu->exclusion_start >= dma_dom->aperture[index]->offset	1708	iommu->exclusion_start >= dma_dom->aperture[index]->offset
1709	&& iommu->exclusion_start < dma_dom->aperture_size) {	1709	&& iommu->exclusion_start < dma_dom->aperture_size) {
1710	unsigned long startpage;	1710	unsigned long startpage;
1711	int pages = iommu_num_pages(iommu->exclusion_start,	1711	int pages = iommu_num_pages(iommu->exclusion_start,
1712	iommu->exclusion_length,	1712	iommu->exclusion_length,
1713	PAGE_SIZE);	1713	PAGE_SIZE);
1714	startpage = iommu->exclusion_start >> PAGE_SHIFT;	1714	startpage = iommu->exclusion_start >> PAGE_SHIFT;
1715	dma_ops_reserve_addresses(dma_dom, startpage, pages);	1715	dma_ops_reserve_addresses(dma_dom, startpage, pages);
1716	}	1716	}
1717	}	1717	}
1718		1718
1719	/*	1719	/*
1720	* Check for areas already mapped as present in the new aperture	1720	* Check for areas already mapped as present in the new aperture
1721	* range and mark those pages as reserved in the allocator. Such	1721	* range and mark those pages as reserved in the allocator. Such
1722	* mappings may already exist as a result of requested unity	1722	* mappings may already exist as a result of requested unity
1723	* mappings for devices.	1723	* mappings for devices.
1724	*/	1724	*/
1725	for (i = dma_dom->aperture[index]->offset;	1725	for (i = dma_dom->aperture[index]->offset;
1726	i < dma_dom->aperture_size;	1726	i < dma_dom->aperture_size;
1727	i += PAGE_SIZE) {	1727	i += PAGE_SIZE) {
1728	u64 *pte = fetch_pte(&dma_dom->domain, i);	1728	u64 *pte = fetch_pte(&dma_dom->domain, i);
1729	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))	1729	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))
1730	continue;	1730	continue;
1731		1731
1732	dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);	1732	dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1733	}	1733	}
1734		1734
1735	update_domain(&dma_dom->domain);	1735	update_domain(&dma_dom->domain);
1736		1736
1737	return 0;	1737	return 0;
1738		1738
1739	out_free:	1739	out_free:
1740	update_domain(&dma_dom->domain);	1740	update_domain(&dma_dom->domain);
1741		1741
1742	free_page((unsigned long)dma_dom->aperture[index]->bitmap);	1742	free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1743		1743
1744	kfree(dma_dom->aperture[index]);	1744	kfree(dma_dom->aperture[index]);
1745	dma_dom->aperture[index] = NULL;	1745	dma_dom->aperture[index] = NULL;
1746		1746
1747	return -ENOMEM;	1747	return -ENOMEM;
1748	}	1748	}
1749		1749
1750	static unsigned long dma_ops_area_alloc(struct device *dev,	1750	static unsigned long dma_ops_area_alloc(struct device *dev,
1751	struct dma_ops_domain *dom,	1751	struct dma_ops_domain *dom,
1752	unsigned int pages,	1752	unsigned int pages,
1753	unsigned long align_mask,	1753	unsigned long align_mask,
1754	u64 dma_mask,	1754	u64 dma_mask,
1755	unsigned long start)	1755	unsigned long start)
1756	{	1756	{
1757	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;	1757	unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1758	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;	1758	int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1759	int i = start >> APERTURE_RANGE_SHIFT;	1759	int i = start >> APERTURE_RANGE_SHIFT;
1760	unsigned long boundary_size;	1760	unsigned long boundary_size;
1761	unsigned long address = -1;	1761	unsigned long address = -1;
1762	unsigned long limit;	1762	unsigned long limit;
1763		1763
1764	next_bit >>= PAGE_SHIFT;	1764	next_bit >>= PAGE_SHIFT;
1765		1765
1766	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,	1766	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1767	PAGE_SIZE) >> PAGE_SHIFT;	1767	PAGE_SIZE) >> PAGE_SHIFT;
1768		1768
1769	for (;i < max_index; ++i) {	1769	for (;i < max_index; ++i) {
1770	unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;	1770	unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1771		1771
1772	if (dom->aperture[i]->offset >= dma_mask)	1772	if (dom->aperture[i]->offset >= dma_mask)
1773	break;	1773	break;
1774		1774
1775	limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,	1775	limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1776	dma_mask >> PAGE_SHIFT);	1776	dma_mask >> PAGE_SHIFT);
1777		1777
1778	address = iommu_area_alloc(dom->aperture[i]->bitmap,	1778	address = iommu_area_alloc(dom->aperture[i]->bitmap,
1779	limit, next_bit, pages, 0,	1779	limit, next_bit, pages, 0,
1780	boundary_size, align_mask);	1780	boundary_size, align_mask);
1781	if (address != -1) {	1781	if (address != -1) {
1782	address = dom->aperture[i]->offset +	1782	address = dom->aperture[i]->offset +
1783	(address << PAGE_SHIFT);	1783	(address << PAGE_SHIFT);
1784	dom->next_address = address + (pages << PAGE_SHIFT);	1784	dom->next_address = address + (pages << PAGE_SHIFT);
1785	break;	1785	break;
1786	}	1786	}
1787		1787
1788	next_bit = 0;	1788	next_bit = 0;
1789	}	1789	}
1790		1790
1791	return address;	1791	return address;
1792	}	1792	}
1793		1793
1794	static unsigned long dma_ops_alloc_addresses(struct device *dev,	1794	static unsigned long dma_ops_alloc_addresses(struct device *dev,
1795	struct dma_ops_domain *dom,	1795	struct dma_ops_domain *dom,
1796	unsigned int pages,	1796	unsigned int pages,
1797	unsigned long align_mask,	1797	unsigned long align_mask,
1798	u64 dma_mask)	1798	u64 dma_mask)
1799	{	1799	{
1800	unsigned long address;	1800	unsigned long address;
1801		1801
1802	#ifdef CONFIG_IOMMU_STRESS	1802	#ifdef CONFIG_IOMMU_STRESS
1803	dom->next_address = 0;	1803	dom->next_address = 0;
1804	dom->need_flush = true;	1804	dom->need_flush = true;
1805	#endif	1805	#endif
1806		1806
1807	address = dma_ops_area_alloc(dev, dom, pages, align_mask,	1807	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1808	dma_mask, dom->next_address);	1808	dma_mask, dom->next_address);
1809		1809
1810	if (address == -1) {	1810	if (address == -1) {
1811	dom->next_address = 0;	1811	dom->next_address = 0;
1812	address = dma_ops_area_alloc(dev, dom, pages, align_mask,	1812	address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1813	dma_mask, 0);	1813	dma_mask, 0);
1814	dom->need_flush = true;	1814	dom->need_flush = true;
1815	}	1815	}
1816		1816
1817	if (unlikely(address == -1))	1817	if (unlikely(address == -1))
1818	address = DMA_ERROR_CODE;	1818	address = DMA_ERROR_CODE;
1819		1819
1820	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);	1820	WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1821		1821
1822	return address;	1822	return address;
1823	}	1823	}
1824		1824
1825	/*	1825	/*
1826	* The address free function.	1826	* The address free function.
1827	*	1827	*
1828	* called with domain->lock held	1828	* called with domain->lock held
1829	*/	1829	*/
1830	static void dma_ops_free_addresses(struct dma_ops_domain *dom,	1830	static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1831	unsigned long address,	1831	unsigned long address,
1832	unsigned int pages)	1832	unsigned int pages)
1833	{	1833	{
1834	unsigned i = address >> APERTURE_RANGE_SHIFT;	1834	unsigned i = address >> APERTURE_RANGE_SHIFT;
1835	struct aperture_range *range = dom->aperture[i];	1835	struct aperture_range *range = dom->aperture[i];
1836		1836
1837	BUG_ON(i >= APERTURE_MAX_RANGES \|\| range == NULL);	1837	BUG_ON(i >= APERTURE_MAX_RANGES \|\| range == NULL);
1838		1838
1839	#ifdef CONFIG_IOMMU_STRESS	1839	#ifdef CONFIG_IOMMU_STRESS
1840	if (i < 4)	1840	if (i < 4)
1841	return;	1841	return;
1842	#endif	1842	#endif
1843		1843
1844	if (address >= dom->next_address)	1844	if (address >= dom->next_address)
1845	dom->need_flush = true;	1845	dom->need_flush = true;
1846		1846
1847	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;	1847	address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1848		1848
1849	bitmap_clear(range->bitmap, address, pages);	1849	bitmap_clear(range->bitmap, address, pages);
1850		1850
1851	}	1851	}
1852		1852
1853	/****************************************************************************	1853	/****************************************************************************
1854	*	1854	*
1855	* The next functions belong to the domain allocation. A domain is	1855	* The next functions belong to the domain allocation. A domain is
1856	* allocated for every IOMMU as the default domain. If device isolation	1856	* allocated for every IOMMU as the default domain. If device isolation
1857	* is enabled, every device get its own domain. The most important thing	1857	* is enabled, every device get its own domain. The most important thing
1858	* about domains is the page table mapping the DMA address space they	1858	* about domains is the page table mapping the DMA address space they
1859	* contain.	1859	* contain.
1860	*	1860	*
1861	****************************************************************************/	1861	****************************************************************************/
1862		1862
1863	/*	1863	/*
1864	* This function adds a protection domain to the global protection domain list	1864	* This function adds a protection domain to the global protection domain list
1865	*/	1865	*/
1866	static void add_domain_to_list(struct protection_domain *domain)	1866	static void add_domain_to_list(struct protection_domain *domain)
1867	{	1867	{
1868	unsigned long flags;	1868	unsigned long flags;
1869		1869
1870	spin_lock_irqsave(&amd_iommu_pd_lock, flags);	1870	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1871	list_add(&domain->list, &amd_iommu_pd_list);	1871	list_add(&domain->list, &amd_iommu_pd_list);
1872	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);	1872	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1873	}	1873	}
1874		1874
1875	/*	1875	/*
1876	* This function removes a protection domain to the global	1876	* This function removes a protection domain to the global
1877	* protection domain list	1877	* protection domain list
1878	*/	1878	*/
1879	static void del_domain_from_list(struct protection_domain *domain)	1879	static void del_domain_from_list(struct protection_domain *domain)
1880	{	1880	{
1881	unsigned long flags;	1881	unsigned long flags;
1882		1882
1883	spin_lock_irqsave(&amd_iommu_pd_lock, flags);	1883	spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1884	list_del(&domain->list);	1884	list_del(&domain->list);
1885	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);	1885	spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1886	}	1886	}
1887		1887
1888	static u16 domain_id_alloc(void)	1888	static u16 domain_id_alloc(void)
1889	{	1889	{
1890	unsigned long flags;	1890	unsigned long flags;
1891	int id;	1891	int id;
1892		1892
1893	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	1893	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1894	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);	1894	id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1895	BUG_ON(id == 0);	1895	BUG_ON(id == 0);
1896	if (id > 0 && id < MAX_DOMAIN_ID)	1896	if (id > 0 && id < MAX_DOMAIN_ID)
1897	__set_bit(id, amd_iommu_pd_alloc_bitmap);	1897	__set_bit(id, amd_iommu_pd_alloc_bitmap);
1898	else	1898	else
1899	id = 0;	1899	id = 0;
1900	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	1900	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1901		1901
1902	return id;	1902	return id;
1903	}	1903	}
1904		1904
1905	static void domain_id_free(int id)	1905	static void domain_id_free(int id)
1906	{	1906	{
1907	unsigned long flags;	1907	unsigned long flags;
1908		1908
1909	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	1909	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1910	if (id > 0 && id < MAX_DOMAIN_ID)	1910	if (id > 0 && id < MAX_DOMAIN_ID)
1911	__clear_bit(id, amd_iommu_pd_alloc_bitmap);	1911	__clear_bit(id, amd_iommu_pd_alloc_bitmap);
1912	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	1912	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1913	}	1913	}
1914		1914
1915	#define DEFINE_FREE_PT_FN(LVL, FN) \	1915	#define DEFINE_FREE_PT_FN(LVL, FN) \
1916	static void free_pt_##LVL (unsigned long __pt) \	1916	static void free_pt_##LVL (unsigned long __pt) \
1917	{ \	1917	{ \
1918	unsigned long p; \	1918	unsigned long p; \
1919	u64 *pt; \	1919	u64 *pt; \
1920	int i; \	1920	int i; \
1921	\	1921	\
1922	pt = (u64 *)__pt; \	1922	pt = (u64 *)__pt; \
1923	\	1923	\
1924	for (i = 0; i < 512; ++i) { \	1924	for (i = 0; i < 512; ++i) { \
1925	if (!IOMMU_PTE_PRESENT(pt[i])) \	1925	if (!IOMMU_PTE_PRESENT(pt[i])) \
1926	continue; \	1926	continue; \
1927	\	1927	\
1928	p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \	1928	p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \
1929	FN(p); \	1929	FN(p); \
1930	} \	1930	} \
1931	free_page((unsigned long)pt); \	1931	free_page((unsigned long)pt); \
1932	}	1932	}
1933		1933
1934	DEFINE_FREE_PT_FN(l2, free_page)	1934	DEFINE_FREE_PT_FN(l2, free_page)
1935	DEFINE_FREE_PT_FN(l3, free_pt_l2)	1935	DEFINE_FREE_PT_FN(l3, free_pt_l2)
1936	DEFINE_FREE_PT_FN(l4, free_pt_l3)	1936	DEFINE_FREE_PT_FN(l4, free_pt_l3)
1937	DEFINE_FREE_PT_FN(l5, free_pt_l4)	1937	DEFINE_FREE_PT_FN(l5, free_pt_l4)
1938	DEFINE_FREE_PT_FN(l6, free_pt_l5)	1938	DEFINE_FREE_PT_FN(l6, free_pt_l5)
1939		1939
1940	static void free_pagetable(struct protection_domain *domain)	1940	static void free_pagetable(struct protection_domain *domain)
1941	{	1941	{
1942	unsigned long root = (unsigned long)domain->pt_root;	1942	unsigned long root = (unsigned long)domain->pt_root;
1943		1943
1944	switch (domain->mode) {	1944	switch (domain->mode) {
1945	case PAGE_MODE_NONE:	1945	case PAGE_MODE_NONE:
1946	break;	1946	break;
1947	case PAGE_MODE_1_LEVEL:	1947	case PAGE_MODE_1_LEVEL:
1948	free_page(root);	1948	free_page(root);
1949	break;	1949	break;
1950	case PAGE_MODE_2_LEVEL:	1950	case PAGE_MODE_2_LEVEL:
1951	free_pt_l2(root);	1951	free_pt_l2(root);
1952	break;	1952	break;
1953	case PAGE_MODE_3_LEVEL:	1953	case PAGE_MODE_3_LEVEL:
1954	free_pt_l3(root);	1954	free_pt_l3(root);
1955	break;	1955	break;
1956	case PAGE_MODE_4_LEVEL:	1956	case PAGE_MODE_4_LEVEL:
1957	free_pt_l4(root);	1957	free_pt_l4(root);
1958	break;	1958	break;
1959	case PAGE_MODE_5_LEVEL:	1959	case PAGE_MODE_5_LEVEL:
1960	free_pt_l5(root);	1960	free_pt_l5(root);
1961	break;	1961	break;
1962	case PAGE_MODE_6_LEVEL:	1962	case PAGE_MODE_6_LEVEL:
1963	free_pt_l6(root);	1963	free_pt_l6(root);
1964	break;	1964	break;
1965	default:	1965	default:
1966	BUG();	1966	BUG();
1967	}	1967	}
1968	}	1968	}
1969		1969
1970	static void free_gcr3_tbl_level1(u64 *tbl)	1970	static void free_gcr3_tbl_level1(u64 *tbl)
1971	{	1971	{
1972	u64 *ptr;	1972	u64 *ptr;
1973	int i;	1973	int i;
1974		1974
1975	for (i = 0; i < 512; ++i) {	1975	for (i = 0; i < 512; ++i) {
1976	if (!(tbl[i] & GCR3_VALID))	1976	if (!(tbl[i] & GCR3_VALID))
1977	continue;	1977	continue;
1978		1978
1979	ptr = __va(tbl[i] & PAGE_MASK);	1979	ptr = __va(tbl[i] & PAGE_MASK);
1980		1980
1981	free_page((unsigned long)ptr);	1981	free_page((unsigned long)ptr);
1982	}	1982	}
1983	}	1983	}
1984		1984
1985	static void free_gcr3_tbl_level2(u64 *tbl)	1985	static void free_gcr3_tbl_level2(u64 *tbl)
1986	{	1986	{
1987	u64 *ptr;	1987	u64 *ptr;
1988	int i;	1988	int i;
1989		1989
1990	for (i = 0; i < 512; ++i) {	1990	for (i = 0; i < 512; ++i) {
1991	if (!(tbl[i] & GCR3_VALID))	1991	if (!(tbl[i] & GCR3_VALID))
1992	continue;	1992	continue;
1993		1993
1994	ptr = __va(tbl[i] & PAGE_MASK);	1994	ptr = __va(tbl[i] & PAGE_MASK);
1995		1995
1996	free_gcr3_tbl_level1(ptr);	1996	free_gcr3_tbl_level1(ptr);
1997	}	1997	}
1998	}	1998	}
1999		1999
2000	static void free_gcr3_table(struct protection_domain *domain)	2000	static void free_gcr3_table(struct protection_domain *domain)
2001	{	2001	{
2002	if (domain->glx == 2)	2002	if (domain->glx == 2)
2003	free_gcr3_tbl_level2(domain->gcr3_tbl);	2003	free_gcr3_tbl_level2(domain->gcr3_tbl);
2004	else if (domain->glx == 1)	2004	else if (domain->glx == 1)
2005	free_gcr3_tbl_level1(domain->gcr3_tbl);	2005	free_gcr3_tbl_level1(domain->gcr3_tbl);
2006	else if (domain->glx != 0)	2006	else if (domain->glx != 0)
2007	BUG();	2007	BUG();
2008		2008
2009	free_page((unsigned long)domain->gcr3_tbl);	2009	free_page((unsigned long)domain->gcr3_tbl);
2010	}	2010	}
2011		2011
2012	/*	2012	/*
2013	* Free a domain, only used if something went wrong in the	2013	* Free a domain, only used if something went wrong in the
2014	* allocation path and we need to free an already allocated page table	2014	* allocation path and we need to free an already allocated page table
2015	*/	2015	*/
2016	static void dma_ops_domain_free(struct dma_ops_domain *dom)	2016	static void dma_ops_domain_free(struct dma_ops_domain *dom)
2017	{	2017	{
2018	int i;	2018	int i;
2019		2019
2020	if (!dom)	2020	if (!dom)
2021	return;	2021	return;
2022		2022
2023	del_domain_from_list(&dom->domain);	2023	del_domain_from_list(&dom->domain);
2024		2024
2025	free_pagetable(&dom->domain);	2025	free_pagetable(&dom->domain);
2026		2026
2027	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {	2027	for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
2028	if (!dom->aperture[i])	2028	if (!dom->aperture[i])
2029	continue;	2029	continue;
2030	free_page((unsigned long)dom->aperture[i]->bitmap);	2030	free_page((unsigned long)dom->aperture[i]->bitmap);
2031	kfree(dom->aperture[i]);	2031	kfree(dom->aperture[i]);
2032	}	2032	}
2033		2033
2034	kfree(dom);	2034	kfree(dom);
2035	}	2035	}
2036		2036
2037	/*	2037	/*
2038	* Allocates a new protection domain usable for the dma_ops functions.	2038	* Allocates a new protection domain usable for the dma_ops functions.
2039	* It also initializes the page table and the address allocator data	2039	* It also initializes the page table and the address allocator data
2040	* structures required for the dma_ops interface	2040	* structures required for the dma_ops interface
2041	*/	2041	*/
2042	static struct dma_ops_domain *dma_ops_domain_alloc(void)	2042	static struct dma_ops_domain *dma_ops_domain_alloc(void)
2043	{	2043	{
2044	struct dma_ops_domain *dma_dom;	2044	struct dma_ops_domain *dma_dom;
2045		2045
2046	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);	2046	dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
2047	if (!dma_dom)	2047	if (!dma_dom)
2048	return NULL;	2048	return NULL;
2049		2049
2050	spin_lock_init(&dma_dom->domain.lock);	2050	spin_lock_init(&dma_dom->domain.lock);
2051		2051
2052	dma_dom->domain.id = domain_id_alloc();	2052	dma_dom->domain.id = domain_id_alloc();
2053	if (dma_dom->domain.id == 0)	2053	if (dma_dom->domain.id == 0)
2054	goto free_dma_dom;	2054	goto free_dma_dom;
2055	INIT_LIST_HEAD(&dma_dom->domain.dev_list);	2055	INIT_LIST_HEAD(&dma_dom->domain.dev_list);
2056	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;	2056	dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2057	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);	2057	dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2058	dma_dom->domain.flags = PD_DMA_OPS_MASK;	2058	dma_dom->domain.flags = PD_DMA_OPS_MASK;
2059	dma_dom->domain.priv = dma_dom;	2059	dma_dom->domain.priv = dma_dom;
2060	if (!dma_dom->domain.pt_root)	2060	if (!dma_dom->domain.pt_root)
2061	goto free_dma_dom;	2061	goto free_dma_dom;
2062		2062
2063	dma_dom->need_flush = false;	2063	dma_dom->need_flush = false;
2064	dma_dom->target_dev = 0xffff;	2064	dma_dom->target_dev = 0xffff;
2065		2065
2066	add_domain_to_list(&dma_dom->domain);	2066	add_domain_to_list(&dma_dom->domain);
2067		2067
2068	if (alloc_new_range(dma_dom, true, GFP_KERNEL))	2068	if (alloc_new_range(dma_dom, true, GFP_KERNEL))
2069	goto free_dma_dom;	2069	goto free_dma_dom;
2070		2070
2071	/*	2071	/*
2072	* mark the first page as allocated so we never return 0 as	2072	* mark the first page as allocated so we never return 0 as
2073	* a valid dma-address. So we can use 0 as error value	2073	* a valid dma-address. So we can use 0 as error value
2074	*/	2074	*/
2075	dma_dom->aperture[0]->bitmap[0] = 1;	2075	dma_dom->aperture[0]->bitmap[0] = 1;
2076	dma_dom->next_address = 0;	2076	dma_dom->next_address = 0;
2077		2077
2078		2078
2079	return dma_dom;	2079	return dma_dom;
2080		2080
2081	free_dma_dom:	2081	free_dma_dom:
2082	dma_ops_domain_free(dma_dom);	2082	dma_ops_domain_free(dma_dom);
2083		2083
2084	return NULL;	2084	return NULL;
2085	}	2085	}
2086		2086
2087	/*	2087	/*
2088	* little helper function to check whether a given protection domain is a	2088	* little helper function to check whether a given protection domain is a
2089	* dma_ops domain	2089	* dma_ops domain
2090	*/	2090	*/
2091	static bool dma_ops_domain(struct protection_domain *domain)	2091	static bool dma_ops_domain(struct protection_domain *domain)
2092	{	2092	{
2093	return domain->flags & PD_DMA_OPS_MASK;	2093	return domain->flags & PD_DMA_OPS_MASK;
2094	}	2094	}
2095		2095
2096	static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)	2096	static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
2097	{	2097	{
2098	u64 pte_root = 0;	2098	u64 pte_root = 0;
2099	u64 flags = 0;	2099	u64 flags = 0;
2100		2100
2101	if (domain->mode != PAGE_MODE_NONE)	2101	if (domain->mode != PAGE_MODE_NONE)
2102	pte_root = virt_to_phys(domain->pt_root);	2102	pte_root = virt_to_phys(domain->pt_root);
2103		2103
2104	pte_root \|= (domain->mode & DEV_ENTRY_MODE_MASK)	2104	pte_root \|= (domain->mode & DEV_ENTRY_MODE_MASK)
2105	<< DEV_ENTRY_MODE_SHIFT;	2105	<< DEV_ENTRY_MODE_SHIFT;
2106	pte_root \|= IOMMU_PTE_IR \| IOMMU_PTE_IW \| IOMMU_PTE_P \| IOMMU_PTE_TV;	2106	pte_root \|= IOMMU_PTE_IR \| IOMMU_PTE_IW \| IOMMU_PTE_P \| IOMMU_PTE_TV;
2107		2107
2108	flags = amd_iommu_dev_table[devid].data[1];	2108	flags = amd_iommu_dev_table[devid].data[1];
2109		2109
2110	if (ats)	2110	if (ats)
2111	flags \|= DTE_FLAG_IOTLB;	2111	flags \|= DTE_FLAG_IOTLB;
2112		2112
2113	if (domain->flags & PD_IOMMUV2_MASK) {	2113	if (domain->flags & PD_IOMMUV2_MASK) {
2114	u64 gcr3 = __pa(domain->gcr3_tbl);	2114	u64 gcr3 = __pa(domain->gcr3_tbl);
2115	u64 glx = domain->glx;	2115	u64 glx = domain->glx;
2116	u64 tmp;	2116	u64 tmp;
2117		2117
2118	pte_root \|= DTE_FLAG_GV;	2118	pte_root \|= DTE_FLAG_GV;
2119	pte_root \|= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;	2119	pte_root \|= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
2120		2120
2121	/* First mask out possible old values for GCR3 table */	2121	/* First mask out possible old values for GCR3 table */
2122	tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;	2122	tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
2123	flags &= ~tmp;	2123	flags &= ~tmp;
2124		2124
2125	tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;	2125	tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
2126	flags &= ~tmp;	2126	flags &= ~tmp;
2127		2127
2128	/* Encode GCR3 table into DTE */	2128	/* Encode GCR3 table into DTE */
2129	tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;	2129	tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
2130	pte_root \|= tmp;	2130	pte_root \|= tmp;
2131		2131
2132	tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;	2132	tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
2133	flags \|= tmp;	2133	flags \|= tmp;
2134		2134
2135	tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;	2135	tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
2136	flags \|= tmp;	2136	flags \|= tmp;
2137	}	2137	}
2138		2138
2139	flags &= ~(0xffffUL);	2139	flags &= ~(0xffffUL);
2140	flags \|= domain->id;	2140	flags \|= domain->id;
2141		2141
2142	amd_iommu_dev_table[devid].data[1] = flags;	2142	amd_iommu_dev_table[devid].data[1] = flags;
2143	amd_iommu_dev_table[devid].data[0] = pte_root;	2143	amd_iommu_dev_table[devid].data[0] = pte_root;
2144	}	2144	}
2145		2145
2146	static void clear_dte_entry(u16 devid)	2146	static void clear_dte_entry(u16 devid)
2147	{	2147	{
2148	/* remove entry from the device table seen by the hardware */	2148	/* remove entry from the device table seen by the hardware */
2149	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P \| IOMMU_PTE_TV;	2149	amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P \| IOMMU_PTE_TV;
2150	amd_iommu_dev_table[devid].data[1] = 0;	2150	amd_iommu_dev_table[devid].data[1] = 0;
2151		2151
2152	amd_iommu_apply_erratum_63(devid);	2152	amd_iommu_apply_erratum_63(devid);
2153	}	2153	}
2154		2154
2155	static void do_attach(struct iommu_dev_data *dev_data,	2155	static void do_attach(struct iommu_dev_data *dev_data,
2156	struct protection_domain *domain)	2156	struct protection_domain *domain)
2157	{	2157	{
2158	struct amd_iommu *iommu;	2158	struct amd_iommu *iommu;
2159	bool ats;	2159	bool ats;
2160		2160
2161	iommu = amd_iommu_rlookup_table[dev_data->devid];	2161	iommu = amd_iommu_rlookup_table[dev_data->devid];
2162	ats = dev_data->ats.enabled;	2162	ats = dev_data->ats.enabled;
2163		2163
2164	/* Update data structures */	2164	/* Update data structures */
2165	dev_data->domain = domain;	2165	dev_data->domain = domain;
2166	list_add(&dev_data->list, &domain->dev_list);	2166	list_add(&dev_data->list, &domain->dev_list);
2167	set_dte_entry(dev_data->devid, domain, ats);	2167	set_dte_entry(dev_data->devid, domain, ats);
2168		2168
2169	/* Do reference counting */	2169	/* Do reference counting */
2170	domain->dev_iommu[iommu->index] += 1;	2170	domain->dev_iommu[iommu->index] += 1;
2171	domain->dev_cnt += 1;	2171	domain->dev_cnt += 1;
2172		2172
2173	/* Flush the DTE entry */	2173	/* Flush the DTE entry */
2174	device_flush_dte(dev_data);	2174	device_flush_dte(dev_data);
2175	}	2175	}
2176		2176
2177	static void do_detach(struct iommu_dev_data *dev_data)	2177	static void do_detach(struct iommu_dev_data *dev_data)
2178	{	2178	{
2179	struct amd_iommu *iommu;	2179	struct amd_iommu *iommu;
2180		2180
2181	iommu = amd_iommu_rlookup_table[dev_data->devid];	2181	iommu = amd_iommu_rlookup_table[dev_data->devid];
2182		2182
2183	/* decrease reference counters */	2183	/* decrease reference counters */
2184	dev_data->domain->dev_iommu[iommu->index] -= 1;	2184	dev_data->domain->dev_iommu[iommu->index] -= 1;
2185	dev_data->domain->dev_cnt -= 1;	2185	dev_data->domain->dev_cnt -= 1;
2186		2186
2187	/* Update data structures */	2187	/* Update data structures */
2188	dev_data->domain = NULL;	2188	dev_data->domain = NULL;
2189	list_del(&dev_data->list);	2189	list_del(&dev_data->list);
2190	clear_dte_entry(dev_data->devid);	2190	clear_dte_entry(dev_data->devid);
2191		2191
2192	/* Flush the DTE entry */	2192	/* Flush the DTE entry */
2193	device_flush_dte(dev_data);	2193	device_flush_dte(dev_data);
2194	}	2194	}
2195		2195
2196	/*	2196	/*
2197	* If a device is not yet associated with a domain, this function does	2197	* If a device is not yet associated with a domain, this function does
2198	* assigns it visible for the hardware	2198	* assigns it visible for the hardware
2199	*/	2199	*/
2200	static int __attach_device(struct iommu_dev_data *dev_data,	2200	static int __attach_device(struct iommu_dev_data *dev_data,
2201	struct protection_domain *domain)	2201	struct protection_domain *domain)
2202	{	2202	{
2203	int ret;	2203	int ret;
2204		2204
2205	/* lock domain */	2205	/* lock domain */
2206	spin_lock(&domain->lock);	2206	spin_lock(&domain->lock);
2207		2207
2208	if (dev_data->alias_data != NULL) {	2208	if (dev_data->alias_data != NULL) {
2209	struct iommu_dev_data *alias_data = dev_data->alias_data;	2209	struct iommu_dev_data *alias_data = dev_data->alias_data;
2210		2210
2211	/* Some sanity checks */	2211	/* Some sanity checks */
2212	ret = -EBUSY;	2212	ret = -EBUSY;
2213	if (alias_data->domain != NULL &&	2213	if (alias_data->domain != NULL &&
2214	alias_data->domain != domain)	2214	alias_data->domain != domain)
2215	goto out_unlock;	2215	goto out_unlock;
2216		2216
2217	if (dev_data->domain != NULL &&	2217	if (dev_data->domain != NULL &&
2218	dev_data->domain != domain)	2218	dev_data->domain != domain)
2219	goto out_unlock;	2219	goto out_unlock;
2220		2220
2221	/* Do real assignment */	2221	/* Do real assignment */
2222	if (alias_data->domain == NULL)	2222	if (alias_data->domain == NULL)
2223	do_attach(alias_data, domain);	2223	do_attach(alias_data, domain);
2224		2224
2225	atomic_inc(&alias_data->bind);	2225	atomic_inc(&alias_data->bind);
2226	}	2226	}
2227		2227
2228	if (dev_data->domain == NULL)	2228	if (dev_data->domain == NULL)
2229	do_attach(dev_data, domain);	2229	do_attach(dev_data, domain);
2230		2230
2231	atomic_inc(&dev_data->bind);	2231	atomic_inc(&dev_data->bind);
2232		2232
2233	ret = 0;	2233	ret = 0;
2234		2234
2235	out_unlock:	2235	out_unlock:
2236		2236
2237	/* ready */	2237	/* ready */
2238	spin_unlock(&domain->lock);	2238	spin_unlock(&domain->lock);
2239		2239
2240	return ret;	2240	return ret;
2241	}	2241	}
2242		2242
2243		2243
2244	static void pdev_iommuv2_disable(struct pci_dev *pdev)	2244	static void pdev_iommuv2_disable(struct pci_dev *pdev)
2245	{	2245	{
2246	pci_disable_ats(pdev);	2246	pci_disable_ats(pdev);
2247	pci_disable_pri(pdev);	2247	pci_disable_pri(pdev);
2248	pci_disable_pasid(pdev);	2248	pci_disable_pasid(pdev);
2249	}	2249	}
2250		2250
2251	/* FIXME: Change generic reset-function to do the same */	2251	/* FIXME: Change generic reset-function to do the same */
2252	static int pri_reset_while_enabled(struct pci_dev *pdev)	2252	static int pri_reset_while_enabled(struct pci_dev *pdev)
2253	{	2253	{
2254	u16 control;	2254	u16 control;
2255	int pos;	2255	int pos;
2256		2256
2257	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);	2257	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2258	if (!pos)	2258	if (!pos)
2259	return -EINVAL;	2259	return -EINVAL;
2260		2260
2261	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);	2261	pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
2262	control \|= PCI_PRI_CTRL_RESET;	2262	control \|= PCI_PRI_CTRL_RESET;
2263	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);	2263	pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
2264		2264
2265	return 0;	2265	return 0;
2266	}	2266	}
2267		2267
2268	static int pdev_iommuv2_enable(struct pci_dev *pdev)	2268	static int pdev_iommuv2_enable(struct pci_dev *pdev)
2269	{	2269	{
2270	bool reset_enable;	2270	bool reset_enable;
2271	int reqs, ret;	2271	int reqs, ret;
2272		2272
2273	/* FIXME: Hardcode number of outstanding requests for now */	2273	/* FIXME: Hardcode number of outstanding requests for now */
2274	reqs = 32;	2274	reqs = 32;
2275	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))	2275	if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
2276	reqs = 1;	2276	reqs = 1;
2277	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);	2277	reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2278		2278
2279	/* Only allow access to user-accessible pages */	2279	/* Only allow access to user-accessible pages */
2280	ret = pci_enable_pasid(pdev, 0);	2280	ret = pci_enable_pasid(pdev, 0);
2281	if (ret)	2281	if (ret)
2282	goto out_err;	2282	goto out_err;
2283		2283
2284	/* First reset the PRI state of the device */	2284	/* First reset the PRI state of the device */
2285	ret = pci_reset_pri(pdev);	2285	ret = pci_reset_pri(pdev);
2286	if (ret)	2286	if (ret)
2287	goto out_err;	2287	goto out_err;
2288		2288
2289	/* Enable PRI */	2289	/* Enable PRI */
2290	ret = pci_enable_pri(pdev, reqs);	2290	ret = pci_enable_pri(pdev, reqs);
2291	if (ret)	2291	if (ret)
2292	goto out_err;	2292	goto out_err;
2293		2293
2294	if (reset_enable) {	2294	if (reset_enable) {
2295	ret = pri_reset_while_enabled(pdev);	2295	ret = pri_reset_while_enabled(pdev);
2296	if (ret)	2296	if (ret)
2297	goto out_err;	2297	goto out_err;
2298	}	2298	}
2299		2299
2300	ret = pci_enable_ats(pdev, PAGE_SHIFT);	2300	ret = pci_enable_ats(pdev, PAGE_SHIFT);
2301	if (ret)	2301	if (ret)
2302	goto out_err;	2302	goto out_err;
2303		2303
2304	return 0;	2304	return 0;
2305		2305
2306	out_err:	2306	out_err:
2307	pci_disable_pri(pdev);	2307	pci_disable_pri(pdev);
2308	pci_disable_pasid(pdev);	2308	pci_disable_pasid(pdev);
2309		2309
2310	return ret;	2310	return ret;
2311	}	2311	}
2312		2312
2313	/* FIXME: Move this to PCI code */	2313	/* FIXME: Move this to PCI code */
2314	#define PCI_PRI_TLP_OFF (1 << 15)	2314	#define PCI_PRI_TLP_OFF (1 << 15)
2315		2315
2316	static bool pci_pri_tlp_required(struct pci_dev *pdev)	2316	static bool pci_pri_tlp_required(struct pci_dev *pdev)
2317	{	2317	{
2318	u16 status;	2318	u16 status;
2319	int pos;	2319	int pos;
2320		2320
2321	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);	2321	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2322	if (!pos)	2322	if (!pos)
2323	return false;	2323	return false;
2324		2324
2325	pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);	2325	pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
2326		2326
2327	return (status & PCI_PRI_TLP_OFF) ? true : false;	2327	return (status & PCI_PRI_TLP_OFF) ? true : false;
2328	}	2328	}
2329		2329
2330	/*	2330	/*
2331	* If a device is not yet associated with a domain, this function	2331	* If a device is not yet associated with a domain, this function
2332	* assigns it visible for the hardware	2332	* assigns it visible for the hardware
2333	*/	2333	*/
2334	static int attach_device(struct device *dev,	2334	static int attach_device(struct device *dev,
2335	struct protection_domain *domain)	2335	struct protection_domain *domain)
2336	{	2336	{
2337	struct pci_dev *pdev = to_pci_dev(dev);	2337	struct pci_dev *pdev = to_pci_dev(dev);
2338	struct iommu_dev_data *dev_data;	2338	struct iommu_dev_data *dev_data;
2339	unsigned long flags;	2339	unsigned long flags;
2340	int ret;	2340	int ret;
2341		2341
2342	dev_data = get_dev_data(dev);	2342	dev_data = get_dev_data(dev);
2343		2343
2344	if (domain->flags & PD_IOMMUV2_MASK) {	2344	if (domain->flags & PD_IOMMUV2_MASK) {
2345	if (!dev_data->iommu_v2 \|\| !dev_data->passthrough)	2345	if (!dev_data->iommu_v2 \|\| !dev_data->passthrough)
2346	return -EINVAL;	2346	return -EINVAL;
2347		2347
2348	if (pdev_iommuv2_enable(pdev) != 0)	2348	if (pdev_iommuv2_enable(pdev) != 0)
2349	return -EINVAL;	2349	return -EINVAL;
2350		2350
2351	dev_data->ats.enabled = true;	2351	dev_data->ats.enabled = true;
2352	dev_data->ats.qdep = pci_ats_queue_depth(pdev);	2352	dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2353	dev_data->pri_tlp = pci_pri_tlp_required(pdev);	2353	dev_data->pri_tlp = pci_pri_tlp_required(pdev);
2354	} else if (amd_iommu_iotlb_sup &&	2354	} else if (amd_iommu_iotlb_sup &&
2355	pci_enable_ats(pdev, PAGE_SHIFT) == 0) {	2355	pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
2356	dev_data->ats.enabled = true;	2356	dev_data->ats.enabled = true;
2357	dev_data->ats.qdep = pci_ats_queue_depth(pdev);	2357	dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2358	}	2358	}
2359		2359
2360	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	2360	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2361	ret = __attach_device(dev_data, domain);	2361	ret = __attach_device(dev_data, domain);
2362	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	2362	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2363		2363
2364	/*	2364	/*
2365	* We might boot into a crash-kernel here. The crashed kernel	2365	* We might boot into a crash-kernel here. The crashed kernel
2366	* left the caches in the IOMMU dirty. So we have to flush	2366	* left the caches in the IOMMU dirty. So we have to flush
2367	* here to evict all dirty stuff.	2367	* here to evict all dirty stuff.
2368	*/	2368	*/
2369	domain_flush_tlb_pde(domain);	2369	domain_flush_tlb_pde(domain);
2370		2370
2371	return ret;	2371	return ret;
2372	}	2372	}
2373		2373
2374	/*	2374	/*
2375	* Removes a device from a protection domain (unlocked)	2375	* Removes a device from a protection domain (unlocked)
2376	*/	2376	*/
2377	static void __detach_device(struct iommu_dev_data *dev_data)	2377	static void __detach_device(struct iommu_dev_data *dev_data)
2378	{	2378	{
2379	struct protection_domain *domain;	2379	struct protection_domain *domain;
2380	unsigned long flags;	2380	unsigned long flags;
2381		2381
2382	BUG_ON(!dev_data->domain);	2382	BUG_ON(!dev_data->domain);
2383		2383
2384	domain = dev_data->domain;	2384	domain = dev_data->domain;
2385		2385
2386	spin_lock_irqsave(&domain->lock, flags);	2386	spin_lock_irqsave(&domain->lock, flags);
2387		2387
2388	if (dev_data->alias_data != NULL) {	2388	if (dev_data->alias_data != NULL) {
2389	struct iommu_dev_data *alias_data = dev_data->alias_data;	2389	struct iommu_dev_data *alias_data = dev_data->alias_data;
2390		2390
2391	if (atomic_dec_and_test(&alias_data->bind))	2391	if (atomic_dec_and_test(&alias_data->bind))
2392	do_detach(alias_data);	2392	do_detach(alias_data);
2393	}	2393	}
2394		2394
2395	if (atomic_dec_and_test(&dev_data->bind))	2395	if (atomic_dec_and_test(&dev_data->bind))
2396	do_detach(dev_data);	2396	do_detach(dev_data);
2397		2397
2398	spin_unlock_irqrestore(&domain->lock, flags);	2398	spin_unlock_irqrestore(&domain->lock, flags);
2399		2399
2400	/*	2400	/*
2401	* If we run in passthrough mode the device must be assigned to the	2401	* If we run in passthrough mode the device must be assigned to the
2402	* passthrough domain if it is detached from any other domain.	2402	* passthrough domain if it is detached from any other domain.
2403	* Make sure we can deassign from the pt_domain itself.	2403	* Make sure we can deassign from the pt_domain itself.
2404	*/	2404	*/
2405	if (dev_data->passthrough &&	2405	if (dev_data->passthrough &&
2406	(dev_data->domain == NULL && domain != pt_domain))	2406	(dev_data->domain == NULL && domain != pt_domain))
2407	__attach_device(dev_data, pt_domain);	2407	__attach_device(dev_data, pt_domain);
2408	}	2408	}
2409		2409
2410	/*	2410	/*
2411	* Removes a device from a protection domain (with devtable_lock held)	2411	* Removes a device from a protection domain (with devtable_lock held)
2412	*/	2412	*/
2413	static void detach_device(struct device *dev)	2413	static void detach_device(struct device *dev)
2414	{	2414	{
2415	struct protection_domain *domain;	2415	struct protection_domain *domain;
2416	struct iommu_dev_data *dev_data;	2416	struct iommu_dev_data *dev_data;
2417	unsigned long flags;	2417	unsigned long flags;
2418		2418
2419	dev_data = get_dev_data(dev);	2419	dev_data = get_dev_data(dev);
2420	domain = dev_data->domain;	2420	domain = dev_data->domain;
2421		2421
2422	/* lock device table */	2422	/* lock device table */
2423	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	2423	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2424	__detach_device(dev_data);	2424	__detach_device(dev_data);
2425	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	2425	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2426		2426
2427	if (domain->flags & PD_IOMMUV2_MASK)	2427	if (domain->flags & PD_IOMMUV2_MASK)
2428	pdev_iommuv2_disable(to_pci_dev(dev));	2428	pdev_iommuv2_disable(to_pci_dev(dev));
2429	else if (dev_data->ats.enabled)	2429	else if (dev_data->ats.enabled)
2430	pci_disable_ats(to_pci_dev(dev));	2430	pci_disable_ats(to_pci_dev(dev));
2431		2431
2432	dev_data->ats.enabled = false;	2432	dev_data->ats.enabled = false;
2433	}	2433	}
2434		2434
2435	/*	2435	/*
2436	* Find out the protection domain structure for a given PCI device. This	2436	* Find out the protection domain structure for a given PCI device. This
2437	* will give us the pointer to the page table root for example.	2437	* will give us the pointer to the page table root for example.
2438	*/	2438	*/
2439	static struct protection_domain domain_for_device(struct device dev)	2439	static struct protection_domain domain_for_device(struct device dev)
2440	{	2440	{
2441	struct iommu_dev_data *dev_data;	2441	struct iommu_dev_data *dev_data;
2442	struct protection_domain *dom = NULL;	2442	struct protection_domain *dom = NULL;
2443	unsigned long flags;	2443	unsigned long flags;
2444		2444
2445	dev_data = get_dev_data(dev);	2445	dev_data = get_dev_data(dev);
2446		2446
2447	if (dev_data->domain)	2447	if (dev_data->domain)
2448	return dev_data->domain;	2448	return dev_data->domain;
2449		2449
2450	if (dev_data->alias_data != NULL) {	2450	if (dev_data->alias_data != NULL) {
2451	struct iommu_dev_data *alias_data = dev_data->alias_data;	2451	struct iommu_dev_data *alias_data = dev_data->alias_data;
2452		2452
2453	read_lock_irqsave(&amd_iommu_devtable_lock, flags);	2453	read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2454	if (alias_data->domain != NULL) {	2454	if (alias_data->domain != NULL) {
2455	__attach_device(dev_data, alias_data->domain);	2455	__attach_device(dev_data, alias_data->domain);
2456	dom = alias_data->domain;	2456	dom = alias_data->domain;
2457	}	2457	}
2458	read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	2458	read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2459	}	2459	}
2460		2460
2461	return dom;	2461	return dom;
2462	}	2462	}
2463		2463
2464	static int device_change_notifier(struct notifier_block *nb,	2464	static int device_change_notifier(struct notifier_block *nb,
2465	unsigned long action, void *data)	2465	unsigned long action, void *data)
2466	{	2466	{
2467	struct dma_ops_domain *dma_domain;	2467	struct dma_ops_domain *dma_domain;
2468	struct protection_domain *domain;	2468	struct protection_domain *domain;
2469	struct iommu_dev_data *dev_data;	2469	struct iommu_dev_data *dev_data;
2470	struct device *dev = data;	2470	struct device *dev = data;
2471	struct amd_iommu *iommu;	2471	struct amd_iommu *iommu;
2472	unsigned long flags;	2472	unsigned long flags;
2473	u16 devid;	2473	u16 devid;
2474		2474
2475	if (!check_device(dev))	2475	if (!check_device(dev))
2476	return 0;	2476	return 0;
2477		2477
2478	devid = get_device_id(dev);	2478	devid = get_device_id(dev);
2479	iommu = amd_iommu_rlookup_table[devid];	2479	iommu = amd_iommu_rlookup_table[devid];
2480	dev_data = get_dev_data(dev);	2480	dev_data = get_dev_data(dev);
2481		2481
2482	switch (action) {	2482	switch (action) {
2483	case BUS_NOTIFY_UNBOUND_DRIVER:	2483	case BUS_NOTIFY_UNBOUND_DRIVER:
2484		2484
2485	domain = domain_for_device(dev);	2485	domain = domain_for_device(dev);
2486		2486
2487	if (!domain)	2487	if (!domain)
2488	goto out;	2488	goto out;
2489	if (dev_data->passthrough)	2489	if (dev_data->passthrough)
2490	break;	2490	break;
2491	detach_device(dev);	2491	detach_device(dev);
2492	break;	2492	break;
2493	case BUS_NOTIFY_ADD_DEVICE:	2493	case BUS_NOTIFY_ADD_DEVICE:
2494		2494
2495	iommu_init_device(dev);	2495	iommu_init_device(dev);
2496		2496
2497	/*	2497	/*
2498	* dev_data is still NULL and	2498	* dev_data is still NULL and
2499	* got initialized in iommu_init_device	2499	* got initialized in iommu_init_device
2500	*/	2500	*/
2501	dev_data = get_dev_data(dev);	2501	dev_data = get_dev_data(dev);
2502		2502
2503	if (iommu_pass_through \|\| dev_data->iommu_v2) {	2503	if (iommu_pass_through \|\| dev_data->iommu_v2) {
2504	dev_data->passthrough = true;	2504	dev_data->passthrough = true;
2505	attach_device(dev, pt_domain);	2505	attach_device(dev, pt_domain);
2506	break;	2506	break;
2507	}	2507	}
2508		2508
2509	domain = domain_for_device(dev);	2509	domain = domain_for_device(dev);
2510		2510
2511	/* allocate a protection domain if a device is added */	2511	/* allocate a protection domain if a device is added */
2512	dma_domain = find_protection_domain(devid);	2512	dma_domain = find_protection_domain(devid);
2513	if (!dma_domain) {	2513	if (!dma_domain) {
2514	dma_domain = dma_ops_domain_alloc();	2514	dma_domain = dma_ops_domain_alloc();
2515	if (!dma_domain)	2515	if (!dma_domain)
2516	goto out;	2516	goto out;
2517	dma_domain->target_dev = devid;	2517	dma_domain->target_dev = devid;
2518		2518
2519	spin_lock_irqsave(&iommu_pd_list_lock, flags);	2519	spin_lock_irqsave(&iommu_pd_list_lock, flags);
2520	list_add_tail(&dma_domain->list, &iommu_pd_list);	2520	list_add_tail(&dma_domain->list, &iommu_pd_list);
2521	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);	2521	spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2522	}	2522	}
2523		2523
2524	dev->archdata.dma_ops = &amd_iommu_dma_ops;	2524	dev->archdata.dma_ops = &amd_iommu_dma_ops;
2525		2525
2526	break;	2526	break;
2527	case BUS_NOTIFY_DEL_DEVICE:	2527	case BUS_NOTIFY_DEL_DEVICE:
2528		2528
2529	iommu_uninit_device(dev);	2529	iommu_uninit_device(dev);
2530		2530
2531	default:	2531	default:
2532	goto out;	2532	goto out;
2533	}	2533	}
2534		2534
2535	iommu_completion_wait(iommu);	2535	iommu_completion_wait(iommu);
2536		2536
2537	out:	2537	out:
2538	return 0;	2538	return 0;
2539	}	2539	}
2540		2540
2541	static struct notifier_block device_nb = {	2541	static struct notifier_block device_nb = {
2542	.notifier_call = device_change_notifier,	2542	.notifier_call = device_change_notifier,
2543	};	2543	};
2544		2544
2545	void amd_iommu_init_notifier(void)	2545	void amd_iommu_init_notifier(void)
2546	{	2546	{
2547	bus_register_notifier(&pci_bus_type, &device_nb);	2547	bus_register_notifier(&pci_bus_type, &device_nb);
2548	}	2548	}
2549		2549
2550	/*****************************************************************************	2550	/*****************************************************************************
2551	*	2551	*
2552	* The next functions belong to the dma_ops mapping/unmapping code.	2552	* The next functions belong to the dma_ops mapping/unmapping code.
2553	*	2553	*
2554	*****************************************************************************/	2554	*****************************************************************************/
2555		2555
2556	/*	2556	/*
2557	* In the dma_ops path we only have the struct device. This function	2557	* In the dma_ops path we only have the struct device. This function
2558	* finds the corresponding IOMMU, the protection domain and the	2558	* finds the corresponding IOMMU, the protection domain and the
2559	* requestor id for a given device.	2559	* requestor id for a given device.
2560	* If the device is not yet associated with a domain this is also done	2560	* If the device is not yet associated with a domain this is also done
2561	* in this function.	2561	* in this function.
2562	*/	2562	*/
2563	static struct protection_domain get_domain(struct device dev)	2563	static struct protection_domain get_domain(struct device dev)
2564	{	2564	{
2565	struct protection_domain *domain;	2565	struct protection_domain *domain;
2566	struct dma_ops_domain *dma_dom;	2566	struct dma_ops_domain *dma_dom;
2567	u16 devid = get_device_id(dev);	2567	u16 devid = get_device_id(dev);
2568		2568
2569	if (!check_device(dev))	2569	if (!check_device(dev))
2570	return ERR_PTR(-EINVAL);	2570	return ERR_PTR(-EINVAL);
2571		2571
2572	domain = domain_for_device(dev);	2572	domain = domain_for_device(dev);
2573	if (domain != NULL && !dma_ops_domain(domain))	2573	if (domain != NULL && !dma_ops_domain(domain))
2574	return ERR_PTR(-EBUSY);	2574	return ERR_PTR(-EBUSY);
2575		2575
2576	if (domain != NULL)	2576	if (domain != NULL)
2577	return domain;	2577	return domain;
2578		2578
2579	/* Device not bound yet - bind it */	2579	/* Device not bound yet - bind it */
2580	dma_dom = find_protection_domain(devid);	2580	dma_dom = find_protection_domain(devid);
2581	if (!dma_dom)	2581	if (!dma_dom)
2582	dma_dom = amd_iommu_rlookup_table[devid]->default_dom;	2582	dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2583	attach_device(dev, &dma_dom->domain);	2583	attach_device(dev, &dma_dom->domain);
2584	DUMP_printk("Using protection domain %d for device %s\n",	2584	DUMP_printk("Using protection domain %d for device %s\n",
2585	dma_dom->domain.id, dev_name(dev));	2585	dma_dom->domain.id, dev_name(dev));
2586		2586
2587	return &dma_dom->domain;	2587	return &dma_dom->domain;
2588	}	2588	}
2589		2589
2590	static void update_device_table(struct protection_domain *domain)	2590	static void update_device_table(struct protection_domain *domain)
2591	{	2591	{
2592	struct iommu_dev_data *dev_data;	2592	struct iommu_dev_data *dev_data;
2593		2593
2594	list_for_each_entry(dev_data, &domain->dev_list, list)	2594	list_for_each_entry(dev_data, &domain->dev_list, list)
2595	set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);	2595	set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2596	}	2596	}
2597		2597
2598	static void update_domain(struct protection_domain *domain)	2598	static void update_domain(struct protection_domain *domain)
2599	{	2599	{
2600	if (!domain->updated)	2600	if (!domain->updated)
2601	return;	2601	return;
2602		2602
2603	update_device_table(domain);	2603	update_device_table(domain);
2604		2604
2605	domain_flush_devices(domain);	2605	domain_flush_devices(domain);
2606	domain_flush_tlb_pde(domain);	2606	domain_flush_tlb_pde(domain);
2607		2607
2608	domain->updated = false;	2608	domain->updated = false;
2609	}	2609	}
2610		2610
2611	/*	2611	/*
2612	* This function fetches the PTE for a given address in the aperture	2612	* This function fetches the PTE for a given address in the aperture
2613	*/	2613	*/
2614	static u64* dma_ops_get_pte(struct dma_ops_domain *dom,	2614	static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2615	unsigned long address)	2615	unsigned long address)
2616	{	2616	{
2617	struct aperture_range *aperture;	2617	struct aperture_range *aperture;
2618	u64 pte, pte_page;	2618	u64 pte, pte_page;
2619		2619
2620	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];	2620	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2621	if (!aperture)	2621	if (!aperture)
2622	return NULL;	2622	return NULL;
2623		2623
2624	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];	2624	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2625	if (!pte) {	2625	if (!pte) {
2626	pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,	2626	pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2627	GFP_ATOMIC);	2627	GFP_ATOMIC);
2628	aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;	2628	aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2629	} else	2629	} else
2630	pte += PM_LEVEL_INDEX(0, address);	2630	pte += PM_LEVEL_INDEX(0, address);
2631		2631
2632	update_domain(&dom->domain);	2632	update_domain(&dom->domain);
2633		2633
2634	return pte;	2634	return pte;
2635	}	2635	}
2636		2636
2637	/*	2637	/*
2638	* This is the generic map function. It maps one 4kb page at paddr to	2638	* This is the generic map function. It maps one 4kb page at paddr to
2639	* the given address in the DMA address space for the domain.	2639	* the given address in the DMA address space for the domain.
2640	*/	2640	*/
2641	static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,	2641	static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2642	unsigned long address,	2642	unsigned long address,
2643	phys_addr_t paddr,	2643	phys_addr_t paddr,
2644	int direction)	2644	int direction)
2645	{	2645	{
2646	u64 *pte, __pte;	2646	u64 *pte, __pte;
2647		2647
2648	WARN_ON(address > dom->aperture_size);	2648	WARN_ON(address > dom->aperture_size);
2649		2649
2650	paddr &= PAGE_MASK;	2650	paddr &= PAGE_MASK;
2651		2651
2652	pte = dma_ops_get_pte(dom, address);	2652	pte = dma_ops_get_pte(dom, address);
2653	if (!pte)	2653	if (!pte)
2654	return DMA_ERROR_CODE;	2654	return DMA_ERROR_CODE;
2655		2655
2656	__pte = paddr \| IOMMU_PTE_P \| IOMMU_PTE_FC;	2656	__pte = paddr \| IOMMU_PTE_P \| IOMMU_PTE_FC;
2657		2657
2658	if (direction == DMA_TO_DEVICE)	2658	if (direction == DMA_TO_DEVICE)
2659	__pte \|= IOMMU_PTE_IR;	2659	__pte \|= IOMMU_PTE_IR;
2660	else if (direction == DMA_FROM_DEVICE)	2660	else if (direction == DMA_FROM_DEVICE)
2661	__pte \|= IOMMU_PTE_IW;	2661	__pte \|= IOMMU_PTE_IW;
2662	else if (direction == DMA_BIDIRECTIONAL)	2662	else if (direction == DMA_BIDIRECTIONAL)
2663	__pte \|= IOMMU_PTE_IR \| IOMMU_PTE_IW;	2663	__pte \|= IOMMU_PTE_IR \| IOMMU_PTE_IW;
2664		2664
2665	WARN_ON(*pte);	2665	WARN_ON(*pte);
2666		2666
2667	*pte = __pte;	2667	*pte = __pte;
2668		2668
2669	return (dma_addr_t)address;	2669	return (dma_addr_t)address;
2670	}	2670	}
2671		2671
2672	/*	2672	/*
2673	* The generic unmapping function for on page in the DMA address space.	2673	* The generic unmapping function for on page in the DMA address space.
2674	*/	2674	*/
2675	static void dma_ops_domain_unmap(struct dma_ops_domain *dom,	2675	static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2676	unsigned long address)	2676	unsigned long address)
2677	{	2677	{
2678	struct aperture_range *aperture;	2678	struct aperture_range *aperture;
2679	u64 *pte;	2679	u64 *pte;
2680		2680
2681	if (address >= dom->aperture_size)	2681	if (address >= dom->aperture_size)
2682	return;	2682	return;
2683		2683
2684	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];	2684	aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2685	if (!aperture)	2685	if (!aperture)
2686	return;	2686	return;
2687		2687
2688	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];	2688	pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2689	if (!pte)	2689	if (!pte)
2690	return;	2690	return;
2691		2691
2692	pte += PM_LEVEL_INDEX(0, address);	2692	pte += PM_LEVEL_INDEX(0, address);
2693		2693
2694	WARN_ON(!*pte);	2694	WARN_ON(!*pte);
2695		2695
2696	*pte = 0ULL;	2696	*pte = 0ULL;
2697	}	2697	}
2698		2698
2699	/*	2699	/*
2700	* This function contains common code for mapping of a physically	2700	* This function contains common code for mapping of a physically
2701	* contiguous memory region into DMA address space. It is used by all	2701	* contiguous memory region into DMA address space. It is used by all
2702	* mapping functions provided with this IOMMU driver.	2702	* mapping functions provided with this IOMMU driver.
2703	* Must be called with the domain lock held.	2703	* Must be called with the domain lock held.
2704	*/	2704	*/
2705	static dma_addr_t __map_single(struct device *dev,	2705	static dma_addr_t __map_single(struct device *dev,
2706	struct dma_ops_domain *dma_dom,	2706	struct dma_ops_domain *dma_dom,
2707	phys_addr_t paddr,	2707	phys_addr_t paddr,
2708	size_t size,	2708	size_t size,
2709	int dir,	2709	int dir,
2710	bool align,	2710	bool align,
2711	u64 dma_mask)	2711	u64 dma_mask)
2712	{	2712	{
2713	dma_addr_t offset = paddr & ~PAGE_MASK;	2713	dma_addr_t offset = paddr & ~PAGE_MASK;
2714	dma_addr_t address, start, ret;	2714	dma_addr_t address, start, ret;
2715	unsigned int pages;	2715	unsigned int pages;
2716	unsigned long align_mask = 0;	2716	unsigned long align_mask = 0;
2717	int i;	2717	int i;
2718		2718
2719	pages = iommu_num_pages(paddr, size, PAGE_SIZE);	2719	pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2720	paddr &= PAGE_MASK;	2720	paddr &= PAGE_MASK;
2721		2721
2722	INC_STATS_COUNTER(total_map_requests);	2722	INC_STATS_COUNTER(total_map_requests);
2723		2723
2724	if (pages > 1)	2724	if (pages > 1)
2725	INC_STATS_COUNTER(cross_page);	2725	INC_STATS_COUNTER(cross_page);
2726		2726
2727	if (align)	2727	if (align)
2728	align_mask = (1UL << get_order(size)) - 1;	2728	align_mask = (1UL << get_order(size)) - 1;
2729		2729
2730	retry:	2730	retry:
2731	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,	2731	address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2732	dma_mask);	2732	dma_mask);
2733	if (unlikely(address == DMA_ERROR_CODE)) {	2733	if (unlikely(address == DMA_ERROR_CODE)) {
2734	/*	2734	/*
2735	* setting next_address here will let the address	2735	* setting next_address here will let the address
2736	* allocator only scan the new allocated range in the	2736	* allocator only scan the new allocated range in the
2737	* first run. This is a small optimization.	2737	* first run. This is a small optimization.
2738	*/	2738	*/
2739	dma_dom->next_address = dma_dom->aperture_size;	2739	dma_dom->next_address = dma_dom->aperture_size;
2740		2740
2741	if (alloc_new_range(dma_dom, false, GFP_ATOMIC))	2741	if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2742	goto out;	2742	goto out;
2743		2743
2744	/*	2744	/*
2745	* aperture was successfully enlarged by 128 MB, try	2745	* aperture was successfully enlarged by 128 MB, try
2746	* allocation again	2746	* allocation again
2747	*/	2747	*/
2748	goto retry;	2748	goto retry;
2749	}	2749	}
2750		2750
2751	start = address;	2751	start = address;
2752	for (i = 0; i < pages; ++i) {	2752	for (i = 0; i < pages; ++i) {
2753	ret = dma_ops_domain_map(dma_dom, start, paddr, dir);	2753	ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2754	if (ret == DMA_ERROR_CODE)	2754	if (ret == DMA_ERROR_CODE)
2755	goto out_unmap;	2755	goto out_unmap;
2756		2756
2757	paddr += PAGE_SIZE;	2757	paddr += PAGE_SIZE;
2758	start += PAGE_SIZE;	2758	start += PAGE_SIZE;
2759	}	2759	}
2760	address += offset;	2760	address += offset;
2761		2761
2762	ADD_STATS_COUNTER(alloced_io_mem, size);	2762	ADD_STATS_COUNTER(alloced_io_mem, size);
2763		2763
2764	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {	2764	if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2765	domain_flush_tlb(&dma_dom->domain);	2765	domain_flush_tlb(&dma_dom->domain);
2766	dma_dom->need_flush = false;	2766	dma_dom->need_flush = false;
2767	} else if (unlikely(amd_iommu_np_cache))	2767	} else if (unlikely(amd_iommu_np_cache))
2768	domain_flush_pages(&dma_dom->domain, address, size);	2768	domain_flush_pages(&dma_dom->domain, address, size);
2769		2769
2770	out:	2770	out:
2771	return address;	2771	return address;
2772		2772
2773	out_unmap:	2773	out_unmap:
2774		2774
2775	for (--i; i >= 0; --i) {	2775	for (--i; i >= 0; --i) {
2776	start -= PAGE_SIZE;	2776	start -= PAGE_SIZE;
2777	dma_ops_domain_unmap(dma_dom, start);	2777	dma_ops_domain_unmap(dma_dom, start);
2778	}	2778	}
2779		2779
2780	dma_ops_free_addresses(dma_dom, address, pages);	2780	dma_ops_free_addresses(dma_dom, address, pages);
2781		2781
2782	return DMA_ERROR_CODE;	2782	return DMA_ERROR_CODE;
2783	}	2783	}
2784		2784
2785	/*	2785	/*
2786	* Does the reverse of the __map_single function. Must be called with	2786	* Does the reverse of the __map_single function. Must be called with
2787	* the domain lock held too	2787	* the domain lock held too
2788	*/	2788	*/
2789	static void __unmap_single(struct dma_ops_domain *dma_dom,	2789	static void __unmap_single(struct dma_ops_domain *dma_dom,
2790	dma_addr_t dma_addr,	2790	dma_addr_t dma_addr,
2791	size_t size,	2791	size_t size,
2792	int dir)	2792	int dir)
2793	{	2793	{
2794	dma_addr_t flush_addr;	2794	dma_addr_t flush_addr;
2795	dma_addr_t i, start;	2795	dma_addr_t i, start;
2796	unsigned int pages;	2796	unsigned int pages;
2797		2797
2798	if ((dma_addr == DMA_ERROR_CODE) \|\|	2798	if ((dma_addr == DMA_ERROR_CODE) \|\|
2799	(dma_addr + size > dma_dom->aperture_size))	2799	(dma_addr + size > dma_dom->aperture_size))
2800	return;	2800	return;
2801		2801
2802	flush_addr = dma_addr;	2802	flush_addr = dma_addr;
2803	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);	2803	pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2804	dma_addr &= PAGE_MASK;	2804	dma_addr &= PAGE_MASK;
2805	start = dma_addr;	2805	start = dma_addr;
2806		2806
2807	for (i = 0; i < pages; ++i) {	2807	for (i = 0; i < pages; ++i) {
2808	dma_ops_domain_unmap(dma_dom, start);	2808	dma_ops_domain_unmap(dma_dom, start);
2809	start += PAGE_SIZE;	2809	start += PAGE_SIZE;
2810	}	2810	}
2811		2811
2812	SUB_STATS_COUNTER(alloced_io_mem, size);	2812	SUB_STATS_COUNTER(alloced_io_mem, size);
2813		2813
2814	dma_ops_free_addresses(dma_dom, dma_addr, pages);	2814	dma_ops_free_addresses(dma_dom, dma_addr, pages);
2815		2815
2816	if (amd_iommu_unmap_flush \|\| dma_dom->need_flush) {	2816	if (amd_iommu_unmap_flush \|\| dma_dom->need_flush) {
2817	domain_flush_pages(&dma_dom->domain, flush_addr, size);	2817	domain_flush_pages(&dma_dom->domain, flush_addr, size);
2818	dma_dom->need_flush = false;	2818	dma_dom->need_flush = false;
2819	}	2819	}
2820	}	2820	}
2821		2821
2822	/*	2822	/*
2823	* The exported map_single function for dma_ops.	2823	* The exported map_single function for dma_ops.
2824	*/	2824	*/
2825	static dma_addr_t map_page(struct device dev, struct page page,	2825	static dma_addr_t map_page(struct device dev, struct page page,
2826	unsigned long offset, size_t size,	2826	unsigned long offset, size_t size,
2827	enum dma_data_direction dir,	2827	enum dma_data_direction dir,
2828	struct dma_attrs *attrs)	2828	struct dma_attrs *attrs)
2829	{	2829	{
2830	unsigned long flags;	2830	unsigned long flags;
2831	struct protection_domain *domain;	2831	struct protection_domain *domain;
2832	dma_addr_t addr;	2832	dma_addr_t addr;
2833	u64 dma_mask;	2833	u64 dma_mask;
2834	phys_addr_t paddr = page_to_phys(page) + offset;	2834	phys_addr_t paddr = page_to_phys(page) + offset;
2835		2835
2836	INC_STATS_COUNTER(cnt_map_single);	2836	INC_STATS_COUNTER(cnt_map_single);
2837		2837
2838	domain = get_domain(dev);	2838	domain = get_domain(dev);
2839	if (PTR_ERR(domain) == -EINVAL)	2839	if (PTR_ERR(domain) == -EINVAL)
2840	return (dma_addr_t)paddr;	2840	return (dma_addr_t)paddr;
2841	else if (IS_ERR(domain))	2841	else if (IS_ERR(domain))
2842	return DMA_ERROR_CODE;	2842	return DMA_ERROR_CODE;
2843		2843
2844	dma_mask = *dev->dma_mask;	2844	dma_mask = *dev->dma_mask;
2845		2845
2846	spin_lock_irqsave(&domain->lock, flags);	2846	spin_lock_irqsave(&domain->lock, flags);
2847		2847
2848	addr = __map_single(dev, domain->priv, paddr, size, dir, false,	2848	addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2849	dma_mask);	2849	dma_mask);
2850	if (addr == DMA_ERROR_CODE)	2850	if (addr == DMA_ERROR_CODE)
2851	goto out;	2851	goto out;
2852		2852
2853	domain_flush_complete(domain);	2853	domain_flush_complete(domain);
2854		2854
2855	out:	2855	out:
2856	spin_unlock_irqrestore(&domain->lock, flags);	2856	spin_unlock_irqrestore(&domain->lock, flags);
2857		2857
2858	return addr;	2858	return addr;
2859	}	2859	}
2860		2860
2861	/*	2861	/*
2862	* The exported unmap_single function for dma_ops.	2862	* The exported unmap_single function for dma_ops.
2863	*/	2863	*/
2864	static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,	2864	static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2865	enum dma_data_direction dir, struct dma_attrs *attrs)	2865	enum dma_data_direction dir, struct dma_attrs *attrs)
2866	{	2866	{
2867	unsigned long flags;	2867	unsigned long flags;
2868	struct protection_domain *domain;	2868	struct protection_domain *domain;
2869		2869
2870	INC_STATS_COUNTER(cnt_unmap_single);	2870	INC_STATS_COUNTER(cnt_unmap_single);
2871		2871
2872	domain = get_domain(dev);	2872	domain = get_domain(dev);
2873	if (IS_ERR(domain))	2873	if (IS_ERR(domain))
2874	return;	2874	return;
2875		2875
2876	spin_lock_irqsave(&domain->lock, flags);	2876	spin_lock_irqsave(&domain->lock, flags);
2877		2877
2878	__unmap_single(domain->priv, dma_addr, size, dir);	2878	__unmap_single(domain->priv, dma_addr, size, dir);
2879		2879
2880	domain_flush_complete(domain);	2880	domain_flush_complete(domain);
2881		2881
2882	spin_unlock_irqrestore(&domain->lock, flags);	2882	spin_unlock_irqrestore(&domain->lock, flags);
2883	}	2883	}
2884		2884
2885	/*	2885	/*
2886	* The exported map_sg function for dma_ops (handles scatter-gather	2886	* The exported map_sg function for dma_ops (handles scatter-gather
2887	* lists).	2887	* lists).
2888	*/	2888	*/
2889	static int map_sg(struct device dev, struct scatterlist sglist,	2889	static int map_sg(struct device dev, struct scatterlist sglist,
2890	int nelems, enum dma_data_direction dir,	2890	int nelems, enum dma_data_direction dir,
2891	struct dma_attrs *attrs)	2891	struct dma_attrs *attrs)
2892	{	2892	{
2893	unsigned long flags;	2893	unsigned long flags;
2894	struct protection_domain *domain;	2894	struct protection_domain *domain;
2895	int i;	2895	int i;
2896	struct scatterlist *s;	2896	struct scatterlist *s;
2897	phys_addr_t paddr;	2897	phys_addr_t paddr;
2898	int mapped_elems = 0;	2898	int mapped_elems = 0;
2899	u64 dma_mask;	2899	u64 dma_mask;
2900		2900
2901	INC_STATS_COUNTER(cnt_map_sg);	2901	INC_STATS_COUNTER(cnt_map_sg);
2902		2902
2903	domain = get_domain(dev);	2903	domain = get_domain(dev);
2904	if (IS_ERR(domain))	2904	if (IS_ERR(domain))
2905	return 0;	2905	return 0;
2906		2906
2907	dma_mask = *dev->dma_mask;	2907	dma_mask = *dev->dma_mask;
2908		2908
2909	spin_lock_irqsave(&domain->lock, flags);	2909	spin_lock_irqsave(&domain->lock, flags);
2910		2910
2911	for_each_sg(sglist, s, nelems, i) {	2911	for_each_sg(sglist, s, nelems, i) {
2912	paddr = sg_phys(s);	2912	paddr = sg_phys(s);
2913		2913
2914	s->dma_address = __map_single(dev, domain->priv,	2914	s->dma_address = __map_single(dev, domain->priv,
2915	paddr, s->length, dir, false,	2915	paddr, s->length, dir, false,
2916	dma_mask);	2916	dma_mask);
2917		2917
2918	if (s->dma_address) {	2918	if (s->dma_address) {
2919	s->dma_length = s->length;	2919	s->dma_length = s->length;
2920	mapped_elems++;	2920	mapped_elems++;
2921	} else	2921	} else
2922	goto unmap;	2922	goto unmap;
2923	}	2923	}
2924		2924
2925	domain_flush_complete(domain);	2925	domain_flush_complete(domain);
2926		2926
2927	out:	2927	out:
2928	spin_unlock_irqrestore(&domain->lock, flags);	2928	spin_unlock_irqrestore(&domain->lock, flags);
2929		2929
2930	return mapped_elems;	2930	return mapped_elems;
2931	unmap:	2931	unmap:
2932	for_each_sg(sglist, s, mapped_elems, i) {	2932	for_each_sg(sglist, s, mapped_elems, i) {
2933	if (s->dma_address)	2933	if (s->dma_address)
2934	__unmap_single(domain->priv, s->dma_address,	2934	__unmap_single(domain->priv, s->dma_address,
2935	s->dma_length, dir);	2935	s->dma_length, dir);
2936	s->dma_address = s->dma_length = 0;	2936	s->dma_address = s->dma_length = 0;
2937	}	2937	}
2938		2938
2939	mapped_elems = 0;	2939	mapped_elems = 0;
2940		2940
2941	goto out;	2941	goto out;
2942	}	2942	}
2943		2943
2944	/*	2944	/*
2945	* The exported map_sg function for dma_ops (handles scatter-gather	2945	* The exported map_sg function for dma_ops (handles scatter-gather
2946	* lists).	2946	* lists).
2947	*/	2947	*/
2948	static void unmap_sg(struct device dev, struct scatterlist sglist,	2948	static void unmap_sg(struct device dev, struct scatterlist sglist,
2949	int nelems, enum dma_data_direction dir,	2949	int nelems, enum dma_data_direction dir,
2950	struct dma_attrs *attrs)	2950	struct dma_attrs *attrs)
2951	{	2951	{
2952	unsigned long flags;	2952	unsigned long flags;
2953	struct protection_domain *domain;	2953	struct protection_domain *domain;
2954	struct scatterlist *s;	2954	struct scatterlist *s;
2955	int i;	2955	int i;
2956		2956
2957	INC_STATS_COUNTER(cnt_unmap_sg);	2957	INC_STATS_COUNTER(cnt_unmap_sg);
2958		2958
2959	domain = get_domain(dev);	2959	domain = get_domain(dev);
2960	if (IS_ERR(domain))	2960	if (IS_ERR(domain))
2961	return;	2961	return;
2962		2962
2963	spin_lock_irqsave(&domain->lock, flags);	2963	spin_lock_irqsave(&domain->lock, flags);
2964		2964
2965	for_each_sg(sglist, s, nelems, i) {	2965	for_each_sg(sglist, s, nelems, i) {
2966	__unmap_single(domain->priv, s->dma_address,	2966	__unmap_single(domain->priv, s->dma_address,
2967	s->dma_length, dir);	2967	s->dma_length, dir);
2968	s->dma_address = s->dma_length = 0;	2968	s->dma_address = s->dma_length = 0;
2969	}	2969	}
2970		2970
2971	domain_flush_complete(domain);	2971	domain_flush_complete(domain);
2972		2972
2973	spin_unlock_irqrestore(&domain->lock, flags);	2973	spin_unlock_irqrestore(&domain->lock, flags);
2974	}	2974	}
2975		2975
2976	/*	2976	/*
2977	* The exported alloc_coherent function for dma_ops.	2977	* The exported alloc_coherent function for dma_ops.
2978	*/	2978	*/
2979	static void alloc_coherent(struct device dev, size_t size,	2979	static void alloc_coherent(struct device dev, size_t size,
2980	dma_addr_t *dma_addr, gfp_t flag,	2980	dma_addr_t *dma_addr, gfp_t flag,
2981	struct dma_attrs *attrs)	2981	struct dma_attrs *attrs)
2982	{	2982	{
2983	unsigned long flags;	2983	unsigned long flags;
2984	void *virt_addr;	2984	void *virt_addr;
2985	struct protection_domain *domain;	2985	struct protection_domain *domain;
2986	phys_addr_t paddr;	2986	phys_addr_t paddr;
2987	u64 dma_mask = dev->coherent_dma_mask;	2987	u64 dma_mask = dev->coherent_dma_mask;
2988		2988
2989	INC_STATS_COUNTER(cnt_alloc_coherent);	2989	INC_STATS_COUNTER(cnt_alloc_coherent);
2990		2990
2991	domain = get_domain(dev);	2991	domain = get_domain(dev);
2992	if (PTR_ERR(domain) == -EINVAL) {	2992	if (PTR_ERR(domain) == -EINVAL) {
2993	virt_addr = (void *)__get_free_pages(flag, get_order(size));	2993	virt_addr = (void *)__get_free_pages(flag, get_order(size));
2994	*dma_addr = __pa(virt_addr);	2994	*dma_addr = __pa(virt_addr);
2995	return virt_addr;	2995	return virt_addr;
2996	} else if (IS_ERR(domain))	2996	} else if (IS_ERR(domain))
2997	return NULL;	2997	return NULL;
2998		2998
2999	dma_mask = dev->coherent_dma_mask;	2999	dma_mask = dev->coherent_dma_mask;
3000	flag &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);	3000	flag &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);
3001	flag \|= __GFP_ZERO;	3001	flag \|= __GFP_ZERO;
3002		3002
3003	virt_addr = (void *)__get_free_pages(flag, get_order(size));	3003	virt_addr = (void *)__get_free_pages(flag, get_order(size));
3004	if (!virt_addr)	3004	if (!virt_addr)
3005	return NULL;	3005	return NULL;
3006		3006
3007	paddr = virt_to_phys(virt_addr);	3007	paddr = virt_to_phys(virt_addr);
3008		3008
3009	if (!dma_mask)	3009	if (!dma_mask)
3010	dma_mask = *dev->dma_mask;	3010	dma_mask = *dev->dma_mask;
3011		3011
3012	spin_lock_irqsave(&domain->lock, flags);	3012	spin_lock_irqsave(&domain->lock, flags);
3013		3013
3014	*dma_addr = __map_single(dev, domain->priv, paddr,	3014	*dma_addr = __map_single(dev, domain->priv, paddr,
3015	size, DMA_BIDIRECTIONAL, true, dma_mask);	3015	size, DMA_BIDIRECTIONAL, true, dma_mask);
3016		3016
3017	if (*dma_addr == DMA_ERROR_CODE) {	3017	if (*dma_addr == DMA_ERROR_CODE) {
3018	spin_unlock_irqrestore(&domain->lock, flags);	3018	spin_unlock_irqrestore(&domain->lock, flags);
3019	goto out_free;	3019	goto out_free;
3020	}	3020	}
3021		3021
3022	domain_flush_complete(domain);	3022	domain_flush_complete(domain);
3023		3023
3024	spin_unlock_irqrestore(&domain->lock, flags);	3024	spin_unlock_irqrestore(&domain->lock, flags);
3025		3025
3026	return virt_addr;	3026	return virt_addr;
3027		3027
3028	out_free:	3028	out_free:
3029		3029
3030	free_pages((unsigned long)virt_addr, get_order(size));	3030	free_pages((unsigned long)virt_addr, get_order(size));
3031		3031
3032	return NULL;	3032	return NULL;
3033	}	3033	}
3034		3034
3035	/*	3035	/*
3036	* The exported free_coherent function for dma_ops.	3036	* The exported free_coherent function for dma_ops.
3037	*/	3037	*/
3038	static void free_coherent(struct device *dev, size_t size,	3038	static void free_coherent(struct device *dev, size_t size,
3039	void *virt_addr, dma_addr_t dma_addr,	3039	void *virt_addr, dma_addr_t dma_addr,
3040	struct dma_attrs *attrs)	3040	struct dma_attrs *attrs)
3041	{	3041	{
3042	unsigned long flags;	3042	unsigned long flags;
3043	struct protection_domain *domain;	3043	struct protection_domain *domain;
3044		3044
3045	INC_STATS_COUNTER(cnt_free_coherent);	3045	INC_STATS_COUNTER(cnt_free_coherent);
3046		3046
3047	domain = get_domain(dev);	3047	domain = get_domain(dev);
3048	if (IS_ERR(domain))	3048	if (IS_ERR(domain))
3049	goto free_mem;	3049	goto free_mem;
3050		3050
3051	spin_lock_irqsave(&domain->lock, flags);	3051	spin_lock_irqsave(&domain->lock, flags);
3052		3052
3053	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);	3053	__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
3054		3054
3055	domain_flush_complete(domain);	3055	domain_flush_complete(domain);
3056		3056
3057	spin_unlock_irqrestore(&domain->lock, flags);	3057	spin_unlock_irqrestore(&domain->lock, flags);
3058		3058
3059	free_mem:	3059	free_mem:
3060	free_pages((unsigned long)virt_addr, get_order(size));	3060	free_pages((unsigned long)virt_addr, get_order(size));
3061	}	3061	}
3062		3062
3063	/*	3063	/*
3064	* This function is called by the DMA layer to find out if we can handle a	3064	* This function is called by the DMA layer to find out if we can handle a
3065	* particular device. It is part of the dma_ops.	3065	* particular device. It is part of the dma_ops.
3066	*/	3066	*/
3067	static int amd_iommu_dma_supported(struct device *dev, u64 mask)	3067	static int amd_iommu_dma_supported(struct device *dev, u64 mask)
3068	{	3068	{
3069	return check_device(dev);	3069	return check_device(dev);
3070	}	3070	}
3071		3071
3072	/*	3072	/*
3073	* The function for pre-allocating protection domains.	3073	* The function for pre-allocating protection domains.
3074	*	3074	*
3075	* If the driver core informs the DMA layer if a driver grabs a device	3075	* If the driver core informs the DMA layer if a driver grabs a device
3076	* we don't need to preallocate the protection domains anymore.	3076	* we don't need to preallocate the protection domains anymore.
3077	* For now we have to.	3077	* For now we have to.
3078	*/	3078	*/
3079	static void __init prealloc_protection_domains(void)	3079	static void __init prealloc_protection_domains(void)
3080	{	3080	{
3081	struct iommu_dev_data *dev_data;	3081	struct iommu_dev_data *dev_data;
3082	struct dma_ops_domain *dma_dom;	3082	struct dma_ops_domain *dma_dom;
3083	struct pci_dev *dev = NULL;	3083	struct pci_dev *dev = NULL;
3084	u16 devid;	3084	u16 devid;
3085		3085
3086	for_each_pci_dev(dev) {	3086	for_each_pci_dev(dev) {
3087		3087
3088	/* Do we handle this device? */	3088	/* Do we handle this device? */
3089	if (!check_device(&dev->dev))	3089	if (!check_device(&dev->dev))
3090	continue;	3090	continue;
3091		3091
3092	dev_data = get_dev_data(&dev->dev);	3092	dev_data = get_dev_data(&dev->dev);
3093	if (!amd_iommu_force_isolation && dev_data->iommu_v2) {	3093	if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
3094	/* Make sure passthrough domain is allocated */	3094	/* Make sure passthrough domain is allocated */
3095	alloc_passthrough_domain();	3095	alloc_passthrough_domain();
3096	dev_data->passthrough = true;	3096	dev_data->passthrough = true;
3097	attach_device(&dev->dev, pt_domain);	3097	attach_device(&dev->dev, pt_domain);
3098	pr_info("AMD-Vi: Using passthrough domain for device %s\n",	3098	pr_info("AMD-Vi: Using passthrough domain for device %s\n",
3099	dev_name(&dev->dev));	3099	dev_name(&dev->dev));
3100	}	3100	}
3101		3101
3102	/* Is there already any domain for it? */	3102	/* Is there already any domain for it? */
3103	if (domain_for_device(&dev->dev))	3103	if (domain_for_device(&dev->dev))
3104	continue;	3104	continue;
3105		3105
3106	devid = get_device_id(&dev->dev);	3106	devid = get_device_id(&dev->dev);
3107		3107
3108	dma_dom = dma_ops_domain_alloc();	3108	dma_dom = dma_ops_domain_alloc();
3109	if (!dma_dom)	3109	if (!dma_dom)
3110	continue;	3110	continue;
3111	init_unity_mappings_for_device(dma_dom, devid);	3111	init_unity_mappings_for_device(dma_dom, devid);
3112	dma_dom->target_dev = devid;	3112	dma_dom->target_dev = devid;
3113		3113
3114	attach_device(&dev->dev, &dma_dom->domain);	3114	attach_device(&dev->dev, &dma_dom->domain);
3115		3115
3116	list_add_tail(&dma_dom->list, &iommu_pd_list);	3116	list_add_tail(&dma_dom->list, &iommu_pd_list);
3117	}	3117	}
3118	}	3118	}
3119		3119
3120	static struct dma_map_ops amd_iommu_dma_ops = {	3120	static struct dma_map_ops amd_iommu_dma_ops = {
3121	.alloc = alloc_coherent,	3121	.alloc = alloc_coherent,
3122	.free = free_coherent,	3122	.free = free_coherent,
3123	.map_page = map_page,	3123	.map_page = map_page,
3124	.unmap_page = unmap_page,	3124	.unmap_page = unmap_page,
3125	.map_sg = map_sg,	3125	.map_sg = map_sg,
3126	.unmap_sg = unmap_sg,	3126	.unmap_sg = unmap_sg,
3127	.dma_supported = amd_iommu_dma_supported,	3127	.dma_supported = amd_iommu_dma_supported,
3128	};	3128	};
3129		3129
3130	static unsigned device_dma_ops_init(void)	3130	static unsigned device_dma_ops_init(void)
3131	{	3131	{
3132	struct iommu_dev_data *dev_data;	3132	struct iommu_dev_data *dev_data;
3133	struct pci_dev *pdev = NULL;	3133	struct pci_dev *pdev = NULL;
3134	unsigned unhandled = 0;	3134	unsigned unhandled = 0;
3135		3135
3136	for_each_pci_dev(pdev) {	3136	for_each_pci_dev(pdev) {
3137	if (!check_device(&pdev->dev)) {	3137	if (!check_device(&pdev->dev)) {
3138		3138
3139	iommu_ignore_device(&pdev->dev);	3139	iommu_ignore_device(&pdev->dev);
3140		3140
3141	unhandled += 1;	3141	unhandled += 1;
3142	continue;	3142	continue;
3143	}	3143	}
3144		3144
3145	dev_data = get_dev_data(&pdev->dev);	3145	dev_data = get_dev_data(&pdev->dev);
3146		3146
3147	if (!dev_data->passthrough)	3147	if (!dev_data->passthrough)
3148	pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;	3148	pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
3149	else	3149	else
3150	pdev->dev.archdata.dma_ops = &nommu_dma_ops;	3150	pdev->dev.archdata.dma_ops = &nommu_dma_ops;
3151	}	3151	}
3152		3152
3153	return unhandled;	3153	return unhandled;
3154	}	3154	}
3155		3155
3156	/*	3156	/*
3157	* The function which clues the AMD IOMMU driver into dma_ops.	3157	* The function which clues the AMD IOMMU driver into dma_ops.
3158	*/	3158	*/
3159		3159
3160	void __init amd_iommu_init_api(void)	3160	void __init amd_iommu_init_api(void)
3161	{	3161	{
3162	bus_set_iommu(&pci_bus_type, &amd_iommu_ops);	3162	bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
3163	}	3163	}
3164		3164
3165	int __init amd_iommu_init_dma_ops(void)	3165	int __init amd_iommu_init_dma_ops(void)
3166	{	3166	{
3167	struct amd_iommu *iommu;	3167	struct amd_iommu *iommu;
3168	int ret, unhandled;	3168	int ret, unhandled;
3169		3169
3170	/*	3170	/*
3171	* first allocate a default protection domain for every IOMMU we	3171	* first allocate a default protection domain for every IOMMU we
3172	* found in the system. Devices not assigned to any other	3172	* found in the system. Devices not assigned to any other
3173	* protection domain will be assigned to the default one.	3173	* protection domain will be assigned to the default one.
3174	*/	3174	*/
3175	for_each_iommu(iommu) {	3175	for_each_iommu(iommu) {
3176	iommu->default_dom = dma_ops_domain_alloc();	3176	iommu->default_dom = dma_ops_domain_alloc();
3177	if (iommu->default_dom == NULL)	3177	if (iommu->default_dom == NULL)
3178	return -ENOMEM;	3178	return -ENOMEM;
3179	iommu->default_dom->domain.flags \|= PD_DEFAULT_MASK;	3179	iommu->default_dom->domain.flags \|= PD_DEFAULT_MASK;
3180	ret = iommu_init_unity_mappings(iommu);	3180	ret = iommu_init_unity_mappings(iommu);
3181	if (ret)	3181	if (ret)
3182	goto free_domains;	3182	goto free_domains;
3183	}	3183	}
3184		3184
3185	/*	3185	/*
3186	* Pre-allocate the protection domains for each device.	3186	* Pre-allocate the protection domains for each device.
3187	*/	3187	*/
3188	prealloc_protection_domains();	3188	prealloc_protection_domains();
3189		3189
3190	iommu_detected = 1;	3190	iommu_detected = 1;
3191	swiotlb = 0;	3191	swiotlb = 0;
3192		3192
3193	/* Make the driver finally visible to the drivers */	3193	/* Make the driver finally visible to the drivers */
3194	unhandled = device_dma_ops_init();	3194	unhandled = device_dma_ops_init();
3195	if (unhandled && max_pfn > MAX_DMA32_PFN) {	3195	if (unhandled && max_pfn > MAX_DMA32_PFN) {
3196	/* There are unhandled devices - initialize swiotlb for them */	3196	/* There are unhandled devices - initialize swiotlb for them */
3197	swiotlb = 1;	3197	swiotlb = 1;
3198	}	3198	}
3199		3199
3200	amd_iommu_stats_init();	3200	amd_iommu_stats_init();
3201		3201
3202	if (amd_iommu_unmap_flush)	3202	if (amd_iommu_unmap_flush)
3203	pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");	3203	pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");
3204	else	3204	else
3205	pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");	3205	pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
3206		3206
3207	return 0;	3207	return 0;
3208		3208
3209	free_domains:	3209	free_domains:
3210		3210
3211	for_each_iommu(iommu) {	3211	for_each_iommu(iommu) {
3212	dma_ops_domain_free(iommu->default_dom);	3212	dma_ops_domain_free(iommu->default_dom);
3213	}	3213	}
3214		3214
3215	return ret;	3215	return ret;
3216	}	3216	}
3217		3217
3218	/*****************************************************************************	3218	/*****************************************************************************
3219	*	3219	*
3220	* The following functions belong to the exported interface of AMD IOMMU	3220	* The following functions belong to the exported interface of AMD IOMMU
3221	*	3221	*
3222	* This interface allows access to lower level functions of the IOMMU	3222	* This interface allows access to lower level functions of the IOMMU
3223	* like protection domain handling and assignement of devices to domains	3223	* like protection domain handling and assignement of devices to domains
3224	* which is not possible with the dma_ops interface.	3224	* which is not possible with the dma_ops interface.
3225	*	3225	*
3226	*****************************************************************************/	3226	*****************************************************************************/
3227		3227
3228	static void cleanup_domain(struct protection_domain *domain)	3228	static void cleanup_domain(struct protection_domain *domain)
3229	{	3229	{
3230	struct iommu_dev_data dev_data, next;	3230	struct iommu_dev_data dev_data, next;
3231	unsigned long flags;	3231	unsigned long flags;
3232		3232
3233	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	3233	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3234		3234
3235	list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {	3235	list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
3236	__detach_device(dev_data);	3236	__detach_device(dev_data);
3237	atomic_set(&dev_data->bind, 0);	3237	atomic_set(&dev_data->bind, 0);
3238	}	3238	}
3239		3239
3240	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	3240	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
3241	}	3241	}
3242		3242
3243	static void protection_domain_free(struct protection_domain *domain)	3243	static void protection_domain_free(struct protection_domain *domain)
3244	{	3244	{
3245	if (!domain)	3245	if (!domain)
3246	return;	3246	return;
3247		3247
3248	del_domain_from_list(domain);	3248	del_domain_from_list(domain);
3249		3249
3250	if (domain->id)	3250	if (domain->id)
3251	domain_id_free(domain->id);	3251	domain_id_free(domain->id);
3252		3252
3253	kfree(domain);	3253	kfree(domain);
3254	}	3254	}
3255		3255
3256	static struct protection_domain *protection_domain_alloc(void)	3256	static struct protection_domain *protection_domain_alloc(void)
3257	{	3257	{
3258	struct protection_domain *domain;	3258	struct protection_domain *domain;
3259		3259
3260	domain = kzalloc(sizeof(*domain), GFP_KERNEL);	3260	domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3261	if (!domain)	3261	if (!domain)
3262	return NULL;	3262	return NULL;
3263		3263
3264	spin_lock_init(&domain->lock);	3264	spin_lock_init(&domain->lock);
3265	mutex_init(&domain->api_lock);	3265	mutex_init(&domain->api_lock);
3266	domain->id = domain_id_alloc();	3266	domain->id = domain_id_alloc();
3267	if (!domain->id)	3267	if (!domain->id)
3268	goto out_err;	3268	goto out_err;
3269	INIT_LIST_HEAD(&domain->dev_list);	3269	INIT_LIST_HEAD(&domain->dev_list);
3270		3270
3271	add_domain_to_list(domain);	3271	add_domain_to_list(domain);
3272		3272
3273	return domain;	3273	return domain;
3274		3274
3275	out_err:	3275	out_err:
3276	kfree(domain);	3276	kfree(domain);
3277		3277
3278	return NULL;	3278	return NULL;
3279	}	3279	}
3280		3280
3281	static int __init alloc_passthrough_domain(void)	3281	static int __init alloc_passthrough_domain(void)
3282	{	3282	{
3283	if (pt_domain != NULL)	3283	if (pt_domain != NULL)
3284	return 0;	3284	return 0;
3285		3285
3286	/* allocate passthrough domain */	3286	/* allocate passthrough domain */
3287	pt_domain = protection_domain_alloc();	3287	pt_domain = protection_domain_alloc();
3288	if (!pt_domain)	3288	if (!pt_domain)
3289	return -ENOMEM;	3289	return -ENOMEM;
3290		3290
3291	pt_domain->mode = PAGE_MODE_NONE;	3291	pt_domain->mode = PAGE_MODE_NONE;
3292		3292
3293	return 0;	3293	return 0;
3294	}	3294	}
3295	static int amd_iommu_domain_init(struct iommu_domain *dom)	3295	static int amd_iommu_domain_init(struct iommu_domain *dom)
3296	{	3296	{
3297	struct protection_domain *domain;	3297	struct protection_domain *domain;
3298		3298
3299	domain = protection_domain_alloc();	3299	domain = protection_domain_alloc();
3300	if (!domain)	3300	if (!domain)
3301	goto out_free;	3301	goto out_free;
3302		3302
3303	domain->mode = PAGE_MODE_3_LEVEL;	3303	domain->mode = PAGE_MODE_3_LEVEL;
3304	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);	3304	domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
3305	if (!domain->pt_root)	3305	if (!domain->pt_root)
3306	goto out_free;	3306	goto out_free;
3307		3307
3308	domain->iommu_domain = dom;	3308	domain->iommu_domain = dom;
3309		3309
3310	dom->priv = domain;	3310	dom->priv = domain;
3311		3311
3312	dom->geometry.aperture_start = 0;	3312	dom->geometry.aperture_start = 0;
3313	dom->geometry.aperture_end = ~0ULL;	3313	dom->geometry.aperture_end = ~0ULL;
3314	dom->geometry.force_aperture = true;	3314	dom->geometry.force_aperture = true;
3315		3315
3316	return 0;	3316	return 0;
3317		3317
3318	out_free:	3318	out_free:
3319	protection_domain_free(domain);	3319	protection_domain_free(domain);
3320		3320
3321	return -ENOMEM;	3321	return -ENOMEM;
3322	}	3322	}
3323		3323
3324	static void amd_iommu_domain_destroy(struct iommu_domain *dom)	3324	static void amd_iommu_domain_destroy(struct iommu_domain *dom)
3325	{	3325	{
3326	struct protection_domain *domain = dom->priv;	3326	struct protection_domain *domain = dom->priv;
3327		3327
3328	if (!domain)	3328	if (!domain)
3329	return;	3329	return;
3330		3330
3331	if (domain->dev_cnt > 0)	3331	if (domain->dev_cnt > 0)
3332	cleanup_domain(domain);	3332	cleanup_domain(domain);
3333		3333
3334	BUG_ON(domain->dev_cnt != 0);	3334	BUG_ON(domain->dev_cnt != 0);
3335		3335
3336	if (domain->mode != PAGE_MODE_NONE)	3336	if (domain->mode != PAGE_MODE_NONE)
3337	free_pagetable(domain);	3337	free_pagetable(domain);
3338		3338
3339	if (domain->flags & PD_IOMMUV2_MASK)	3339	if (domain->flags & PD_IOMMUV2_MASK)
3340	free_gcr3_table(domain);	3340	free_gcr3_table(domain);
3341		3341
3342	protection_domain_free(domain);	3342	protection_domain_free(domain);
3343		3343
3344	dom->priv = NULL;	3344	dom->priv = NULL;
3345	}	3345	}
3346		3346
3347	static void amd_iommu_detach_device(struct iommu_domain *dom,	3347	static void amd_iommu_detach_device(struct iommu_domain *dom,
3348	struct device *dev)	3348	struct device *dev)
3349	{	3349	{
3350	struct iommu_dev_data *dev_data = dev->archdata.iommu;	3350	struct iommu_dev_data *dev_data = dev->archdata.iommu;
3351	struct amd_iommu *iommu;	3351	struct amd_iommu *iommu;
3352	u16 devid;	3352	u16 devid;
3353		3353
3354	if (!check_device(dev))	3354	if (!check_device(dev))
3355	return;	3355	return;
3356		3356
3357	devid = get_device_id(dev);	3357	devid = get_device_id(dev);
3358		3358
3359	if (dev_data->domain != NULL)	3359	if (dev_data->domain != NULL)
3360	detach_device(dev);	3360	detach_device(dev);
3361		3361
3362	iommu = amd_iommu_rlookup_table[devid];	3362	iommu = amd_iommu_rlookup_table[devid];
3363	if (!iommu)	3363	if (!iommu)
3364	return;	3364	return;
3365		3365
3366	iommu_completion_wait(iommu);	3366	iommu_completion_wait(iommu);
3367	}	3367	}
3368		3368
3369	static int amd_iommu_attach_device(struct iommu_domain *dom,	3369	static int amd_iommu_attach_device(struct iommu_domain *dom,
3370	struct device *dev)	3370	struct device *dev)
3371	{	3371	{
3372	struct protection_domain *domain = dom->priv;	3372	struct protection_domain *domain = dom->priv;
3373	struct iommu_dev_data *dev_data;	3373	struct iommu_dev_data *dev_data;
3374	struct amd_iommu *iommu;	3374	struct amd_iommu *iommu;
3375	int ret;	3375	int ret;
3376		3376
3377	if (!check_device(dev))	3377	if (!check_device(dev))
3378	return -EINVAL;	3378	return -EINVAL;
3379		3379
3380	dev_data = dev->archdata.iommu;	3380	dev_data = dev->archdata.iommu;
3381		3381
3382	iommu = amd_iommu_rlookup_table[dev_data->devid];	3382	iommu = amd_iommu_rlookup_table[dev_data->devid];
3383	if (!iommu)	3383	if (!iommu)
3384	return -EINVAL;	3384	return -EINVAL;
3385		3385
3386	if (dev_data->domain)	3386	if (dev_data->domain)
3387	detach_device(dev);	3387	detach_device(dev);
3388		3388
3389	ret = attach_device(dev, domain);	3389	ret = attach_device(dev, domain);
3390		3390
3391	iommu_completion_wait(iommu);	3391	iommu_completion_wait(iommu);
3392		3392
3393	return ret;	3393	return ret;
3394	}	3394	}
3395		3395
3396	static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,	3396	static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3397	phys_addr_t paddr, size_t page_size, int iommu_prot)	3397	phys_addr_t paddr, size_t page_size, int iommu_prot)
3398	{	3398	{
3399	struct protection_domain *domain = dom->priv;	3399	struct protection_domain *domain = dom->priv;
3400	int prot = 0;	3400	int prot = 0;
3401	int ret;	3401	int ret;
3402		3402
3403	if (domain->mode == PAGE_MODE_NONE)	3403	if (domain->mode == PAGE_MODE_NONE)
3404	return -EINVAL;	3404	return -EINVAL;
3405		3405
3406	if (iommu_prot & IOMMU_READ)	3406	if (iommu_prot & IOMMU_READ)
3407	prot \|= IOMMU_PROT_IR;	3407	prot \|= IOMMU_PROT_IR;
3408	if (iommu_prot & IOMMU_WRITE)	3408	if (iommu_prot & IOMMU_WRITE)
3409	prot \|= IOMMU_PROT_IW;	3409	prot \|= IOMMU_PROT_IW;
3410		3410
3411	mutex_lock(&domain->api_lock);	3411	mutex_lock(&domain->api_lock);
3412	ret = iommu_map_page(domain, iova, paddr, prot, page_size);	3412	ret = iommu_map_page(domain, iova, paddr, prot, page_size);
3413	mutex_unlock(&domain->api_lock);	3413	mutex_unlock(&domain->api_lock);
3414		3414
3415	return ret;	3415	return ret;
3416	}	3416	}
3417		3417
3418	static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,	3418	static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
3419	size_t page_size)	3419	size_t page_size)
3420	{	3420	{
3421	struct protection_domain *domain = dom->priv;	3421	struct protection_domain *domain = dom->priv;
3422	size_t unmap_size;	3422	size_t unmap_size;
3423		3423
3424	if (domain->mode == PAGE_MODE_NONE)	3424	if (domain->mode == PAGE_MODE_NONE)
3425	return -EINVAL;	3425	return -EINVAL;
3426		3426
3427	mutex_lock(&domain->api_lock);	3427	mutex_lock(&domain->api_lock);
3428	unmap_size = iommu_unmap_page(domain, iova, page_size);	3428	unmap_size = iommu_unmap_page(domain, iova, page_size);
3429	mutex_unlock(&domain->api_lock);	3429	mutex_unlock(&domain->api_lock);
3430		3430
3431	domain_flush_tlb_pde(domain);	3431	domain_flush_tlb_pde(domain);
3432		3432
3433	return unmap_size;	3433	return unmap_size;
3434	}	3434	}
3435		3435
3436	static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,	3436	static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
3437	dma_addr_t iova)	3437	dma_addr_t iova)
3438	{	3438	{
3439	struct protection_domain *domain = dom->priv;	3439	struct protection_domain *domain = dom->priv;
3440	unsigned long offset_mask;	3440	unsigned long offset_mask;
3441	phys_addr_t paddr;	3441	phys_addr_t paddr;
3442	u64 *pte, __pte;	3442	u64 *pte, __pte;
3443		3443
3444	if (domain->mode == PAGE_MODE_NONE)	3444	if (domain->mode == PAGE_MODE_NONE)
3445	return iova;	3445	return iova;
3446		3446
3447	pte = fetch_pte(domain, iova);	3447	pte = fetch_pte(domain, iova);
3448		3448
3449	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))	3449	if (!pte \|\| !IOMMU_PTE_PRESENT(*pte))
3450	return 0;	3450	return 0;
3451		3451
3452	if (PM_PTE_LEVEL(*pte) == 0)	3452	if (PM_PTE_LEVEL(*pte) == 0)
3453	offset_mask = PAGE_SIZE - 1;	3453	offset_mask = PAGE_SIZE - 1;
3454	else	3454	else
3455	offset_mask = PTE_PAGE_SIZE(*pte) - 1;	3455	offset_mask = PTE_PAGE_SIZE(*pte) - 1;
3456		3456
3457	__pte = *pte & PM_ADDR_MASK;	3457	__pte = *pte & PM_ADDR_MASK;
3458	paddr = (__pte & ~offset_mask) \| (iova & offset_mask);	3458	paddr = (__pte & ~offset_mask) \| (iova & offset_mask);
3459		3459
3460	return paddr;	3460	return paddr;
3461	}	3461	}
3462		3462
3463	static int amd_iommu_domain_has_cap(struct iommu_domain *domain,	3463	static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
3464	unsigned long cap)	3464	unsigned long cap)
3465	{	3465	{
3466	switch (cap) {	3466	switch (cap) {
3467	case IOMMU_CAP_CACHE_COHERENCY:	3467	case IOMMU_CAP_CACHE_COHERENCY:
3468	return 1;	3468	return 1;
3469	case IOMMU_CAP_INTR_REMAP:	3469	case IOMMU_CAP_INTR_REMAP:
3470	return irq_remapping_enabled;	3470	return irq_remapping_enabled;
3471	}	3471	}
3472		3472
3473	return 0;	3473	return 0;
3474	}	3474	}
3475		3475
3476	static struct iommu_ops amd_iommu_ops = {	3476	static struct iommu_ops amd_iommu_ops = {
3477	.domain_init = amd_iommu_domain_init,	3477	.domain_init = amd_iommu_domain_init,
3478	.domain_destroy = amd_iommu_domain_destroy,	3478	.domain_destroy = amd_iommu_domain_destroy,
3479	.attach_dev = amd_iommu_attach_device,	3479	.attach_dev = amd_iommu_attach_device,
3480	.detach_dev = amd_iommu_detach_device,	3480	.detach_dev = amd_iommu_detach_device,
3481	.map = amd_iommu_map,	3481	.map = amd_iommu_map,
3482	.unmap = amd_iommu_unmap,	3482	.unmap = amd_iommu_unmap,
3483	.iova_to_phys = amd_iommu_iova_to_phys,	3483	.iova_to_phys = amd_iommu_iova_to_phys,
3484	.domain_has_cap = amd_iommu_domain_has_cap,	3484	.domain_has_cap = amd_iommu_domain_has_cap,
3485	.pgsize_bitmap = AMD_IOMMU_PGSIZES,	3485	.pgsize_bitmap = AMD_IOMMU_PGSIZES,
3486	};	3486	};
3487		3487
3488	/*****************************************************************************	3488	/*****************************************************************************
3489	*	3489	*
3490	* The next functions do a basic initialization of IOMMU for pass through	3490	* The next functions do a basic initialization of IOMMU for pass through
3491	* mode	3491	* mode
3492	*	3492	*
3493	* In passthrough mode the IOMMU is initialized and enabled but not used for	3493	* In passthrough mode the IOMMU is initialized and enabled but not used for
3494	* DMA-API translation.	3494	* DMA-API translation.
3495	*	3495	*
3496	*****************************************************************************/	3496	*****************************************************************************/
3497		3497
3498	int __init amd_iommu_init_passthrough(void)	3498	int __init amd_iommu_init_passthrough(void)
3499	{	3499	{
3500	struct iommu_dev_data *dev_data;	3500	struct iommu_dev_data *dev_data;
3501	struct pci_dev *dev = NULL;	3501	struct pci_dev *dev = NULL;
3502	struct amd_iommu *iommu;	3502	struct amd_iommu *iommu;
3503	u16 devid;	3503	u16 devid;
3504	int ret;	3504	int ret;
3505		3505
3506	ret = alloc_passthrough_domain();	3506	ret = alloc_passthrough_domain();
3507	if (ret)	3507	if (ret)
3508	return ret;	3508	return ret;
3509		3509
3510	for_each_pci_dev(dev) {	3510	for_each_pci_dev(dev) {
3511	if (!check_device(&dev->dev))	3511	if (!check_device(&dev->dev))
3512	continue;	3512	continue;
3513		3513
3514	dev_data = get_dev_data(&dev->dev);	3514	dev_data = get_dev_data(&dev->dev);
3515	dev_data->passthrough = true;	3515	dev_data->passthrough = true;
3516		3516
3517	devid = get_device_id(&dev->dev);	3517	devid = get_device_id(&dev->dev);
3518		3518
3519	iommu = amd_iommu_rlookup_table[devid];	3519	iommu = amd_iommu_rlookup_table[devid];
3520	if (!iommu)	3520	if (!iommu)
3521	continue;	3521	continue;
3522		3522
3523	attach_device(&dev->dev, pt_domain);	3523	attach_device(&dev->dev, pt_domain);
3524	}	3524	}
3525		3525
3526	amd_iommu_stats_init();	3526	amd_iommu_stats_init();
3527		3527
3528	pr_info("AMD-Vi: Initialized for Passthrough Mode\n");	3528	pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3529		3529
3530	return 0;	3530	return 0;
3531	}	3531	}
3532		3532
3533	/* IOMMUv2 specific functions */	3533	/* IOMMUv2 specific functions */
3534	int amd_iommu_register_ppr_notifier(struct notifier_block *nb)	3534	int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3535	{	3535	{
3536	return atomic_notifier_chain_register(&ppr_notifier, nb);	3536	return atomic_notifier_chain_register(&ppr_notifier, nb);
3537	}	3537	}
3538	EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);	3538	EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3539		3539
3540	int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)	3540	int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3541	{	3541	{
3542	return atomic_notifier_chain_unregister(&ppr_notifier, nb);	3542	return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3543	}	3543	}
3544	EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);	3544	EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3545		3545
3546	void amd_iommu_domain_direct_map(struct iommu_domain *dom)	3546	void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3547	{	3547	{
3548	struct protection_domain *domain = dom->priv;	3548	struct protection_domain *domain = dom->priv;
3549	unsigned long flags;	3549	unsigned long flags;
3550		3550
3551	spin_lock_irqsave(&domain->lock, flags);	3551	spin_lock_irqsave(&domain->lock, flags);
3552		3552
3553	/* Update data structure */	3553	/* Update data structure */
3554	domain->mode = PAGE_MODE_NONE;	3554	domain->mode = PAGE_MODE_NONE;
3555	domain->updated = true;	3555	domain->updated = true;
3556		3556
3557	/* Make changes visible to IOMMUs */	3557	/* Make changes visible to IOMMUs */
3558	update_domain(domain);	3558	update_domain(domain);
3559		3559
3560	/* Page-table is not visible to IOMMU anymore, so free it */	3560	/* Page-table is not visible to IOMMU anymore, so free it */
3561	free_pagetable(domain);	3561	free_pagetable(domain);
3562		3562
3563	spin_unlock_irqrestore(&domain->lock, flags);	3563	spin_unlock_irqrestore(&domain->lock, flags);
3564	}	3564	}
3565	EXPORT_SYMBOL(amd_iommu_domain_direct_map);	3565	EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3566		3566
3567	int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)	3567	int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3568	{	3568	{
3569	struct protection_domain *domain = dom->priv;	3569	struct protection_domain *domain = dom->priv;
3570	unsigned long flags;	3570	unsigned long flags;
3571	int levels, ret;	3571	int levels, ret;
3572		3572
3573	if (pasids <= 0 \|\| pasids > (PASID_MASK + 1))	3573	if (pasids <= 0 \|\| pasids > (PASID_MASK + 1))
3574	return -EINVAL;	3574	return -EINVAL;
3575		3575
3576	/* Number of GCR3 table levels required */	3576	/* Number of GCR3 table levels required */
3577	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)	3577	for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3578	levels += 1;	3578	levels += 1;
3579		3579
3580	if (levels > amd_iommu_max_glx_val)	3580	if (levels > amd_iommu_max_glx_val)
3581	return -EINVAL;	3581	return -EINVAL;
3582		3582
3583	spin_lock_irqsave(&domain->lock, flags);	3583	spin_lock_irqsave(&domain->lock, flags);
3584		3584
3585	/*	3585	/*
3586	* Save us all sanity checks whether devices already in the	3586	* Save us all sanity checks whether devices already in the
3587	* domain support IOMMUv2. Just force that the domain has no	3587	* domain support IOMMUv2. Just force that the domain has no
3588	* devices attached when it is switched into IOMMUv2 mode.	3588	* devices attached when it is switched into IOMMUv2 mode.
3589	*/	3589	*/
3590	ret = -EBUSY;	3590	ret = -EBUSY;
3591	if (domain->dev_cnt > 0 \|\| domain->flags & PD_IOMMUV2_MASK)	3591	if (domain->dev_cnt > 0 \|\| domain->flags & PD_IOMMUV2_MASK)
3592	goto out;	3592	goto out;
3593		3593
3594	ret = -ENOMEM;	3594	ret = -ENOMEM;
3595	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);	3595	domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3596	if (domain->gcr3_tbl == NULL)	3596	if (domain->gcr3_tbl == NULL)
3597	goto out;	3597	goto out;
3598		3598
3599	domain->glx = levels;	3599	domain->glx = levels;
3600	domain->flags \|= PD_IOMMUV2_MASK;	3600	domain->flags \|= PD_IOMMUV2_MASK;
3601	domain->updated = true;	3601	domain->updated = true;
3602		3602
3603	update_domain(domain);	3603	update_domain(domain);
3604		3604
3605	ret = 0;	3605	ret = 0;
3606		3606
3607	out:	3607	out:
3608	spin_unlock_irqrestore(&domain->lock, flags);	3608	spin_unlock_irqrestore(&domain->lock, flags);
3609		3609
3610	return ret;	3610	return ret;
3611	}	3611	}
3612	EXPORT_SYMBOL(amd_iommu_domain_enable_v2);	3612	EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3613		3613
3614	static int __flush_pasid(struct protection_domain *domain, int pasid,	3614	static int __flush_pasid(struct protection_domain *domain, int pasid,
3615	u64 address, bool size)	3615	u64 address, bool size)
3616	{	3616	{
3617	struct iommu_dev_data *dev_data;	3617	struct iommu_dev_data *dev_data;
3618	struct iommu_cmd cmd;	3618	struct iommu_cmd cmd;
3619	int i, ret;	3619	int i, ret;
3620		3620
3621	if (!(domain->flags & PD_IOMMUV2_MASK))	3621	if (!(domain->flags & PD_IOMMUV2_MASK))
3622	return -EINVAL;	3622	return -EINVAL;
3623		3623
3624	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);	3624	build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
3625		3625
3626	/*	3626	/*
3627	* IOMMU TLB needs to be flushed before Device TLB to	3627	* IOMMU TLB needs to be flushed before Device TLB to
3628	* prevent device TLB refill from IOMMU TLB	3628	* prevent device TLB refill from IOMMU TLB
3629	*/	3629	*/
3630	for (i = 0; i < amd_iommus_present; ++i) {	3630	for (i = 0; i < amd_iommus_present; ++i) {
3631	if (domain->dev_iommu[i] == 0)	3631	if (domain->dev_iommu[i] == 0)
3632	continue;	3632	continue;
3633		3633
3634	ret = iommu_queue_command(amd_iommus[i], &cmd);	3634	ret = iommu_queue_command(amd_iommus[i], &cmd);
3635	if (ret != 0)	3635	if (ret != 0)
3636	goto out;	3636	goto out;
3637	}	3637	}
3638		3638
3639	/* Wait until IOMMU TLB flushes are complete */	3639	/* Wait until IOMMU TLB flushes are complete */
3640	domain_flush_complete(domain);	3640	domain_flush_complete(domain);
3641		3641
3642	/* Now flush device TLBs */	3642	/* Now flush device TLBs */
3643	list_for_each_entry(dev_data, &domain->dev_list, list) {	3643	list_for_each_entry(dev_data, &domain->dev_list, list) {
3644	struct amd_iommu *iommu;	3644	struct amd_iommu *iommu;
3645	int qdep;	3645	int qdep;
3646		3646
3647	BUG_ON(!dev_data->ats.enabled);	3647	BUG_ON(!dev_data->ats.enabled);
3648		3648
3649	qdep = dev_data->ats.qdep;	3649	qdep = dev_data->ats.qdep;
3650	iommu = amd_iommu_rlookup_table[dev_data->devid];	3650	iommu = amd_iommu_rlookup_table[dev_data->devid];
3651		3651
3652	build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,	3652	build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
3653	qdep, address, size);	3653	qdep, address, size);
3654		3654
3655	ret = iommu_queue_command(iommu, &cmd);	3655	ret = iommu_queue_command(iommu, &cmd);
3656	if (ret != 0)	3656	if (ret != 0)
3657	goto out;	3657	goto out;
3658	}	3658	}
3659		3659
3660	/* Wait until all device TLBs are flushed */	3660	/* Wait until all device TLBs are flushed */
3661	domain_flush_complete(domain);	3661	domain_flush_complete(domain);
3662		3662
3663	ret = 0;	3663	ret = 0;
3664		3664
3665	out:	3665	out:
3666		3666
3667	return ret;	3667	return ret;
3668	}	3668	}
3669		3669
3670	static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,	3670	static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
3671	u64 address)	3671	u64 address)
3672	{	3672	{
3673	INC_STATS_COUNTER(invalidate_iotlb);	3673	INC_STATS_COUNTER(invalidate_iotlb);
3674		3674
3675	return __flush_pasid(domain, pasid, address, false);	3675	return __flush_pasid(domain, pasid, address, false);
3676	}	3676	}
3677		3677
3678	int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,	3678	int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
3679	u64 address)	3679	u64 address)
3680	{	3680	{
3681	struct protection_domain *domain = dom->priv;	3681	struct protection_domain *domain = dom->priv;
3682	unsigned long flags;	3682	unsigned long flags;
3683	int ret;	3683	int ret;
3684		3684
3685	spin_lock_irqsave(&domain->lock, flags);	3685	spin_lock_irqsave(&domain->lock, flags);
3686	ret = __amd_iommu_flush_page(domain, pasid, address);	3686	ret = __amd_iommu_flush_page(domain, pasid, address);
3687	spin_unlock_irqrestore(&domain->lock, flags);	3687	spin_unlock_irqrestore(&domain->lock, flags);
3688		3688
3689	return ret;	3689	return ret;
3690	}	3690	}
3691	EXPORT_SYMBOL(amd_iommu_flush_page);	3691	EXPORT_SYMBOL(amd_iommu_flush_page);
3692		3692
3693	static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)	3693	static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
3694	{	3694	{
3695	INC_STATS_COUNTER(invalidate_iotlb_all);	3695	INC_STATS_COUNTER(invalidate_iotlb_all);
3696		3696
3697	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,	3697	return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
3698	true);	3698	true);
3699	}	3699	}
3700		3700
3701	int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)	3701	int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
3702	{	3702	{
3703	struct protection_domain *domain = dom->priv;	3703	struct protection_domain *domain = dom->priv;
3704	unsigned long flags;	3704	unsigned long flags;
3705	int ret;	3705	int ret;
3706		3706
3707	spin_lock_irqsave(&domain->lock, flags);	3707	spin_lock_irqsave(&domain->lock, flags);
3708	ret = __amd_iommu_flush_tlb(domain, pasid);	3708	ret = __amd_iommu_flush_tlb(domain, pasid);
3709	spin_unlock_irqrestore(&domain->lock, flags);	3709	spin_unlock_irqrestore(&domain->lock, flags);
3710		3710
3711	return ret;	3711	return ret;
3712	}	3712	}
3713	EXPORT_SYMBOL(amd_iommu_flush_tlb);	3713	EXPORT_SYMBOL(amd_iommu_flush_tlb);
3714		3714
3715	static u64 __get_gcr3_pte(u64 root, int level, int pasid, bool alloc)	3715	static u64 __get_gcr3_pte(u64 root, int level, int pasid, bool alloc)
3716	{	3716	{
3717	int index;	3717	int index;
3718	u64 *pte;	3718	u64 *pte;
3719		3719
3720	while (true) {	3720	while (true) {
3721		3721
3722	index = (pasid >> (9 * level)) & 0x1ff;	3722	index = (pasid >> (9 * level)) & 0x1ff;
3723	pte = &root[index];	3723	pte = &root[index];
3724		3724
3725	if (level == 0)	3725	if (level == 0)
3726	break;	3726	break;
3727		3727
3728	if (!(*pte & GCR3_VALID)) {	3728	if (!(*pte & GCR3_VALID)) {
3729	if (!alloc)	3729	if (!alloc)
3730	return NULL;	3730	return NULL;
3731		3731
3732	root = (void *)get_zeroed_page(GFP_ATOMIC);	3732	root = (void *)get_zeroed_page(GFP_ATOMIC);
3733	if (root == NULL)	3733	if (root == NULL)
3734	return NULL;	3734	return NULL;
3735		3735
3736	*pte = __pa(root) \| GCR3_VALID;	3736	*pte = __pa(root) \| GCR3_VALID;
3737	}	3737	}
3738		3738
3739	root = __va(*pte & PAGE_MASK);	3739	root = __va(*pte & PAGE_MASK);
3740		3740
3741	level -= 1;	3741	level -= 1;
3742	}	3742	}
3743		3743
3744	return pte;	3744	return pte;
3745	}	3745	}
3746		3746
3747	static int __set_gcr3(struct protection_domain *domain, int pasid,	3747	static int __set_gcr3(struct protection_domain *domain, int pasid,
3748	unsigned long cr3)	3748	unsigned long cr3)
3749	{	3749	{
3750	u64 *pte;	3750	u64 *pte;
3751		3751
3752	if (domain->mode != PAGE_MODE_NONE)	3752	if (domain->mode != PAGE_MODE_NONE)
3753	return -EINVAL;	3753	return -EINVAL;
3754		3754
3755	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);	3755	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
3756	if (pte == NULL)	3756	if (pte == NULL)
3757	return -ENOMEM;	3757	return -ENOMEM;
3758		3758
3759	*pte = (cr3 & PAGE_MASK) \| GCR3_VALID;	3759	*pte = (cr3 & PAGE_MASK) \| GCR3_VALID;
3760		3760
3761	return __amd_iommu_flush_tlb(domain, pasid);	3761	return __amd_iommu_flush_tlb(domain, pasid);
3762	}	3762	}
3763		3763
3764	static int __clear_gcr3(struct protection_domain *domain, int pasid)	3764	static int __clear_gcr3(struct protection_domain *domain, int pasid)
3765	{	3765	{
3766	u64 *pte;	3766	u64 *pte;
3767		3767
3768	if (domain->mode != PAGE_MODE_NONE)	3768	if (domain->mode != PAGE_MODE_NONE)
3769	return -EINVAL;	3769	return -EINVAL;
3770		3770
3771	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);	3771	pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
3772	if (pte == NULL)	3772	if (pte == NULL)
3773	return 0;	3773	return 0;
3774		3774
3775	*pte = 0;	3775	*pte = 0;
3776		3776
3777	return __amd_iommu_flush_tlb(domain, pasid);	3777	return __amd_iommu_flush_tlb(domain, pasid);
3778	}	3778	}
3779		3779
3780	int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,	3780	int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
3781	unsigned long cr3)	3781	unsigned long cr3)
3782	{	3782	{
3783	struct protection_domain *domain = dom->priv;	3783	struct protection_domain *domain = dom->priv;
3784	unsigned long flags;	3784	unsigned long flags;
3785	int ret;	3785	int ret;
3786		3786
3787	spin_lock_irqsave(&domain->lock, flags);	3787	spin_lock_irqsave(&domain->lock, flags);
3788	ret = __set_gcr3(domain, pasid, cr3);	3788	ret = __set_gcr3(domain, pasid, cr3);
3789	spin_unlock_irqrestore(&domain->lock, flags);	3789	spin_unlock_irqrestore(&domain->lock, flags);
3790		3790
3791	return ret;	3791	return ret;
3792	}	3792	}
3793	EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);	3793	EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
3794		3794
3795	int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)	3795	int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
3796	{	3796	{
3797	struct protection_domain *domain = dom->priv;	3797	struct protection_domain *domain = dom->priv;
3798	unsigned long flags;	3798	unsigned long flags;
3799	int ret;	3799	int ret;
3800		3800
3801	spin_lock_irqsave(&domain->lock, flags);	3801	spin_lock_irqsave(&domain->lock, flags);
3802	ret = __clear_gcr3(domain, pasid);	3802	ret = __clear_gcr3(domain, pasid);
3803	spin_unlock_irqrestore(&domain->lock, flags);	3803	spin_unlock_irqrestore(&domain->lock, flags);
3804		3804
3805	return ret;	3805	return ret;
3806	}	3806	}
3807	EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);	3807	EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3808		3808
3809	int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,	3809	int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
3810	int status, int tag)	3810	int status, int tag)
3811	{	3811	{
3812	struct iommu_dev_data *dev_data;	3812	struct iommu_dev_data *dev_data;
3813	struct amd_iommu *iommu;	3813	struct amd_iommu *iommu;
3814	struct iommu_cmd cmd;	3814	struct iommu_cmd cmd;
3815		3815
3816	INC_STATS_COUNTER(complete_ppr);	3816	INC_STATS_COUNTER(complete_ppr);
3817		3817
3818	dev_data = get_dev_data(&pdev->dev);	3818	dev_data = get_dev_data(&pdev->dev);
3819	iommu = amd_iommu_rlookup_table[dev_data->devid];	3819	iommu = amd_iommu_rlookup_table[dev_data->devid];
3820		3820
3821	build_complete_ppr(&cmd, dev_data->devid, pasid, status,	3821	build_complete_ppr(&cmd, dev_data->devid, pasid, status,
3822	tag, dev_data->pri_tlp);	3822	tag, dev_data->pri_tlp);
3823		3823
3824	return iommu_queue_command(iommu, &cmd);	3824	return iommu_queue_command(iommu, &cmd);
3825	}	3825	}
3826	EXPORT_SYMBOL(amd_iommu_complete_ppr);	3826	EXPORT_SYMBOL(amd_iommu_complete_ppr);
3827		3827
3828	struct iommu_domain amd_iommu_get_v2_domain(struct pci_dev pdev)	3828	struct iommu_domain amd_iommu_get_v2_domain(struct pci_dev pdev)
3829	{	3829	{
3830	struct protection_domain *domain;	3830	struct protection_domain *domain;
3831		3831
3832	domain = get_domain(&pdev->dev);	3832	domain = get_domain(&pdev->dev);
3833	if (IS_ERR(domain))	3833	if (IS_ERR(domain))
3834	return NULL;	3834	return NULL;
3835		3835
3836	/* Only return IOMMUv2 domains */	3836	/* Only return IOMMUv2 domains */
3837	if (!(domain->flags & PD_IOMMUV2_MASK))	3837	if (!(domain->flags & PD_IOMMUV2_MASK))
3838	return NULL;	3838	return NULL;
3839		3839
3840	return domain->iommu_domain;	3840	return domain->iommu_domain;
3841	}	3841	}
3842	EXPORT_SYMBOL(amd_iommu_get_v2_domain);	3842	EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3843		3843
3844	void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)	3844	void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
3845	{	3845	{
3846	struct iommu_dev_data *dev_data;	3846	struct iommu_dev_data *dev_data;
3847		3847
3848	if (!amd_iommu_v2_supported())	3848	if (!amd_iommu_v2_supported())
3849	return;	3849	return;
3850		3850
3851	dev_data = get_dev_data(&pdev->dev);	3851	dev_data = get_dev_data(&pdev->dev);
3852	dev_data->errata \|= (1 << erratum);	3852	dev_data->errata \|= (1 << erratum);
3853	}	3853	}
3854	EXPORT_SYMBOL(amd_iommu_enable_device_erratum);	3854	EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3855		3855
3856	int amd_iommu_device_info(struct pci_dev *pdev,	3856	int amd_iommu_device_info(struct pci_dev *pdev,
3857	struct amd_iommu_device_info *info)	3857	struct amd_iommu_device_info *info)
3858	{	3858	{
3859	int max_pasids;	3859	int max_pasids;
3860	int pos;	3860	int pos;
3861		3861
3862	if (pdev == NULL \|\| info == NULL)	3862	if (pdev == NULL \|\| info == NULL)
3863	return -EINVAL;	3863	return -EINVAL;
3864		3864
3865	if (!amd_iommu_v2_supported())	3865	if (!amd_iommu_v2_supported())
3866	return -EINVAL;	3866	return -EINVAL;
3867		3867
3868	memset(info, 0, sizeof(*info));	3868	memset(info, 0, sizeof(*info));
3869		3869
3870	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);	3870	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
3871	if (pos)	3871	if (pos)
3872	info->flags \|= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;	3872	info->flags \|= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
3873		3873
3874	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);	3874	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
3875	if (pos)	3875	if (pos)
3876	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;	3876	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
3877		3877
3878	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);	3878	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
3879	if (pos) {	3879	if (pos) {
3880	int features;	3880	int features;
3881		3881
3882	max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));	3882	max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
3883	max_pasids = min(max_pasids, (1 << 20));	3883	max_pasids = min(max_pasids, (1 << 20));
3884		3884
3885	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;	3885	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
3886	info->max_pasids = min(pci_max_pasids(pdev), max_pasids);	3886	info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
3887		3887
3888	features = pci_pasid_features(pdev);	3888	features = pci_pasid_features(pdev);
3889	if (features & PCI_PASID_CAP_EXEC)	3889	if (features & PCI_PASID_CAP_EXEC)
3890	info->flags \|= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;	3890	info->flags \|= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
3891	if (features & PCI_PASID_CAP_PRIV)	3891	if (features & PCI_PASID_CAP_PRIV)
3892	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;	3892	info->flags \|= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
3893	}	3893	}
3894		3894
3895	return 0;	3895	return 0;
3896	}	3896	}
3897	EXPORT_SYMBOL(amd_iommu_device_info);	3897	EXPORT_SYMBOL(amd_iommu_device_info);
3898		3898
3899	#ifdef CONFIG_IRQ_REMAP	3899	#ifdef CONFIG_IRQ_REMAP
3900		3900
3901	/*****************************************************************************	3901	/*****************************************************************************
3902	*	3902	*
3903	* Interrupt Remapping Implementation	3903	* Interrupt Remapping Implementation
3904	*	3904	*
3905	*****************************************************************************/	3905	*****************************************************************************/
3906		3906
3907	union irte {	3907	union irte {
3908	u32 val;	3908	u32 val;
3909	struct {	3909	struct {
3910	u32 valid : 1,	3910	u32 valid : 1,
3911	no_fault : 1,	3911	no_fault : 1,
3912	int_type : 3,	3912	int_type : 3,
3913	rq_eoi : 1,	3913	rq_eoi : 1,
3914	dm : 1,	3914	dm : 1,
3915	rsvd_1 : 1,	3915	rsvd_1 : 1,
3916	destination : 8,	3916	destination : 8,
3917	vector : 8,	3917	vector : 8,
3918	rsvd_2 : 8;	3918	rsvd_2 : 8;
3919	} fields;	3919	} fields;
3920	};	3920	};
3921		3921
3922	#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)	3922	#define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6)
3923	#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)	3923	#define DTE_IRQ_REMAP_INTCTL (2ULL << 60)
3924	#define DTE_IRQ_TABLE_LEN (8ULL << 1)	3924	#define DTE_IRQ_TABLE_LEN (8ULL << 1)
3925	#define DTE_IRQ_REMAP_ENABLE 1ULL	3925	#define DTE_IRQ_REMAP_ENABLE 1ULL
3926		3926
3927	static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)	3927	static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table)
3928	{	3928	{
3929	u64 dte;	3929	u64 dte;
3930		3930
3931	dte = amd_iommu_dev_table[devid].data[2];	3931	dte = amd_iommu_dev_table[devid].data[2];
3932	dte &= ~DTE_IRQ_PHYS_ADDR_MASK;	3932	dte &= ~DTE_IRQ_PHYS_ADDR_MASK;
3933	dte \|= virt_to_phys(table->table);	3933	dte \|= virt_to_phys(table->table);
3934	dte \|= DTE_IRQ_REMAP_INTCTL;	3934	dte \|= DTE_IRQ_REMAP_INTCTL;
3935	dte \|= DTE_IRQ_TABLE_LEN;	3935	dte \|= DTE_IRQ_TABLE_LEN;
3936	dte \|= DTE_IRQ_REMAP_ENABLE;	3936	dte \|= DTE_IRQ_REMAP_ENABLE;
3937		3937
3938	amd_iommu_dev_table[devid].data[2] = dte;	3938	amd_iommu_dev_table[devid].data[2] = dte;
3939	}	3939	}
3940		3940
3941	#define IRTE_ALLOCATED (~1U)	3941	#define IRTE_ALLOCATED (~1U)
3942		3942
3943	static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)	3943	static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic)
3944	{	3944	{
3945	struct irq_remap_table *table = NULL;	3945	struct irq_remap_table *table = NULL;
3946	struct amd_iommu *iommu;	3946	struct amd_iommu *iommu;
3947	unsigned long flags;	3947	unsigned long flags;
3948	u16 alias;	3948	u16 alias;
3949		3949
3950	write_lock_irqsave(&amd_iommu_devtable_lock, flags);	3950	write_lock_irqsave(&amd_iommu_devtable_lock, flags);
3951		3951
3952	iommu = amd_iommu_rlookup_table[devid];	3952	iommu = amd_iommu_rlookup_table[devid];
3953	if (!iommu)	3953	if (!iommu)
3954	goto out_unlock;	3954	goto out_unlock;
3955		3955
3956	table = irq_lookup_table[devid];	3956	table = irq_lookup_table[devid];
3957	if (table)	3957	if (table)
3958	goto out;	3958	goto out;
3959		3959
3960	alias = amd_iommu_alias_table[devid];	3960	alias = amd_iommu_alias_table[devid];
3961	table = irq_lookup_table[alias];	3961	table = irq_lookup_table[alias];
3962	if (table) {	3962	if (table) {
3963	irq_lookup_table[devid] = table;	3963	irq_lookup_table[devid] = table;
3964	set_dte_irq_entry(devid, table);	3964	set_dte_irq_entry(devid, table);
3965	iommu_flush_dte(iommu, devid);	3965	iommu_flush_dte(iommu, devid);
3966	goto out;	3966	goto out;
3967	}	3967	}
3968		3968
3969	/* Nothing there yet, allocate new irq remapping table */	3969	/* Nothing there yet, allocate new irq remapping table */
3970	table = kzalloc(sizeof(*table), GFP_ATOMIC);	3970	table = kzalloc(sizeof(*table), GFP_ATOMIC);
3971	if (!table)	3971	if (!table)
3972	goto out;	3972	goto out;
3973		3973
3974	/* Initialize table spin-lock */	3974	/* Initialize table spin-lock */
3975	spin_lock_init(&table->lock);	3975	spin_lock_init(&table->lock);
3976		3976
3977	if (ioapic)	3977	if (ioapic)
3978	/* Keep the first 32 indexes free for IOAPIC interrupts */	3978	/* Keep the first 32 indexes free for IOAPIC interrupts */
3979	table->min_index = 32;	3979	table->min_index = 32;
3980		3980
3981	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);	3981	table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_ATOMIC);
3982	if (!table->table) {	3982	if (!table->table) {
3983	kfree(table);	3983	kfree(table);
3984	table = NULL;	3984	table = NULL;
3985	goto out;	3985	goto out;
3986	}	3986	}
3987		3987
3988	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));	3988	memset(table->table, 0, MAX_IRQS_PER_TABLE * sizeof(u32));
3989		3989
3990	if (ioapic) {	3990	if (ioapic) {
3991	int i;	3991	int i;
3992		3992
3993	for (i = 0; i < 32; ++i)	3993	for (i = 0; i < 32; ++i)
3994	table->table[i] = IRTE_ALLOCATED;	3994	table->table[i] = IRTE_ALLOCATED;
3995	}	3995	}
3996		3996
3997	irq_lookup_table[devid] = table;	3997	irq_lookup_table[devid] = table;
3998	set_dte_irq_entry(devid, table);	3998	set_dte_irq_entry(devid, table);
3999	iommu_flush_dte(iommu, devid);	3999	iommu_flush_dte(iommu, devid);
4000	if (devid != alias) {	4000	if (devid != alias) {
4001	irq_lookup_table[alias] = table;	4001	irq_lookup_table[alias] = table;
4002	set_dte_irq_entry(devid, table);	4002	set_dte_irq_entry(alias, table);
4003	iommu_flush_dte(iommu, alias);	4003	iommu_flush_dte(iommu, alias);
4004	}	4004	}
4005		4005
4006	out:	4006	out:
4007	iommu_completion_wait(iommu);	4007	iommu_completion_wait(iommu);
4008		4008
4009	out_unlock:	4009	out_unlock:
4010	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);	4010	write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
4011		4011
4012	return table;	4012	return table;
4013	}	4013	}
4014		4014
4015	static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count)	4015	static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count)
4016	{	4016	{
4017	struct irq_remap_table *table;	4017	struct irq_remap_table *table;
4018	unsigned long flags;	4018	unsigned long flags;
4019	int index, c;	4019	int index, c;
4020		4020
4021	table = get_irq_table(devid, false);	4021	table = get_irq_table(devid, false);
4022	if (!table)	4022	if (!table)
4023	return -ENODEV;	4023	return -ENODEV;
4024		4024
4025	spin_lock_irqsave(&table->lock, flags);	4025	spin_lock_irqsave(&table->lock, flags);
4026		4026
4027	/* Scan table for free entries */	4027	/* Scan table for free entries */
4028	for (c = 0, index = table->min_index;	4028	for (c = 0, index = table->min_index;
4029	index < MAX_IRQS_PER_TABLE;	4029	index < MAX_IRQS_PER_TABLE;
4030	++index) {	4030	++index) {
4031	if (table->table[index] == 0)	4031	if (table->table[index] == 0)
4032	c += 1;	4032	c += 1;
4033	else	4033	else
4034	c = 0;	4034	c = 0;
4035		4035
4036	if (c == count) {	4036	if (c == count) {
4037	struct irq_2_irte *irte_info;	4037	struct irq_2_irte *irte_info;
4038		4038
4039	for (; c != 0; --c)	4039	for (; c != 0; --c)
4040	table->table[index - c + 1] = IRTE_ALLOCATED;	4040	table->table[index - c + 1] = IRTE_ALLOCATED;
4041		4041
4042	index -= count - 1;	4042	index -= count - 1;
4043		4043
4044	cfg->remapped = 1;	4044	cfg->remapped = 1;
4045	irte_info = &cfg->irq_2_irte;	4045	irte_info = &cfg->irq_2_irte;
4046	irte_info->devid = devid;	4046	irte_info->devid = devid;
4047	irte_info->index = index;	4047	irte_info->index = index;
4048		4048
4049	goto out;	4049	goto out;
4050	}	4050	}
4051	}	4051	}
4052		4052
4053	index = -ENOSPC;	4053	index = -ENOSPC;
4054		4054
4055	out:	4055	out:
4056	spin_unlock_irqrestore(&table->lock, flags);	4056	spin_unlock_irqrestore(&table->lock, flags);
4057		4057
4058	return index;	4058	return index;
4059	}	4059	}
4060		4060
4061	static int get_irte(u16 devid, int index, union irte *irte)	4061	static int get_irte(u16 devid, int index, union irte *irte)
4062	{	4062	{
4063	struct irq_remap_table *table;	4063	struct irq_remap_table *table;
4064	unsigned long flags;	4064	unsigned long flags;
4065		4065
4066	table = get_irq_table(devid, false);	4066	table = get_irq_table(devid, false);
4067	if (!table)	4067	if (!table)
4068	return -ENOMEM;	4068	return -ENOMEM;
4069		4069
4070	spin_lock_irqsave(&table->lock, flags);	4070	spin_lock_irqsave(&table->lock, flags);
4071	irte->val = table->table[index];	4071	irte->val = table->table[index];
4072	spin_unlock_irqrestore(&table->lock, flags);	4072	spin_unlock_irqrestore(&table->lock, flags);
4073		4073
4074	return 0;	4074	return 0;
4075	}	4075	}
4076		4076
4077	static int modify_irte(u16 devid, int index, union irte irte)	4077	static int modify_irte(u16 devid, int index, union irte irte)
4078	{	4078	{
4079	struct irq_remap_table *table;	4079	struct irq_remap_table *table;
4080	struct amd_iommu *iommu;	4080	struct amd_iommu *iommu;
4081	unsigned long flags;	4081	unsigned long flags;
4082		4082
4083	iommu = amd_iommu_rlookup_table[devid];	4083	iommu = amd_iommu_rlookup_table[devid];
4084	if (iommu == NULL)	4084	if (iommu == NULL)
4085	return -EINVAL;	4085	return -EINVAL;
4086		4086
4087	table = get_irq_table(devid, false);	4087	table = get_irq_table(devid, false);
4088	if (!table)	4088	if (!table)
4089	return -ENOMEM;	4089	return -ENOMEM;
4090		4090
4091	spin_lock_irqsave(&table->lock, flags);	4091	spin_lock_irqsave(&table->lock, flags);
4092	table->table[index] = irte.val;	4092	table->table[index] = irte.val;
4093	spin_unlock_irqrestore(&table->lock, flags);	4093	spin_unlock_irqrestore(&table->lock, flags);
4094		4094
4095	iommu_flush_irt(iommu, devid);	4095	iommu_flush_irt(iommu, devid);
4096	iommu_completion_wait(iommu);	4096	iommu_completion_wait(iommu);
4097		4097
4098	return 0;	4098	return 0;
4099	}	4099	}
4100		4100
4101	static void free_irte(u16 devid, int index)	4101	static void free_irte(u16 devid, int index)
4102	{	4102	{
4103	struct irq_remap_table *table;	4103	struct irq_remap_table *table;
4104	struct amd_iommu *iommu;	4104	struct amd_iommu *iommu;
4105	unsigned long flags;	4105	unsigned long flags;
4106		4106
4107	iommu = amd_iommu_rlookup_table[devid];	4107	iommu = amd_iommu_rlookup_table[devid];
4108	if (iommu == NULL)	4108	if (iommu == NULL)
4109	return;	4109	return;
4110		4110
4111	table = get_irq_table(devid, false);	4111	table = get_irq_table(devid, false);
4112	if (!table)	4112	if (!table)
4113	return;	4113	return;
4114		4114
4115	spin_lock_irqsave(&table->lock, flags);	4115	spin_lock_irqsave(&table->lock, flags);
4116	table->table[index] = 0;	4116	table->table[index] = 0;
4117	spin_unlock_irqrestore(&table->lock, flags);	4117	spin_unlock_irqrestore(&table->lock, flags);
4118		4118
4119	iommu_flush_irt(iommu, devid);	4119	iommu_flush_irt(iommu, devid);
4120	iommu_completion_wait(iommu);	4120	iommu_completion_wait(iommu);
4121	}	4121	}
4122		4122
4123	static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,	4123	static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry,
4124	unsigned int destination, int vector,	4124	unsigned int destination, int vector,
4125	struct io_apic_irq_attr *attr)	4125	struct io_apic_irq_attr *attr)
4126	{	4126	{
4127	struct irq_remap_table *table;	4127	struct irq_remap_table *table;
4128	struct irq_2_irte *irte_info;	4128	struct irq_2_irte *irte_info;
4129	struct irq_cfg *cfg;	4129	struct irq_cfg *cfg;
4130	union irte irte;	4130	union irte irte;
4131	int ioapic_id;	4131	int ioapic_id;
4132	int index;	4132	int index;
4133	int devid;	4133	int devid;
4134	int ret;	4134	int ret;
4135		4135
4136	cfg = irq_get_chip_data(irq);	4136	cfg = irq_get_chip_data(irq);
4137	if (!cfg)	4137	if (!cfg)
4138	return -EINVAL;	4138	return -EINVAL;
4139		4139
4140	irte_info = &cfg->irq_2_irte;	4140	irte_info = &cfg->irq_2_irte;
4141	ioapic_id = mpc_ioapic_id(attr->ioapic);	4141	ioapic_id = mpc_ioapic_id(attr->ioapic);
4142	devid = get_ioapic_devid(ioapic_id);	4142	devid = get_ioapic_devid(ioapic_id);
4143		4143
4144	if (devid < 0)	4144	if (devid < 0)
4145	return devid;	4145	return devid;
4146		4146
4147	table = get_irq_table(devid, true);	4147	table = get_irq_table(devid, true);
4148	if (table == NULL)	4148	if (table == NULL)
4149	return -ENOMEM;	4149	return -ENOMEM;
4150		4150
4151	index = attr->ioapic_pin;	4151	index = attr->ioapic_pin;
4152		4152
4153	/* Setup IRQ remapping info */	4153	/* Setup IRQ remapping info */
4154	cfg->remapped = 1;	4154	cfg->remapped = 1;
4155	irte_info->devid = devid;	4155	irte_info->devid = devid;
4156	irte_info->index = index;	4156	irte_info->index = index;
4157		4157
4158	/* Setup IRTE for IOMMU */	4158	/* Setup IRTE for IOMMU */
4159	irte.val = 0;	4159	irte.val = 0;
4160	irte.fields.vector = vector;	4160	irte.fields.vector = vector;
4161	irte.fields.int_type = apic->irq_delivery_mode;	4161	irte.fields.int_type = apic->irq_delivery_mode;
4162	irte.fields.destination = destination;	4162	irte.fields.destination = destination;
4163	irte.fields.dm = apic->irq_dest_mode;	4163	irte.fields.dm = apic->irq_dest_mode;
4164	irte.fields.valid = 1;	4164	irte.fields.valid = 1;
4165		4165
4166	ret = modify_irte(devid, index, irte);	4166	ret = modify_irte(devid, index, irte);
4167	if (ret)	4167	if (ret)
4168	return ret;	4168	return ret;
4169		4169
4170	/* Setup IOAPIC entry */	4170	/* Setup IOAPIC entry */
4171	memset(entry, 0, sizeof(*entry));	4171	memset(entry, 0, sizeof(*entry));
4172		4172
4173	entry->vector = index;	4173	entry->vector = index;
4174	entry->mask = 0;	4174	entry->mask = 0;
4175	entry->trigger = attr->trigger;	4175	entry->trigger = attr->trigger;
4176	entry->polarity = attr->polarity;	4176	entry->polarity = attr->polarity;
4177		4177
4178	/*	4178	/*
4179	* Mask level triggered irqs.	4179	* Mask level triggered irqs.
4180	*/	4180	*/
4181	if (attr->trigger)	4181	if (attr->trigger)
4182	entry->mask = 1;	4182	entry->mask = 1;
4183		4183
4184	return 0;	4184	return 0;
4185	}	4185	}
4186		4186
4187	static int set_affinity(struct irq_data data, const struct cpumask mask,	4187	static int set_affinity(struct irq_data data, const struct cpumask mask,
4188	bool force)	4188	bool force)
4189	{	4189	{
4190	struct irq_2_irte *irte_info;	4190	struct irq_2_irte *irte_info;
4191	unsigned int dest, irq;	4191	unsigned int dest, irq;
4192	struct irq_cfg *cfg;	4192	struct irq_cfg *cfg;
4193	union irte irte;	4193	union irte irte;
4194	int err;	4194	int err;
4195		4195
4196	if (!config_enabled(CONFIG_SMP))	4196	if (!config_enabled(CONFIG_SMP))
4197	return -1;	4197	return -1;
4198		4198
4199	cfg = data->chip_data;	4199	cfg = data->chip_data;
4200	irq = data->irq;	4200	irq = data->irq;
4201	irte_info = &cfg->irq_2_irte;	4201	irte_info = &cfg->irq_2_irte;
4202		4202
4203	if (!cpumask_intersects(mask, cpu_online_mask))	4203	if (!cpumask_intersects(mask, cpu_online_mask))
4204	return -EINVAL;	4204	return -EINVAL;
4205		4205
4206	if (get_irte(irte_info->devid, irte_info->index, &irte))	4206	if (get_irte(irte_info->devid, irte_info->index, &irte))
4207	return -EBUSY;	4207	return -EBUSY;
4208		4208
4209	if (assign_irq_vector(irq, cfg, mask))	4209	if (assign_irq_vector(irq, cfg, mask))
4210	return -EBUSY;	4210	return -EBUSY;
4211		4211
4212	err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);	4212	err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest);
4213	if (err) {	4213	if (err) {
4214	if (assign_irq_vector(irq, cfg, data->affinity))	4214	if (assign_irq_vector(irq, cfg, data->affinity))
4215	pr_err("AMD-Vi: Failed to recover vector for irq %d\n", irq);	4215	pr_err("AMD-Vi: Failed to recover vector for irq %d\n", irq);
4216	return err;	4216	return err;
4217	}	4217	}
4218		4218
4219	irte.fields.vector = cfg->vector;	4219	irte.fields.vector = cfg->vector;
4220	irte.fields.destination = dest;	4220	irte.fields.destination = dest;
4221		4221
4222	modify_irte(irte_info->devid, irte_info->index, irte);	4222	modify_irte(irte_info->devid, irte_info->index, irte);
4223		4223
4224	if (cfg->move_in_progress)	4224	if (cfg->move_in_progress)
4225	send_cleanup_vector(cfg);	4225	send_cleanup_vector(cfg);
4226		4226
4227	cpumask_copy(data->affinity, mask);	4227	cpumask_copy(data->affinity, mask);
4228		4228
4229	return 0;	4229	return 0;
4230	}	4230	}
4231		4231
4232	static int free_irq(int irq)	4232	static int free_irq(int irq)
4233	{	4233	{
4234	struct irq_2_irte *irte_info;	4234	struct irq_2_irte *irte_info;
4235	struct irq_cfg *cfg;	4235	struct irq_cfg *cfg;
4236		4236
4237	cfg = irq_get_chip_data(irq);	4237	cfg = irq_get_chip_data(irq);
4238	if (!cfg)	4238	if (!cfg)
4239	return -EINVAL;	4239	return -EINVAL;
4240		4240
4241	irte_info = &cfg->irq_2_irte;	4241	irte_info = &cfg->irq_2_irte;
4242		4242
4243	free_irte(irte_info->devid, irte_info->index);	4243	free_irte(irte_info->devid, irte_info->index);
4244		4244
4245	return 0;	4245	return 0;
4246	}	4246	}
4247		4247
4248	static void compose_msi_msg(struct pci_dev *pdev,	4248	static void compose_msi_msg(struct pci_dev *pdev,
4249	unsigned int irq, unsigned int dest,	4249	unsigned int irq, unsigned int dest,
4250	struct msi_msg *msg, u8 hpet_id)	4250	struct msi_msg *msg, u8 hpet_id)
4251	{	4251	{
4252	struct irq_2_irte *irte_info;	4252	struct irq_2_irte *irte_info;
4253	struct irq_cfg *cfg;	4253	struct irq_cfg *cfg;
4254	union irte irte;	4254	union irte irte;
4255		4255
4256	cfg = irq_get_chip_data(irq);	4256	cfg = irq_get_chip_data(irq);
4257	if (!cfg)	4257	if (!cfg)
4258	return;	4258	return;
4259		4259
4260	irte_info = &cfg->irq_2_irte;	4260	irte_info = &cfg->irq_2_irte;
4261		4261
4262	irte.val = 0;	4262	irte.val = 0;
4263	irte.fields.vector = cfg->vector;	4263	irte.fields.vector = cfg->vector;
4264	irte.fields.int_type = apic->irq_delivery_mode;	4264	irte.fields.int_type = apic->irq_delivery_mode;
4265	irte.fields.destination = dest;	4265	irte.fields.destination = dest;
4266	irte.fields.dm = apic->irq_dest_mode;	4266	irte.fields.dm = apic->irq_dest_mode;
4267	irte.fields.valid = 1;	4267	irte.fields.valid = 1;
4268		4268
4269	modify_irte(irte_info->devid, irte_info->index, irte);	4269	modify_irte(irte_info->devid, irte_info->index, irte);
4270		4270
4271	msg->address_hi = MSI_ADDR_BASE_HI;	4271	msg->address_hi = MSI_ADDR_BASE_HI;
4272	msg->address_lo = MSI_ADDR_BASE_LO;	4272	msg->address_lo = MSI_ADDR_BASE_LO;
4273	msg->data = irte_info->index;	4273	msg->data = irte_info->index;
4274	}	4274	}
4275		4275
4276	static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec)	4276	static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec)
4277	{	4277	{
4278	struct irq_cfg *cfg;	4278	struct irq_cfg *cfg;
4279	int index;	4279	int index;
4280	u16 devid;	4280	u16 devid;
4281		4281
4282	if (!pdev)	4282	if (!pdev)
4283	return -EINVAL;	4283	return -EINVAL;
4284		4284
4285	cfg = irq_get_chip_data(irq);	4285	cfg = irq_get_chip_data(irq);
4286	if (!cfg)	4286	if (!cfg)
4287	return -EINVAL;	4287	return -EINVAL;
4288		4288
4289	devid = get_device_id(&pdev->dev);	4289	devid = get_device_id(&pdev->dev);
4290	index = alloc_irq_index(cfg, devid, nvec);	4290	index = alloc_irq_index(cfg, devid, nvec);
4291		4291
4292	return index < 0 ? MAX_IRQS_PER_TABLE : index;	4292	return index < 0 ? MAX_IRQS_PER_TABLE : index;
4293	}	4293	}
4294		4294
4295	static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,	4295	static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
4296	int index, int offset)	4296	int index, int offset)
4297	{	4297	{
4298	struct irq_2_irte *irte_info;	4298	struct irq_2_irte *irte_info;
4299	struct irq_cfg *cfg;	4299	struct irq_cfg *cfg;
4300	u16 devid;	4300	u16 devid;
4301		4301
4302	if (!pdev)	4302	if (!pdev)
4303	return -EINVAL;	4303	return -EINVAL;
4304		4304
4305	cfg = irq_get_chip_data(irq);	4305	cfg = irq_get_chip_data(irq);
4306	if (!cfg)	4306	if (!cfg)
4307	return -EINVAL;	4307	return -EINVAL;
4308		4308
4309	if (index >= MAX_IRQS_PER_TABLE)	4309	if (index >= MAX_IRQS_PER_TABLE)
4310	return 0;	4310	return 0;
4311		4311
4312	devid = get_device_id(&pdev->dev);	4312	devid = get_device_id(&pdev->dev);
4313	irte_info = &cfg->irq_2_irte;	4313	irte_info = &cfg->irq_2_irte;
4314		4314
4315	cfg->remapped = 1;	4315	cfg->remapped = 1;
4316	irte_info->devid = devid;	4316	irte_info->devid = devid;
4317	irte_info->index = index + offset;	4317	irte_info->index = index + offset;
4318		4318
4319	return 0;	4319	return 0;
4320	}	4320	}
4321		4321
4322	static int setup_hpet_msi(unsigned int irq, unsigned int id)	4322	static int setup_hpet_msi(unsigned int irq, unsigned int id)
4323	{	4323	{
4324	struct irq_2_irte *irte_info;	4324	struct irq_2_irte *irte_info;
4325	struct irq_cfg *cfg;	4325	struct irq_cfg *cfg;
4326	int index, devid;	4326	int index, devid;
4327		4327
4328	cfg = irq_get_chip_data(irq);	4328	cfg = irq_get_chip_data(irq);
4329	if (!cfg)	4329	if (!cfg)
4330	return -EINVAL;	4330	return -EINVAL;
4331		4331
4332	irte_info = &cfg->irq_2_irte;	4332	irte_info = &cfg->irq_2_irte;
4333	devid = get_hpet_devid(id);	4333	devid = get_hpet_devid(id);
4334	if (devid < 0)	4334	if (devid < 0)
4335	return devid;	4335	return devid;
4336		4336
4337	index = alloc_irq_index(cfg, devid, 1);	4337	index = alloc_irq_index(cfg, devid, 1);
4338	if (index < 0)	4338	if (index < 0)
4339	return index;	4339	return index;
4340		4340
4341	cfg->remapped = 1;	4341	cfg->remapped = 1;
4342	irte_info->devid = devid;	4342	irte_info->devid = devid;
4343	irte_info->index = index;	4343	irte_info->index = index;
4344		4344
4345	return 0;	4345	return 0;
4346	}	4346	}
4347		4347
4348	struct irq_remap_ops amd_iommu_irq_ops = {	4348	struct irq_remap_ops amd_iommu_irq_ops = {
4349	.supported = amd_iommu_supported,	4349	.supported = amd_iommu_supported,
4350	.prepare = amd_iommu_prepare,	4350	.prepare = amd_iommu_prepare,
4351	.enable = amd_iommu_enable,	4351	.enable = amd_iommu_enable,
4352	.disable = amd_iommu_disable,	4352	.disable = amd_iommu_disable,
4353	.reenable = amd_iommu_reenable,	4353	.reenable = amd_iommu_reenable,
4354	.enable_faulting = amd_iommu_enable_faulting,	4354	.enable_faulting = amd_iommu_enable_faulting,
4355	.setup_ioapic_entry = setup_ioapic_entry,	4355	.setup_ioapic_entry = setup_ioapic_entry,
4356	.set_affinity = set_affinity,	4356	.set_affinity = set_affinity,
4357	.free_irq = free_irq,	4357	.free_irq = free_irq,
4358	.compose_msi_msg = compose_msi_msg,	4358	.compose_msi_msg = compose_msi_msg,
4359	.msi_alloc_irq = msi_alloc_irq,	4359	.msi_alloc_irq = msi_alloc_irq,
4360	.msi_setup_irq = msi_setup_irq,	4360	.msi_setup_irq = msi_setup_irq,
4361	.setup_hpet_msi = setup_hpet_msi,	4361	.setup_hpet_msi = setup_hpet_msi,
4362	};	4362	};
4363	#endif	4363	#endif
4364		4364

drivers/iommu/amd_iommu_init.c

Diff comments View file @ 989d216

1	/*	1	/*
2	* Copyright (C) 2007-2010 Advanced Micro Devices, Inc.	2	* Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3	* Author: Joerg Roedel <joerg.roedel@amd.com>	3	* Author: Joerg Roedel <joerg.roedel@amd.com>
4	* Leo Duran <leo.duran@amd.com>	4	* Leo Duran <leo.duran@amd.com>
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify it	6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms of the GNU General Public License version 2 as published	7	* under the terms of the GNU General Public License version 2 as published
8	* by the Free Software Foundation.	8	* by the Free Software Foundation.
9	*	9	*
10	* This program is distributed in the hope that it will be useful,	10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.	13	* GNU General Public License for more details.
14	*	14	*
15	* You should have received a copy of the GNU General Public License	15	* You should have received a copy of the GNU General Public License
16	* along with this program; if not, write to the Free Software	16	* along with this program; if not, write to the Free Software
17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA	17	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18	*/	18	*/
19		19
20	#include <linux/pci.h>	20	#include <linux/pci.h>
21	#include <linux/acpi.h>	21	#include <linux/acpi.h>
22	#include <linux/list.h>	22	#include <linux/list.h>
23	#include <linux/slab.h>	23	#include <linux/slab.h>
24	#include <linux/syscore_ops.h>	24	#include <linux/syscore_ops.h>
25	#include <linux/interrupt.h>	25	#include <linux/interrupt.h>
26	#include <linux/msi.h>	26	#include <linux/msi.h>
27	#include <linux/amd-iommu.h>	27	#include <linux/amd-iommu.h>
28	#include <linux/export.h>	28	#include <linux/export.h>
29	#include <asm/pci-direct.h>	29	#include <asm/pci-direct.h>
30	#include <asm/iommu.h>	30	#include <asm/iommu.h>
31	#include <asm/gart.h>	31	#include <asm/gart.h>
32	#include <asm/x86_init.h>	32	#include <asm/x86_init.h>
33	#include <asm/iommu_table.h>	33	#include <asm/iommu_table.h>
34	#include <asm/io_apic.h>	34	#include <asm/io_apic.h>
35	#include <asm/irq_remapping.h>	35	#include <asm/irq_remapping.h>
36		36
37	#include "amd_iommu_proto.h"	37	#include "amd_iommu_proto.h"
38	#include "amd_iommu_types.h"	38	#include "amd_iommu_types.h"
39	#include "irq_remapping.h"	39	#include "irq_remapping.h"
40		40
41	/*	41	/*
42	* definitions for the ACPI scanning code	42	* definitions for the ACPI scanning code
43	*/	43	*/
44	#define IVRS_HEADER_LENGTH 48	44	#define IVRS_HEADER_LENGTH 48
45		45
46	#define ACPI_IVHD_TYPE 0x10	46	#define ACPI_IVHD_TYPE 0x10
47	#define ACPI_IVMD_TYPE_ALL 0x20	47	#define ACPI_IVMD_TYPE_ALL 0x20
48	#define ACPI_IVMD_TYPE 0x21	48	#define ACPI_IVMD_TYPE 0x21
49	#define ACPI_IVMD_TYPE_RANGE 0x22	49	#define ACPI_IVMD_TYPE_RANGE 0x22
50		50
51	#define IVHD_DEV_ALL 0x01	51	#define IVHD_DEV_ALL 0x01
52	#define IVHD_DEV_SELECT 0x02	52	#define IVHD_DEV_SELECT 0x02
53	#define IVHD_DEV_SELECT_RANGE_START 0x03	53	#define IVHD_DEV_SELECT_RANGE_START 0x03
54	#define IVHD_DEV_RANGE_END 0x04	54	#define IVHD_DEV_RANGE_END 0x04
55	#define IVHD_DEV_ALIAS 0x42	55	#define IVHD_DEV_ALIAS 0x42
56	#define IVHD_DEV_ALIAS_RANGE 0x43	56	#define IVHD_DEV_ALIAS_RANGE 0x43
57	#define IVHD_DEV_EXT_SELECT 0x46	57	#define IVHD_DEV_EXT_SELECT 0x46
58	#define IVHD_DEV_EXT_SELECT_RANGE 0x47	58	#define IVHD_DEV_EXT_SELECT_RANGE 0x47
59	#define IVHD_DEV_SPECIAL 0x48	59	#define IVHD_DEV_SPECIAL 0x48
60		60
61	#define IVHD_SPECIAL_IOAPIC 1	61	#define IVHD_SPECIAL_IOAPIC 1
62	#define IVHD_SPECIAL_HPET 2	62	#define IVHD_SPECIAL_HPET 2
63		63
64	#define IVHD_FLAG_HT_TUN_EN_MASK 0x01	64	#define IVHD_FLAG_HT_TUN_EN_MASK 0x01
65	#define IVHD_FLAG_PASSPW_EN_MASK 0x02	65	#define IVHD_FLAG_PASSPW_EN_MASK 0x02
66	#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04	66	#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
67	#define IVHD_FLAG_ISOC_EN_MASK 0x08	67	#define IVHD_FLAG_ISOC_EN_MASK 0x08
68		68
69	#define IVMD_FLAG_EXCL_RANGE 0x08	69	#define IVMD_FLAG_EXCL_RANGE 0x08
70	#define IVMD_FLAG_UNITY_MAP 0x01	70	#define IVMD_FLAG_UNITY_MAP 0x01
71		71
72	#define ACPI_DEVFLAG_INITPASS 0x01	72	#define ACPI_DEVFLAG_INITPASS 0x01
73	#define ACPI_DEVFLAG_EXTINT 0x02	73	#define ACPI_DEVFLAG_EXTINT 0x02
74	#define ACPI_DEVFLAG_NMI 0x04	74	#define ACPI_DEVFLAG_NMI 0x04
75	#define ACPI_DEVFLAG_SYSMGT1 0x10	75	#define ACPI_DEVFLAG_SYSMGT1 0x10
76	#define ACPI_DEVFLAG_SYSMGT2 0x20	76	#define ACPI_DEVFLAG_SYSMGT2 0x20
77	#define ACPI_DEVFLAG_LINT0 0x40	77	#define ACPI_DEVFLAG_LINT0 0x40
78	#define ACPI_DEVFLAG_LINT1 0x80	78	#define ACPI_DEVFLAG_LINT1 0x80
79	#define ACPI_DEVFLAG_ATSDIS 0x10000000	79	#define ACPI_DEVFLAG_ATSDIS 0x10000000
80		80
81	/*	81	/*
82	* ACPI table definitions	82	* ACPI table definitions
83	*	83	*
84	* These data structures are laid over the table to parse the important values	84	* These data structures are laid over the table to parse the important values
85	* out of it.	85	* out of it.
86	*/	86	*/
87		87
88	/*	88	/*
89	* structure describing one IOMMU in the ACPI table. Typically followed by one	89	* structure describing one IOMMU in the ACPI table. Typically followed by one
90	* or more ivhd_entrys.	90	* or more ivhd_entrys.
91	*/	91	*/
92	struct ivhd_header {	92	struct ivhd_header {
93	u8 type;	93	u8 type;
94	u8 flags;	94	u8 flags;
95	u16 length;	95	u16 length;
96	u16 devid;	96	u16 devid;
97	u16 cap_ptr;	97	u16 cap_ptr;
98	u64 mmio_phys;	98	u64 mmio_phys;
99	u16 pci_seg;	99	u16 pci_seg;
100	u16 info;	100	u16 info;
101	u32 efr;	101	u32 efr;
102	} __attribute__((packed));	102	} __attribute__((packed));
103		103
104	/*	104	/*
105	* A device entry describing which devices a specific IOMMU translates and	105	* A device entry describing which devices a specific IOMMU translates and
106	* which requestor ids they use.	106	* which requestor ids they use.
107	*/	107	*/
108	struct ivhd_entry {	108	struct ivhd_entry {
109	u8 type;	109	u8 type;
110	u16 devid;	110	u16 devid;
111	u8 flags;	111	u8 flags;
112	u32 ext;	112	u32 ext;
113	} __attribute__((packed));	113	} __attribute__((packed));
114		114
115	/*	115	/*
116	* An AMD IOMMU memory definition structure. It defines things like exclusion	116	* An AMD IOMMU memory definition structure. It defines things like exclusion
117	* ranges for devices and regions that should be unity mapped.	117	* ranges for devices and regions that should be unity mapped.
118	*/	118	*/
119	struct ivmd_header {	119	struct ivmd_header {
120	u8 type;	120	u8 type;
121	u8 flags;	121	u8 flags;
122	u16 length;	122	u16 length;
123	u16 devid;	123	u16 devid;
124	u16 aux;	124	u16 aux;
125	u64 resv;	125	u64 resv;
126	u64 range_start;	126	u64 range_start;
127	u64 range_length;	127	u64 range_length;
128	} __attribute__((packed));	128	} __attribute__((packed));
129		129
130	bool amd_iommu_dump;	130	bool amd_iommu_dump;
131	bool amd_iommu_irq_remap __read_mostly;	131	bool amd_iommu_irq_remap __read_mostly;
132		132
133	static bool amd_iommu_detected;	133	static bool amd_iommu_detected;
134	static bool __initdata amd_iommu_disabled;	134	static bool __initdata amd_iommu_disabled;
135		135
136	u16 amd_iommu_last_bdf; /* largest PCI device id we have	136	u16 amd_iommu_last_bdf; /* largest PCI device id we have
137	to handle */	137	to handle */
138	LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings	138	LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
139	we find in ACPI */	139	we find in ACPI */
140	u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */	140	u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */
141		141
142	LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the	142	LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
143	system */	143	system */
144		144
145	/* Array to assign indices to IOMMUs*/	145	/* Array to assign indices to IOMMUs*/
146	struct amd_iommu *amd_iommus[MAX_IOMMUS];	146	struct amd_iommu *amd_iommus[MAX_IOMMUS];
147	int amd_iommus_present;	147	int amd_iommus_present;
148		148
149	/* IOMMUs have a non-present cache? */	149	/* IOMMUs have a non-present cache? */
150	bool amd_iommu_np_cache __read_mostly;	150	bool amd_iommu_np_cache __read_mostly;
151	bool amd_iommu_iotlb_sup __read_mostly = true;	151	bool amd_iommu_iotlb_sup __read_mostly = true;
152		152
153	u32 amd_iommu_max_pasid __read_mostly = ~0;	153	u32 amd_iommu_max_pasid __read_mostly = ~0;
154		154
155	bool amd_iommu_v2_present __read_mostly;	155	bool amd_iommu_v2_present __read_mostly;
156	bool amd_iommu_pc_present __read_mostly;	156	bool amd_iommu_pc_present __read_mostly;
157		157
158	bool amd_iommu_force_isolation __read_mostly;	158	bool amd_iommu_force_isolation __read_mostly;
159		159
160	/*	160	/*
161	* List of protection domains - used during resume	161	* List of protection domains - used during resume
162	*/	162	*/
163	LIST_HEAD(amd_iommu_pd_list);	163	LIST_HEAD(amd_iommu_pd_list);
164	spinlock_t amd_iommu_pd_lock;	164	spinlock_t amd_iommu_pd_lock;
165		165
166	/*	166	/*
167	* Pointer to the device table which is shared by all AMD IOMMUs	167	* Pointer to the device table which is shared by all AMD IOMMUs
168	* it is indexed by the PCI device id or the HT unit id and contains	168	* it is indexed by the PCI device id or the HT unit id and contains
169	* information about the domain the device belongs to as well as the	169	* information about the domain the device belongs to as well as the
170	* page table root pointer.	170	* page table root pointer.
171	*/	171	*/
172	struct dev_table_entry *amd_iommu_dev_table;	172	struct dev_table_entry *amd_iommu_dev_table;
173		173
174	/*	174	/*
175	* The alias table is a driver specific data structure which contains the	175	* The alias table is a driver specific data structure which contains the
176	* mappings of the PCI device ids to the actual requestor ids on the IOMMU.	176	* mappings of the PCI device ids to the actual requestor ids on the IOMMU.
177	* More than one device can share the same requestor id.	177	* More than one device can share the same requestor id.
178	*/	178	*/
179	u16 *amd_iommu_alias_table;	179	u16 *amd_iommu_alias_table;
180		180
181	/*	181	/*
182	* The rlookup table is used to find the IOMMU which is responsible	182	* The rlookup table is used to find the IOMMU which is responsible
183	* for a specific device. It is also indexed by the PCI device id.	183	* for a specific device. It is also indexed by the PCI device id.
184	*/	184	*/
185	struct amd_iommu **amd_iommu_rlookup_table;	185	struct amd_iommu **amd_iommu_rlookup_table;
186		186
187	/*	187	/*
188	* This table is used to find the irq remapping table for a given device id	188	* This table is used to find the irq remapping table for a given device id
189	* quickly.	189	* quickly.
190	*/	190	*/
191	struct irq_remap_table **irq_lookup_table;	191	struct irq_remap_table **irq_lookup_table;
192		192
193	/*	193	/*
194	* AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap	194	* AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
195	* to know which ones are already in use.	195	* to know which ones are already in use.
196	*/	196	*/
197	unsigned long *amd_iommu_pd_alloc_bitmap;	197	unsigned long *amd_iommu_pd_alloc_bitmap;
198		198
199	static u32 dev_table_size; /* size of the device table */	199	static u32 dev_table_size; /* size of the device table */
200	static u32 alias_table_size; /* size of the alias table */	200	static u32 alias_table_size; /* size of the alias table */
201	static u32 rlookup_table_size; /* size if the rlookup table */	201	static u32 rlookup_table_size; /* size if the rlookup table */
202		202
203	enum iommu_init_state {	203	enum iommu_init_state {
204	IOMMU_START_STATE,	204	IOMMU_START_STATE,
205	IOMMU_IVRS_DETECTED,	205	IOMMU_IVRS_DETECTED,
206	IOMMU_ACPI_FINISHED,	206	IOMMU_ACPI_FINISHED,
207	IOMMU_ENABLED,	207	IOMMU_ENABLED,
208	IOMMU_PCI_INIT,	208	IOMMU_PCI_INIT,
209	IOMMU_INTERRUPTS_EN,	209	IOMMU_INTERRUPTS_EN,
210	IOMMU_DMA_OPS,	210	IOMMU_DMA_OPS,
211	IOMMU_INITIALIZED,	211	IOMMU_INITIALIZED,
212	IOMMU_NOT_FOUND,	212	IOMMU_NOT_FOUND,
213	IOMMU_INIT_ERROR,	213	IOMMU_INIT_ERROR,
214	};	214	};
215		215
216	/* Early ioapic and hpet maps from kernel command line */	216	/* Early ioapic and hpet maps from kernel command line */
217	#define EARLY_MAP_SIZE 4	217	#define EARLY_MAP_SIZE 4
218	static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];	218	static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
219	static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];	219	static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
220	static int __initdata early_ioapic_map_size;	220	static int __initdata early_ioapic_map_size;
221	static int __initdata early_hpet_map_size;	221	static int __initdata early_hpet_map_size;
222	static bool __initdata cmdline_maps;	222	static bool __initdata cmdline_maps;
223		223
224	static enum iommu_init_state init_state = IOMMU_START_STATE;	224	static enum iommu_init_state init_state = IOMMU_START_STATE;
225		225
226	static int amd_iommu_enable_interrupts(void);	226	static int amd_iommu_enable_interrupts(void);
227	static int __init iommu_go_to_state(enum iommu_init_state state);	227	static int __init iommu_go_to_state(enum iommu_init_state state);
228		228
229	static inline void update_last_devid(u16 devid)	229	static inline void update_last_devid(u16 devid)
230	{	230	{
231	if (devid > amd_iommu_last_bdf)	231	if (devid > amd_iommu_last_bdf)
232	amd_iommu_last_bdf = devid;	232	amd_iommu_last_bdf = devid;
233	}	233	}
234		234
235	static inline unsigned long tbl_size(int entry_size)	235	static inline unsigned long tbl_size(int entry_size)
236	{	236	{
237	unsigned shift = PAGE_SHIFT +	237	unsigned shift = PAGE_SHIFT +
238	get_order(((int)amd_iommu_last_bdf + 1) * entry_size);	238	get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
239		239
240	return 1UL << shift;	240	return 1UL << shift;
241	}	241	}
242		242
243	/* Access to l1 and l2 indexed register spaces */	243	/* Access to l1 and l2 indexed register spaces */
244		244
245	static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)	245	static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
246	{	246	{
247	u32 val;	247	u32 val;
248		248
249	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16));	249	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16));
250	pci_read_config_dword(iommu->dev, 0xfc, &val);	250	pci_read_config_dword(iommu->dev, 0xfc, &val);
251	return val;	251	return val;
252	}	252	}
253		253
254	static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)	254	static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
255	{	255	{
256	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16 \| 1 << 31));	256	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16 \| 1 << 31));
257	pci_write_config_dword(iommu->dev, 0xfc, val);	257	pci_write_config_dword(iommu->dev, 0xfc, val);
258	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16));	258	pci_write_config_dword(iommu->dev, 0xf8, (address \| l1 << 16));
259	}	259	}
260		260
261	static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)	261	static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
262	{	262	{
263	u32 val;	263	u32 val;
264		264
265	pci_write_config_dword(iommu->dev, 0xf0, address);	265	pci_write_config_dword(iommu->dev, 0xf0, address);
266	pci_read_config_dword(iommu->dev, 0xf4, &val);	266	pci_read_config_dword(iommu->dev, 0xf4, &val);
267	return val;	267	return val;
268	}	268	}
269		269
270	static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)	270	static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
271	{	271	{
272	pci_write_config_dword(iommu->dev, 0xf0, (address \| 1 << 8));	272	pci_write_config_dword(iommu->dev, 0xf0, (address \| 1 << 8));
273	pci_write_config_dword(iommu->dev, 0xf4, val);	273	pci_write_config_dword(iommu->dev, 0xf4, val);
274	}	274	}
275		275
276	/****************************************************************************	276	/****************************************************************************
277	*	277	*
278	* AMD IOMMU MMIO register space handling functions	278	* AMD IOMMU MMIO register space handling functions
279	*	279	*
280	* These functions are used to program the IOMMU device registers in	280	* These functions are used to program the IOMMU device registers in
281	* MMIO space required for that driver.	281	* MMIO space required for that driver.
282	*	282	*
283	****************************************************************************/	283	****************************************************************************/
284		284
285	/*	285	/*
286	* This function set the exclusion range in the IOMMU. DMA accesses to the	286	* This function set the exclusion range in the IOMMU. DMA accesses to the
287	* exclusion range are passed through untranslated	287	* exclusion range are passed through untranslated
288	*/	288	*/
289	static void iommu_set_exclusion_range(struct amd_iommu *iommu)	289	static void iommu_set_exclusion_range(struct amd_iommu *iommu)
290	{	290	{
291	u64 start = iommu->exclusion_start & PAGE_MASK;	291	u64 start = iommu->exclusion_start & PAGE_MASK;
292	u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;	292	u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
293	u64 entry;	293	u64 entry;
294		294
295	if (!iommu->exclusion_start)	295	if (!iommu->exclusion_start)
296	return;	296	return;
297		297
298	entry = start \| MMIO_EXCL_ENABLE_MASK;	298	entry = start \| MMIO_EXCL_ENABLE_MASK;
299	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,	299	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
300	&entry, sizeof(entry));	300	&entry, sizeof(entry));
301		301
302	entry = limit;	302	entry = limit;
303	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,	303	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
304	&entry, sizeof(entry));	304	&entry, sizeof(entry));
305	}	305	}
306		306
307	/* Programs the physical address of the device table into the IOMMU hardware */	307	/* Programs the physical address of the device table into the IOMMU hardware */
308	static void iommu_set_device_table(struct amd_iommu *iommu)	308	static void iommu_set_device_table(struct amd_iommu *iommu)
309	{	309	{
310	u64 entry;	310	u64 entry;
311		311
312	BUG_ON(iommu->mmio_base == NULL);	312	BUG_ON(iommu->mmio_base == NULL);
313		313
314	entry = virt_to_phys(amd_iommu_dev_table);	314	entry = virt_to_phys(amd_iommu_dev_table);
315	entry \|= (dev_table_size >> 12) - 1;	315	entry \|= (dev_table_size >> 12) - 1;
316	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,	316	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
317	&entry, sizeof(entry));	317	&entry, sizeof(entry));
318	}	318	}
319		319
320	/* Generic functions to enable/disable certain features of the IOMMU. */	320	/* Generic functions to enable/disable certain features of the IOMMU. */
321	static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)	321	static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
322	{	322	{
323	u32 ctrl;	323	u32 ctrl;
324		324
325	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);	325	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
326	ctrl \|= (1 << bit);	326	ctrl \|= (1 << bit);
327	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);	327	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
328	}	328	}
329		329
330	static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)	330	static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
331	{	331	{
332	u32 ctrl;	332	u32 ctrl;
333		333
334	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);	334	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
335	ctrl &= ~(1 << bit);	335	ctrl &= ~(1 << bit);
336	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);	336	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
337	}	337	}
338		338
339	static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)	339	static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
340	{	340	{
341	u32 ctrl;	341	u32 ctrl;
342		342
343	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);	343	ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
344	ctrl &= ~CTRL_INV_TO_MASK;	344	ctrl &= ~CTRL_INV_TO_MASK;
345	ctrl \|= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;	345	ctrl \|= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
346	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);	346	writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
347	}	347	}
348		348
349	/* Function to enable the hardware */	349	/* Function to enable the hardware */
350	static void iommu_enable(struct amd_iommu *iommu)	350	static void iommu_enable(struct amd_iommu *iommu)
351	{	351	{
352	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);	352	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
353	}	353	}
354		354
355	static void iommu_disable(struct amd_iommu *iommu)	355	static void iommu_disable(struct amd_iommu *iommu)
356	{	356	{
357	/* Disable command buffer */	357	/* Disable command buffer */
358	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);	358	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
359		359
360	/* Disable event logging and event interrupts */	360	/* Disable event logging and event interrupts */
361	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);	361	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
362	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);	362	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
363		363
364	/* Disable IOMMU hardware itself */	364	/* Disable IOMMU hardware itself */
365	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);	365	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
366	}	366	}
367		367
368	/*	368	/*
369	* mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in	369	* mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
370	* the system has one.	370	* the system has one.
371	*/	371	*/
372	static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)	372	static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
373	{	373	{
374	if (!request_mem_region(address, end, "amd_iommu")) {	374	if (!request_mem_region(address, end, "amd_iommu")) {
375	pr_err("AMD-Vi: Can not reserve memory region %llx-%llx for mmio\n",	375	pr_err("AMD-Vi: Can not reserve memory region %llx-%llx for mmio\n",
376	address, end);	376	address, end);
377	pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");	377	pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
378	return NULL;	378	return NULL;
379	}	379	}
380		380
381	return (u8 __iomem *)ioremap_nocache(address, end);	381	return (u8 __iomem *)ioremap_nocache(address, end);
382	}	382	}
383		383
384	static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)	384	static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
385	{	385	{
386	if (iommu->mmio_base)	386	if (iommu->mmio_base)
387	iounmap(iommu->mmio_base);	387	iounmap(iommu->mmio_base);
388	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);	388	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
389	}	389	}
390		390
391	/****************************************************************************	391	/****************************************************************************
392	*	392	*
393	* The functions below belong to the first pass of AMD IOMMU ACPI table	393	* The functions below belong to the first pass of AMD IOMMU ACPI table
394	* parsing. In this pass we try to find out the highest device id this	394	* parsing. In this pass we try to find out the highest device id this
395	* code has to handle. Upon this information the size of the shared data	395	* code has to handle. Upon this information the size of the shared data
396	* structures is determined later.	396	* structures is determined later.
397	*	397	*
398	****************************************************************************/	398	****************************************************************************/
399		399
400	/*	400	/*
401	* This function calculates the length of a given IVHD entry	401	* This function calculates the length of a given IVHD entry
402	*/	402	*/
403	static inline int ivhd_entry_length(u8 *ivhd)	403	static inline int ivhd_entry_length(u8 *ivhd)
404	{	404	{
405	return 0x04 << (*ivhd >> 6);	405	return 0x04 << (*ivhd >> 6);
406	}	406	}
407		407
408	/*	408	/*
409	* This function reads the last device id the IOMMU has to handle from the PCI	409	* This function reads the last device id the IOMMU has to handle from the PCI
410	* capability header for this IOMMU	410	* capability header for this IOMMU
411	*/	411	*/
412	static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)	412	static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
413	{	413	{
414	u32 cap;	414	u32 cap;
415		415
416	cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);	416	cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
417	update_last_devid(PCI_DEVID(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));	417	update_last_devid(PCI_DEVID(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
418		418
419	return 0;	419	return 0;
420	}	420	}
421		421
422	/*	422	/*
423	* After reading the highest device id from the IOMMU PCI capability header	423	* After reading the highest device id from the IOMMU PCI capability header
424	* this function looks if there is a higher device id defined in the ACPI table	424	* this function looks if there is a higher device id defined in the ACPI table
425	*/	425	*/
426	static int __init find_last_devid_from_ivhd(struct ivhd_header *h)	426	static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
427	{	427	{
428	u8 p = (void )h, end = (void )h;	428	u8 p = (void )h, end = (void )h;
429	struct ivhd_entry *dev;	429	struct ivhd_entry *dev;
430		430
431	p += sizeof(*h);	431	p += sizeof(*h);
432	end += h->length;	432	end += h->length;
433		433
434	find_last_devid_on_pci(PCI_BUS_NUM(h->devid),	434	find_last_devid_on_pci(PCI_BUS_NUM(h->devid),
435	PCI_SLOT(h->devid),	435	PCI_SLOT(h->devid),
436	PCI_FUNC(h->devid),	436	PCI_FUNC(h->devid),
437	h->cap_ptr);	437	h->cap_ptr);
438		438
439	while (p < end) {	439	while (p < end) {
440	dev = (struct ivhd_entry *)p;	440	dev = (struct ivhd_entry *)p;
441	switch (dev->type) {	441	switch (dev->type) {
442	case IVHD_DEV_SELECT:	442	case IVHD_DEV_SELECT:
443	case IVHD_DEV_RANGE_END:	443	case IVHD_DEV_RANGE_END:
444	case IVHD_DEV_ALIAS:	444	case IVHD_DEV_ALIAS:
445	case IVHD_DEV_EXT_SELECT:	445	case IVHD_DEV_EXT_SELECT:
446	/* all the above subfield types refer to device ids */	446	/* all the above subfield types refer to device ids */
447	update_last_devid(dev->devid);	447	update_last_devid(dev->devid);
448	break;	448	break;
449	default:	449	default:
450	break;	450	break;
451	}	451	}
452	p += ivhd_entry_length(p);	452	p += ivhd_entry_length(p);
453	}	453	}
454		454
455	WARN_ON(p != end);	455	WARN_ON(p != end);
456		456
457	return 0;	457	return 0;
458	}	458	}
459		459
460	/*	460	/*
461	* Iterate over all IVHD entries in the ACPI table and find the highest device	461	* Iterate over all IVHD entries in the ACPI table and find the highest device
462	* id which we need to handle. This is the first of three functions which parse	462	* id which we need to handle. This is the first of three functions which parse
463	* the ACPI table. So we check the checksum here.	463	* the ACPI table. So we check the checksum here.
464	*/	464	*/
465	static int __init find_last_devid_acpi(struct acpi_table_header *table)	465	static int __init find_last_devid_acpi(struct acpi_table_header *table)
466	{	466	{
467	int i;	467	int i;
468	u8 checksum = 0, p = (u8 )table, end = (u8 )table;	468	u8 checksum = 0, p = (u8 )table, end = (u8 )table;
469	struct ivhd_header *h;	469	struct ivhd_header *h;
470		470
471	/*	471	/*
472	* Validate checksum here so we don't need to do it when	472	* Validate checksum here so we don't need to do it when
473	* we actually parse the table	473	* we actually parse the table
474	*/	474	*/
475	for (i = 0; i < table->length; ++i)	475	for (i = 0; i < table->length; ++i)
476	checksum += p[i];	476	checksum += p[i];
477	if (checksum != 0)	477	if (checksum != 0)
478	/* ACPI table corrupt */	478	/* ACPI table corrupt */
479	return -ENODEV;	479	return -ENODEV;
480		480
481	p += IVRS_HEADER_LENGTH;	481	p += IVRS_HEADER_LENGTH;
482		482
483	end += table->length;	483	end += table->length;
484	while (p < end) {	484	while (p < end) {
485	h = (struct ivhd_header *)p;	485	h = (struct ivhd_header *)p;
486	switch (h->type) {	486	switch (h->type) {
487	case ACPI_IVHD_TYPE:	487	case ACPI_IVHD_TYPE:
488	find_last_devid_from_ivhd(h);	488	find_last_devid_from_ivhd(h);
489	break;	489	break;
490	default:	490	default:
491	break;	491	break;
492	}	492	}
493	p += h->length;	493	p += h->length;
494	}	494	}
495	WARN_ON(p != end);	495	WARN_ON(p != end);
496		496
497	return 0;	497	return 0;
498	}	498	}
499		499
500	/****************************************************************************	500	/****************************************************************************
501	*	501	*
502	* The following functions belong to the code path which parses the ACPI table	502	* The following functions belong to the code path which parses the ACPI table
503	* the second time. In this ACPI parsing iteration we allocate IOMMU specific	503	* the second time. In this ACPI parsing iteration we allocate IOMMU specific
504	* data structures, initialize the device/alias/rlookup table and also	504	* data structures, initialize the device/alias/rlookup table and also
505	* basically initialize the hardware.	505	* basically initialize the hardware.
506	*	506	*
507	****************************************************************************/	507	****************************************************************************/
508		508
509	/*	509	/*
510	* Allocates the command buffer. This buffer is per AMD IOMMU. We can	510	* Allocates the command buffer. This buffer is per AMD IOMMU. We can
511	* write commands to that buffer later and the IOMMU will execute them	511	* write commands to that buffer later and the IOMMU will execute them
512	* asynchronously	512	* asynchronously
513	*/	513	*/
514	static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)	514	static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
515	{	515	{
516	u8 cmd_buf = (u8 )__get_free_pages(GFP_KERNEL \| __GFP_ZERO,	516	u8 cmd_buf = (u8 )__get_free_pages(GFP_KERNEL \| __GFP_ZERO,
517	get_order(CMD_BUFFER_SIZE));	517	get_order(CMD_BUFFER_SIZE));
518		518
519	if (cmd_buf == NULL)	519	if (cmd_buf == NULL)
520	return NULL;	520	return NULL;
521		521
522	iommu->cmd_buf_size = CMD_BUFFER_SIZE \| CMD_BUFFER_UNINITIALIZED;	522	iommu->cmd_buf_size = CMD_BUFFER_SIZE \| CMD_BUFFER_UNINITIALIZED;
523		523
524	return cmd_buf;	524	return cmd_buf;
525	}	525	}
526		526
527	/*	527	/*
528	* This function resets the command buffer if the IOMMU stopped fetching	528	* This function resets the command buffer if the IOMMU stopped fetching
529	* commands from it.	529	* commands from it.
530	*/	530	*/
531	void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)	531	void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
532	{	532	{
533	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);	533	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
534		534
535	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);	535	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
536	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);	536	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
537		537
538	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);	538	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
539	}	539	}
540		540
541	/*	541	/*
542	* This function writes the command buffer address to the hardware and	542	* This function writes the command buffer address to the hardware and
543	* enables it.	543	* enables it.
544	*/	544	*/
545	static void iommu_enable_command_buffer(struct amd_iommu *iommu)	545	static void iommu_enable_command_buffer(struct amd_iommu *iommu)
546	{	546	{
547	u64 entry;	547	u64 entry;
548		548
549	BUG_ON(iommu->cmd_buf == NULL);	549	BUG_ON(iommu->cmd_buf == NULL);
550		550
551	entry = (u64)virt_to_phys(iommu->cmd_buf);	551	entry = (u64)virt_to_phys(iommu->cmd_buf);
552	entry \|= MMIO_CMD_SIZE_512;	552	entry \|= MMIO_CMD_SIZE_512;
553		553
554	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,	554	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
555	&entry, sizeof(entry));	555	&entry, sizeof(entry));
556		556
557	amd_iommu_reset_cmd_buffer(iommu);	557	amd_iommu_reset_cmd_buffer(iommu);
558	iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);	558	iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);
559	}	559	}
560		560
561	static void __init free_command_buffer(struct amd_iommu *iommu)	561	static void __init free_command_buffer(struct amd_iommu *iommu)
562	{	562	{
563	free_pages((unsigned long)iommu->cmd_buf,	563	free_pages((unsigned long)iommu->cmd_buf,
564	get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));	564	get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));
565	}	565	}
566		566
567	/* allocates the memory where the IOMMU will log its events to */	567	/* allocates the memory where the IOMMU will log its events to */
568	static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)	568	static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
569	{	569	{
570	iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,	570	iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,
571	get_order(EVT_BUFFER_SIZE));	571	get_order(EVT_BUFFER_SIZE));
572		572
573	if (iommu->evt_buf == NULL)	573	if (iommu->evt_buf == NULL)
574	return NULL;	574	return NULL;
575		575
576	iommu->evt_buf_size = EVT_BUFFER_SIZE;	576	iommu->evt_buf_size = EVT_BUFFER_SIZE;
577		577
578	return iommu->evt_buf;	578	return iommu->evt_buf;
579	}	579	}
580		580
581	static void iommu_enable_event_buffer(struct amd_iommu *iommu)	581	static void iommu_enable_event_buffer(struct amd_iommu *iommu)
582	{	582	{
583	u64 entry;	583	u64 entry;
584		584
585	BUG_ON(iommu->evt_buf == NULL);	585	BUG_ON(iommu->evt_buf == NULL);
586		586
587	entry = (u64)virt_to_phys(iommu->evt_buf) \| EVT_LEN_MASK;	587	entry = (u64)virt_to_phys(iommu->evt_buf) \| EVT_LEN_MASK;
588		588
589	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,	589	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
590	&entry, sizeof(entry));	590	&entry, sizeof(entry));
591		591
592	/* set head and tail to zero manually */	592	/* set head and tail to zero manually */
593	writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);	593	writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
594	writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);	594	writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
595		595
596	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);	596	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
597	}	597	}
598		598
599	static void __init free_event_buffer(struct amd_iommu *iommu)	599	static void __init free_event_buffer(struct amd_iommu *iommu)
600	{	600	{
601	free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));	601	free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
602	}	602	}
603		603
604	/* allocates the memory where the IOMMU will log its events to */	604	/* allocates the memory where the IOMMU will log its events to */
605	static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)	605	static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
606	{	606	{
607	iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,	607	iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,
608	get_order(PPR_LOG_SIZE));	608	get_order(PPR_LOG_SIZE));
609		609
610	if (iommu->ppr_log == NULL)	610	if (iommu->ppr_log == NULL)
611	return NULL;	611	return NULL;
612		612
613	return iommu->ppr_log;	613	return iommu->ppr_log;
614	}	614	}
615		615
616	static void iommu_enable_ppr_log(struct amd_iommu *iommu)	616	static void iommu_enable_ppr_log(struct amd_iommu *iommu)
617	{	617	{
618	u64 entry;	618	u64 entry;
619		619
620	if (iommu->ppr_log == NULL)	620	if (iommu->ppr_log == NULL)
621	return;	621	return;
622		622
623	entry = (u64)virt_to_phys(iommu->ppr_log) \| PPR_LOG_SIZE_512;	623	entry = (u64)virt_to_phys(iommu->ppr_log) \| PPR_LOG_SIZE_512;
624		624
625	memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,	625	memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
626	&entry, sizeof(entry));	626	&entry, sizeof(entry));
627		627
628	/* set head and tail to zero manually */	628	/* set head and tail to zero manually */
629	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);	629	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
630	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);	630	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
631		631
632	iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);	632	iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
633	iommu_feature_enable(iommu, CONTROL_PPR_EN);	633	iommu_feature_enable(iommu, CONTROL_PPR_EN);
634	}	634	}
635		635
636	static void __init free_ppr_log(struct amd_iommu *iommu)	636	static void __init free_ppr_log(struct amd_iommu *iommu)
637	{	637	{
638	if (iommu->ppr_log == NULL)	638	if (iommu->ppr_log == NULL)
639	return;	639	return;
640		640
641	free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));	641	free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
642	}	642	}
643		643
644	static void iommu_enable_gt(struct amd_iommu *iommu)	644	static void iommu_enable_gt(struct amd_iommu *iommu)
645	{	645	{
646	if (!iommu_feature(iommu, FEATURE_GT))	646	if (!iommu_feature(iommu, FEATURE_GT))
647	return;	647	return;
648		648
649	iommu_feature_enable(iommu, CONTROL_GT_EN);	649	iommu_feature_enable(iommu, CONTROL_GT_EN);
650	}	650	}
651		651
652	/* sets a specific bit in the device table entry. */	652	/* sets a specific bit in the device table entry. */
653	static void set_dev_entry_bit(u16 devid, u8 bit)	653	static void set_dev_entry_bit(u16 devid, u8 bit)
654	{	654	{
655	int i = (bit >> 6) & 0x03;	655	int i = (bit >> 6) & 0x03;
656	int _bit = bit & 0x3f;	656	int _bit = bit & 0x3f;
657		657
658	amd_iommu_dev_table[devid].data[i] \|= (1UL << _bit);	658	amd_iommu_dev_table[devid].data[i] \|= (1UL << _bit);
659	}	659	}
660		660
661	static int get_dev_entry_bit(u16 devid, u8 bit)	661	static int get_dev_entry_bit(u16 devid, u8 bit)
662	{	662	{
663	int i = (bit >> 6) & 0x03;	663	int i = (bit >> 6) & 0x03;
664	int _bit = bit & 0x3f;	664	int _bit = bit & 0x3f;
665		665
666	return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;	666	return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
667	}	667	}
668		668
669		669
670	void amd_iommu_apply_erratum_63(u16 devid)	670	void amd_iommu_apply_erratum_63(u16 devid)
671	{	671	{
672	int sysmgt;	672	int sysmgt;
673		673
674	sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) \|	674	sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) \|
675	(get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);	675	(get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
676		676
677	if (sysmgt == 0x01)	677	if (sysmgt == 0x01)
678	set_dev_entry_bit(devid, DEV_ENTRY_IW);	678	set_dev_entry_bit(devid, DEV_ENTRY_IW);
679	}	679	}
680		680
681	/* Writes the specific IOMMU for a device into the rlookup table */	681	/* Writes the specific IOMMU for a device into the rlookup table */
682	static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)	682	static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
683	{	683	{
684	amd_iommu_rlookup_table[devid] = iommu;	684	amd_iommu_rlookup_table[devid] = iommu;
685	}	685	}
686		686
687	/*	687	/*
688	* This function takes the device specific flags read from the ACPI	688	* This function takes the device specific flags read from the ACPI
689	* table and sets up the device table entry with that information	689	* table and sets up the device table entry with that information
690	*/	690	*/
691	static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,	691	static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
692	u16 devid, u32 flags, u32 ext_flags)	692	u16 devid, u32 flags, u32 ext_flags)
693	{	693	{
694	if (flags & ACPI_DEVFLAG_INITPASS)	694	if (flags & ACPI_DEVFLAG_INITPASS)
695	set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);	695	set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
696	if (flags & ACPI_DEVFLAG_EXTINT)	696	if (flags & ACPI_DEVFLAG_EXTINT)
697	set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);	697	set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
698	if (flags & ACPI_DEVFLAG_NMI)	698	if (flags & ACPI_DEVFLAG_NMI)
699	set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);	699	set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
700	if (flags & ACPI_DEVFLAG_SYSMGT1)	700	if (flags & ACPI_DEVFLAG_SYSMGT1)
701	set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);	701	set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
702	if (flags & ACPI_DEVFLAG_SYSMGT2)	702	if (flags & ACPI_DEVFLAG_SYSMGT2)
703	set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);	703	set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
704	if (flags & ACPI_DEVFLAG_LINT0)	704	if (flags & ACPI_DEVFLAG_LINT0)
705	set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);	705	set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
706	if (flags & ACPI_DEVFLAG_LINT1)	706	if (flags & ACPI_DEVFLAG_LINT1)
707	set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);	707	set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
708		708
709	amd_iommu_apply_erratum_63(devid);	709	amd_iommu_apply_erratum_63(devid);
710		710
711	set_iommu_for_device(iommu, devid);	711	set_iommu_for_device(iommu, devid);
712	}	712	}
713		713
714	static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line)	714	static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line)
715	{	715	{
716	struct devid_map *entry;	716	struct devid_map *entry;
717	struct list_head *list;	717	struct list_head *list;
718		718
719	if (type == IVHD_SPECIAL_IOAPIC)	719	if (type == IVHD_SPECIAL_IOAPIC)
720	list = &ioapic_map;	720	list = &ioapic_map;
721	else if (type == IVHD_SPECIAL_HPET)	721	else if (type == IVHD_SPECIAL_HPET)
722	list = &hpet_map;	722	list = &hpet_map;
723	else	723	else
724	return -EINVAL;	724	return -EINVAL;
725		725
726	list_for_each_entry(entry, list, list) {	726	list_for_each_entry(entry, list, list) {
727	if (!(entry->id == id && entry->cmd_line))	727	if (!(entry->id == id && entry->cmd_line))
728	continue;	728	continue;
729		729
730	pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n",	730	pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n",
731	type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);	731	type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
732		732
733	return 0;	733	return 0;
734	}	734	}
735		735
736	entry = kzalloc(sizeof(*entry), GFP_KERNEL);	736	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
737	if (!entry)	737	if (!entry)
738	return -ENOMEM;	738	return -ENOMEM;
739		739
740	entry->id = id;	740	entry->id = id;
741	entry->devid = devid;	741	entry->devid = devid;
742	entry->cmd_line = cmd_line;	742	entry->cmd_line = cmd_line;
743		743
744	list_add_tail(&entry->list, list);	744	list_add_tail(&entry->list, list);
745		745
746	return 0;	746	return 0;
747	}	747	}
748		748
749	static int __init add_early_maps(void)	749	static int __init add_early_maps(void)
750	{	750	{
751	int i, ret;	751	int i, ret;
752		752
753	for (i = 0; i < early_ioapic_map_size; ++i) {	753	for (i = 0; i < early_ioapic_map_size; ++i) {
754	ret = add_special_device(IVHD_SPECIAL_IOAPIC,	754	ret = add_special_device(IVHD_SPECIAL_IOAPIC,
755	early_ioapic_map[i].id,	755	early_ioapic_map[i].id,
756	early_ioapic_map[i].devid,	756	early_ioapic_map[i].devid,
757	early_ioapic_map[i].cmd_line);	757	early_ioapic_map[i].cmd_line);
758	if (ret)	758	if (ret)
759	return ret;	759	return ret;
760	}	760	}
761		761
762	for (i = 0; i < early_hpet_map_size; ++i) {	762	for (i = 0; i < early_hpet_map_size; ++i) {
763	ret = add_special_device(IVHD_SPECIAL_HPET,	763	ret = add_special_device(IVHD_SPECIAL_HPET,
764	early_hpet_map[i].id,	764	early_hpet_map[i].id,
765	early_hpet_map[i].devid,	765	early_hpet_map[i].devid,
766	early_hpet_map[i].cmd_line);	766	early_hpet_map[i].cmd_line);
767	if (ret)	767	if (ret)
768	return ret;	768	return ret;
769	}	769	}
770		770
771	return 0;	771	return 0;
772	}	772	}
773		773
774	/*	774	/*
775	* Reads the device exclusion range from ACPI and initializes the IOMMU with	775	* Reads the device exclusion range from ACPI and initializes the IOMMU with
776	* it	776	* it
777	*/	777	*/
778	static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)	778	static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
779	{	779	{
780	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];	780	struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
781		781
782	if (!(m->flags & IVMD_FLAG_EXCL_RANGE))	782	if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
783	return;	783	return;
784		784
785	if (iommu) {	785	if (iommu) {
786	/*	786	/*
787	* We only can configure exclusion ranges per IOMMU, not	787	* We only can configure exclusion ranges per IOMMU, not
788	* per device. But we can enable the exclusion range per	788	* per device. But we can enable the exclusion range per
789	* device. This is done here	789	* device. This is done here
790	*/	790	*/
791	set_dev_entry_bit(m->devid, DEV_ENTRY_EX);	791	set_dev_entry_bit(devid, DEV_ENTRY_EX);
792	iommu->exclusion_start = m->range_start;	792	iommu->exclusion_start = m->range_start;
793	iommu->exclusion_length = m->range_length;	793	iommu->exclusion_length = m->range_length;
794	}	794	}
795	}	795	}
796		796
797	/*	797	/*
798	* Takes a pointer to an AMD IOMMU entry in the ACPI table and	798	* Takes a pointer to an AMD IOMMU entry in the ACPI table and
799	* initializes the hardware and our data structures with it.	799	* initializes the hardware and our data structures with it.
800	*/	800	*/
801	static int __init init_iommu_from_acpi(struct amd_iommu *iommu,	801	static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
802	struct ivhd_header *h)	802	struct ivhd_header *h)
803	{	803	{
804	u8 p = (u8 )h;	804	u8 p = (u8 )h;
805	u8 *end = p, flags = 0;	805	u8 *end = p, flags = 0;
806	u16 devid = 0, devid_start = 0, devid_to = 0;	806	u16 devid = 0, devid_start = 0, devid_to = 0;
807	u32 dev_i, ext_flags = 0;	807	u32 dev_i, ext_flags = 0;
808	bool alias = false;	808	bool alias = false;
809	struct ivhd_entry *e;	809	struct ivhd_entry *e;
810	int ret;	810	int ret;
811		811
812		812
813	ret = add_early_maps();	813	ret = add_early_maps();
814	if (ret)	814	if (ret)
815	return ret;	815	return ret;
816		816
817	/*	817	/*
818	* First save the recommended feature enable bits from ACPI	818	* First save the recommended feature enable bits from ACPI
819	*/	819	*/
820	iommu->acpi_flags = h->flags;	820	iommu->acpi_flags = h->flags;
821		821
822	/*	822	/*
823	* Done. Now parse the device entries	823	* Done. Now parse the device entries
824	*/	824	*/
825	p += sizeof(struct ivhd_header);	825	p += sizeof(struct ivhd_header);
826	end += h->length;	826	end += h->length;
827		827
828		828
829	while (p < end) {	829	while (p < end) {
830	e = (struct ivhd_entry *)p;	830	e = (struct ivhd_entry *)p;
831	switch (e->type) {	831	switch (e->type) {
832	case IVHD_DEV_ALL:	832	case IVHD_DEV_ALL:
833		833
834	DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"	834	DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
835	" last device %02x:%02x.%x flags: %02x\n",	835	" last device %02x:%02x.%x flags: %02x\n",
836	PCI_BUS_NUM(iommu->first_device),	836	PCI_BUS_NUM(iommu->first_device),
837	PCI_SLOT(iommu->first_device),	837	PCI_SLOT(iommu->first_device),
838	PCI_FUNC(iommu->first_device),	838	PCI_FUNC(iommu->first_device),
839	PCI_BUS_NUM(iommu->last_device),	839	PCI_BUS_NUM(iommu->last_device),
840	PCI_SLOT(iommu->last_device),	840	PCI_SLOT(iommu->last_device),
841	PCI_FUNC(iommu->last_device),	841	PCI_FUNC(iommu->last_device),
842	e->flags);	842	e->flags);
843		843
844	for (dev_i = iommu->first_device;	844	for (dev_i = iommu->first_device;
845	dev_i <= iommu->last_device; ++dev_i)	845	dev_i <= iommu->last_device; ++dev_i)
846	set_dev_entry_from_acpi(iommu, dev_i,	846	set_dev_entry_from_acpi(iommu, dev_i,
847	e->flags, 0);	847	e->flags, 0);
848	break;	848	break;
849	case IVHD_DEV_SELECT:	849	case IVHD_DEV_SELECT:
850		850
851	DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "	851	DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
852	"flags: %02x\n",	852	"flags: %02x\n",
853	PCI_BUS_NUM(e->devid),	853	PCI_BUS_NUM(e->devid),
854	PCI_SLOT(e->devid),	854	PCI_SLOT(e->devid),
855	PCI_FUNC(e->devid),	855	PCI_FUNC(e->devid),
856	e->flags);	856	e->flags);
857		857
858	devid = e->devid;	858	devid = e->devid;
859	set_dev_entry_from_acpi(iommu, devid, e->flags, 0);	859	set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
860	break;	860	break;
861	case IVHD_DEV_SELECT_RANGE_START:	861	case IVHD_DEV_SELECT_RANGE_START:
862		862
863	DUMP_printk(" DEV_SELECT_RANGE_START\t "	863	DUMP_printk(" DEV_SELECT_RANGE_START\t "
864	"devid: %02x:%02x.%x flags: %02x\n",	864	"devid: %02x:%02x.%x flags: %02x\n",
865	PCI_BUS_NUM(e->devid),	865	PCI_BUS_NUM(e->devid),
866	PCI_SLOT(e->devid),	866	PCI_SLOT(e->devid),
867	PCI_FUNC(e->devid),	867	PCI_FUNC(e->devid),
868	e->flags);	868	e->flags);
869		869
870	devid_start = e->devid;	870	devid_start = e->devid;
871	flags = e->flags;	871	flags = e->flags;
872	ext_flags = 0;	872	ext_flags = 0;
873	alias = false;	873	alias = false;
874	break;	874	break;
875	case IVHD_DEV_ALIAS:	875	case IVHD_DEV_ALIAS:
876		876
877	DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "	877	DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
878	"flags: %02x devid_to: %02x:%02x.%x\n",	878	"flags: %02x devid_to: %02x:%02x.%x\n",
879	PCI_BUS_NUM(e->devid),	879	PCI_BUS_NUM(e->devid),
880	PCI_SLOT(e->devid),	880	PCI_SLOT(e->devid),
881	PCI_FUNC(e->devid),	881	PCI_FUNC(e->devid),
882	e->flags,	882	e->flags,
883	PCI_BUS_NUM(e->ext >> 8),	883	PCI_BUS_NUM(e->ext >> 8),
884	PCI_SLOT(e->ext >> 8),	884	PCI_SLOT(e->ext >> 8),
885	PCI_FUNC(e->ext >> 8));	885	PCI_FUNC(e->ext >> 8));
886		886
887	devid = e->devid;	887	devid = e->devid;
888	devid_to = e->ext >> 8;	888	devid_to = e->ext >> 8;
889	set_dev_entry_from_acpi(iommu, devid , e->flags, 0);	889	set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
890	set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);	890	set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
891	amd_iommu_alias_table[devid] = devid_to;	891	amd_iommu_alias_table[devid] = devid_to;
892	break;	892	break;
893	case IVHD_DEV_ALIAS_RANGE:	893	case IVHD_DEV_ALIAS_RANGE:
894		894
895	DUMP_printk(" DEV_ALIAS_RANGE\t\t "	895	DUMP_printk(" DEV_ALIAS_RANGE\t\t "
896	"devid: %02x:%02x.%x flags: %02x "	896	"devid: %02x:%02x.%x flags: %02x "
897	"devid_to: %02x:%02x.%x\n",	897	"devid_to: %02x:%02x.%x\n",
898	PCI_BUS_NUM(e->devid),	898	PCI_BUS_NUM(e->devid),
899	PCI_SLOT(e->devid),	899	PCI_SLOT(e->devid),
900	PCI_FUNC(e->devid),	900	PCI_FUNC(e->devid),
901	e->flags,	901	e->flags,
902	PCI_BUS_NUM(e->ext >> 8),	902	PCI_BUS_NUM(e->ext >> 8),
903	PCI_SLOT(e->ext >> 8),	903	PCI_SLOT(e->ext >> 8),
904	PCI_FUNC(e->ext >> 8));	904	PCI_FUNC(e->ext >> 8));
905		905
906	devid_start = e->devid;	906	devid_start = e->devid;
907	flags = e->flags;	907	flags = e->flags;
908	devid_to = e->ext >> 8;	908	devid_to = e->ext >> 8;
909	ext_flags = 0;	909	ext_flags = 0;
910	alias = true;	910	alias = true;
911	break;	911	break;
912	case IVHD_DEV_EXT_SELECT:	912	case IVHD_DEV_EXT_SELECT:
913		913
914	DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "	914	DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
915	"flags: %02x ext: %08x\n",	915	"flags: %02x ext: %08x\n",
916	PCI_BUS_NUM(e->devid),	916	PCI_BUS_NUM(e->devid),
917	PCI_SLOT(e->devid),	917	PCI_SLOT(e->devid),
918	PCI_FUNC(e->devid),	918	PCI_FUNC(e->devid),
919	e->flags, e->ext);	919	e->flags, e->ext);
920		920
921	devid = e->devid;	921	devid = e->devid;
922	set_dev_entry_from_acpi(iommu, devid, e->flags,	922	set_dev_entry_from_acpi(iommu, devid, e->flags,
923	e->ext);	923	e->ext);
924	break;	924	break;
925	case IVHD_DEV_EXT_SELECT_RANGE:	925	case IVHD_DEV_EXT_SELECT_RANGE:
926		926
927	DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "	927	DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
928	"%02x:%02x.%x flags: %02x ext: %08x\n",	928	"%02x:%02x.%x flags: %02x ext: %08x\n",
929	PCI_BUS_NUM(e->devid),	929	PCI_BUS_NUM(e->devid),
930	PCI_SLOT(e->devid),	930	PCI_SLOT(e->devid),
931	PCI_FUNC(e->devid),	931	PCI_FUNC(e->devid),
932	e->flags, e->ext);	932	e->flags, e->ext);
933		933
934	devid_start = e->devid;	934	devid_start = e->devid;
935	flags = e->flags;	935	flags = e->flags;
936	ext_flags = e->ext;	936	ext_flags = e->ext;
937	alias = false;	937	alias = false;
938	break;	938	break;
939	case IVHD_DEV_RANGE_END:	939	case IVHD_DEV_RANGE_END:
940		940
941	DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",	941	DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
942	PCI_BUS_NUM(e->devid),	942	PCI_BUS_NUM(e->devid),
943	PCI_SLOT(e->devid),	943	PCI_SLOT(e->devid),
944	PCI_FUNC(e->devid));	944	PCI_FUNC(e->devid));
945		945
946	devid = e->devid;	946	devid = e->devid;
947	for (dev_i = devid_start; dev_i <= devid; ++dev_i) {	947	for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
948	if (alias) {	948	if (alias) {
949	amd_iommu_alias_table[dev_i] = devid_to;	949	amd_iommu_alias_table[dev_i] = devid_to;
950	set_dev_entry_from_acpi(iommu,	950	set_dev_entry_from_acpi(iommu,
951	devid_to, flags, ext_flags);	951	devid_to, flags, ext_flags);
952	}	952	}
953	set_dev_entry_from_acpi(iommu, dev_i,	953	set_dev_entry_from_acpi(iommu, dev_i,
954	flags, ext_flags);	954	flags, ext_flags);
955	}	955	}
956	break;	956	break;
957	case IVHD_DEV_SPECIAL: {	957	case IVHD_DEV_SPECIAL: {
958	u8 handle, type;	958	u8 handle, type;
959	const char *var;	959	const char *var;
960	u16 devid;	960	u16 devid;
961	int ret;	961	int ret;
962		962
963	handle = e->ext & 0xff;	963	handle = e->ext & 0xff;
964	devid = (e->ext >> 8) & 0xffff;	964	devid = (e->ext >> 8) & 0xffff;
965	type = (e->ext >> 24) & 0xff;	965	type = (e->ext >> 24) & 0xff;
966		966
967	if (type == IVHD_SPECIAL_IOAPIC)	967	if (type == IVHD_SPECIAL_IOAPIC)
968	var = "IOAPIC";	968	var = "IOAPIC";
969	else if (type == IVHD_SPECIAL_HPET)	969	else if (type == IVHD_SPECIAL_HPET)
970	var = "HPET";	970	var = "HPET";
971	else	971	else
972	var = "UNKNOWN";	972	var = "UNKNOWN";
973		973
974	DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",	974	DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
975	var, (int)handle,	975	var, (int)handle,
976	PCI_BUS_NUM(devid),	976	PCI_BUS_NUM(devid),
977	PCI_SLOT(devid),	977	PCI_SLOT(devid),
978	PCI_FUNC(devid));	978	PCI_FUNC(devid));
979		979
980	set_dev_entry_from_acpi(iommu, devid, e->flags, 0);	980	set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
981	ret = add_special_device(type, handle, devid, false);	981	ret = add_special_device(type, handle, devid, false);
982	if (ret)	982	if (ret)
983	return ret;	983	return ret;
984	break;	984	break;
985	}	985	}
986	default:	986	default:
987	break;	987	break;
988	}	988	}
989		989
990	p += ivhd_entry_length(p);	990	p += ivhd_entry_length(p);
991	}	991	}
992		992
993	return 0;	993	return 0;
994	}	994	}
995		995
996	/* Initializes the device->iommu mapping for the driver */	996	/* Initializes the device->iommu mapping for the driver */
997	static int __init init_iommu_devices(struct amd_iommu *iommu)	997	static int __init init_iommu_devices(struct amd_iommu *iommu)
998	{	998	{
999	u32 i;	999	u32 i;
1000		1000
1001	for (i = iommu->first_device; i <= iommu->last_device; ++i)	1001	for (i = iommu->first_device; i <= iommu->last_device; ++i)
1002	set_iommu_for_device(iommu, i);	1002	set_iommu_for_device(iommu, i);
1003		1003
1004	return 0;	1004	return 0;
1005	}	1005	}
1006		1006
1007	static void __init free_iommu_one(struct amd_iommu *iommu)	1007	static void __init free_iommu_one(struct amd_iommu *iommu)
1008	{	1008	{
1009	free_command_buffer(iommu);	1009	free_command_buffer(iommu);
1010	free_event_buffer(iommu);	1010	free_event_buffer(iommu);
1011	free_ppr_log(iommu);	1011	free_ppr_log(iommu);
1012	iommu_unmap_mmio_space(iommu);	1012	iommu_unmap_mmio_space(iommu);
1013	}	1013	}
1014		1014
1015	static void __init free_iommu_all(void)	1015	static void __init free_iommu_all(void)
1016	{	1016	{
1017	struct amd_iommu iommu, next;	1017	struct amd_iommu iommu, next;
1018		1018
1019	for_each_iommu_safe(iommu, next) {	1019	for_each_iommu_safe(iommu, next) {
1020	list_del(&iommu->list);	1020	list_del(&iommu->list);
1021	free_iommu_one(iommu);	1021	free_iommu_one(iommu);
1022	kfree(iommu);	1022	kfree(iommu);
1023	}	1023	}
1024	}	1024	}
1025		1025
1026	/*	1026	/*
1027	* Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)	1027	* Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1028	* Workaround:	1028	* Workaround:
1029	* BIOS should disable L2B micellaneous clock gating by setting	1029	* BIOS should disable L2B micellaneous clock gating by setting
1030	* L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b	1030	* L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1031	*/	1031	*/
1032	static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)	1032	static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1033	{	1033	{
1034	u32 value;	1034	u32 value;
1035		1035
1036	if ((boot_cpu_data.x86 != 0x15) \|\|	1036	if ((boot_cpu_data.x86 != 0x15) \|\|
1037	(boot_cpu_data.x86_model < 0x10) \|\|	1037	(boot_cpu_data.x86_model < 0x10) \|\|
1038	(boot_cpu_data.x86_model > 0x1f))	1038	(boot_cpu_data.x86_model > 0x1f))
1039	return;	1039	return;
1040		1040
1041	pci_write_config_dword(iommu->dev, 0xf0, 0x90);	1041	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1042	pci_read_config_dword(iommu->dev, 0xf4, &value);	1042	pci_read_config_dword(iommu->dev, 0xf4, &value);
1043		1043
1044	if (value & BIT(2))	1044	if (value & BIT(2))
1045	return;	1045	return;
1046		1046
1047	/* Select NB indirect register 0x90 and enable writing */	1047	/* Select NB indirect register 0x90 and enable writing */
1048	pci_write_config_dword(iommu->dev, 0xf0, 0x90 \| (1 << 8));	1048	pci_write_config_dword(iommu->dev, 0xf0, 0x90 \| (1 << 8));
1049		1049
1050	pci_write_config_dword(iommu->dev, 0xf4, value \| 0x4);	1050	pci_write_config_dword(iommu->dev, 0xf4, value \| 0x4);
1051	pr_info("AMD-Vi: Applying erratum 746 workaround for IOMMU at %s\n",	1051	pr_info("AMD-Vi: Applying erratum 746 workaround for IOMMU at %s\n",
1052	dev_name(&iommu->dev->dev));	1052	dev_name(&iommu->dev->dev));
1053		1053
1054	/* Clear the enable writing bit */	1054	/* Clear the enable writing bit */
1055	pci_write_config_dword(iommu->dev, 0xf0, 0x90);	1055	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1056	}	1056	}
1057		1057
1058	/*	1058	/*
1059	* This function clues the initialization function for one IOMMU	1059	* This function clues the initialization function for one IOMMU
1060	* together and also allocates the command buffer and programs the	1060	* together and also allocates the command buffer and programs the
1061	* hardware. It does NOT enable the IOMMU. This is done afterwards.	1061	* hardware. It does NOT enable the IOMMU. This is done afterwards.
1062	*/	1062	*/
1063	static int __init init_iommu_one(struct amd_iommu iommu, struct ivhd_header h)	1063	static int __init init_iommu_one(struct amd_iommu iommu, struct ivhd_header h)
1064	{	1064	{
1065	int ret;	1065	int ret;
1066		1066
1067	spin_lock_init(&iommu->lock);	1067	spin_lock_init(&iommu->lock);
1068		1068
1069	/* Add IOMMU to internal data structures */	1069	/* Add IOMMU to internal data structures */
1070	list_add_tail(&iommu->list, &amd_iommu_list);	1070	list_add_tail(&iommu->list, &amd_iommu_list);
1071	iommu->index = amd_iommus_present++;	1071	iommu->index = amd_iommus_present++;
1072		1072
1073	if (unlikely(iommu->index >= MAX_IOMMUS)) {	1073	if (unlikely(iommu->index >= MAX_IOMMUS)) {
1074	WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");	1074	WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
1075	return -ENOSYS;	1075	return -ENOSYS;
1076	}	1076	}
1077		1077
1078	/* Index is fine - add IOMMU to the array */	1078	/* Index is fine - add IOMMU to the array */
1079	amd_iommus[iommu->index] = iommu;	1079	amd_iommus[iommu->index] = iommu;
1080		1080
1081	/*	1081	/*
1082	* Copy data from ACPI table entry to the iommu struct	1082	* Copy data from ACPI table entry to the iommu struct
1083	*/	1083	*/
1084	iommu->devid = h->devid;	1084	iommu->devid = h->devid;
1085	iommu->cap_ptr = h->cap_ptr;	1085	iommu->cap_ptr = h->cap_ptr;
1086	iommu->pci_seg = h->pci_seg;	1086	iommu->pci_seg = h->pci_seg;
1087	iommu->mmio_phys = h->mmio_phys;	1087	iommu->mmio_phys = h->mmio_phys;
1088		1088
1089	/* Check if IVHD EFR contains proper max banks/counters */	1089	/* Check if IVHD EFR contains proper max banks/counters */
1090	if ((h->efr != 0) &&	1090	if ((h->efr != 0) &&
1091	((h->efr & (0xF << 13)) != 0) &&	1091	((h->efr & (0xF << 13)) != 0) &&
1092	((h->efr & (0x3F << 17)) != 0)) {	1092	((h->efr & (0x3F << 17)) != 0)) {
1093	iommu->mmio_phys_end = MMIO_REG_END_OFFSET;	1093	iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1094	} else {	1094	} else {
1095	iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;	1095	iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1096	}	1096	}
1097		1097
1098	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,	1098	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
1099	iommu->mmio_phys_end);	1099	iommu->mmio_phys_end);
1100	if (!iommu->mmio_base)	1100	if (!iommu->mmio_base)
1101	return -ENOMEM;	1101	return -ENOMEM;
1102		1102
1103	iommu->cmd_buf = alloc_command_buffer(iommu);	1103	iommu->cmd_buf = alloc_command_buffer(iommu);
1104	if (!iommu->cmd_buf)	1104	if (!iommu->cmd_buf)
1105	return -ENOMEM;	1105	return -ENOMEM;
1106		1106
1107	iommu->evt_buf = alloc_event_buffer(iommu);	1107	iommu->evt_buf = alloc_event_buffer(iommu);
1108	if (!iommu->evt_buf)	1108	if (!iommu->evt_buf)
1109	return -ENOMEM;	1109	return -ENOMEM;
1110		1110
1111	iommu->int_enabled = false;	1111	iommu->int_enabled = false;
1112		1112
1113	ret = init_iommu_from_acpi(iommu, h);	1113	ret = init_iommu_from_acpi(iommu, h);
1114	if (ret)	1114	if (ret)
1115	return ret;	1115	return ret;
1116		1116
1117	/*	1117	/*
1118	* Make sure IOMMU is not considered to translate itself. The IVRS	1118	* Make sure IOMMU is not considered to translate itself. The IVRS
1119	* table tells us so, but this is a lie!	1119	* table tells us so, but this is a lie!
1120	*/	1120	*/
1121	amd_iommu_rlookup_table[iommu->devid] = NULL;	1121	amd_iommu_rlookup_table[iommu->devid] = NULL;
1122		1122
1123	init_iommu_devices(iommu);	1123	init_iommu_devices(iommu);
1124		1124
1125	return 0;	1125	return 0;
1126	}	1126	}
1127		1127
1128	/*	1128	/*
1129	* Iterates over all IOMMU entries in the ACPI table, allocates the	1129	* Iterates over all IOMMU entries in the ACPI table, allocates the
1130	* IOMMU structure and initializes it with init_iommu_one()	1130	* IOMMU structure and initializes it with init_iommu_one()
1131	*/	1131	*/
1132	static int __init init_iommu_all(struct acpi_table_header *table)	1132	static int __init init_iommu_all(struct acpi_table_header *table)
1133	{	1133	{
1134	u8 p = (u8 )table, end = (u8 )table;	1134	u8 p = (u8 )table, end = (u8 )table;
1135	struct ivhd_header *h;	1135	struct ivhd_header *h;
1136	struct amd_iommu *iommu;	1136	struct amd_iommu *iommu;
1137	int ret;	1137	int ret;
1138		1138
1139	end += table->length;	1139	end += table->length;
1140	p += IVRS_HEADER_LENGTH;	1140	p += IVRS_HEADER_LENGTH;
1141		1141
1142	while (p < end) {	1142	while (p < end) {
1143	h = (struct ivhd_header *)p;	1143	h = (struct ivhd_header *)p;
1144	switch (*p) {	1144	switch (*p) {
1145	case ACPI_IVHD_TYPE:	1145	case ACPI_IVHD_TYPE:
1146		1146
1147	DUMP_printk("device: %02x:%02x.%01x cap: %04x "	1147	DUMP_printk("device: %02x:%02x.%01x cap: %04x "
1148	"seg: %d flags: %01x info %04x\n",	1148	"seg: %d flags: %01x info %04x\n",
1149	PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid),	1149	PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid),
1150	PCI_FUNC(h->devid), h->cap_ptr,	1150	PCI_FUNC(h->devid), h->cap_ptr,
1151	h->pci_seg, h->flags, h->info);	1151	h->pci_seg, h->flags, h->info);
1152	DUMP_printk(" mmio-addr: %016llx\n",	1152	DUMP_printk(" mmio-addr: %016llx\n",
1153	h->mmio_phys);	1153	h->mmio_phys);
1154		1154
1155	iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);	1155	iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1156	if (iommu == NULL)	1156	if (iommu == NULL)
1157	return -ENOMEM;	1157	return -ENOMEM;
1158		1158
1159	ret = init_iommu_one(iommu, h);	1159	ret = init_iommu_one(iommu, h);
1160	if (ret)	1160	if (ret)
1161	return ret;	1161	return ret;
1162	break;	1162	break;
1163	default:	1163	default:
1164	break;	1164	break;
1165	}	1165	}
1166	p += h->length;	1166	p += h->length;
1167		1167
1168	}	1168	}
1169	WARN_ON(p != end);	1169	WARN_ON(p != end);
1170		1170
1171	return 0;	1171	return 0;
1172	}	1172	}
1173		1173
1174		1174
1175	static void init_iommu_perf_ctr(struct amd_iommu *iommu)	1175	static void init_iommu_perf_ctr(struct amd_iommu *iommu)
1176	{	1176	{
1177	u64 val = 0xabcd, val2 = 0;	1177	u64 val = 0xabcd, val2 = 0;
1178		1178
1179	if (!iommu_feature(iommu, FEATURE_PC))	1179	if (!iommu_feature(iommu, FEATURE_PC))
1180	return;	1180	return;
1181		1181
1182	amd_iommu_pc_present = true;	1182	amd_iommu_pc_present = true;
1183		1183
1184	/* Check if the performance counters can be written to */	1184	/* Check if the performance counters can be written to */
1185	if ((0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val, true)) \|\|	1185	if ((0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val, true)) \|\|
1186	(0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val2, false)) \|\|	1186	(0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val2, false)) \|\|
1187	(val != val2)) {	1187	(val != val2)) {
1188	pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n");	1188	pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n");
1189	amd_iommu_pc_present = false;	1189	amd_iommu_pc_present = false;
1190	return;	1190	return;
1191	}	1191	}
1192		1192
1193	pr_info("AMD-Vi: IOMMU performance counters supported\n");	1193	pr_info("AMD-Vi: IOMMU performance counters supported\n");
1194		1194
1195	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);	1195	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
1196	iommu->max_banks = (u8) ((val >> 12) & 0x3f);	1196	iommu->max_banks = (u8) ((val >> 12) & 0x3f);
1197	iommu->max_counters = (u8) ((val >> 7) & 0xf);	1197	iommu->max_counters = (u8) ((val >> 7) & 0xf);
1198	}	1198	}
1199		1199
1200		1200
1201	static int iommu_init_pci(struct amd_iommu *iommu)	1201	static int iommu_init_pci(struct amd_iommu *iommu)
1202	{	1202	{
1203	int cap_ptr = iommu->cap_ptr;	1203	int cap_ptr = iommu->cap_ptr;
1204	u32 range, misc, low, high;	1204	u32 range, misc, low, high;
1205		1205
1206	iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),	1206	iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),
1207	iommu->devid & 0xff);	1207	iommu->devid & 0xff);
1208	if (!iommu->dev)	1208	if (!iommu->dev)
1209	return -ENODEV;	1209	return -ENODEV;
1210		1210
1211	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,	1211	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
1212	&iommu->cap);	1212	&iommu->cap);
1213	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,	1213	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
1214	&range);	1214	&range);
1215	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,	1215	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
1216	&misc);	1216	&misc);
1217		1217
1218	iommu->first_device = PCI_DEVID(MMIO_GET_BUS(range),	1218	iommu->first_device = PCI_DEVID(MMIO_GET_BUS(range),
1219	MMIO_GET_FD(range));	1219	MMIO_GET_FD(range));
1220	iommu->last_device = PCI_DEVID(MMIO_GET_BUS(range),	1220	iommu->last_device = PCI_DEVID(MMIO_GET_BUS(range),
1221	MMIO_GET_LD(range));	1221	MMIO_GET_LD(range));
1222		1222
1223	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))	1223	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
1224	amd_iommu_iotlb_sup = false;	1224	amd_iommu_iotlb_sup = false;
1225		1225
1226	/* read extended feature bits */	1226	/* read extended feature bits */
1227	low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);	1227	low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
1228	high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);	1228	high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
1229		1229
1230	iommu->features = ((u64)high << 32) \| low;	1230	iommu->features = ((u64)high << 32) \| low;
1231		1231
1232	if (iommu_feature(iommu, FEATURE_GT)) {	1232	if (iommu_feature(iommu, FEATURE_GT)) {
1233	int glxval;	1233	int glxval;
1234	u32 max_pasid;	1234	u32 max_pasid;
1235	u64 pasmax;	1235	u64 pasmax;
1236		1236
1237	pasmax = iommu->features & FEATURE_PASID_MASK;	1237	pasmax = iommu->features & FEATURE_PASID_MASK;
1238	pasmax >>= FEATURE_PASID_SHIFT;	1238	pasmax >>= FEATURE_PASID_SHIFT;
1239	max_pasid = (1 << (pasmax + 1)) - 1;	1239	max_pasid = (1 << (pasmax + 1)) - 1;
1240		1240
1241	amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);	1241	amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
1242		1242
1243	BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);	1243	BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
1244		1244
1245	glxval = iommu->features & FEATURE_GLXVAL_MASK;	1245	glxval = iommu->features & FEATURE_GLXVAL_MASK;
1246	glxval >>= FEATURE_GLXVAL_SHIFT;	1246	glxval >>= FEATURE_GLXVAL_SHIFT;
1247		1247
1248	if (amd_iommu_max_glx_val == -1)	1248	if (amd_iommu_max_glx_val == -1)
1249	amd_iommu_max_glx_val = glxval;	1249	amd_iommu_max_glx_val = glxval;
1250	else	1250	else
1251	amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);	1251	amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
1252	}	1252	}
1253		1253
1254	if (iommu_feature(iommu, FEATURE_GT) &&	1254	if (iommu_feature(iommu, FEATURE_GT) &&
1255	iommu_feature(iommu, FEATURE_PPR)) {	1255	iommu_feature(iommu, FEATURE_PPR)) {
1256	iommu->is_iommu_v2 = true;	1256	iommu->is_iommu_v2 = true;
1257	amd_iommu_v2_present = true;	1257	amd_iommu_v2_present = true;
1258	}	1258	}
1259		1259
1260	if (iommu_feature(iommu, FEATURE_PPR)) {	1260	if (iommu_feature(iommu, FEATURE_PPR)) {
1261	iommu->ppr_log = alloc_ppr_log(iommu);	1261	iommu->ppr_log = alloc_ppr_log(iommu);
1262	if (!iommu->ppr_log)	1262	if (!iommu->ppr_log)
1263	return -ENOMEM;	1263	return -ENOMEM;
1264	}	1264	}
1265		1265
1266	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))	1266	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
1267	amd_iommu_np_cache = true;	1267	amd_iommu_np_cache = true;
1268		1268
1269	init_iommu_perf_ctr(iommu);	1269	init_iommu_perf_ctr(iommu);
1270		1270
1271	if (is_rd890_iommu(iommu->dev)) {	1271	if (is_rd890_iommu(iommu->dev)) {
1272	int i, j;	1272	int i, j;
1273		1273
1274	iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,	1274	iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
1275	PCI_DEVFN(0, 0));	1275	PCI_DEVFN(0, 0));
1276		1276
1277	/*	1277	/*
1278	* Some rd890 systems may not be fully reconfigured by the	1278	* Some rd890 systems may not be fully reconfigured by the
1279	* BIOS, so it's necessary for us to store this information so	1279	* BIOS, so it's necessary for us to store this information so
1280	* it can be reprogrammed on resume	1280	* it can be reprogrammed on resume
1281	*/	1281	*/
1282	pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,	1282	pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
1283	&iommu->stored_addr_lo);	1283	&iommu->stored_addr_lo);
1284	pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,	1284	pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
1285	&iommu->stored_addr_hi);	1285	&iommu->stored_addr_hi);
1286		1286
1287	/* Low bit locks writes to configuration space */	1287	/* Low bit locks writes to configuration space */
1288	iommu->stored_addr_lo &= ~1;	1288	iommu->stored_addr_lo &= ~1;
1289		1289
1290	for (i = 0; i < 6; i++)	1290	for (i = 0; i < 6; i++)
1291	for (j = 0; j < 0x12; j++)	1291	for (j = 0; j < 0x12; j++)
1292	iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);	1292	iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
1293		1293
1294	for (i = 0; i < 0x83; i++)	1294	for (i = 0; i < 0x83; i++)
1295	iommu->stored_l2[i] = iommu_read_l2(iommu, i);	1295	iommu->stored_l2[i] = iommu_read_l2(iommu, i);
1296	}	1296	}
1297		1297
1298	amd_iommu_erratum_746_workaround(iommu);	1298	amd_iommu_erratum_746_workaround(iommu);
1299		1299
1300	return pci_enable_device(iommu->dev);	1300	return pci_enable_device(iommu->dev);
1301	}	1301	}
1302		1302
1303	static void print_iommu_info(void)	1303	static void print_iommu_info(void)
1304	{	1304	{
1305	static const char * const feat_str[] = {	1305	static const char * const feat_str[] = {
1306	"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",	1306	"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
1307	"IA", "GA", "HE", "PC"	1307	"IA", "GA", "HE", "PC"
1308	};	1308	};
1309	struct amd_iommu *iommu;	1309	struct amd_iommu *iommu;
1310		1310
1311	for_each_iommu(iommu) {	1311	for_each_iommu(iommu) {
1312	int i;	1312	int i;
1313		1313
1314	pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",	1314	pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
1315	dev_name(&iommu->dev->dev), iommu->cap_ptr);	1315	dev_name(&iommu->dev->dev), iommu->cap_ptr);
1316		1316
1317	if (iommu->cap & (1 << IOMMU_CAP_EFR)) {	1317	if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
1318	pr_info("AMD-Vi: Extended features: ");	1318	pr_info("AMD-Vi: Extended features: ");
1319	for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {	1319	for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
1320	if (iommu_feature(iommu, (1ULL << i)))	1320	if (iommu_feature(iommu, (1ULL << i)))
1321	pr_cont(" %s", feat_str[i]);	1321	pr_cont(" %s", feat_str[i]);
1322	}	1322	}
1323	pr_cont("\n");	1323	pr_cont("\n");
1324	}	1324	}
1325	}	1325	}
1326	if (irq_remapping_enabled)	1326	if (irq_remapping_enabled)
1327	pr_info("AMD-Vi: Interrupt remapping enabled\n");	1327	pr_info("AMD-Vi: Interrupt remapping enabled\n");
1328	}	1328	}
1329		1329
1330	static int __init amd_iommu_init_pci(void)	1330	static int __init amd_iommu_init_pci(void)
1331	{	1331	{
1332	struct amd_iommu *iommu;	1332	struct amd_iommu *iommu;
1333	int ret = 0;	1333	int ret = 0;
1334		1334
1335	for_each_iommu(iommu) {	1335	for_each_iommu(iommu) {
1336	ret = iommu_init_pci(iommu);	1336	ret = iommu_init_pci(iommu);
1337	if (ret)	1337	if (ret)
1338	break;	1338	break;
1339	}	1339	}
1340		1340
1341	ret = amd_iommu_init_devices();	1341	ret = amd_iommu_init_devices();
1342		1342
1343	print_iommu_info();	1343	print_iommu_info();
1344		1344
1345	return ret;	1345	return ret;
1346	}	1346	}
1347		1347
1348	/****************************************************************************	1348	/****************************************************************************
1349	*	1349	*
1350	* The following functions initialize the MSI interrupts for all IOMMUs	1350	* The following functions initialize the MSI interrupts for all IOMMUs
1351	* in the system. It's a bit challenging because there could be multiple	1351	* in the system. It's a bit challenging because there could be multiple
1352	* IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per	1352	* IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
1353	* pci_dev.	1353	* pci_dev.
1354	*	1354	*
1355	****************************************************************************/	1355	****************************************************************************/
1356		1356
1357	static int iommu_setup_msi(struct amd_iommu *iommu)	1357	static int iommu_setup_msi(struct amd_iommu *iommu)
1358	{	1358	{
1359	int r;	1359	int r;
1360		1360
1361	r = pci_enable_msi(iommu->dev);	1361	r = pci_enable_msi(iommu->dev);
1362	if (r)	1362	if (r)
1363	return r;	1363	return r;
1364		1364
1365	r = request_threaded_irq(iommu->dev->irq,	1365	r = request_threaded_irq(iommu->dev->irq,
1366	amd_iommu_int_handler,	1366	amd_iommu_int_handler,
1367	amd_iommu_int_thread,	1367	amd_iommu_int_thread,
1368	0, "AMD-Vi",	1368	0, "AMD-Vi",
1369	iommu);	1369	iommu);
1370		1370
1371	if (r) {	1371	if (r) {
1372	pci_disable_msi(iommu->dev);	1372	pci_disable_msi(iommu->dev);
1373	return r;	1373	return r;
1374	}	1374	}
1375		1375
1376	iommu->int_enabled = true;	1376	iommu->int_enabled = true;
1377		1377
1378	return 0;	1378	return 0;
1379	}	1379	}
1380		1380
1381	static int iommu_init_msi(struct amd_iommu *iommu)	1381	static int iommu_init_msi(struct amd_iommu *iommu)
1382	{	1382	{
1383	int ret;	1383	int ret;
1384		1384
1385	if (iommu->int_enabled)	1385	if (iommu->int_enabled)
1386	goto enable_faults;	1386	goto enable_faults;
1387		1387
1388	if (iommu->dev->msi_cap)	1388	if (iommu->dev->msi_cap)
1389	ret = iommu_setup_msi(iommu);	1389	ret = iommu_setup_msi(iommu);
1390	else	1390	else
1391	ret = -ENODEV;	1391	ret = -ENODEV;
1392		1392
1393	if (ret)	1393	if (ret)
1394	return ret;	1394	return ret;
1395		1395
1396	enable_faults:	1396	enable_faults:
1397	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);	1397	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
1398		1398
1399	if (iommu->ppr_log != NULL)	1399	if (iommu->ppr_log != NULL)
1400	iommu_feature_enable(iommu, CONTROL_PPFINT_EN);	1400	iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
1401		1401
1402	return 0;	1402	return 0;
1403	}	1403	}
1404		1404
1405	/****************************************************************************	1405	/****************************************************************************
1406	*	1406	*
1407	* The next functions belong to the third pass of parsing the ACPI	1407	* The next functions belong to the third pass of parsing the ACPI
1408	* table. In this last pass the memory mapping requirements are	1408	* table. In this last pass the memory mapping requirements are
1409	* gathered (like exclusion and unity mapping ranges).	1409	* gathered (like exclusion and unity mapping ranges).
1410	*	1410	*
1411	****************************************************************************/	1411	****************************************************************************/
1412		1412
1413	static void __init free_unity_maps(void)	1413	static void __init free_unity_maps(void)
1414	{	1414	{
1415	struct unity_map_entry entry, next;	1415	struct unity_map_entry entry, next;
1416		1416
1417	list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {	1417	list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
1418	list_del(&entry->list);	1418	list_del(&entry->list);
1419	kfree(entry);	1419	kfree(entry);
1420	}	1420	}
1421	}	1421	}
1422		1422
1423	/* called when we find an exclusion range definition in ACPI */	1423	/* called when we find an exclusion range definition in ACPI */
1424	static int __init init_exclusion_range(struct ivmd_header *m)	1424	static int __init init_exclusion_range(struct ivmd_header *m)
1425	{	1425	{
1426	int i;	1426	int i;
1427		1427
1428	switch (m->type) {	1428	switch (m->type) {
1429	case ACPI_IVMD_TYPE:	1429	case ACPI_IVMD_TYPE:
1430	set_device_exclusion_range(m->devid, m);	1430	set_device_exclusion_range(m->devid, m);
1431	break;	1431	break;
1432	case ACPI_IVMD_TYPE_ALL:	1432	case ACPI_IVMD_TYPE_ALL:
1433	for (i = 0; i <= amd_iommu_last_bdf; ++i)	1433	for (i = 0; i <= amd_iommu_last_bdf; ++i)
1434	set_device_exclusion_range(i, m);	1434	set_device_exclusion_range(i, m);
1435	break;	1435	break;
1436	case ACPI_IVMD_TYPE_RANGE:	1436	case ACPI_IVMD_TYPE_RANGE:
1437	for (i = m->devid; i <= m->aux; ++i)	1437	for (i = m->devid; i <= m->aux; ++i)
1438	set_device_exclusion_range(i, m);	1438	set_device_exclusion_range(i, m);
1439	break;	1439	break;
1440	default:	1440	default:
1441	break;	1441	break;
1442	}	1442	}
1443		1443
1444	return 0;	1444	return 0;
1445	}	1445	}
1446		1446
1447	/* called for unity map ACPI definition */	1447	/* called for unity map ACPI definition */
1448	static int __init init_unity_map_range(struct ivmd_header *m)	1448	static int __init init_unity_map_range(struct ivmd_header *m)
1449	{	1449	{
1450	struct unity_map_entry *e = NULL;	1450	struct unity_map_entry *e = NULL;
1451	char *s;	1451	char *s;
1452		1452
1453	e = kzalloc(sizeof(*e), GFP_KERNEL);	1453	e = kzalloc(sizeof(*e), GFP_KERNEL);
1454	if (e == NULL)	1454	if (e == NULL)
1455	return -ENOMEM;	1455	return -ENOMEM;
1456		1456
1457	switch (m->type) {	1457	switch (m->type) {
1458	default:	1458	default:
1459	kfree(e);	1459	kfree(e);
1460	return 0;	1460	return 0;
1461	case ACPI_IVMD_TYPE:	1461	case ACPI_IVMD_TYPE:
1462	s = "IVMD_TYPEi\t\t\t";	1462	s = "IVMD_TYPEi\t\t\t";
1463	e->devid_start = e->devid_end = m->devid;	1463	e->devid_start = e->devid_end = m->devid;
1464	break;	1464	break;
1465	case ACPI_IVMD_TYPE_ALL:	1465	case ACPI_IVMD_TYPE_ALL:
1466	s = "IVMD_TYPE_ALL\t\t";	1466	s = "IVMD_TYPE_ALL\t\t";
1467	e->devid_start = 0;	1467	e->devid_start = 0;
1468	e->devid_end = amd_iommu_last_bdf;	1468	e->devid_end = amd_iommu_last_bdf;
1469	break;	1469	break;
1470	case ACPI_IVMD_TYPE_RANGE:	1470	case ACPI_IVMD_TYPE_RANGE:
1471	s = "IVMD_TYPE_RANGE\t\t";	1471	s = "IVMD_TYPE_RANGE\t\t";
1472	e->devid_start = m->devid;	1472	e->devid_start = m->devid;
1473	e->devid_end = m->aux;	1473	e->devid_end = m->aux;
1474	break;	1474	break;
1475	}	1475	}
1476	e->address_start = PAGE_ALIGN(m->range_start);	1476	e->address_start = PAGE_ALIGN(m->range_start);
1477	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);	1477	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
1478	e->prot = m->flags >> 1;	1478	e->prot = m->flags >> 1;
1479		1479
1480	DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"	1480	DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
1481	" range_start: %016llx range_end: %016llx flags: %x\n", s,	1481	" range_start: %016llx range_end: %016llx flags: %x\n", s,
1482	PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),	1482	PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
1483	PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end),	1483	PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end),
1484	PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),	1484	PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
1485	e->address_start, e->address_end, m->flags);	1485	e->address_start, e->address_end, m->flags);
1486		1486
1487	list_add_tail(&e->list, &amd_iommu_unity_map);	1487	list_add_tail(&e->list, &amd_iommu_unity_map);
1488		1488
1489	return 0;	1489	return 0;
1490	}	1490	}
1491		1491
1492	/* iterates over all memory definitions we find in the ACPI table */	1492	/* iterates over all memory definitions we find in the ACPI table */
1493	static int __init init_memory_definitions(struct acpi_table_header *table)	1493	static int __init init_memory_definitions(struct acpi_table_header *table)
1494	{	1494	{
1495	u8 p = (u8 )table, end = (u8 )table;	1495	u8 p = (u8 )table, end = (u8 )table;
1496	struct ivmd_header *m;	1496	struct ivmd_header *m;
1497		1497
1498	end += table->length;	1498	end += table->length;
1499	p += IVRS_HEADER_LENGTH;	1499	p += IVRS_HEADER_LENGTH;
1500		1500
1501	while (p < end) {	1501	while (p < end) {
1502	m = (struct ivmd_header *)p;	1502	m = (struct ivmd_header *)p;
1503	if (m->flags & IVMD_FLAG_EXCL_RANGE)	1503	if (m->flags & IVMD_FLAG_EXCL_RANGE)
1504	init_exclusion_range(m);	1504	init_exclusion_range(m);
1505	else if (m->flags & IVMD_FLAG_UNITY_MAP)	1505	else if (m->flags & IVMD_FLAG_UNITY_MAP)
1506	init_unity_map_range(m);	1506	init_unity_map_range(m);
1507		1507
1508	p += m->length;	1508	p += m->length;
1509	}	1509	}
1510		1510
1511	return 0;	1511	return 0;
1512	}	1512	}
1513		1513
1514	/*	1514	/*
1515	* Init the device table to not allow DMA access for devices and	1515	* Init the device table to not allow DMA access for devices and
1516	* suppress all page faults	1516	* suppress all page faults
1517	*/	1517	*/
1518	static void init_device_table_dma(void)	1518	static void init_device_table_dma(void)
1519	{	1519	{
1520	u32 devid;	1520	u32 devid;
1521		1521
1522	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {	1522	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1523	set_dev_entry_bit(devid, DEV_ENTRY_VALID);	1523	set_dev_entry_bit(devid, DEV_ENTRY_VALID);
1524	set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);	1524	set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
1525	}	1525	}
1526	}	1526	}
1527		1527
1528	static void __init uninit_device_table_dma(void)	1528	static void __init uninit_device_table_dma(void)
1529	{	1529	{
1530	u32 devid;	1530	u32 devid;
1531		1531
1532	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {	1532	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1533	amd_iommu_dev_table[devid].data[0] = 0ULL;	1533	amd_iommu_dev_table[devid].data[0] = 0ULL;
1534	amd_iommu_dev_table[devid].data[1] = 0ULL;	1534	amd_iommu_dev_table[devid].data[1] = 0ULL;
1535	}	1535	}
1536	}	1536	}
1537		1537
1538	static void init_device_table(void)	1538	static void init_device_table(void)
1539	{	1539	{
1540	u32 devid;	1540	u32 devid;
1541		1541
1542	if (!amd_iommu_irq_remap)	1542	if (!amd_iommu_irq_remap)
1543	return;	1543	return;
1544		1544
1545	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)	1545	for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
1546	set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN);	1546	set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN);
1547	}	1547	}
1548		1548
1549	static void iommu_init_flags(struct amd_iommu *iommu)	1549	static void iommu_init_flags(struct amd_iommu *iommu)
1550	{	1550	{
1551	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?	1551	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
1552	iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :	1552	iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
1553	iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);	1553	iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
1554		1554
1555	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?	1555	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
1556	iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :	1556	iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
1557	iommu_feature_disable(iommu, CONTROL_PASSPW_EN);	1557	iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
1558		1558
1559	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?	1559	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
1560	iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :	1560	iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
1561	iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);	1561	iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
1562		1562
1563	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?	1563	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
1564	iommu_feature_enable(iommu, CONTROL_ISOC_EN) :	1564	iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
1565	iommu_feature_disable(iommu, CONTROL_ISOC_EN);	1565	iommu_feature_disable(iommu, CONTROL_ISOC_EN);
1566		1566
1567	/*	1567	/*
1568	* make IOMMU memory accesses cache coherent	1568	* make IOMMU memory accesses cache coherent
1569	*/	1569	*/
1570	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);	1570	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
1571		1571
1572	/* Set IOTLB invalidation timeout to 1s */	1572	/* Set IOTLB invalidation timeout to 1s */
1573	iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);	1573	iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
1574	}	1574	}
1575		1575
1576	static void iommu_apply_resume_quirks(struct amd_iommu *iommu)	1576	static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
1577	{	1577	{
1578	int i, j;	1578	int i, j;
1579	u32 ioc_feature_control;	1579	u32 ioc_feature_control;
1580	struct pci_dev *pdev = iommu->root_pdev;	1580	struct pci_dev *pdev = iommu->root_pdev;
1581		1581
1582	/* RD890 BIOSes may not have completely reconfigured the iommu */	1582	/* RD890 BIOSes may not have completely reconfigured the iommu */
1583	if (!is_rd890_iommu(iommu->dev) \|\| !pdev)	1583	if (!is_rd890_iommu(iommu->dev) \|\| !pdev)
1584	return;	1584	return;
1585		1585
1586	/*	1586	/*
1587	* First, we need to ensure that the iommu is enabled. This is	1587	* First, we need to ensure that the iommu is enabled. This is
1588	* controlled by a register in the northbridge	1588	* controlled by a register in the northbridge
1589	*/	1589	*/
1590		1590
1591	/* Select Northbridge indirect register 0x75 and enable writing */	1591	/* Select Northbridge indirect register 0x75 and enable writing */
1592	pci_write_config_dword(pdev, 0x60, 0x75 \| (1 << 7));	1592	pci_write_config_dword(pdev, 0x60, 0x75 \| (1 << 7));
1593	pci_read_config_dword(pdev, 0x64, &ioc_feature_control);	1593	pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
1594		1594
1595	/* Enable the iommu */	1595	/* Enable the iommu */
1596	if (!(ioc_feature_control & 0x1))	1596	if (!(ioc_feature_control & 0x1))
1597	pci_write_config_dword(pdev, 0x64, ioc_feature_control \| 1);	1597	pci_write_config_dword(pdev, 0x64, ioc_feature_control \| 1);
1598		1598
1599	/* Restore the iommu BAR */	1599	/* Restore the iommu BAR */
1600	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,	1600	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1601	iommu->stored_addr_lo);	1601	iommu->stored_addr_lo);
1602	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,	1602	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
1603	iommu->stored_addr_hi);	1603	iommu->stored_addr_hi);
1604		1604
1605	/* Restore the l1 indirect regs for each of the 6 l1s */	1605	/* Restore the l1 indirect regs for each of the 6 l1s */
1606	for (i = 0; i < 6; i++)	1606	for (i = 0; i < 6; i++)
1607	for (j = 0; j < 0x12; j++)	1607	for (j = 0; j < 0x12; j++)
1608	iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);	1608	iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
1609		1609
1610	/* Restore the l2 indirect regs */	1610	/* Restore the l2 indirect regs */
1611	for (i = 0; i < 0x83; i++)	1611	for (i = 0; i < 0x83; i++)
1612	iommu_write_l2(iommu, i, iommu->stored_l2[i]);	1612	iommu_write_l2(iommu, i, iommu->stored_l2[i]);
1613		1613
1614	/* Lock PCI setup registers */	1614	/* Lock PCI setup registers */
1615	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,	1615	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1616	iommu->stored_addr_lo \| 1);	1616	iommu->stored_addr_lo \| 1);
1617	}	1617	}
1618		1618
1619	/*	1619	/*
1620	* This function finally enables all IOMMUs found in the system after	1620	* This function finally enables all IOMMUs found in the system after
1621	* they have been initialized	1621	* they have been initialized
1622	*/	1622	*/
1623	static void early_enable_iommus(void)	1623	static void early_enable_iommus(void)
1624	{	1624	{
1625	struct amd_iommu *iommu;	1625	struct amd_iommu *iommu;
1626		1626
1627	for_each_iommu(iommu) {	1627	for_each_iommu(iommu) {
1628	iommu_disable(iommu);	1628	iommu_disable(iommu);
1629	iommu_init_flags(iommu);	1629	iommu_init_flags(iommu);
1630	iommu_set_device_table(iommu);	1630	iommu_set_device_table(iommu);
1631	iommu_enable_command_buffer(iommu);	1631	iommu_enable_command_buffer(iommu);
1632	iommu_enable_event_buffer(iommu);	1632	iommu_enable_event_buffer(iommu);
1633	iommu_set_exclusion_range(iommu);	1633	iommu_set_exclusion_range(iommu);
1634	iommu_enable(iommu);	1634	iommu_enable(iommu);
1635	iommu_flush_all_caches(iommu);	1635	iommu_flush_all_caches(iommu);
1636	}	1636	}
1637	}	1637	}
1638		1638
1639	static void enable_iommus_v2(void)	1639	static void enable_iommus_v2(void)
1640	{	1640	{
1641	struct amd_iommu *iommu;	1641	struct amd_iommu *iommu;
1642		1642
1643	for_each_iommu(iommu) {	1643	for_each_iommu(iommu) {
1644	iommu_enable_ppr_log(iommu);	1644	iommu_enable_ppr_log(iommu);
1645	iommu_enable_gt(iommu);	1645	iommu_enable_gt(iommu);
1646	}	1646	}
1647	}	1647	}
1648		1648
1649	static void enable_iommus(void)	1649	static void enable_iommus(void)
1650	{	1650	{
1651	early_enable_iommus();	1651	early_enable_iommus();
1652		1652
1653	enable_iommus_v2();	1653	enable_iommus_v2();
1654	}	1654	}
1655		1655
1656	static void disable_iommus(void)	1656	static void disable_iommus(void)
1657	{	1657	{
1658	struct amd_iommu *iommu;	1658	struct amd_iommu *iommu;
1659		1659
1660	for_each_iommu(iommu)	1660	for_each_iommu(iommu)
1661	iommu_disable(iommu);	1661	iommu_disable(iommu);
1662	}	1662	}
1663		1663
1664	/*	1664	/*
1665	* Suspend/Resume support	1665	* Suspend/Resume support
1666	* disable suspend until real resume implemented	1666	* disable suspend until real resume implemented
1667	*/	1667	*/
1668		1668
1669	static void amd_iommu_resume(void)	1669	static void amd_iommu_resume(void)
1670	{	1670	{
1671	struct amd_iommu *iommu;	1671	struct amd_iommu *iommu;
1672		1672
1673	for_each_iommu(iommu)	1673	for_each_iommu(iommu)
1674	iommu_apply_resume_quirks(iommu);	1674	iommu_apply_resume_quirks(iommu);
1675		1675
1676	/* re-load the hardware */	1676	/* re-load the hardware */
1677	enable_iommus();	1677	enable_iommus();
1678		1678
1679	amd_iommu_enable_interrupts();	1679	amd_iommu_enable_interrupts();
1680	}	1680	}
1681		1681
1682	static int amd_iommu_suspend(void)	1682	static int amd_iommu_suspend(void)
1683	{	1683	{
1684	/* disable IOMMUs to go out of the way for BIOS */	1684	/* disable IOMMUs to go out of the way for BIOS */
1685	disable_iommus();	1685	disable_iommus();
1686		1686
1687	return 0;	1687	return 0;
1688	}	1688	}
1689		1689
1690	static struct syscore_ops amd_iommu_syscore_ops = {	1690	static struct syscore_ops amd_iommu_syscore_ops = {
1691	.suspend = amd_iommu_suspend,	1691	.suspend = amd_iommu_suspend,
1692	.resume = amd_iommu_resume,	1692	.resume = amd_iommu_resume,
1693	};	1693	};
1694		1694
1695	static void __init free_on_init_error(void)	1695	static void __init free_on_init_error(void)
1696	{	1696	{
1697	free_pages((unsigned long)irq_lookup_table,	1697	free_pages((unsigned long)irq_lookup_table,
1698	get_order(rlookup_table_size));	1698	get_order(rlookup_table_size));
1699		1699
1700	if (amd_iommu_irq_cache) {	1700	if (amd_iommu_irq_cache) {
1701	kmem_cache_destroy(amd_iommu_irq_cache);	1701	kmem_cache_destroy(amd_iommu_irq_cache);
1702	amd_iommu_irq_cache = NULL;	1702	amd_iommu_irq_cache = NULL;
1703		1703
1704	}	1704	}
1705		1705
1706	free_pages((unsigned long)amd_iommu_rlookup_table,	1706	free_pages((unsigned long)amd_iommu_rlookup_table,
1707	get_order(rlookup_table_size));	1707	get_order(rlookup_table_size));
1708		1708
1709	free_pages((unsigned long)amd_iommu_alias_table,	1709	free_pages((unsigned long)amd_iommu_alias_table,
1710	get_order(alias_table_size));	1710	get_order(alias_table_size));
1711		1711
1712	free_pages((unsigned long)amd_iommu_dev_table,	1712	free_pages((unsigned long)amd_iommu_dev_table,
1713	get_order(dev_table_size));	1713	get_order(dev_table_size));
1714		1714
1715	free_iommu_all();	1715	free_iommu_all();
1716		1716
1717	#ifdef CONFIG_GART_IOMMU	1717	#ifdef CONFIG_GART_IOMMU
1718	/*	1718	/*
1719	* We failed to initialize the AMD IOMMU - try fallback to GART	1719	* We failed to initialize the AMD IOMMU - try fallback to GART
1720	* if possible.	1720	* if possible.
1721	*/	1721	*/
1722	gart_iommu_init();	1722	gart_iommu_init();
1723		1723
1724	#endif	1724	#endif
1725	}	1725	}
1726		1726
1727	/* SB IOAPIC is always on this device in AMD systems */	1727	/* SB IOAPIC is always on this device in AMD systems */
1728	#define IOAPIC_SB_DEVID ((0x00 << 8) \| PCI_DEVFN(0x14, 0))	1728	#define IOAPIC_SB_DEVID ((0x00 << 8) \| PCI_DEVFN(0x14, 0))
1729		1729
1730	static bool __init check_ioapic_information(void)	1730	static bool __init check_ioapic_information(void)
1731	{	1731	{
1732	const char *fw_bug = FW_BUG;	1732	const char *fw_bug = FW_BUG;
1733	bool ret, has_sb_ioapic;	1733	bool ret, has_sb_ioapic;
1734	int idx;	1734	int idx;
1735		1735
1736	has_sb_ioapic = false;	1736	has_sb_ioapic = false;
1737	ret = false;	1737	ret = false;
1738		1738
1739	/*	1739	/*
1740	* If we have map overrides on the kernel command line the	1740	* If we have map overrides on the kernel command line the
1741	* messages in this function might not describe firmware bugs	1741	* messages in this function might not describe firmware bugs
1742	* anymore - so be careful	1742	* anymore - so be careful
1743	*/	1743	*/
1744	if (cmdline_maps)	1744	if (cmdline_maps)
1745	fw_bug = "";	1745	fw_bug = "";
1746		1746
1747	for (idx = 0; idx < nr_ioapics; idx++) {	1747	for (idx = 0; idx < nr_ioapics; idx++) {
1748	int devid, id = mpc_ioapic_id(idx);	1748	int devid, id = mpc_ioapic_id(idx);
1749		1749
1750	devid = get_ioapic_devid(id);	1750	devid = get_ioapic_devid(id);
1751	if (devid < 0) {	1751	if (devid < 0) {
1752	pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n",	1752	pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n",
1753	fw_bug, id);	1753	fw_bug, id);
1754	ret = false;	1754	ret = false;
1755	} else if (devid == IOAPIC_SB_DEVID) {	1755	} else if (devid == IOAPIC_SB_DEVID) {
1756	has_sb_ioapic = true;	1756	has_sb_ioapic = true;
1757	ret = true;	1757	ret = true;
1758	}	1758	}
1759	}	1759	}
1760		1760
1761	if (!has_sb_ioapic) {	1761	if (!has_sb_ioapic) {
1762	/*	1762	/*
1763	* We expect the SB IOAPIC to be listed in the IVRS	1763	* We expect the SB IOAPIC to be listed in the IVRS
1764	* table. The system timer is connected to the SB IOAPIC	1764	* table. The system timer is connected to the SB IOAPIC
1765	* and if we don't have it in the list the system will	1765	* and if we don't have it in the list the system will
1766	* panic at boot time. This situation usually happens	1766	* panic at boot time. This situation usually happens
1767	* when the BIOS is buggy and provides us the wrong	1767	* when the BIOS is buggy and provides us the wrong
1768	* device id for the IOAPIC in the system.	1768	* device id for the IOAPIC in the system.
1769	*/	1769	*/
1770	pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug);	1770	pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug);
1771	}	1771	}
1772		1772
1773	if (!ret)	1773	if (!ret)
1774	pr_err("AMD-Vi: Disabling interrupt remapping\n");	1774	pr_err("AMD-Vi: Disabling interrupt remapping\n");
1775		1775
1776	return ret;	1776	return ret;
1777	}	1777	}
1778		1778
1779	static void __init free_dma_resources(void)	1779	static void __init free_dma_resources(void)
1780	{	1780	{
1781	amd_iommu_uninit_devices();	1781	amd_iommu_uninit_devices();
1782		1782
1783	free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,	1783	free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1784	get_order(MAX_DOMAIN_ID/8));	1784	get_order(MAX_DOMAIN_ID/8));
1785		1785
1786	free_unity_maps();	1786	free_unity_maps();
1787	}	1787	}
1788		1788
1789	/*	1789	/*
1790	* This is the hardware init function for AMD IOMMU in the system.	1790	* This is the hardware init function for AMD IOMMU in the system.
1791	* This function is called either from amd_iommu_init or from the interrupt	1791	* This function is called either from amd_iommu_init or from the interrupt
1792	* remapping setup code.	1792	* remapping setup code.
1793	*	1793	*
1794	* This function basically parses the ACPI table for AMD IOMMU (IVRS)	1794	* This function basically parses the ACPI table for AMD IOMMU (IVRS)
1795	* three times:	1795	* three times:
1796	*	1796	*
1797	* 1 pass) Find the highest PCI device id the driver has to handle.	1797	* 1 pass) Find the highest PCI device id the driver has to handle.
1798	* Upon this information the size of the data structures is	1798	* Upon this information the size of the data structures is
1799	* determined that needs to be allocated.	1799	* determined that needs to be allocated.
1800	*	1800	*
1801	* 2 pass) Initialize the data structures just allocated with the	1801	* 2 pass) Initialize the data structures just allocated with the
1802	* information in the ACPI table about available AMD IOMMUs	1802	* information in the ACPI table about available AMD IOMMUs
1803	* in the system. It also maps the PCI devices in the	1803	* in the system. It also maps the PCI devices in the
1804	* system to specific IOMMUs	1804	* system to specific IOMMUs
1805	*	1805	*
1806	* 3 pass) After the basic data structures are allocated and	1806	* 3 pass) After the basic data structures are allocated and
1807	* initialized we update them with information about memory	1807	* initialized we update them with information about memory
1808	* remapping requirements parsed out of the ACPI table in	1808	* remapping requirements parsed out of the ACPI table in
1809	* this last pass.	1809	* this last pass.
1810	*	1810	*
1811	* After everything is set up the IOMMUs are enabled and the necessary	1811	* After everything is set up the IOMMUs are enabled and the necessary
1812	* hotplug and suspend notifiers are registered.	1812	* hotplug and suspend notifiers are registered.
1813	*/	1813	*/
1814	static int __init early_amd_iommu_init(void)	1814	static int __init early_amd_iommu_init(void)
1815	{	1815	{
1816	struct acpi_table_header *ivrs_base;	1816	struct acpi_table_header *ivrs_base;
1817	acpi_size ivrs_size;	1817	acpi_size ivrs_size;
1818	acpi_status status;	1818	acpi_status status;
1819	int i, ret = 0;	1819	int i, ret = 0;
1820		1820
1821	if (!amd_iommu_detected)	1821	if (!amd_iommu_detected)
1822	return -ENODEV;	1822	return -ENODEV;
1823		1823
1824	status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);	1824	status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1825	if (status == AE_NOT_FOUND)	1825	if (status == AE_NOT_FOUND)
1826	return -ENODEV;	1826	return -ENODEV;
1827	else if (ACPI_FAILURE(status)) {	1827	else if (ACPI_FAILURE(status)) {
1828	const char *err = acpi_format_exception(status);	1828	const char *err = acpi_format_exception(status);
1829	pr_err("AMD-Vi: IVRS table error: %s\n", err);	1829	pr_err("AMD-Vi: IVRS table error: %s\n", err);
1830	return -EINVAL;	1830	return -EINVAL;
1831	}	1831	}
1832		1832
1833	/*	1833	/*
1834	* First parse ACPI tables to find the largest Bus/Dev/Func	1834	* First parse ACPI tables to find the largest Bus/Dev/Func
1835	* we need to handle. Upon this information the shared data	1835	* we need to handle. Upon this information the shared data
1836	* structures for the IOMMUs in the system will be allocated	1836	* structures for the IOMMUs in the system will be allocated
1837	*/	1837	*/
1838	ret = find_last_devid_acpi(ivrs_base);	1838	ret = find_last_devid_acpi(ivrs_base);
1839	if (ret)	1839	if (ret)
1840	goto out;	1840	goto out;
1841		1841
1842	dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);	1842	dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
1843	alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);	1843	alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
1844	rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);	1844	rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
1845		1845
1846	/* Device table - directly used by all IOMMUs */	1846	/* Device table - directly used by all IOMMUs */
1847	ret = -ENOMEM;	1847	ret = -ENOMEM;
1848	amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,	1848	amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO,
1849	get_order(dev_table_size));	1849	get_order(dev_table_size));
1850	if (amd_iommu_dev_table == NULL)	1850	if (amd_iommu_dev_table == NULL)
1851	goto out;	1851	goto out;
1852		1852
1853	/*	1853	/*
1854	* Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the	1854	* Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
1855	* IOMMU see for that device	1855	* IOMMU see for that device
1856	*/	1856	*/
1857	amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,	1857	amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
1858	get_order(alias_table_size));	1858	get_order(alias_table_size));
1859	if (amd_iommu_alias_table == NULL)	1859	if (amd_iommu_alias_table == NULL)
1860	goto out;	1860	goto out;
1861		1861
1862	/* IOMMU rlookup table - find the IOMMU for a specific device */	1862	/* IOMMU rlookup table - find the IOMMU for a specific device */
1863	amd_iommu_rlookup_table = (void *)__get_free_pages(	1863	amd_iommu_rlookup_table = (void *)__get_free_pages(
1864	GFP_KERNEL \| __GFP_ZERO,	1864	GFP_KERNEL \| __GFP_ZERO,
1865	get_order(rlookup_table_size));	1865	get_order(rlookup_table_size));
1866	if (amd_iommu_rlookup_table == NULL)	1866	if (amd_iommu_rlookup_table == NULL)
1867	goto out;	1867	goto out;
1868		1868
1869	amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(	1869	amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
1870	GFP_KERNEL \| __GFP_ZERO,	1870	GFP_KERNEL \| __GFP_ZERO,
1871	get_order(MAX_DOMAIN_ID/8));	1871	get_order(MAX_DOMAIN_ID/8));
1872	if (amd_iommu_pd_alloc_bitmap == NULL)	1872	if (amd_iommu_pd_alloc_bitmap == NULL)
1873	goto out;	1873	goto out;
1874		1874
1875	/*	1875	/*
1876	* let all alias entries point to itself	1876	* let all alias entries point to itself
1877	*/	1877	*/
1878	for (i = 0; i <= amd_iommu_last_bdf; ++i)	1878	for (i = 0; i <= amd_iommu_last_bdf; ++i)
1879	amd_iommu_alias_table[i] = i;	1879	amd_iommu_alias_table[i] = i;
1880		1880
1881	/*	1881	/*
1882	* never allocate domain 0 because its used as the non-allocated and	1882	* never allocate domain 0 because its used as the non-allocated and
1883	* error value placeholder	1883	* error value placeholder
1884	*/	1884	*/
1885	amd_iommu_pd_alloc_bitmap[0] = 1;	1885	amd_iommu_pd_alloc_bitmap[0] = 1;
1886		1886
1887	spin_lock_init(&amd_iommu_pd_lock);	1887	spin_lock_init(&amd_iommu_pd_lock);
1888		1888
1889	/*	1889	/*
1890	* now the data structures are allocated and basically initialized	1890	* now the data structures are allocated and basically initialized
1891	* start the real acpi table scan	1891	* start the real acpi table scan
1892	*/	1892	*/
1893	ret = init_iommu_all(ivrs_base);	1893	ret = init_iommu_all(ivrs_base);
1894	if (ret)	1894	if (ret)
1895	goto out;	1895	goto out;
1896		1896
1897	if (amd_iommu_irq_remap)	1897	if (amd_iommu_irq_remap)
1898	amd_iommu_irq_remap = check_ioapic_information();	1898	amd_iommu_irq_remap = check_ioapic_information();
1899		1899
1900	if (amd_iommu_irq_remap) {	1900	if (amd_iommu_irq_remap) {
1901	/*	1901	/*
1902	* Interrupt remapping enabled, create kmem_cache for the	1902	* Interrupt remapping enabled, create kmem_cache for the
1903	* remapping tables.	1903	* remapping tables.
1904	*/	1904	*/
1905	ret = -ENOMEM;	1905	ret = -ENOMEM;
1906	amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",	1906	amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
1907	MAX_IRQS_PER_TABLE * sizeof(u32),	1907	MAX_IRQS_PER_TABLE * sizeof(u32),
1908	IRQ_TABLE_ALIGNMENT,	1908	IRQ_TABLE_ALIGNMENT,
1909	0, NULL);	1909	0, NULL);
1910	if (!amd_iommu_irq_cache)	1910	if (!amd_iommu_irq_cache)
1911	goto out;	1911	goto out;
1912		1912
1913	irq_lookup_table = (void *)__get_free_pages(	1913	irq_lookup_table = (void *)__get_free_pages(
1914	GFP_KERNEL \| __GFP_ZERO,	1914	GFP_KERNEL \| __GFP_ZERO,
1915	get_order(rlookup_table_size));	1915	get_order(rlookup_table_size));
1916	if (!irq_lookup_table)	1916	if (!irq_lookup_table)
1917	goto out;	1917	goto out;
1918	}	1918	}
1919		1919
1920	ret = init_memory_definitions(ivrs_base);	1920	ret = init_memory_definitions(ivrs_base);
1921	if (ret)	1921	if (ret)
1922	goto out;	1922	goto out;
1923		1923
1924	/* init the device table */	1924	/* init the device table */
1925	init_device_table();	1925	init_device_table();
1926		1926
1927	out:	1927	out:
1928	/* Don't leak any ACPI memory */	1928	/* Don't leak any ACPI memory */
1929	early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);	1929	early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1930	ivrs_base = NULL;	1930	ivrs_base = NULL;
1931		1931
1932	return ret;	1932	return ret;
1933	}	1933	}
1934		1934
1935	static int amd_iommu_enable_interrupts(void)	1935	static int amd_iommu_enable_interrupts(void)
1936	{	1936	{
1937	struct amd_iommu *iommu;	1937	struct amd_iommu *iommu;
1938	int ret = 0;	1938	int ret = 0;
1939		1939
1940	for_each_iommu(iommu) {	1940	for_each_iommu(iommu) {
1941	ret = iommu_init_msi(iommu);	1941	ret = iommu_init_msi(iommu);
1942	if (ret)	1942	if (ret)
1943	goto out;	1943	goto out;
1944	}	1944	}
1945		1945
1946	out:	1946	out:
1947	return ret;	1947	return ret;
1948	}	1948	}
1949		1949
1950	static bool detect_ivrs(void)	1950	static bool detect_ivrs(void)
1951	{	1951	{
1952	struct acpi_table_header *ivrs_base;	1952	struct acpi_table_header *ivrs_base;
1953	acpi_size ivrs_size;	1953	acpi_size ivrs_size;
1954	acpi_status status;	1954	acpi_status status;
1955		1955
1956	status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);	1956	status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1957	if (status == AE_NOT_FOUND)	1957	if (status == AE_NOT_FOUND)
1958	return false;	1958	return false;
1959	else if (ACPI_FAILURE(status)) {	1959	else if (ACPI_FAILURE(status)) {
1960	const char *err = acpi_format_exception(status);	1960	const char *err = acpi_format_exception(status);
1961	pr_err("AMD-Vi: IVRS table error: %s\n", err);	1961	pr_err("AMD-Vi: IVRS table error: %s\n", err);
1962	return false;	1962	return false;
1963	}	1963	}
1964		1964
1965	early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);	1965	early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1966		1966
1967	/* Make sure ACS will be enabled during PCI probe */	1967	/* Make sure ACS will be enabled during PCI probe */
1968	pci_request_acs();	1968	pci_request_acs();
1969		1969
1970	if (!disable_irq_remap)	1970	if (!disable_irq_remap)
1971	amd_iommu_irq_remap = true;	1971	amd_iommu_irq_remap = true;
1972		1972
1973	return true;	1973	return true;
1974	}	1974	}
1975		1975
1976	static int amd_iommu_init_dma(void)	1976	static int amd_iommu_init_dma(void)
1977	{	1977	{
1978	struct amd_iommu *iommu;	1978	struct amd_iommu *iommu;
1979	int ret;	1979	int ret;
1980		1980
1981	if (iommu_pass_through)	1981	if (iommu_pass_through)
1982	ret = amd_iommu_init_passthrough();	1982	ret = amd_iommu_init_passthrough();
1983	else	1983	else
1984	ret = amd_iommu_init_dma_ops();	1984	ret = amd_iommu_init_dma_ops();
1985		1985
1986	if (ret)	1986	if (ret)
1987	return ret;	1987	return ret;
1988		1988
1989	init_device_table_dma();	1989	init_device_table_dma();
1990		1990
1991	for_each_iommu(iommu)	1991	for_each_iommu(iommu)
1992	iommu_flush_all_caches(iommu);	1992	iommu_flush_all_caches(iommu);
1993		1993
1994	amd_iommu_init_api();	1994	amd_iommu_init_api();
1995		1995
1996	amd_iommu_init_notifier();	1996	amd_iommu_init_notifier();
1997		1997
1998	return 0;	1998	return 0;
1999	}	1999	}
2000		2000
2001	/****************************************************************************	2001	/****************************************************************************
2002	*	2002	*
2003	* AMD IOMMU Initialization State Machine	2003	* AMD IOMMU Initialization State Machine
2004	*	2004	*
2005	****************************************************************************/	2005	****************************************************************************/
2006		2006
2007	static int __init state_next(void)	2007	static int __init state_next(void)
2008	{	2008	{
2009	int ret = 0;	2009	int ret = 0;
2010		2010
2011	switch (init_state) {	2011	switch (init_state) {
2012	case IOMMU_START_STATE:	2012	case IOMMU_START_STATE:
2013	if (!detect_ivrs()) {	2013	if (!detect_ivrs()) {
2014	init_state = IOMMU_NOT_FOUND;	2014	init_state = IOMMU_NOT_FOUND;
2015	ret = -ENODEV;	2015	ret = -ENODEV;
2016	} else {	2016	} else {
2017	init_state = IOMMU_IVRS_DETECTED;	2017	init_state = IOMMU_IVRS_DETECTED;
2018	}	2018	}
2019	break;	2019	break;
2020	case IOMMU_IVRS_DETECTED:	2020	case IOMMU_IVRS_DETECTED:
2021	ret = early_amd_iommu_init();	2021	ret = early_amd_iommu_init();
2022	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;	2022	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
2023	break;	2023	break;
2024	case IOMMU_ACPI_FINISHED:	2024	case IOMMU_ACPI_FINISHED:
2025	early_enable_iommus();	2025	early_enable_iommus();
2026	register_syscore_ops(&amd_iommu_syscore_ops);	2026	register_syscore_ops(&amd_iommu_syscore_ops);
2027	x86_platform.iommu_shutdown = disable_iommus;	2027	x86_platform.iommu_shutdown = disable_iommus;
2028	init_state = IOMMU_ENABLED;	2028	init_state = IOMMU_ENABLED;
2029	break;	2029	break;
2030	case IOMMU_ENABLED:	2030	case IOMMU_ENABLED:
2031	ret = amd_iommu_init_pci();	2031	ret = amd_iommu_init_pci();
2032	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;	2032	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
2033	enable_iommus_v2();	2033	enable_iommus_v2();
2034	break;	2034	break;
2035	case IOMMU_PCI_INIT:	2035	case IOMMU_PCI_INIT:
2036	ret = amd_iommu_enable_interrupts();	2036	ret = amd_iommu_enable_interrupts();
2037	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;	2037	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
2038	break;	2038	break;
2039	case IOMMU_INTERRUPTS_EN:	2039	case IOMMU_INTERRUPTS_EN:
2040	ret = amd_iommu_init_dma();	2040	ret = amd_iommu_init_dma();
2041	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;	2041	init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
2042	break;	2042	break;
2043	case IOMMU_DMA_OPS:	2043	case IOMMU_DMA_OPS:
2044	init_state = IOMMU_INITIALIZED;	2044	init_state = IOMMU_INITIALIZED;
2045	break;	2045	break;
2046	case IOMMU_INITIALIZED:	2046	case IOMMU_INITIALIZED:
2047	/* Nothing to do */	2047	/* Nothing to do */
2048	break;	2048	break;
2049	case IOMMU_NOT_FOUND:	2049	case IOMMU_NOT_FOUND:
2050	case IOMMU_INIT_ERROR:	2050	case IOMMU_INIT_ERROR:
2051	/* Error states => do nothing */	2051	/* Error states => do nothing */
2052	ret = -EINVAL;	2052	ret = -EINVAL;
2053	break;	2053	break;
2054	default:	2054	default:
2055	/* Unknown state */	2055	/* Unknown state */
2056	BUG();	2056	BUG();
2057	}	2057	}
2058		2058
2059	return ret;	2059	return ret;
2060	}	2060	}
2061		2061
2062	static int __init iommu_go_to_state(enum iommu_init_state state)	2062	static int __init iommu_go_to_state(enum iommu_init_state state)
2063	{	2063	{
2064	int ret = 0;	2064	int ret = 0;
2065		2065
2066	while (init_state != state) {	2066	while (init_state != state) {
2067	ret = state_next();	2067	ret = state_next();
2068	if (init_state == IOMMU_NOT_FOUND \|\|	2068	if (init_state == IOMMU_NOT_FOUND \|\|
2069	init_state == IOMMU_INIT_ERROR)	2069	init_state == IOMMU_INIT_ERROR)
2070	break;	2070	break;
2071	}	2071	}
2072		2072
2073	return ret;	2073	return ret;
2074	}	2074	}
2075		2075
2076	#ifdef CONFIG_IRQ_REMAP	2076	#ifdef CONFIG_IRQ_REMAP
2077	int __init amd_iommu_prepare(void)	2077	int __init amd_iommu_prepare(void)
2078	{	2078	{
2079	return iommu_go_to_state(IOMMU_ACPI_FINISHED);	2079	return iommu_go_to_state(IOMMU_ACPI_FINISHED);
2080	}	2080	}
2081		2081
2082	int __init amd_iommu_supported(void)	2082	int __init amd_iommu_supported(void)
2083	{	2083	{
2084	return amd_iommu_irq_remap ? 1 : 0;	2084	return amd_iommu_irq_remap ? 1 : 0;
2085	}	2085	}
2086		2086
2087	int __init amd_iommu_enable(void)	2087	int __init amd_iommu_enable(void)
2088	{	2088	{
2089	int ret;	2089	int ret;
2090		2090
2091	ret = iommu_go_to_state(IOMMU_ENABLED);	2091	ret = iommu_go_to_state(IOMMU_ENABLED);
2092	if (ret)	2092	if (ret)
2093	return ret;	2093	return ret;
2094		2094
2095	irq_remapping_enabled = 1;	2095	irq_remapping_enabled = 1;
2096		2096
2097	return 0;	2097	return 0;
2098	}	2098	}
2099		2099
2100	void amd_iommu_disable(void)	2100	void amd_iommu_disable(void)
2101	{	2101	{
2102	amd_iommu_suspend();	2102	amd_iommu_suspend();
2103	}	2103	}
2104		2104
2105	int amd_iommu_reenable(int mode)	2105	int amd_iommu_reenable(int mode)
2106	{	2106	{
2107	amd_iommu_resume();	2107	amd_iommu_resume();
2108		2108
2109	return 0;	2109	return 0;
2110	}	2110	}
2111		2111
2112	int __init amd_iommu_enable_faulting(void)	2112	int __init amd_iommu_enable_faulting(void)
2113	{	2113	{
2114	/* We enable MSI later when PCI is initialized */	2114	/* We enable MSI later when PCI is initialized */
2115	return 0;	2115	return 0;
2116	}	2116	}
2117	#endif	2117	#endif
2118		2118
2119	/*	2119	/*
2120	* This is the core init function for AMD IOMMU hardware in the system.	2120	* This is the core init function for AMD IOMMU hardware in the system.
2121	* This function is called from the generic x86 DMA layer initialization	2121	* This function is called from the generic x86 DMA layer initialization
2122	* code.	2122	* code.
2123	*/	2123	*/
2124	static int __init amd_iommu_init(void)	2124	static int __init amd_iommu_init(void)
2125	{	2125	{
2126	int ret;	2126	int ret;
2127		2127
2128	ret = iommu_go_to_state(IOMMU_INITIALIZED);	2128	ret = iommu_go_to_state(IOMMU_INITIALIZED);
2129	if (ret) {	2129	if (ret) {
2130	free_dma_resources();	2130	free_dma_resources();
2131	if (!irq_remapping_enabled) {	2131	if (!irq_remapping_enabled) {
2132	disable_iommus();	2132	disable_iommus();
2133	free_on_init_error();	2133	free_on_init_error();
2134	} else {	2134	} else {
2135	struct amd_iommu *iommu;	2135	struct amd_iommu *iommu;
2136		2136
2137	uninit_device_table_dma();	2137	uninit_device_table_dma();
2138	for_each_iommu(iommu)	2138	for_each_iommu(iommu)
2139	iommu_flush_all_caches(iommu);	2139	iommu_flush_all_caches(iommu);
2140	}	2140	}
2141	}	2141	}
2142		2142
2143	return ret;	2143	return ret;
2144	}	2144	}
2145		2145
2146	/****************************************************************************	2146	/****************************************************************************
2147	*	2147	*
2148	* Early detect code. This code runs at IOMMU detection time in the DMA	2148	* Early detect code. This code runs at IOMMU detection time in the DMA
2149	* layer. It just looks if there is an IVRS ACPI table to detect AMD	2149	* layer. It just looks if there is an IVRS ACPI table to detect AMD
2150	* IOMMUs	2150	* IOMMUs
2151	*	2151	*
2152	****************************************************************************/	2152	****************************************************************************/
2153	int __init amd_iommu_detect(void)	2153	int __init amd_iommu_detect(void)
2154	{	2154	{
2155	int ret;	2155	int ret;
2156		2156
2157	if (no_iommu \|\| (iommu_detected && !gart_iommu_aperture))	2157	if (no_iommu \|\| (iommu_detected && !gart_iommu_aperture))
2158	return -ENODEV;	2158	return -ENODEV;
2159		2159
2160	if (amd_iommu_disabled)	2160	if (amd_iommu_disabled)
2161	return -ENODEV;	2161	return -ENODEV;
2162		2162
2163	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);	2163	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
2164	if (ret)	2164	if (ret)
2165	return ret;	2165	return ret;
2166		2166
2167	amd_iommu_detected = true;	2167	amd_iommu_detected = true;
2168	iommu_detected = 1;	2168	iommu_detected = 1;
2169	x86_init.iommu.iommu_init = amd_iommu_init;	2169	x86_init.iommu.iommu_init = amd_iommu_init;
2170		2170
2171	return 0;	2171	return 0;
2172	}	2172	}
2173		2173
2174	/****************************************************************************	2174	/****************************************************************************
2175	*	2175	*
2176	* Parsing functions for the AMD IOMMU specific kernel command line	2176	* Parsing functions for the AMD IOMMU specific kernel command line
2177	* options.	2177	* options.
2178	*	2178	*
2179	****************************************************************************/	2179	****************************************************************************/
2180		2180
2181	static int __init parse_amd_iommu_dump(char *str)	2181	static int __init parse_amd_iommu_dump(char *str)
2182	{	2182	{
2183	amd_iommu_dump = true;	2183	amd_iommu_dump = true;
2184		2184
2185	return 1;	2185	return 1;
2186	}	2186	}
2187		2187
2188	static int __init parse_amd_iommu_options(char *str)	2188	static int __init parse_amd_iommu_options(char *str)
2189	{	2189	{
2190	for (; *str; ++str) {	2190	for (; *str; ++str) {
2191	if (strncmp(str, "fullflush", 9) == 0)	2191	if (strncmp(str, "fullflush", 9) == 0)
2192	amd_iommu_unmap_flush = true;	2192	amd_iommu_unmap_flush = true;
2193	if (strncmp(str, "off", 3) == 0)	2193	if (strncmp(str, "off", 3) == 0)
2194	amd_iommu_disabled = true;	2194	amd_iommu_disabled = true;
2195	if (strncmp(str, "force_isolation", 15) == 0)	2195	if (strncmp(str, "force_isolation", 15) == 0)
2196	amd_iommu_force_isolation = true;	2196	amd_iommu_force_isolation = true;
2197	}	2197	}
2198		2198
2199	return 1;	2199	return 1;
2200	}	2200	}
2201		2201
2202	static int __init parse_ivrs_ioapic(char *str)	2202	static int __init parse_ivrs_ioapic(char *str)
2203	{	2203	{
2204	unsigned int bus, dev, fn;	2204	unsigned int bus, dev, fn;
2205	int ret, id, i;	2205	int ret, id, i;
2206	u16 devid;	2206	u16 devid;
2207		2207
2208	ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);	2208	ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
2209		2209
2210	if (ret != 4) {	2210	if (ret != 4) {
2211	pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str);	2211	pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str);
2212	return 1;	2212	return 1;
2213	}	2213	}
2214		2214
2215	if (early_ioapic_map_size == EARLY_MAP_SIZE) {	2215	if (early_ioapic_map_size == EARLY_MAP_SIZE) {
2216	pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",	2216	pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
2217	str);	2217	str);
2218	return 1;	2218	return 1;
2219	}	2219	}
2220		2220
2221	devid = ((bus & 0xff) << 8) \| ((dev & 0x1f) << 3) \| (fn & 0x7);	2221	devid = ((bus & 0xff) << 8) \| ((dev & 0x1f) << 3) \| (fn & 0x7);
2222		2222
2223	cmdline_maps = true;	2223	cmdline_maps = true;
2224	i = early_ioapic_map_size++;	2224	i = early_ioapic_map_size++;
2225	early_ioapic_map[i].id = id;	2225	early_ioapic_map[i].id = id;
2226	early_ioapic_map[i].devid = devid;	2226	early_ioapic_map[i].devid = devid;
2227	early_ioapic_map[i].cmd_line = true;	2227	early_ioapic_map[i].cmd_line = true;
2228		2228
2229	return 1;	2229	return 1;
2230	}	2230	}
2231		2231
2232	static int __init parse_ivrs_hpet(char *str)	2232	static int __init parse_ivrs_hpet(char *str)
2233	{	2233	{
2234	unsigned int bus, dev, fn;	2234	unsigned int bus, dev, fn;
2235	int ret, id, i;	2235	int ret, id, i;
2236	u16 devid;	2236	u16 devid;
2237		2237
2238	ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);	2238	ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn);
2239		2239
2240	if (ret != 4) {	2240	if (ret != 4) {
2241	pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str);	2241	pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str);
2242	return 1;	2242	return 1;
2243	}	2243	}
2244		2244
2245	if (early_hpet_map_size == EARLY_MAP_SIZE) {	2245	if (early_hpet_map_size == EARLY_MAP_SIZE) {
2246	pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n",	2246	pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n",
2247	str);	2247	str);
2248	return 1;	2248	return 1;
2249	}	2249	}
2250		2250
2251	devid = ((bus & 0xff) << 8) \| ((dev & 0x1f) << 3) \| (fn & 0x7);	2251	devid = ((bus & 0xff) << 8) \| ((dev & 0x1f) << 3) \| (fn & 0x7);
2252		2252
2253	cmdline_maps = true;	2253	cmdline_maps = true;
2254	i = early_hpet_map_size++;	2254	i = early_hpet_map_size++;
2255	early_hpet_map[i].id = id;	2255	early_hpet_map[i].id = id;
2256	early_hpet_map[i].devid = devid;	2256	early_hpet_map[i].devid = devid;
2257	early_hpet_map[i].cmd_line = true;	2257	early_hpet_map[i].cmd_line = true;
2258		2258
2259	return 1;	2259	return 1;
2260	}	2260	}
2261		2261
2262	__setup("amd_iommu_dump", parse_amd_iommu_dump);	2262	__setup("amd_iommu_dump", parse_amd_iommu_dump);
2263	__setup("amd_iommu=", parse_amd_iommu_options);	2263	__setup("amd_iommu=", parse_amd_iommu_options);
2264	__setup("ivrs_ioapic", parse_ivrs_ioapic);	2264	__setup("ivrs_ioapic", parse_ivrs_ioapic);
2265	__setup("ivrs_hpet", parse_ivrs_hpet);	2265	__setup("ivrs_hpet", parse_ivrs_hpet);
2266		2266
2267	IOMMU_INIT_FINISH(amd_iommu_detect,	2267	IOMMU_INIT_FINISH(amd_iommu_detect,
2268	gart_iommu_hole_init,	2268	gart_iommu_hole_init,
2269	NULL,	2269	NULL,
2270	NULL);	2270	NULL);
2271		2271
2272	bool amd_iommu_v2_supported(void)	2272	bool amd_iommu_v2_supported(void)
2273	{	2273	{
2274	return amd_iommu_v2_present;	2274	return amd_iommu_v2_present;
2275	}	2275	}
2276	EXPORT_SYMBOL(amd_iommu_v2_supported);	2276	EXPORT_SYMBOL(amd_iommu_v2_supported);
2277		2277
2278	/****************************************************************************	2278	/****************************************************************************
2279	*	2279	*
2280	* IOMMU EFR Performance Counter support functionality. This code allows	2280	* IOMMU EFR Performance Counter support functionality. This code allows
2281	* access to the IOMMU PC functionality.	2281	* access to the IOMMU PC functionality.
2282	*	2282	*
2283	****************************************************************************/	2283	****************************************************************************/
2284		2284
2285	u8 amd_iommu_pc_get_max_banks(u16 devid)	2285	u8 amd_iommu_pc_get_max_banks(u16 devid)
2286	{	2286	{
2287	struct amd_iommu *iommu;	2287	struct amd_iommu *iommu;
2288	u8 ret = 0;	2288	u8 ret = 0;
2289		2289
2290	/* locate the iommu governing the devid */	2290	/* locate the iommu governing the devid */
2291	iommu = amd_iommu_rlookup_table[devid];	2291	iommu = amd_iommu_rlookup_table[devid];
2292	if (iommu)	2292	if (iommu)
2293	ret = iommu->max_banks;	2293	ret = iommu->max_banks;
2294		2294
2295	return ret;	2295	return ret;
2296	}	2296	}
2297	EXPORT_SYMBOL(amd_iommu_pc_get_max_banks);	2297	EXPORT_SYMBOL(amd_iommu_pc_get_max_banks);
2298		2298
2299	bool amd_iommu_pc_supported(void)	2299	bool amd_iommu_pc_supported(void)
2300	{	2300	{
2301	return amd_iommu_pc_present;	2301	return amd_iommu_pc_present;
2302	}	2302	}
2303	EXPORT_SYMBOL(amd_iommu_pc_supported);	2303	EXPORT_SYMBOL(amd_iommu_pc_supported);
2304		2304
2305	u8 amd_iommu_pc_get_max_counters(u16 devid)	2305	u8 amd_iommu_pc_get_max_counters(u16 devid)
2306	{	2306	{
2307	struct amd_iommu *iommu;	2307	struct amd_iommu *iommu;
2308	u8 ret = 0;	2308	u8 ret = 0;
2309		2309
2310	/* locate the iommu governing the devid */	2310	/* locate the iommu governing the devid */
2311	iommu = amd_iommu_rlookup_table[devid];	2311	iommu = amd_iommu_rlookup_table[devid];
2312	if (iommu)	2312	if (iommu)
2313	ret = iommu->max_counters;	2313	ret = iommu->max_counters;
2314		2314
2315	return ret;	2315	return ret;
2316	}	2316	}
2317	EXPORT_SYMBOL(amd_iommu_pc_get_max_counters);	2317	EXPORT_SYMBOL(amd_iommu_pc_get_max_counters);
2318		2318
2319	int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn,	2319	int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn,
2320	u64 *value, bool is_write)	2320	u64 *value, bool is_write)
2321	{	2321	{
2322	struct amd_iommu *iommu;	2322	struct amd_iommu *iommu;
2323	u32 offset;	2323	u32 offset;
2324	u32 max_offset_lim;	2324	u32 max_offset_lim;
2325		2325
2326	/* Make sure the IOMMU PC resource is available */	2326	/* Make sure the IOMMU PC resource is available */
2327	if (!amd_iommu_pc_present)	2327	if (!amd_iommu_pc_present)
2328	return -ENODEV;	2328	return -ENODEV;
2329		2329
2330	/* Locate the iommu associated with the device ID */	2330	/* Locate the iommu associated with the device ID */
2331	iommu = amd_iommu_rlookup_table[devid];	2331	iommu = amd_iommu_rlookup_table[devid];
2332		2332
2333	/* Check for valid iommu and pc register indexing */	2333	/* Check for valid iommu and pc register indexing */
2334	if (WARN_ON((iommu == NULL) \|\| (fxn > 0x28) \|\| (fxn & 7)))	2334	if (WARN_ON((iommu == NULL) \|\| (fxn > 0x28) \|\| (fxn & 7)))
2335	return -ENODEV;	2335	return -ENODEV;
2336		2336
2337	offset = (u32)(((0x40\|bank) << 12) \| (cntr << 8) \| fxn);	2337	offset = (u32)(((0x40\|bank) << 12) \| (cntr << 8) \| fxn);
2338		2338
2339	/* Limit the offset to the hw defined mmio region aperture */	2339	/* Limit the offset to the hw defined mmio region aperture */
2340	max_offset_lim = (u32)(((0x40\|iommu->max_banks) << 12) \|	2340	max_offset_lim = (u32)(((0x40\|iommu->max_banks) << 12) \|
2341	(iommu->max_counters << 8) \| 0x28);	2341	(iommu->max_counters << 8) \| 0x28);
2342	if ((offset < MMIO_CNTR_REG_OFFSET) \|\|	2342	if ((offset < MMIO_CNTR_REG_OFFSET) \|\|
2343	(offset > max_offset_lim))	2343	(offset > max_offset_lim))
2344	return -EINVAL;	2344	return -EINVAL;
2345		2345
2346	if (is_write) {	2346	if (is_write) {
2347	writel((u32)*value, iommu->mmio_base + offset);	2347	writel((u32)*value, iommu->mmio_base + offset);
2348	writel((*value >> 32), iommu->mmio_base + offset + 4);	2348	writel((*value >> 32), iommu->mmio_base + offset + 4);
2349	} else {	2349	} else {
2350	*value = readl(iommu->mmio_base + offset + 4);	2350	*value = readl(iommu->mmio_base + offset + 4);
2351	*value <<= 32;	2351	*value <<= 32;
2352	*value = readl(iommu->mmio_base + offset);	2352	*value = readl(iommu->mmio_base + offset);
2353	}	2353	}
2354		2354
2355	return 0;	2355	return 0;
2356	}	2356	}
2357	EXPORT_SYMBOL(amd_iommu_pc_get_set_reg_val);	2357	EXPORT_SYMBOL(amd_iommu_pc_get_set_reg_val);
2358		2358

drivers/iommu/amd_iommu_v2.c

Diff comments View file @ 989d216

 /*
  * Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 #include <linux/mmu_notifier.h>
 #include <linux/amd-iommu.h>
 #include <linux/mm_types.h>
 #include <linux/profile.h>
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/iommu.h>
 #include <linux/wait.h>
 #include <linux/pci.h>
 #include <linux/gfp.h>
 #include "amd_iommu_types.h"
 #include "amd_iommu_proto.h"
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Joerg Roedel <joerg.roedel@amd.com>");
 #define MAX_DEVICES		0x10000
 #define PRI_QUEUE_SIZE		512
 struct pri_queue {
 	atomic_t inflight;
 	bool finish;
 	int status;
 };
 struct pasid_state {
 	struct list_head list;			/* For global state-list */
 	atomic_t count;				/* Reference count */
 	struct task_struct *task;		/* Task bound to this PASID */
 	struct mm_struct *mm;			/* mm_struct for the faults */
 	struct mmu_notifier mn;                 /* mmu_otifier handle */
 	struct pri_queue pri[PRI_QUEUE_SIZE];	/* PRI tag states */
 	struct device_state *device_state;	/* Link to our device_state */
 	int pasid;				/* PASID index */
 	spinlock_t lock;			/* Protect pri_queues */
 	wait_queue_head_t wq;			/* To wait for count == 0 */
 };
 struct device_state {
 	atomic_t count;
 	struct pci_dev *pdev;
 	struct pasid_state **states;
 	struct iommu_domain *domain;
 	int pasid_levels;
 	int max_pasids;
 	amd_iommu_invalid_ppr_cb inv_ppr_cb;
 	amd_iommu_invalidate_ctx inv_ctx_cb;
 	spinlock_t lock;
 	wait_queue_head_t wq;
 };
 struct fault {
 	struct work_struct work;
 	struct device_state *dev_state;
 	struct pasid_state *state;
 	struct mm_struct *mm;
 	u64 address;
 	u16 devid;
 	u16 pasid;
 	u16 tag;
 	u16 finish;
 	u16 flags;
 };
 static struct device_state **state_table;
 static spinlock_t state_lock;
 /* List and lock for all pasid_states */
 static LIST_HEAD(pasid_state_list);
 static DEFINE_SPINLOCK(ps_lock);
 static struct workqueue_struct *iommu_wq;
 /*
  * Empty page table - Used between
  * mmu_notifier_invalidate_range_start and
  * mmu_notifier_invalidate_range_end
  */
 static u64 *empty_page_table;
 static void free_pasid_states(struct device_state *dev_state);
 static void unbind_pasid(struct device_state *dev_state, int pasid);
 static int task_exit(struct notifier_block *nb, unsigned long e, void *data);
 static u16 device_id(struct pci_dev *pdev)
 {
 	u16 devid;
 	devid = pdev->bus->number;
 	devid = (devid << 8) | pdev->devfn;
 	return devid;
 }
 static struct device_state *get_device_state(u16 devid)
 {
 	struct device_state *dev_state;
 	unsigned long flags;
 	spin_lock_irqsave(&state_lock, flags);
 	dev_state = state_table[devid];
 	if (dev_state != NULL)
 		atomic_inc(&dev_state->count);
 	spin_unlock_irqrestore(&state_lock, flags);
 	return dev_state;
 }
 static void free_device_state(struct device_state *dev_state)
 {
 	/*
 	 * First detach device from domain - No more PRI requests will arrive
 	 * from that device after it is unbound from the IOMMUv2 domain.
 	 */
 	iommu_detach_device(dev_state->domain, &dev_state->pdev->dev);
 	/* Everything is down now, free the IOMMUv2 domain */
 	iommu_domain_free(dev_state->domain);
 	/* Finally get rid of the device-state */
 	kfree(dev_state);
 }
 static void put_device_state(struct device_state *dev_state)
 {
 	if (atomic_dec_and_test(&dev_state->count))
 		wake_up(&dev_state->wq);
 }
 static void put_device_state_wait(struct device_state *dev_state)
 {
 	DEFINE_WAIT(wait);
 	prepare_to_wait(&dev_state->wq, &wait, TASK_UNINTERRUPTIBLE);
 	if (!atomic_dec_and_test(&dev_state->count))
 		schedule();
 	finish_wait(&dev_state->wq, &wait);
 	free_device_state(dev_state);
 }
 static struct notifier_block profile_nb = {
 	.notifier_call = task_exit,
 };
 static void link_pasid_state(struct pasid_state *pasid_state)
 {
 	spin_lock(&ps_lock);
 	list_add_tail(&pasid_state->list, &pasid_state_list);
 	spin_unlock(&ps_lock);
 }
 static void __unlink_pasid_state(struct pasid_state *pasid_state)
 {
 	list_del(&pasid_state->list);
 }
 static void unlink_pasid_state(struct pasid_state *pasid_state)
 {
 	spin_lock(&ps_lock);
 	__unlink_pasid_state(pasid_state);
 	spin_unlock(&ps_lock);
 }
 /* Must be called under dev_state->lock */
 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
 						  int pasid, bool alloc)
 {
 	struct pasid_state **root, **ptr;
 	int level, index;
 	level = dev_state->pasid_levels;
 	root  = dev_state->states;
 	while (true) {
 		index = (pasid >> (9 * level)) & 0x1ff;
 		ptr   = &root[index];
 		if (level == 0)
 			break;
 		if (*ptr == NULL) {
 			if (!alloc)
 				return NULL;
 			*ptr = (void *)get_zeroed_page(GFP_ATOMIC);
 			if (*ptr == NULL)
 				return NULL;
 		}
 		root   = (struct pasid_state **)*ptr;
 		level -= 1;
 	}
 	return ptr;
 }
 static int set_pasid_state(struct device_state *dev_state,
 			   struct pasid_state *pasid_state,
 			   int pasid)
 {
 	struct pasid_state **ptr;
 	unsigned long flags;
 	int ret;
 	spin_lock_irqsave(&dev_state->lock, flags);
 	ptr = __get_pasid_state_ptr(dev_state, pasid, true);
 	ret = -ENOMEM;
 	if (ptr == NULL)
 		goto out_unlock;
 	ret = -ENOMEM;
 	if (*ptr != NULL)
 		goto out_unlock;
 	*ptr = pasid_state;
 	ret = 0;
 out_unlock:
 	spin_unlock_irqrestore(&dev_state->lock, flags);
 	return ret;
 }
 static void clear_pasid_state(struct device_state *dev_state, int pasid)
 {
 	struct pasid_state **ptr;
 	unsigned long flags;
 	spin_lock_irqsave(&dev_state->lock, flags);
 	ptr = __get_pasid_state_ptr(dev_state, pasid, true);
 	if (ptr == NULL)
 		goto out_unlock;
 	*ptr = NULL;
 out_unlock:
 	spin_unlock_irqrestore(&dev_state->lock, flags);
 }
 static struct pasid_state *get_pasid_state(struct device_state *dev_state,
 					   int pasid)
 {
 	struct pasid_state **ptr, *ret = NULL;
 	unsigned long flags;
 	spin_lock_irqsave(&dev_state->lock, flags);
 	ptr = __get_pasid_state_ptr(dev_state, pasid, false);
 	if (ptr == NULL)
 		goto out_unlock;
 	ret = *ptr;
 	if (ret)
 		atomic_inc(&ret->count);
 out_unlock:
 	spin_unlock_irqrestore(&dev_state->lock, flags);
 	return ret;
 }
 static void free_pasid_state(struct pasid_state *pasid_state)
 {
 	kfree(pasid_state);
 }
 static void put_pasid_state(struct pasid_state *pasid_state)
 {
 	if (atomic_dec_and_test(&pasid_state->count)) {
 		put_device_state(pasid_state->device_state);
 		wake_up(&pasid_state->wq);
 	}
 }
 static void put_pasid_state_wait(struct pasid_state *pasid_state)
 {
 	DEFINE_WAIT(wait);
 	prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);
 	if (atomic_dec_and_test(&pasid_state->count))
 		put_device_state(pasid_state->device_state);
 	else
 		schedule();
 	finish_wait(&pasid_state->wq, &wait);
 	mmput(pasid_state->mm);
 	free_pasid_state(pasid_state);
 }
 static void __unbind_pasid(struct pasid_state *pasid_state)
 {
 	struct iommu_domain *domain;
 	domain = pasid_state->device_state->domain;
 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
 	clear_pasid_state(pasid_state->device_state, pasid_state->pasid);
 	/* Make sure no more pending faults are in the queue */
 	flush_workqueue(iommu_wq);
 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
 	put_pasid_state(pasid_state); /* Reference taken in bind() function */
 }
 static void unbind_pasid(struct device_state *dev_state, int pasid)
 {
 	struct pasid_state *pasid_state;
 	pasid_state = get_pasid_state(dev_state, pasid);
 	if (pasid_state == NULL)
 		return;
 	unlink_pasid_state(pasid_state);
 	__unbind_pasid(pasid_state);
 	put_pasid_state_wait(pasid_state); /* Reference taken in this function */
 }
 static void free_pasid_states_level1(struct pasid_state **tbl)
 {
 	int i;
 	for (i = 0; i < 512; ++i) {
 		if (tbl[i] == NULL)
 			continue;
 		free_page((unsigned long)tbl[i]);
 	}
 }
 static void free_pasid_states_level2(struct pasid_state **tbl)
 {
 	struct pasid_state **ptr;
 	int i;
 	for (i = 0; i < 512; ++i) {
 		if (tbl[i] == NULL)
 			continue;
 		ptr = (struct pasid_state **)tbl[i];
 		free_pasid_states_level1(ptr);
 	}
 }
 static void free_pasid_states(struct device_state *dev_state)
 {
 	struct pasid_state *pasid_state;
 	int i;
 	for (i = 0; i < dev_state->max_pasids; ++i) {
 		pasid_state = get_pasid_state(dev_state, i);
 		if (pasid_state == NULL)
 			continue;
 		put_pasid_state(pasid_state);
 		unbind_pasid(dev_state, i);
 	}
 	if (dev_state->pasid_levels == 2)
 		free_pasid_states_level2(dev_state->states);
 	else if (dev_state->pasid_levels == 1)
 		free_pasid_states_level1(dev_state->states);
 	else if (dev_state->pasid_levels != 0)
 		BUG();
 	free_page((unsigned long)dev_state->states);
 }
 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
 {
 	return container_of(mn, struct pasid_state, mn);
 }
 static void __mn_flush_page(struct mmu_notifier *mn,
 			    unsigned long address)
 {
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
 	pasid_state = mn_to_state(mn);
 	dev_state   = pasid_state->device_state;
 	amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);
 }
 static int mn_clear_flush_young(struct mmu_notifier *mn,
 				struct mm_struct *mm,
 				unsigned long address)
 {
 	__mn_flush_page(mn, address);
 	return 0;
 }
 static void mn_change_pte(struct mmu_notifier *mn,
 			  struct mm_struct *mm,
 			  unsigned long address,
 			  pte_t pte)
 {
 	__mn_flush_page(mn, address);
 }
 static void mn_invalidate_page(struct mmu_notifier *mn,
 			       struct mm_struct *mm,
 			       unsigned long address)
 {
 	__mn_flush_page(mn, address);
 }
 static void mn_invalidate_range_start(struct mmu_notifier *mn,
 				      struct mm_struct *mm,
 				      unsigned long start, unsigned long end)
 {
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
 	pasid_state = mn_to_state(mn);
 	dev_state   = pasid_state->device_state;
 	amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
 				  __pa(empty_page_table));
 }
 static void mn_invalidate_range_end(struct mmu_notifier *mn,
 				    struct mm_struct *mm,
 				    unsigned long start, unsigned long end)
 {
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
 	pasid_state = mn_to_state(mn);
 	dev_state   = pasid_state->device_state;
 	amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
 				  __pa(pasid_state->mm->pgd));
 }
 static struct mmu_notifier_ops iommu_mn = {
 	.clear_flush_young      = mn_clear_flush_young,
 	.change_pte             = mn_change_pte,
 	.invalidate_page        = mn_invalidate_page,
 	.invalidate_range_start = mn_invalidate_range_start,
 	.invalidate_range_end   = mn_invalidate_range_end,
 };
 static void set_pri_tag_status(struct pasid_state *pasid_state,
 			       u16 tag, int status)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&pasid_state->lock, flags);
 	pasid_state->pri[tag].status = status;
 	spin_unlock_irqrestore(&pasid_state->lock, flags);
 }
 static void finish_pri_tag(struct device_state *dev_state,
 			   struct pasid_state *pasid_state,
 			   u16 tag)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&pasid_state->lock, flags);
 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
 	    pasid_state->pri[tag].finish) {
 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
 				       pasid_state->pri[tag].status, tag);
 		pasid_state->pri[tag].finish = false;
 		pasid_state->pri[tag].status = PPR_SUCCESS;
 	}
 	spin_unlock_irqrestore(&pasid_state->lock, flags);
 }
 static void do_fault(struct work_struct *work)
 {
 	struct fault *fault = container_of(work, struct fault, work);
 	int npages, write;
 	struct page *page;
 	write = !!(fault->flags & PPR_FAULT_WRITE);
+	down_read(&fault->state->mm->mmap_sem);
 	npages = get_user_pages(fault->state->task, fault->state->mm,
 				fault->address, 1, write, 0, &page, NULL);
+	up_read(&fault->state->mm->mmap_sem);
 	if (npages == 1) {
 		put_page(page);
 	} else if (fault->dev_state->inv_ppr_cb) {
 		int status;
 		status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
 						      fault->pasid,
 						      fault->address,
 						      fault->flags);
 		switch (status) {
 		case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
 			set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
 			break;
 		case AMD_IOMMU_INV_PRI_RSP_INVALID:
 			set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
 			break;
 		case AMD_IOMMU_INV_PRI_RSP_FAIL:
 			set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
 			break;
 		default:
 			BUG();
 		}
 	} else {
 		set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
 	}
 	finish_pri_tag(fault->dev_state, fault->state, fault->tag);
 	put_pasid_state(fault->state);
 	kfree(fault);
 }
 static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 {
 	struct amd_iommu_fault *iommu_fault;
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
 	unsigned long flags;
 	struct fault *fault;
 	bool finish;
 	u16 tag;
 	int ret;
 	iommu_fault = data;
 	tag         = iommu_fault->tag & 0x1ff;
 	finish      = (iommu_fault->tag >> 9) & 1;
 	ret = NOTIFY_DONE;
 	dev_state = get_device_state(iommu_fault->device_id);
 	if (dev_state == NULL)
 		goto out;
 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
 	if (pasid_state == NULL) {
 		/* We know the device but not the PASID -> send INVALID */
 		amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
 				       PPR_INVALID, tag);
 		goto out_drop_state;
 	}
 	spin_lock_irqsave(&pasid_state->lock, flags);
 	atomic_inc(&pasid_state->pri[tag].inflight);
 	if (finish)
 		pasid_state->pri[tag].finish = true;
 	spin_unlock_irqrestore(&pasid_state->lock, flags);
 	fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
 	if (fault == NULL) {
 		/* We are OOM - send success and let the device re-fault */
 		finish_pri_tag(dev_state, pasid_state, tag);
 		goto out_drop_state;
 	}
 	fault->dev_state = dev_state;
 	fault->address   = iommu_fault->address;
 	fault->state     = pasid_state;
 	fault->tag       = tag;
 	fault->finish    = finish;
 	fault->flags     = iommu_fault->flags;
 	INIT_WORK(&fault->work, do_fault);
 	queue_work(iommu_wq, &fault->work);
 	ret = NOTIFY_OK;
 out_drop_state:
 	put_device_state(dev_state);
 out:
 	return ret;
 }
 static struct notifier_block ppr_nb = {
 	.notifier_call = ppr_notifier,
 };
 static int task_exit(struct notifier_block *nb, unsigned long e, void *data)
 {
 	struct pasid_state *pasid_state;
 	struct task_struct *task;
 	task = data;
 	/*
 	 * Using this notifier is a hack - but there is no other choice
 	 * at the moment. What I really want is a sleeping notifier that
 	 * is called when an MM goes down. But such a notifier doesn't
 	 * exist yet. The notifier needs to sleep because it has to make
 	 * sure that the device does not use the PASID and the address
 	 * space anymore before it is destroyed. This includes waiting
 	 * for pending PRI requests to pass the workqueue. The
 	 * MMU-Notifiers would be a good fit, but they use RCU and so
 	 * they are not allowed to sleep. Lets see how we can solve this
 	 * in a more intelligent way in the future.
 	 */
 again:
 	spin_lock(&ps_lock);
 	list_for_each_entry(pasid_state, &pasid_state_list, list) {
 		struct device_state *dev_state;
 		int pasid;
 		if (pasid_state->task != task)
 			continue;
 		/* Drop Lock and unbind */
 		spin_unlock(&ps_lock);
 		dev_state = pasid_state->device_state;
 		pasid     = pasid_state->pasid;
 		if (pasid_state->device_state->inv_ctx_cb)
 			dev_state->inv_ctx_cb(dev_state->pdev, pasid);
 		unbind_pasid(dev_state, pasid);
 		/* Task may be in the list multiple times */
 		goto again;
 	}
 	spin_unlock(&ps_lock);
 	return NOTIFY_OK;
 }
 int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
 			 struct task_struct *task)
 {
 	struct pasid_state *pasid_state;
 	struct device_state *dev_state;
 	u16 devid;
 	int ret;
 	might_sleep();
 	if (!amd_iommu_v2_supported())
 		return -ENODEV;
 	devid     = device_id(pdev);
 	dev_state = get_device_state(devid);
 	if (dev_state == NULL)
 		return -EINVAL;
 	ret = -EINVAL;
 	if (pasid < 0 || pasid >= dev_state->max_pasids)
 		goto out;
 	ret = -ENOMEM;
 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
 	if (pasid_state == NULL)
 		goto out;
 	atomic_set(&pasid_state->count, 1);
 	init_waitqueue_head(&pasid_state->wq);
 	spin_lock_init(&pasid_state->lock);
 	pasid_state->task         = task;
 	pasid_state->mm           = get_task_mm(task);
 	pasid_state->device_state = dev_state;
 	pasid_state->pasid        = pasid;
 	pasid_state->mn.ops       = &iommu_mn;
 	if (pasid_state->mm == NULL)
 		goto out_free;
 	mmu_notifier_register(&pasid_state->mn, pasid_state->mm);
 	ret = set_pasid_state(dev_state, pasid_state, pasid);
 	if (ret)
 		goto out_unregister;
 	ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
 					__pa(pasid_state->mm->pgd));
 	if (ret)
 		goto out_clear_state;
 	link_pasid_state(pasid_state);
 	return 0;
 out_clear_state:
 	clear_pasid_state(dev_state, pasid);
 out_unregister:
 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
 out_free:
 	free_pasid_state(pasid_state);
 out:
 	put_device_state(dev_state);
 	return ret;
 }
 EXPORT_SYMBOL(amd_iommu_bind_pasid);
 void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
 {
 	struct device_state *dev_state;
 	u16 devid;
 	might_sleep();
 	if (!amd_iommu_v2_supported())
 		return;
 	devid = device_id(pdev);
 	dev_state = get_device_state(devid);
 	if (dev_state == NULL)
 		return;
 	if (pasid < 0 || pasid >= dev_state->max_pasids)
 		goto out;
 	unbind_pasid(dev_state, pasid);
 out:
 	put_device_state(dev_state);
 }
 EXPORT_SYMBOL(amd_iommu_unbind_pasid);
 int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
 {
 	struct device_state *dev_state;
 	unsigned long flags;
 	int ret, tmp;
 	u16 devid;
 	might_sleep();
 	if (!amd_iommu_v2_supported())
 		return -ENODEV;
 	if (pasids <= 0 || pasids > (PASID_MASK + 1))
 		return -EINVAL;
 	devid = device_id(pdev);
 	dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
 	if (dev_state == NULL)
 		return -ENOMEM;
 	spin_lock_init(&dev_state->lock);
 	init_waitqueue_head(&dev_state->wq);
 	dev_state->pdev = pdev;
 	tmp = pasids;
 	for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
 		dev_state->pasid_levels += 1;
 	atomic_set(&dev_state->count, 1);
 	dev_state->max_pasids = pasids;
 	ret = -ENOMEM;
 	dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
 	if (dev_state->states == NULL)
 		goto out_free_dev_state;
 	dev_state->domain = iommu_domain_alloc(&pci_bus_type);
 	if (dev_state->domain == NULL)
 		goto out_free_states;
 	amd_iommu_domain_direct_map(dev_state->domain);
 	ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
 	if (ret)
 		goto out_free_domain;
 	ret = iommu_attach_device(dev_state->domain, &pdev->dev);
 	if (ret != 0)
 		goto out_free_domain;
 	spin_lock_irqsave(&state_lock, flags);
 	if (state_table[devid] != NULL) {
 		spin_unlock_irqrestore(&state_lock, flags);
 		ret = -EBUSY;
 		goto out_free_domain;
 	}
 	state_table[devid] = dev_state;
 	spin_unlock_irqrestore(&state_lock, flags);
 	return 0;
 out_free_domain:
 	iommu_domain_free(dev_state->domain);
 out_free_states:
 	free_page((unsigned long)dev_state->states);
 out_free_dev_state:
 	kfree(dev_state);
 	return ret;
 }
 EXPORT_SYMBOL(amd_iommu_init_device);
 void amd_iommu_free_device(struct pci_dev *pdev)
 {
 	struct device_state *dev_state;
 	unsigned long flags;
 	u16 devid;
 	if (!amd_iommu_v2_supported())
 		return;
 	devid = device_id(pdev);
 	spin_lock_irqsave(&state_lock, flags);
 	dev_state = state_table[devid];
 	if (dev_state == NULL) {
 		spin_unlock_irqrestore(&state_lock, flags);
 		return;
 	}
 	state_table[devid] = NULL;
 	spin_unlock_irqrestore(&state_lock, flags);
 	/* Get rid of any remaining pasid states */
 	free_pasid_states(dev_state);
 	put_device_state_wait(dev_state);
 }
 EXPORT_SYMBOL(amd_iommu_free_device);
 int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
 				 amd_iommu_invalid_ppr_cb cb)
 {
 	struct device_state *dev_state;
 	unsigned long flags;
 	u16 devid;
 	int ret;
 	if (!amd_iommu_v2_supported())
 		return -ENODEV;
 	devid = device_id(pdev);
 	spin_lock_irqsave(&state_lock, flags);
 	ret = -EINVAL;
 	dev_state = state_table[devid];
 	if (dev_state == NULL)
 		goto out_unlock;
 	dev_state->inv_ppr_cb = cb;
 	ret = 0;
 out_unlock:
 	spin_unlock_irqrestore(&state_lock, flags);
 	return ret;
 }
 EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
 int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
 				    amd_iommu_invalidate_ctx cb)
 {
 	struct device_state *dev_state;
 	unsigned long flags;
 	u16 devid;
 	int ret;
 	if (!amd_iommu_v2_supported())
 		return -ENODEV;
 	devid = device_id(pdev);
 	spin_lock_irqsave(&state_lock, flags);
 	ret = -EINVAL;
 	dev_state = state_table[devid];
 	if (dev_state == NULL)
 		goto out_unlock;
 	dev_state->inv_ctx_cb = cb;
 	ret = 0;
 out_unlock:
 	spin_unlock_irqrestore(&state_lock, flags);
 	return ret;
 }
 EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
 static int __init amd_iommu_v2_init(void)
 {
 	size_t state_table_size;
 	int ret;
 	pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n");
 	if (!amd_iommu_v2_supported()) {
 		pr_info("AMD IOMMUv2 functionality not available on this system\n");
 		/*
 		 * Load anyway to provide the symbols to other modules
 		 * which may use AMD IOMMUv2 optionally.
 		 */
 		return 0;
 	}
 	spin_lock_init(&state_lock);
 	state_table_size = MAX_DEVICES * sizeof(struct device_state *);
 	state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 					       get_order(state_table_size));
 	if (state_table == NULL)
 		return -ENOMEM;
 	ret = -ENOMEM;
 	iommu_wq = create_workqueue("amd_iommu_v2");
 	if (iommu_wq == NULL)
 		goto out_free;
 	ret = -ENOMEM;
 	empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
 	if (empty_page_table == NULL)
 		goto out_destroy_wq;
 	amd_iommu_register_ppr_notifier(&ppr_nb);
 	profile_event_register(PROFILE_TASK_EXIT, &profile_nb);
 	return 0;
 out_destroy_wq:
 	destroy_workqueue(iommu_wq);
 out_free:
 	free_pages((unsigned long)state_table, get_order(state_table_size));
 	return ret;
 }
 static void __exit amd_iommu_v2_exit(void)
 {
 	struct device_state *dev_state;
 	size_t state_table_size;
 	int i;
 	if (!amd_iommu_v2_supported())
 		return;
 	profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb);
 	amd_iommu_unregister_ppr_notifier(&ppr_nb);
 	flush_workqueue(iommu_wq);
 	/*
 	 * The loop below might call flush_workqueue(), so call
 	 * destroy_workqueue() after it
 	 */
 	for (i = 0; i < MAX_DEVICES; ++i) {
 		dev_state = get_device_state(i);
 		if (dev_state == NULL)
 			continue;
 		WARN_ON_ONCE(1);
 		put_device_state(dev_state);
 		amd_iommu_free_device(dev_state->pdev);
 	}
 	destroy_workqueue(iommu_wq);
 	state_table_size = MAX_DEVICES * sizeof(struct device_state *);
 	free_pages((unsigned long)state_table, get_order(state_table_size));
 	free_page((unsigned long)empty_page_table);
 }
 module_init(amd_iommu_v2_init);
 module_exit(amd_iommu_v2_exit);