Doug / smarc-fsl-linux-kernel

1

/*

1

/*

2

3

* Author: Joerg Roedel <joerg.roedel@amd.com>

3

* Author: Joerg Roedel <joerg.roedel@amd.com>

4

* Leo Duran <leo.duran@amd.com>

4

* Leo Duran <leo.duran@amd.com>

5

*

5

*

6

* This program is free software; you can redistribute it and/or modify it

6

* This program is free software; you can redistribute it and/or modify it

7

* under the terms of the GNU General Public License version 2 as published

7

* under the terms of the GNU General Public License version 2 as published

8

* by the Free Software Foundation.

8

* by the Free Software Foundation.

9

*

9

*

10

* This program is distributed in the hope that it will be useful,

10

* This program is distributed in the hope that it will be useful,

11

* but WITHOUT ANY WARRANTY; without even the implied warranty of

11

* but WITHOUT ANY WARRANTY; without even the implied warranty of

12

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

13

* GNU General Public License for more details.

13

* GNU General Public License for more details.

14

*

14

*

15

* You should have received a copy of the GNU General Public License

15

* You should have received a copy of the GNU General Public License

16

* along with this program; if not, write to the Free Software

16

* along with this program; if not, write to the Free Software

17

* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

17

* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

18

*/

18

*/

19

20

#include <linux/pci.h>

20

#include <linux/pci.h>

21

#include <linux/acpi.h>

21

#include <linux/acpi.h>

22

#include <linux/list.h>

22

#include <linux/list.h>

23

#include <linux/slab.h>

23

#include <linux/slab.h>

24

#include <linux/syscore_ops.h>

24

#include <linux/syscore_ops.h>

25

#include <linux/interrupt.h>

25

#include <linux/interrupt.h>

26

#include <linux/msi.h>

26

#include <linux/msi.h>

27

#include <linux/amd-iommu.h>

27

#include <linux/amd-iommu.h>

28

#include <linux/export.h>

28

#include <linux/export.h>

29

#include <asm/pci-direct.h>

29

#include <asm/pci-direct.h>

30

#include <asm/iommu.h>

30

#include <asm/iommu.h>

31

#include <asm/gart.h>

31

#include <asm/gart.h>

32

#include <asm/x86_init.h>

32

#include <asm/x86_init.h>

33

#include <asm/iommu_table.h>

33

#include <asm/iommu_table.h>

34

35

#include "amd_iommu_proto.h"

35

#include "amd_iommu_proto.h"

36

#include "amd_iommu_types.h"

36

#include "amd_iommu_types.h"

37

38

/*

38

/*

39

* definitions for the ACPI scanning code

39

* definitions for the ACPI scanning code

40

*/

40

*/

41

#define IVRS_HEADER_LENGTH 48

41

#define IVRS_HEADER_LENGTH 48

42

43

#define ACPI_IVHD_TYPE 0x10

43

#define ACPI_IVHD_TYPE 0x10

44

#define ACPI_IVMD_TYPE_ALL 0x20

44

#define ACPI_IVMD_TYPE_ALL 0x20

45

#define ACPI_IVMD_TYPE 0x21

45

#define ACPI_IVMD_TYPE 0x21

46

#define ACPI_IVMD_TYPE_RANGE 0x22

46

#define ACPI_IVMD_TYPE_RANGE 0x22

47

48

#define IVHD_DEV_ALL 0x01

48

#define IVHD_DEV_ALL 0x01

49

#define IVHD_DEV_SELECT 0x02

49

#define IVHD_DEV_SELECT 0x02

50

#define IVHD_DEV_SELECT_RANGE_START 0x03

50

#define IVHD_DEV_SELECT_RANGE_START 0x03

51

#define IVHD_DEV_RANGE_END 0x04

51

#define IVHD_DEV_RANGE_END 0x04

52

#define IVHD_DEV_ALIAS 0x42

52

#define IVHD_DEV_ALIAS 0x42

53

#define IVHD_DEV_ALIAS_RANGE 0x43

53

#define IVHD_DEV_ALIAS_RANGE 0x43

54

#define IVHD_DEV_EXT_SELECT 0x46

54

#define IVHD_DEV_EXT_SELECT 0x46

55

#define IVHD_DEV_EXT_SELECT_RANGE 0x47

55

#define IVHD_DEV_EXT_SELECT_RANGE 0x47

56

57

#define IVHD_FLAG_HT_TUN_EN_MASK 0x01

57

#define IVHD_FLAG_HT_TUN_EN_MASK 0x01

58

#define IVHD_FLAG_PASSPW_EN_MASK 0x02

58

#define IVHD_FLAG_PASSPW_EN_MASK 0x02

59

#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04

59

#define IVHD_FLAG_RESPASSPW_EN_MASK 0x04

60

#define IVHD_FLAG_ISOC_EN_MASK 0x08

60

#define IVHD_FLAG_ISOC_EN_MASK 0x08

61

62

#define IVMD_FLAG_EXCL_RANGE 0x08

62

#define IVMD_FLAG_EXCL_RANGE 0x08

63

#define IVMD_FLAG_UNITY_MAP 0x01

63

#define IVMD_FLAG_UNITY_MAP 0x01

64

65

#define ACPI_DEVFLAG_INITPASS 0x01

65

#define ACPI_DEVFLAG_INITPASS 0x01

66

#define ACPI_DEVFLAG_EXTINT 0x02

66

#define ACPI_DEVFLAG_EXTINT 0x02

67

#define ACPI_DEVFLAG_NMI 0x04

67

#define ACPI_DEVFLAG_NMI 0x04

68

#define ACPI_DEVFLAG_SYSMGT1 0x10

68

#define ACPI_DEVFLAG_SYSMGT1 0x10

69

#define ACPI_DEVFLAG_SYSMGT2 0x20

69

#define ACPI_DEVFLAG_SYSMGT2 0x20

70

#define ACPI_DEVFLAG_LINT0 0x40

70

#define ACPI_DEVFLAG_LINT0 0x40

71

#define ACPI_DEVFLAG_LINT1 0x80

71

#define ACPI_DEVFLAG_LINT1 0x80

72

#define ACPI_DEVFLAG_ATSDIS 0x10000000

72

#define ACPI_DEVFLAG_ATSDIS 0x10000000

73

74

/*

74

/*

75

* ACPI table definitions

75

* ACPI table definitions

76

*

76

*

77

* These data structures are laid over the table to parse the important values

77

* These data structures are laid over the table to parse the important values

78

* out of it.

78

* out of it.

79

*/

79

*/

80

81

/*

81

/*

82

* structure describing one IOMMU in the ACPI table. Typically followed by one

82

* structure describing one IOMMU in the ACPI table. Typically followed by one

83

* or more ivhd_entrys.

83

* or more ivhd_entrys.

84

*/

84

*/

85

struct ivhd_header {

85

struct ivhd_header {

86

u8 type;

86

u8 type;

87

u8 flags;

87

u8 flags;

88

u16 length;

88

u16 length;

89

u16 devid;

89

u16 devid;

90

u16 cap_ptr;

90

u16 cap_ptr;

91

u64 mmio_phys;

91

u64 mmio_phys;

92

u16 pci_seg;

92

u16 pci_seg;

93

u16 info;

93

u16 info;

94

u32 reserved;

94

u32 reserved;

95

} __attribute__((packed));

95

} __attribute__((packed));

96

97

/*

97

/*

98

* A device entry describing which devices a specific IOMMU translates and

98

* A device entry describing which devices a specific IOMMU translates and

99

* which requestor ids they use.

99

* which requestor ids they use.

100

*/

100

*/

101

struct ivhd_entry {

101

struct ivhd_entry {

102

u8 type;

102

u8 type;

103

u16 devid;

103

u16 devid;

104

u8 flags;

104

u8 flags;

105

u32 ext;

105

u32 ext;

106

} __attribute__((packed));

106

} __attribute__((packed));

107

108

/*

108

/*

109

* An AMD IOMMU memory definition structure. It defines things like exclusion

109

* An AMD IOMMU memory definition structure. It defines things like exclusion

110

* ranges for devices and regions that should be unity mapped.

110

* ranges for devices and regions that should be unity mapped.

111

*/

111

*/

112

struct ivmd_header {

112

struct ivmd_header {

113

u8 type;

113

u8 type;

114

u8 flags;

114

u8 flags;

115

u16 length;

115

u16 length;

116

u16 devid;

116

u16 devid;

117

u16 aux;

117

u16 aux;

118

u64 resv;

118

u64 resv;

119

u64 range_start;

119

u64 range_start;

120

u64 range_length;

120

u64 range_length;

121

} __attribute__((packed));

121

} __attribute__((packed));

122

123

bool amd_iommu_dump;

123

bool amd_iommu_dump;

124

125

static int __initdata amd_iommu_detected;

125

static int __initdata amd_iommu_detected;

126

static bool __initdata amd_iommu_disabled;

126

static bool __initdata amd_iommu_disabled;

127

128

u16 amd_iommu_last_bdf; /* largest PCI device id we have

128

u16 amd_iommu_last_bdf; /* largest PCI device id we have

129

to handle */

129

to handle */

130

LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings

130

LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings

131

we find in ACPI */

131

we find in ACPI */

132

bool amd_iommu_unmap_flush; /* if true, flush on every unmap */

132

bool amd_iommu_unmap_flush; /* if true, flush on every unmap */

133

134

LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the

134

LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the

135

system */

135

system */

136

137

/* Array to assign indices to IOMMUs*/

137

/* Array to assign indices to IOMMUs*/

138

struct amd_iommu *amd_iommus[MAX_IOMMUS];

138

struct amd_iommu *amd_iommus[MAX_IOMMUS];

139

int amd_iommus_present;

139

int amd_iommus_present;

140

141

/* IOMMUs have a non-present cache? */

141

/* IOMMUs have a non-present cache? */

142

bool amd_iommu_np_cache __read_mostly;

142

bool amd_iommu_np_cache __read_mostly;

143

bool amd_iommu_iotlb_sup __read_mostly = true;

143

bool amd_iommu_iotlb_sup __read_mostly = true;

144

145

u32 amd_iommu_max_pasids __read_mostly = ~0;

145

u32 amd_iommu_max_pasids __read_mostly = ~0;

146

147

bool amd_iommu_v2_present __read_mostly;

147

bool amd_iommu_v2_present __read_mostly;

148

149

bool amd_iommu_force_isolation __read_mostly;

149

bool amd_iommu_force_isolation __read_mostly;

150

151

/*

151

/*

152

* The ACPI table parsing functions set this variable on an error

152

* The ACPI table parsing functions set this variable on an error

153

*/

153

*/

154

static int __initdata amd_iommu_init_err;

154

static int __initdata amd_iommu_init_err;

155

156

/*

156

/*

157

* List of protection domains - used during resume

157

* List of protection domains - used during resume

158

*/

158

*/

159

LIST_HEAD(amd_iommu_pd_list);

159

LIST_HEAD(amd_iommu_pd_list);

160

spinlock_t amd_iommu_pd_lock;

160

spinlock_t amd_iommu_pd_lock;

161

162

/*

162

/*

163

* Pointer to the device table which is shared by all AMD IOMMUs

163

* Pointer to the device table which is shared by all AMD IOMMUs

164

* it is indexed by the PCI device id or the HT unit id and contains

164

* it is indexed by the PCI device id or the HT unit id and contains

165

* information about the domain the device belongs to as well as the

165

* information about the domain the device belongs to as well as the

166

* page table root pointer.

166

* page table root pointer.

167

*/

167

*/

168

struct dev_table_entry *amd_iommu_dev_table;

168

struct dev_table_entry *amd_iommu_dev_table;

169

170

/*

170

/*

171

* The alias table is a driver specific data structure which contains the

171

* The alias table is a driver specific data structure which contains the

172

* mappings of the PCI device ids to the actual requestor ids on the IOMMU.

172

* mappings of the PCI device ids to the actual requestor ids on the IOMMU.

173

* More than one device can share the same requestor id.

173

* More than one device can share the same requestor id.

174

*/

174

*/

175

u16 *amd_iommu_alias_table;

175

u16 *amd_iommu_alias_table;

176

177

/*

177

/*

178

* The rlookup table is used to find the IOMMU which is responsible

178

* The rlookup table is used to find the IOMMU which is responsible

179

* for a specific device. It is also indexed by the PCI device id.

179

* for a specific device. It is also indexed by the PCI device id.

180

*/

180

*/

181

struct amd_iommu **amd_iommu_rlookup_table;

181

struct amd_iommu **amd_iommu_rlookup_table;

182

183

/*

183

/*

184

* AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap

184

* AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap

185

* to know which ones are already in use.

185

* to know which ones are already in use.

186

*/

186

*/

187

unsigned long *amd_iommu_pd_alloc_bitmap;

187

unsigned long *amd_iommu_pd_alloc_bitmap;

188

189

static u32 dev_table_size; /* size of the device table */

189

static u32 dev_table_size; /* size of the device table */

190

static u32 alias_table_size; /* size of the alias table */

190

static u32 alias_table_size; /* size of the alias table */

191

static u32 rlookup_table_size; /* size if the rlookup table */

191

static u32 rlookup_table_size; /* size if the rlookup table */

192

193

/*

193

/*

194

* This function flushes all internal caches of

194

* This function flushes all internal caches of

195

* the IOMMU used by this driver.

195

* the IOMMU used by this driver.

196

*/

196

*/

197

extern void iommu_flush_all_caches(struct amd_iommu *iommu);

197

extern void iommu_flush_all_caches(struct amd_iommu *iommu);

198

199

static inline void update_last_devid(u16 devid)

199

static inline void update_last_devid(u16 devid)

200

{

200

{

201

if (devid > amd_iommu_last_bdf)

201

if (devid > amd_iommu_last_bdf)

202

amd_iommu_last_bdf = devid;

202

amd_iommu_last_bdf = devid;

203

}

203

}

204

205

static inline unsigned long tbl_size(int entry_size)

205

static inline unsigned long tbl_size(int entry_size)

206

{

206

{

207

unsigned shift = PAGE_SHIFT +

207

unsigned shift = PAGE_SHIFT +

208

get_order(((int)amd_iommu_last_bdf + 1) * entry_size);

208

get_order(((int)amd_iommu_last_bdf + 1) * entry_size);

209

210

return 1UL << shift;

210

return 1UL << shift;

211

}

211

}

212

213

/* Access to l1 and l2 indexed register spaces */

213

/* Access to l1 and l2 indexed register spaces */

214

215

static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)

215

static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)

216

{

216

{

217

u32 val;

217

u32 val;

218

219

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

219

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

220

pci_read_config_dword(iommu->dev, 0xfc, &val);

220

pci_read_config_dword(iommu->dev, 0xfc, &val);

221

return val;

221

return val;

222

}

222

}

223

224

static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)

224

static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)

225

{

225

{

226

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));

226

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));

227

pci_write_config_dword(iommu->dev, 0xfc, val);

227

pci_write_config_dword(iommu->dev, 0xfc, val);

228

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

228

pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));

229

}

229

}

230

231

static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)

231

static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)

232

{

232

{

233

u32 val;

233

u32 val;

234

235

pci_write_config_dword(iommu->dev, 0xf0, address);

235

pci_write_config_dword(iommu->dev, 0xf0, address);

236

pci_read_config_dword(iommu->dev, 0xf4, &val);

236

pci_read_config_dword(iommu->dev, 0xf4, &val);

237

return val;

237

return val;

238

}

238

}

239

240

static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)

240

static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)

241

{

241

{

242

pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));

242

pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));

243

pci_write_config_dword(iommu->dev, 0xf4, val);

243

pci_write_config_dword(iommu->dev, 0xf4, val);

244

}

244

}

245

246

/****************************************************************************

246

/****************************************************************************

247

*

247

*

248

* AMD IOMMU MMIO register space handling functions

248

* AMD IOMMU MMIO register space handling functions

249

*

249

*

250

* These functions are used to program the IOMMU device registers in

250

* These functions are used to program the IOMMU device registers in

251

* MMIO space required for that driver.

251

* MMIO space required for that driver.

252

*

252

*

253

****************************************************************************/

253

****************************************************************************/

254

255

/*

255

/*

256

* This function set the exclusion range in the IOMMU. DMA accesses to the

256

* This function set the exclusion range in the IOMMU. DMA accesses to the

257

* exclusion range are passed through untranslated

257

* exclusion range are passed through untranslated

258

*/

258

*/

259

static void iommu_set_exclusion_range(struct amd_iommu *iommu)

259

static void iommu_set_exclusion_range(struct amd_iommu *iommu)

260

{

260

{

261

u64 start = iommu->exclusion_start & PAGE_MASK;

261

u64 start = iommu->exclusion_start & PAGE_MASK;

262

u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;

262

u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;

263

u64 entry;

263

u64 entry;

264

265

if (!iommu->exclusion_start)

265

if (!iommu->exclusion_start)

266

return;

266

return;

267

268

entry = start | MMIO_EXCL_ENABLE_MASK;

268

entry = start | MMIO_EXCL_ENABLE_MASK;

269

memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,

269

memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,

270

&entry, sizeof(entry));

270

&entry, sizeof(entry));

271

272

entry = limit;

272

entry = limit;

273

memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,

273

memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,

274

&entry, sizeof(entry));

274

&entry, sizeof(entry));

275

}

275

}

276

277

/* Programs the physical address of the device table into the IOMMU hardware */

277

/* Programs the physical address of the device table into the IOMMU hardware */

278

static void __init iommu_set_device_table(struct amd_iommu *iommu)

278

static void iommu_set_device_table(struct amd_iommu *iommu)

279

{

279

{

280

u64 entry;

280

u64 entry;

281

282

BUG_ON(iommu->mmio_base == NULL);

282

BUG_ON(iommu->mmio_base == NULL);

283

284

entry = virt_to_phys(amd_iommu_dev_table);

284

entry = virt_to_phys(amd_iommu_dev_table);

285

entry |= (dev_table_size >> 12) - 1;

285

entry |= (dev_table_size >> 12) - 1;

286

memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,

286

memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,

287

&entry, sizeof(entry));

287

&entry, sizeof(entry));

288

}

288

}

289

290

/* Generic functions to enable/disable certain features of the IOMMU. */

290

/* Generic functions to enable/disable certain features of the IOMMU. */

291

static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)

291

static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)

292

{

292

{

293

u32 ctrl;

293

u32 ctrl;

294

295

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

295

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

296

ctrl |= (1 << bit);

296

ctrl |= (1 << bit);

297

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

297

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

298

}

298

}

299

300

static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)

300

static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)

301

{

301

{

302

u32 ctrl;

302

u32 ctrl;

303

304

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

304

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

305

ctrl &= ~(1 << bit);

305

ctrl &= ~(1 << bit);

306

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

306

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

307

}

307

}

308

309

static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)

309

static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)

310

{

310

{

311

u32 ctrl;

311

u32 ctrl;

312

313

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

313

ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);

314

ctrl &= ~CTRL_INV_TO_MASK;

314

ctrl &= ~CTRL_INV_TO_MASK;

315

ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;

315

ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;

316

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

316

writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);

317

}

317

}

318

319

/* Function to enable the hardware */

319

/* Function to enable the hardware */

320

static void iommu_enable(struct amd_iommu *iommu)

320

static void iommu_enable(struct amd_iommu *iommu)

321

{

321

{

322

static const char * const feat_str[] = {

322

static const char * const feat_str[] = {

323

"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",

323

"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",

324

"IA", "GA", "HE", "PC", NULL

324

"IA", "GA", "HE", "PC", NULL

325

};

325

};

326

int i;

326

int i;

327

328

printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx",

328

printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx",

329

dev_name(&iommu->dev->dev), iommu->cap_ptr);

329

dev_name(&iommu->dev->dev), iommu->cap_ptr);

330

331

if (iommu->cap & (1 << IOMMU_CAP_EFR)) {

331

if (iommu->cap & (1 << IOMMU_CAP_EFR)) {

332

printk(KERN_CONT " extended features: ");

332

printk(KERN_CONT " extended features: ");

333

for (i = 0; feat_str[i]; ++i)

333

for (i = 0; feat_str[i]; ++i)

334

if (iommu_feature(iommu, (1ULL << i)))

334

if (iommu_feature(iommu, (1ULL << i)))

335

printk(KERN_CONT " %s", feat_str[i]);

335

printk(KERN_CONT " %s", feat_str[i]);

336

}

336

}

337

printk(KERN_CONT "\n");

337

printk(KERN_CONT "\n");

338

339

iommu_feature_enable(iommu, CONTROL_IOMMU_EN);

339

iommu_feature_enable(iommu, CONTROL_IOMMU_EN);

340

}

340

}

341

342

static void iommu_disable(struct amd_iommu *iommu)

342

static void iommu_disable(struct amd_iommu *iommu)

343

{

343

{

344

/* Disable command buffer */

344

/* Disable command buffer */

345

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

345

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

346

347

/* Disable event logging and event interrupts */

347

/* Disable event logging and event interrupts */

348

iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);

348

iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);

349

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

349

iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);

350

351

/* Disable IOMMU hardware itself */

351

/* Disable IOMMU hardware itself */

352

iommu_feature_disable(iommu, CONTROL_IOMMU_EN);

352

iommu_feature_disable(iommu, CONTROL_IOMMU_EN);

353

}

353

}

354

355

/*

355

/*

356

* mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in

356

* mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in

357

* the system has one.

357

* the system has one.

358

*/

358

*/

359

static u8 * __init iommu_map_mmio_space(u64 address)

359

static u8 * __init iommu_map_mmio_space(u64 address)

360

{

360

{

361

u8 *ret;

361

u8 *ret;

362

363

if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {

363

if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {

364

pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",

364

pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",

365

address);

365

address);

366

pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");

366

pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");

367

return NULL;

367

return NULL;

368

}

368

}

369

370

ret = ioremap_nocache(address, MMIO_REGION_LENGTH);

370

ret = ioremap_nocache(address, MMIO_REGION_LENGTH);

371

if (ret != NULL)

371

if (ret != NULL)

372

return ret;

372

return ret;

373

374

release_mem_region(address, MMIO_REGION_LENGTH);

374

release_mem_region(address, MMIO_REGION_LENGTH);

375

376

return NULL;

376

return NULL;

377

}

377

}

378

379

static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)

379

static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)

380

{

380

{

381

if (iommu->mmio_base)

381

if (iommu->mmio_base)

382

iounmap(iommu->mmio_base);

382

iounmap(iommu->mmio_base);

383

release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);

383

release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);

384

}

384

}

385

386

/****************************************************************************

386

/****************************************************************************

387

*

387

*

388

* The functions below belong to the first pass of AMD IOMMU ACPI table

388

* The functions below belong to the first pass of AMD IOMMU ACPI table

389

* parsing. In this pass we try to find out the highest device id this

389

* parsing. In this pass we try to find out the highest device id this

390

* code has to handle. Upon this information the size of the shared data

390

* code has to handle. Upon this information the size of the shared data

391

* structures is determined later.

391

* structures is determined later.

392

*

392

*

393

****************************************************************************/

393

****************************************************************************/

394

395

/*

395

/*

396

* This function calculates the length of a given IVHD entry

396

* This function calculates the length of a given IVHD entry

397

*/

397

*/

398

static inline int ivhd_entry_length(u8 *ivhd)

398

static inline int ivhd_entry_length(u8 *ivhd)

399

{

399

{

400

return 0x04 << (*ivhd >> 6);

400

return 0x04 << (*ivhd >> 6);

401

}

401

}

402

403

/*

403

/*

404

* This function reads the last device id the IOMMU has to handle from the PCI

404

* This function reads the last device id the IOMMU has to handle from the PCI

405

* capability header for this IOMMU

405

* capability header for this IOMMU

406

*/

406

*/

407

static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)

407

static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)

408

{

408

{

409

u32 cap;

409

u32 cap;

410

411

cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);

411

cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);

412

update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));

412

update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));

413

414

return 0;

414

return 0;

415

}

415

}

416

417

/*

417

/*

418

* After reading the highest device id from the IOMMU PCI capability header

418

* After reading the highest device id from the IOMMU PCI capability header

419

* this function looks if there is a higher device id defined in the ACPI table

419

* this function looks if there is a higher device id defined in the ACPI table

420

*/

420

*/

421

static int __init find_last_devid_from_ivhd(struct ivhd_header *h)

421

static int __init find_last_devid_from_ivhd(struct ivhd_header *h)

422

{

422

{

423

u8 *p = (void *)h, *end = (void *)h;

423

u8 *p = (void *)h, *end = (void *)h;

424

struct ivhd_entry *dev;

424

struct ivhd_entry *dev;

425

426

p += sizeof(*h);

426

p += sizeof(*h);

427

end += h->length;

427

end += h->length;

428

429

find_last_devid_on_pci(PCI_BUS(h->devid),

429

find_last_devid_on_pci(PCI_BUS(h->devid),

430

PCI_SLOT(h->devid),

430

PCI_SLOT(h->devid),

431

PCI_FUNC(h->devid),

431

PCI_FUNC(h->devid),

432

h->cap_ptr);

432

h->cap_ptr);

433

434

while (p < end) {

434

while (p < end) {

435

dev = (struct ivhd_entry *)p;

435

dev = (struct ivhd_entry *)p;

436

switch (dev->type) {

436

switch (dev->type) {

437

case IVHD_DEV_SELECT:

437

case IVHD_DEV_SELECT:

438

case IVHD_DEV_RANGE_END:

438

case IVHD_DEV_RANGE_END:

439

case IVHD_DEV_ALIAS:

439

case IVHD_DEV_ALIAS:

440

case IVHD_DEV_EXT_SELECT:

440

case IVHD_DEV_EXT_SELECT:

441

/* all the above subfield types refer to device ids */

441

/* all the above subfield types refer to device ids */

442

update_last_devid(dev->devid);

442

update_last_devid(dev->devid);

443

break;

443

break;

444

default:

444

default:

445

break;

445

break;

446

}

446

}

447

p += ivhd_entry_length(p);

447

p += ivhd_entry_length(p);

448

}

448

}

449

450

WARN_ON(p != end);

450

WARN_ON(p != end);

451

452

return 0;

452

return 0;

453

}

453

}

454

455

/*

455

/*

456

* Iterate over all IVHD entries in the ACPI table and find the highest device

456

* Iterate over all IVHD entries in the ACPI table and find the highest device

457

* id which we need to handle. This is the first of three functions which parse

457

* id which we need to handle. This is the first of three functions which parse

458

* the ACPI table. So we check the checksum here.

458

* the ACPI table. So we check the checksum here.

459

*/

459

*/

460

static int __init find_last_devid_acpi(struct acpi_table_header *table)

460

static int __init find_last_devid_acpi(struct acpi_table_header *table)

461

{

461

{

462

int i;

462

int i;

463

u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;

463

u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;

464

struct ivhd_header *h;

464

struct ivhd_header *h;

465

466

/*

466

/*

467

* Validate checksum here so we don't need to do it when

467

* Validate checksum here so we don't need to do it when

468

* we actually parse the table

468

* we actually parse the table

469

*/

469

*/

470

for (i = 0; i < table->length; ++i)

470

for (i = 0; i < table->length; ++i)

471

checksum += p[i];

471

checksum += p[i];

472

if (checksum != 0) {

472

if (checksum != 0) {

473

/* ACPI table corrupt */

473

/* ACPI table corrupt */

474

amd_iommu_init_err = -ENODEV;

474

amd_iommu_init_err = -ENODEV;

475

return 0;

475

return 0;

476

}

476

}

477

478

p += IVRS_HEADER_LENGTH;

478

p += IVRS_HEADER_LENGTH;

479

480

end += table->length;

480

end += table->length;

481

while (p < end) {

481

while (p < end) {

482

h = (struct ivhd_header *)p;

482

h = (struct ivhd_header *)p;

483

switch (h->type) {

483

switch (h->type) {

484

case ACPI_IVHD_TYPE:

484

case ACPI_IVHD_TYPE:

485

find_last_devid_from_ivhd(h);

485

find_last_devid_from_ivhd(h);

486

break;

486

break;

487

default:

487

default:

488

break;

488

break;

489

}

489

}

490

p += h->length;

490

p += h->length;

491

}

491

}

492

WARN_ON(p != end);

492

WARN_ON(p != end);

493

494

return 0;

494

return 0;

495

}

495

}

496

497

/****************************************************************************

497

/****************************************************************************

498

*

498

*

499

* The following functions belong the the code path which parses the ACPI table

499

* The following functions belong the the code path which parses the ACPI table

500

* the second time. In this ACPI parsing iteration we allocate IOMMU specific

500

* the second time. In this ACPI parsing iteration we allocate IOMMU specific

501

* data structures, initialize the device/alias/rlookup table and also

501

* data structures, initialize the device/alias/rlookup table and also

502

* basically initialize the hardware.

502

* basically initialize the hardware.

503

*

503

*

504

****************************************************************************/

504

****************************************************************************/

505

506

/*

506

/*

507

* Allocates the command buffer. This buffer is per AMD IOMMU. We can

507

* Allocates the command buffer. This buffer is per AMD IOMMU. We can

508

* write commands to that buffer later and the IOMMU will execute them

508

* write commands to that buffer later and the IOMMU will execute them

509

* asynchronously

509

* asynchronously

510

*/

510

*/

511

static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)

511

static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)

512

{

512

{

513

u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

513

u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

514

get_order(CMD_BUFFER_SIZE));

514

get_order(CMD_BUFFER_SIZE));

515

516

if (cmd_buf == NULL)

516

if (cmd_buf == NULL)

517

return NULL;

517

return NULL;

518

519

iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;

519

iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;

520

521

return cmd_buf;

521

return cmd_buf;

522

}

522

}

523

524

/*

524

/*

525

* This function resets the command buffer if the IOMMU stopped fetching

525

* This function resets the command buffer if the IOMMU stopped fetching

526

* commands from it.

526

* commands from it.

527

*/

527

*/

528

void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)

528

void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)

529

{

529

{

530

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

530

iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);

531

532

writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

532

writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);

533

writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

533

writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);

534

535

iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);

535

iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);

536

}

536

}

537

538

/*

538

/*

539

* This function writes the command buffer address to the hardware and

539

* This function writes the command buffer address to the hardware and

540

* enables it.

540

* enables it.

541

*/

541

*/

542

static void iommu_enable_command_buffer(struct amd_iommu *iommu)

542

static void iommu_enable_command_buffer(struct amd_iommu *iommu)

543

{

543

{

544

u64 entry;

544

u64 entry;

545

546

BUG_ON(iommu->cmd_buf == NULL);

546

BUG_ON(iommu->cmd_buf == NULL);

547

548

entry = (u64)virt_to_phys(iommu->cmd_buf);

548

entry = (u64)virt_to_phys(iommu->cmd_buf);

549

entry |= MMIO_CMD_SIZE_512;

549

entry |= MMIO_CMD_SIZE_512;

550

551

memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,

551

memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,

552

&entry, sizeof(entry));

552

&entry, sizeof(entry));

553

554

amd_iommu_reset_cmd_buffer(iommu);

554

amd_iommu_reset_cmd_buffer(iommu);

555

iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);

555

iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);

556

}

556

}

557

558

static void __init free_command_buffer(struct amd_iommu *iommu)

558

static void __init free_command_buffer(struct amd_iommu *iommu)

559

{

559

{

560

free_pages((unsigned long)iommu->cmd_buf,

560

free_pages((unsigned long)iommu->cmd_buf,

561

get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));

561

get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));

562

}

562

}

563

564

/* allocates the memory where the IOMMU will log its events to */

564

/* allocates the memory where the IOMMU will log its events to */

565

static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)

565

static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)

566

{

566

{

567

iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

567

iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

568

get_order(EVT_BUFFER_SIZE));

568

get_order(EVT_BUFFER_SIZE));

569

570

if (iommu->evt_buf == NULL)

570

if (iommu->evt_buf == NULL)

571

return NULL;

571

return NULL;

572

573

iommu->evt_buf_size = EVT_BUFFER_SIZE;

573

iommu->evt_buf_size = EVT_BUFFER_SIZE;

574

575

return iommu->evt_buf;

575

return iommu->evt_buf;

576

}

576

}

577

578

static void iommu_enable_event_buffer(struct amd_iommu *iommu)

578

static void iommu_enable_event_buffer(struct amd_iommu *iommu)

579

{

579

{

580

u64 entry;

580

u64 entry;

581

582

BUG_ON(iommu->evt_buf == NULL);

582

BUG_ON(iommu->evt_buf == NULL);

583

584

entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;

584

entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;

585

586

memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,

586

memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,

587

&entry, sizeof(entry));

587

&entry, sizeof(entry));

588

589

/* set head and tail to zero manually */

589

/* set head and tail to zero manually */

590

writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

590

writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);

591

writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

591

writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);

592

593

iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);

593

iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);

594

}

594

}

595

596

static void __init free_event_buffer(struct amd_iommu *iommu)

596

static void __init free_event_buffer(struct amd_iommu *iommu)

597

{

597

{

598

free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));

598

free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));

599

}

599

}

600

601

/* allocates the memory where the IOMMU will log its events to */

601

/* allocates the memory where the IOMMU will log its events to */

602

static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)

602

static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)

603

{

603

{

604

iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

604

iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

605

get_order(PPR_LOG_SIZE));

605

get_order(PPR_LOG_SIZE));

606

607

if (iommu->ppr_log == NULL)

607

if (iommu->ppr_log == NULL)

608

return NULL;

608

return NULL;

609

610

return iommu->ppr_log;

610

return iommu->ppr_log;

611

}

611

}

612

613

static void iommu_enable_ppr_log(struct amd_iommu *iommu)

613

static void iommu_enable_ppr_log(struct amd_iommu *iommu)

614

{

614

{

615

u64 entry;

615

u64 entry;

616

617

if (iommu->ppr_log == NULL)

617

if (iommu->ppr_log == NULL)

618

return;

618

return;

619

620

entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;

620

entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;

621

622

memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,

622

memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,

623

&entry, sizeof(entry));

623

&entry, sizeof(entry));

624

625

/* set head and tail to zero manually */

625

/* set head and tail to zero manually */

626

writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

626

writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);

627

writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

627

writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);

628

629

iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);

629

iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);

630

iommu_feature_enable(iommu, CONTROL_PPR_EN);

630

iommu_feature_enable(iommu, CONTROL_PPR_EN);

631

}

631

}

632

633

static void __init free_ppr_log(struct amd_iommu *iommu)

633

static void __init free_ppr_log(struct amd_iommu *iommu)

634

{

634

{

635

if (iommu->ppr_log == NULL)

635

if (iommu->ppr_log == NULL)

636

return;

636

return;

637

638

free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));

638

free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));

639

}

639

}

640

641

static void iommu_enable_gt(struct amd_iommu *iommu)

641

static void iommu_enable_gt(struct amd_iommu *iommu)

642

{

642

{

643

if (!iommu_feature(iommu, FEATURE_GT))

643

if (!iommu_feature(iommu, FEATURE_GT))

644

return;

644

return;

645

646

iommu_feature_enable(iommu, CONTROL_GT_EN);

646

iommu_feature_enable(iommu, CONTROL_GT_EN);

647

}

647

}

648

649

/* sets a specific bit in the device table entry. */

649

/* sets a specific bit in the device table entry. */

650

static void set_dev_entry_bit(u16 devid, u8 bit)

650

static void set_dev_entry_bit(u16 devid, u8 bit)

651

{

651

{

652

int i = (bit >> 6) & 0x03;

652

int i = (bit >> 6) & 0x03;

653

int _bit = bit & 0x3f;

653

int _bit = bit & 0x3f;

654

655

amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);

655

amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);

656

}

656

}

657

658

static int get_dev_entry_bit(u16 devid, u8 bit)

658

static int get_dev_entry_bit(u16 devid, u8 bit)

659

{

659

{

660

int i = (bit >> 6) & 0x03;

660

int i = (bit >> 6) & 0x03;

661

int _bit = bit & 0x3f;

661

int _bit = bit & 0x3f;

662

663

return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;

663

return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;

664

}

664

}

665

666

667

void amd_iommu_apply_erratum_63(u16 devid)

667

void amd_iommu_apply_erratum_63(u16 devid)

668

{

668

{

669

int sysmgt;

669

int sysmgt;

670

671

sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |

671

sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |

672

(get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);

672

(get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);

673

674

if (sysmgt == 0x01)

674

if (sysmgt == 0x01)

675

set_dev_entry_bit(devid, DEV_ENTRY_IW);

675

set_dev_entry_bit(devid, DEV_ENTRY_IW);

676

}

676

}

677

678

/* Writes the specific IOMMU for a device into the rlookup table */

678

/* Writes the specific IOMMU for a device into the rlookup table */

679

static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)

679

static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)

680

{

680

{

681

amd_iommu_rlookup_table[devid] = iommu;

681

amd_iommu_rlookup_table[devid] = iommu;

682

}

682

}

683

684

/*

684

/*

685

* This function takes the device specific flags read from the ACPI

685

* This function takes the device specific flags read from the ACPI

686

* table and sets up the device table entry with that information

686

* table and sets up the device table entry with that information

687

*/

687

*/

688

static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,

688

static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,

689

u16 devid, u32 flags, u32 ext_flags)

689

u16 devid, u32 flags, u32 ext_flags)

690

{

690

{

691

if (flags & ACPI_DEVFLAG_INITPASS)

691

if (flags & ACPI_DEVFLAG_INITPASS)

692

set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);

692

set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);

693

if (flags & ACPI_DEVFLAG_EXTINT)

693

if (flags & ACPI_DEVFLAG_EXTINT)

694

set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);

694

set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);

695

if (flags & ACPI_DEVFLAG_NMI)

695

if (flags & ACPI_DEVFLAG_NMI)

696

set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);

696

set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);

697

if (flags & ACPI_DEVFLAG_SYSMGT1)

697

if (flags & ACPI_DEVFLAG_SYSMGT1)

698

set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);

698

set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);

699

if (flags & ACPI_DEVFLAG_SYSMGT2)

699

if (flags & ACPI_DEVFLAG_SYSMGT2)

700

set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);

700

set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);

701

if (flags & ACPI_DEVFLAG_LINT0)

701

if (flags & ACPI_DEVFLAG_LINT0)

702

set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);

702

set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);

703

if (flags & ACPI_DEVFLAG_LINT1)

703

if (flags & ACPI_DEVFLAG_LINT1)

704

set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);

704

set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);

705

706

amd_iommu_apply_erratum_63(devid);

706

amd_iommu_apply_erratum_63(devid);

707

708

set_iommu_for_device(iommu, devid);

708

set_iommu_for_device(iommu, devid);

709

}

709

}

710

711

/*

711

/*

712

* Reads the device exclusion range from ACPI and initialize IOMMU with

712

* Reads the device exclusion range from ACPI and initialize IOMMU with

713

* it

713

* it

714

*/

714

*/

715

static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)

715

static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)

716

{

716

{

717

struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];

717

struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];

718

719

if (!(m->flags & IVMD_FLAG_EXCL_RANGE))

719

if (!(m->flags & IVMD_FLAG_EXCL_RANGE))

720

return;

720

return;

721

722

if (iommu) {

722

if (iommu) {

723

/*

723

/*

724

* We only can configure exclusion ranges per IOMMU, not

724

* We only can configure exclusion ranges per IOMMU, not

725

* per device. But we can enable the exclusion range per

725

* per device. But we can enable the exclusion range per

726

* device. This is done here

726

* device. This is done here

727

*/

727

*/

728

set_dev_entry_bit(m->devid, DEV_ENTRY_EX);

728

set_dev_entry_bit(m->devid, DEV_ENTRY_EX);

729

iommu->exclusion_start = m->range_start;

729

iommu->exclusion_start = m->range_start;

730

iommu->exclusion_length = m->range_length;

730

iommu->exclusion_length = m->range_length;

731

}

731

}

732

}

732

}

733

734

/*

734

/*

735

* This function reads some important data from the IOMMU PCI space and

735

* This function reads some important data from the IOMMU PCI space and

736

* initializes the driver data structure with it. It reads the hardware

736

* initializes the driver data structure with it. It reads the hardware

737

* capabilities and the first/last device entries

737

* capabilities and the first/last device entries

738

*/

738

*/

739

static void __init init_iommu_from_pci(struct amd_iommu *iommu)

739

static void __init init_iommu_from_pci(struct amd_iommu *iommu)

740

{

740

{

741

int cap_ptr = iommu->cap_ptr;

741

int cap_ptr = iommu->cap_ptr;

742

u32 range, misc, low, high;

742

u32 range, misc, low, high;

743

int i, j;

743

int i, j;

744

745

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,

745

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,

746

&iommu->cap);

746

&iommu->cap);

747

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,

747

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,

748

&range);

748

&range);

749

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,

749

pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,

750

&misc);

750

&misc);

751

752

iommu->first_device = calc_devid(MMIO_GET_BUS(range),

752

iommu->first_device = calc_devid(MMIO_GET_BUS(range),

753

MMIO_GET_FD(range));

753

MMIO_GET_FD(range));

754

iommu->last_device = calc_devid(MMIO_GET_BUS(range),

754

iommu->last_device = calc_devid(MMIO_GET_BUS(range),

755

MMIO_GET_LD(range));

755

MMIO_GET_LD(range));

756

iommu->evt_msi_num = MMIO_MSI_NUM(misc);

756

iommu->evt_msi_num = MMIO_MSI_NUM(misc);

757

758

if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))

758

if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))

759

amd_iommu_iotlb_sup = false;

759

amd_iommu_iotlb_sup = false;

760

761

/* read extended feature bits */

761

/* read extended feature bits */

762

low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);

762

low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);

763

high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);

763

high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);

764

765

iommu->features = ((u64)high << 32) | low;

765

iommu->features = ((u64)high << 32) | low;

766

767

if (iommu_feature(iommu, FEATURE_GT)) {

767

if (iommu_feature(iommu, FEATURE_GT)) {

768

int glxval;

768

int glxval;

769

u32 pasids;

769

u32 pasids;

770

u64 shift;

770

u64 shift;

771

772

shift = iommu->features & FEATURE_PASID_MASK;

772

shift = iommu->features & FEATURE_PASID_MASK;

773

shift >>= FEATURE_PASID_SHIFT;

773

shift >>= FEATURE_PASID_SHIFT;

774

pasids = (1 << shift);

774

pasids = (1 << shift);

775

776

amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);

776

amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);

777

778

glxval = iommu->features & FEATURE_GLXVAL_MASK;

778

glxval = iommu->features & FEATURE_GLXVAL_MASK;

779

glxval >>= FEATURE_GLXVAL_SHIFT;

779

glxval >>= FEATURE_GLXVAL_SHIFT;

780

781

if (amd_iommu_max_glx_val == -1)

781

if (amd_iommu_max_glx_val == -1)

782

amd_iommu_max_glx_val = glxval;

782

amd_iommu_max_glx_val = glxval;

783

else

783

else

784

amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);

784

amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);

785

}

785

}

786

787

if (iommu_feature(iommu, FEATURE_GT) &&

787

if (iommu_feature(iommu, FEATURE_GT) &&

788

iommu_feature(iommu, FEATURE_PPR)) {

788

iommu_feature(iommu, FEATURE_PPR)) {

789

iommu->is_iommu_v2 = true;

789

iommu->is_iommu_v2 = true;

790

amd_iommu_v2_present = true;

790

amd_iommu_v2_present = true;

791

}

791

}

792

793

if (!is_rd890_iommu(iommu->dev))

793

if (!is_rd890_iommu(iommu->dev))

794

return;

794

return;

795

796

/*

796

/*

797

* Some rd890 systems may not be fully reconfigured by the BIOS, so

797

* Some rd890 systems may not be fully reconfigured by the BIOS, so

798

* it's necessary for us to store this information so it can be

798

* it's necessary for us to store this information so it can be

799

* reprogrammed on resume

799

* reprogrammed on resume

800

*/

800

*/

801

802

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,

802

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,

803

&iommu->stored_addr_lo);

803

&iommu->stored_addr_lo);

804

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,

804

pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,

805

&iommu->stored_addr_hi);

805

&iommu->stored_addr_hi);

806

807

/* Low bit locks writes to configuration space */

807

/* Low bit locks writes to configuration space */

808

iommu->stored_addr_lo &= ~1;

808

iommu->stored_addr_lo &= ~1;

809

810

for (i = 0; i < 6; i++)

810

for (i = 0; i < 6; i++)

811

for (j = 0; j < 0x12; j++)

811

for (j = 0; j < 0x12; j++)

812

iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);

812

iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);

813

814

for (i = 0; i < 0x83; i++)

814

for (i = 0; i < 0x83; i++)

815

iommu->stored_l2[i] = iommu_read_l2(iommu, i);

815

iommu->stored_l2[i] = iommu_read_l2(iommu, i);

816

}

816

}

817

818

/*

818

/*

819

* Takes a pointer to an AMD IOMMU entry in the ACPI table and

819

* Takes a pointer to an AMD IOMMU entry in the ACPI table and

820

* initializes the hardware and our data structures with it.

820

* initializes the hardware and our data structures with it.

821

*/

821

*/

822

static void __init init_iommu_from_acpi(struct amd_iommu *iommu,

822

static void __init init_iommu_from_acpi(struct amd_iommu *iommu,

823

struct ivhd_header *h)

823

struct ivhd_header *h)

824

{

824

{

825

u8 *p = (u8 *)h;

825

u8 *p = (u8 *)h;

826

u8 *end = p, flags = 0;

826

u8 *end = p, flags = 0;

827

u16 devid = 0, devid_start = 0, devid_to = 0;

827

u16 devid = 0, devid_start = 0, devid_to = 0;

828

u32 dev_i, ext_flags = 0;

828

u32 dev_i, ext_flags = 0;

829

bool alias = false;

829

bool alias = false;

830

struct ivhd_entry *e;

830

struct ivhd_entry *e;

831

832

/*

832

/*

833

* First save the recommended feature enable bits from ACPI

833

* First save the recommended feature enable bits from ACPI

834

*/

834

*/

835

iommu->acpi_flags = h->flags;

835

iommu->acpi_flags = h->flags;

836

837

/*

837

/*

838

* Done. Now parse the device entries

838

* Done. Now parse the device entries

839

*/

839

*/

840

p += sizeof(struct ivhd_header);

840

p += sizeof(struct ivhd_header);

841

end += h->length;

841

end += h->length;

842

843

844

while (p < end) {

844

while (p < end) {

845

e = (struct ivhd_entry *)p;

845

e = (struct ivhd_entry *)p;

846

switch (e->type) {

846

switch (e->type) {

847

case IVHD_DEV_ALL:

847

case IVHD_DEV_ALL:

848

849

DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"

849

DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"

850

" last device %02x:%02x.%x flags: %02x\n",

850

" last device %02x:%02x.%x flags: %02x\n",

851

PCI_BUS(iommu->first_device),

851

PCI_BUS(iommu->first_device),

852

PCI_SLOT(iommu->first_device),

852

PCI_SLOT(iommu->first_device),

853

PCI_FUNC(iommu->first_device),

853

PCI_FUNC(iommu->first_device),

854

PCI_BUS(iommu->last_device),

854

PCI_BUS(iommu->last_device),

855

PCI_SLOT(iommu->last_device),

855

PCI_SLOT(iommu->last_device),

856

PCI_FUNC(iommu->last_device),

856

PCI_FUNC(iommu->last_device),

857

e->flags);

857

e->flags);

858

859

for (dev_i = iommu->first_device;

859

for (dev_i = iommu->first_device;

860

dev_i <= iommu->last_device; ++dev_i)

860

dev_i <= iommu->last_device; ++dev_i)

861

set_dev_entry_from_acpi(iommu, dev_i,

861

set_dev_entry_from_acpi(iommu, dev_i,

862

e->flags, 0);

862

e->flags, 0);

863

break;

863

break;

864

case IVHD_DEV_SELECT:

864

case IVHD_DEV_SELECT:

865

866

DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "

866

DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "

867

"flags: %02x\n",

867

"flags: %02x\n",

868

PCI_BUS(e->devid),

868

PCI_BUS(e->devid),

869

PCI_SLOT(e->devid),

869

PCI_SLOT(e->devid),

870

PCI_FUNC(e->devid),

870

PCI_FUNC(e->devid),

871

e->flags);

871

e->flags);

872

873

devid = e->devid;

873

devid = e->devid;

874

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

874

set_dev_entry_from_acpi(iommu, devid, e->flags, 0);

875

break;

875

break;

876

case IVHD_DEV_SELECT_RANGE_START:

876

case IVHD_DEV_SELECT_RANGE_START:

877

878

DUMP_printk(" DEV_SELECT_RANGE_START\t "

878

DUMP_printk(" DEV_SELECT_RANGE_START\t "

879

"devid: %02x:%02x.%x flags: %02x\n",

879

"devid: %02x:%02x.%x flags: %02x\n",

880

PCI_BUS(e->devid),

880

PCI_BUS(e->devid),

881

PCI_SLOT(e->devid),

881

PCI_SLOT(e->devid),

882

PCI_FUNC(e->devid),

882

PCI_FUNC(e->devid),

883

e->flags);

883

e->flags);

884

885

devid_start = e->devid;

885

devid_start = e->devid;

886

flags = e->flags;

886

flags = e->flags;

887

ext_flags = 0;

887

ext_flags = 0;

888

alias = false;

888

alias = false;

889

break;

889

break;

890

case IVHD_DEV_ALIAS:

890

case IVHD_DEV_ALIAS:

891

892

DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "

892

DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "

893

"flags: %02x devid_to: %02x:%02x.%x\n",

893

"flags: %02x devid_to: %02x:%02x.%x\n",

894

PCI_BUS(e->devid),

894

PCI_BUS(e->devid),

895

PCI_SLOT(e->devid),

895

PCI_SLOT(e->devid),

896

PCI_FUNC(e->devid),

896

PCI_FUNC(e->devid),

897

e->flags,

897

e->flags,

898

PCI_BUS(e->ext >> 8),

898

PCI_BUS(e->ext >> 8),

899

PCI_SLOT(e->ext >> 8),

899

PCI_SLOT(e->ext >> 8),

900

PCI_FUNC(e->ext >> 8));

900

PCI_FUNC(e->ext >> 8));

901

902

devid = e->devid;

902

devid = e->devid;

903

devid_to = e->ext >> 8;

903

devid_to = e->ext >> 8;

904

set_dev_entry_from_acpi(iommu, devid , e->flags, 0);

904

set_dev_entry_from_acpi(iommu, devid , e->flags, 0);

905

set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);

905

set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);

906

amd_iommu_alias_table[devid] = devid_to;

906

amd_iommu_alias_table[devid] = devid_to;

907

break;

907

break;

908

case IVHD_DEV_ALIAS_RANGE:

908

case IVHD_DEV_ALIAS_RANGE:

909

910

DUMP_printk(" DEV_ALIAS_RANGE\t\t "

910

DUMP_printk(" DEV_ALIAS_RANGE\t\t "

911

"devid: %02x:%02x.%x flags: %02x "

911

"devid: %02x:%02x.%x flags: %02x "

912

"devid_to: %02x:%02x.%x\n",

912

"devid_to: %02x:%02x.%x\n",

913

PCI_BUS(e->devid),

913

PCI_BUS(e->devid),

914

PCI_SLOT(e->devid),

914

PCI_SLOT(e->devid),

915

PCI_FUNC(e->devid),

915

PCI_FUNC(e->devid),

916

e->flags,

916

e->flags,

917

PCI_BUS(e->ext >> 8),

917

PCI_BUS(e->ext >> 8),

918

PCI_SLOT(e->ext >> 8),

918

PCI_SLOT(e->ext >> 8),

919

PCI_FUNC(e->ext >> 8));

919

PCI_FUNC(e->ext >> 8));

920

921

devid_start = e->devid;

921

devid_start = e->devid;

922

flags = e->flags;

922

flags = e->flags;

923

devid_to = e->ext >> 8;

923

devid_to = e->ext >> 8;

924

ext_flags = 0;

924

ext_flags = 0;

925

alias = true;

925

alias = true;

926

break;

926

break;

927

case IVHD_DEV_EXT_SELECT:

927

case IVHD_DEV_EXT_SELECT:

928

929

DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "

929

DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "

930

"flags: %02x ext: %08x\n",

930

"flags: %02x ext: %08x\n",

931

PCI_BUS(e->devid),

931

PCI_BUS(e->devid),

932

PCI_SLOT(e->devid),

932

PCI_SLOT(e->devid),

933

PCI_FUNC(e->devid),

933

PCI_FUNC(e->devid),

934

e->flags, e->ext);

934

e->flags, e->ext);

935

936

devid = e->devid;

936

devid = e->devid;

937

set_dev_entry_from_acpi(iommu, devid, e->flags,

937

set_dev_entry_from_acpi(iommu, devid, e->flags,

938

e->ext);

938

e->ext);

939

break;

939

break;

940

case IVHD_DEV_EXT_SELECT_RANGE:

940

case IVHD_DEV_EXT_SELECT_RANGE:

941

942

DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "

942

DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "

943

"%02x:%02x.%x flags: %02x ext: %08x\n",

943

"%02x:%02x.%x flags: %02x ext: %08x\n",

944

PCI_BUS(e->devid),

944

PCI_BUS(e->devid),

945

PCI_SLOT(e->devid),

945

PCI_SLOT(e->devid),

946

PCI_FUNC(e->devid),

946

PCI_FUNC(e->devid),

947

e->flags, e->ext);

947

e->flags, e->ext);

948

949

devid_start = e->devid;

949

devid_start = e->devid;

950

flags = e->flags;

950

flags = e->flags;

951

ext_flags = e->ext;

951

ext_flags = e->ext;

952

alias = false;

952

alias = false;

953

break;

953

break;

954

case IVHD_DEV_RANGE_END:

954

case IVHD_DEV_RANGE_END:

955

956

DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",

956

DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",

957

PCI_BUS(e->devid),

957

PCI_BUS(e->devid),

958

PCI_SLOT(e->devid),

958

PCI_SLOT(e->devid),

959

PCI_FUNC(e->devid));

959

PCI_FUNC(e->devid));

960

961

devid = e->devid;

961

devid = e->devid;

962

for (dev_i = devid_start; dev_i <= devid; ++dev_i) {

962

for (dev_i = devid_start; dev_i <= devid; ++dev_i) {

963

if (alias) {

963

if (alias) {

964

amd_iommu_alias_table[dev_i] = devid_to;

964

amd_iommu_alias_table[dev_i] = devid_to;

965

set_dev_entry_from_acpi(iommu,

965

set_dev_entry_from_acpi(iommu,

966

devid_to, flags, ext_flags);

966

devid_to, flags, ext_flags);

967

}

967

}

968

set_dev_entry_from_acpi(iommu, dev_i,

968

set_dev_entry_from_acpi(iommu, dev_i,

969

flags, ext_flags);

969

flags, ext_flags);

970

}

970

}

971

break;

971

break;

972

default:

972

default:

973

break;

973

break;

974

}

974

}

975

976

p += ivhd_entry_length(p);

976

p += ivhd_entry_length(p);

977

}

977

}

978

}

978

}

979

980

/* Initializes the device->iommu mapping for the driver */

980

/* Initializes the device->iommu mapping for the driver */

981

static int __init init_iommu_devices(struct amd_iommu *iommu)

981

static int __init init_iommu_devices(struct amd_iommu *iommu)

982

{

982

{

983

u32 i;

983

u32 i;

984

985

for (i = iommu->first_device; i <= iommu->last_device; ++i)

985

for (i = iommu->first_device; i <= iommu->last_device; ++i)

986

set_iommu_for_device(iommu, i);

986

set_iommu_for_device(iommu, i);

987

988

return 0;

988

return 0;

989

}

989

}

990

991

static void __init free_iommu_one(struct amd_iommu *iommu)

991

static void __init free_iommu_one(struct amd_iommu *iommu)

992

{

992

{

993

free_command_buffer(iommu);

993

free_command_buffer(iommu);

994

free_event_buffer(iommu);

994

free_event_buffer(iommu);

995

free_ppr_log(iommu);

995

free_ppr_log(iommu);

996

iommu_unmap_mmio_space(iommu);

996

iommu_unmap_mmio_space(iommu);

997

}

997

}

998

999

static void __init free_iommu_all(void)

999

static void __init free_iommu_all(void)

1000

{

1000

{

1001

struct amd_iommu *iommu, *next;

1001

struct amd_iommu *iommu, *next;

1002

1003

for_each_iommu_safe(iommu, next) {

1003

for_each_iommu_safe(iommu, next) {

1004

list_del(&iommu->list);

1004

list_del(&iommu->list);

1005

free_iommu_one(iommu);

1005

free_iommu_one(iommu);

1006

kfree(iommu);

1006

kfree(iommu);

1007

}

1007

}

1008

}

1008

}

1009

1010

/*

1010

/*

1011

* This function clues the initialization function for one IOMMU

1011

* This function clues the initialization function for one IOMMU

1012

* together and also allocates the command buffer and programs the

1012

* together and also allocates the command buffer and programs the

1013

* hardware. It does NOT enable the IOMMU. This is done afterwards.

1013

* hardware. It does NOT enable the IOMMU. This is done afterwards.

1014

*/

1014

*/

1015

static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)

1015

static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)

1016

{

1016

{

1017

spin_lock_init(&iommu->lock);

1017

spin_lock_init(&iommu->lock);

1018

1019

/* Add IOMMU to internal data structures */

1019

/* Add IOMMU to internal data structures */

1020

list_add_tail(&iommu->list, &amd_iommu_list);

1020

list_add_tail(&iommu->list, &amd_iommu_list);

1021

iommu->index = amd_iommus_present++;

1021

iommu->index = amd_iommus_present++;

1022

1023

if (unlikely(iommu->index >= MAX_IOMMUS)) {

1023

if (unlikely(iommu->index >= MAX_IOMMUS)) {

1024

WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");

1024

WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");

1025

return -ENOSYS;

1025

return -ENOSYS;

1026

}

1026

}

1027

1028

/* Index is fine - add IOMMU to the array */

1028

/* Index is fine - add IOMMU to the array */

1029

amd_iommus[iommu->index] = iommu;

1029

amd_iommus[iommu->index] = iommu;

1030

1031

/*

1031

/*

1032

* Copy data from ACPI table entry to the iommu struct

1032

* Copy data from ACPI table entry to the iommu struct

1033

*/

1033

*/

1034

iommu->dev = pci_get_bus_and_slot(PCI_BUS(h->devid), h->devid & 0xff);

1034

iommu->dev = pci_get_bus_and_slot(PCI_BUS(h->devid), h->devid & 0xff);

1035

if (!iommu->dev)

1035

if (!iommu->dev)

1036

return 1;

1036

return 1;

1037

1038

iommu->cap_ptr = h->cap_ptr;

1038

iommu->cap_ptr = h->cap_ptr;

1039

iommu->pci_seg = h->pci_seg;

1039

iommu->pci_seg = h->pci_seg;

1040

iommu->mmio_phys = h->mmio_phys;

1040

iommu->mmio_phys = h->mmio_phys;

1041

iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);

1041

iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);

1042

if (!iommu->mmio_base)

1042

if (!iommu->mmio_base)

1043

return -ENOMEM;

1043

return -ENOMEM;

1044

1045

iommu->cmd_buf = alloc_command_buffer(iommu);

1045

iommu->cmd_buf = alloc_command_buffer(iommu);

1046

if (!iommu->cmd_buf)

1046

if (!iommu->cmd_buf)

1047

return -ENOMEM;

1047

return -ENOMEM;

1048

1049

iommu->evt_buf = alloc_event_buffer(iommu);

1049

iommu->evt_buf = alloc_event_buffer(iommu);

1050

if (!iommu->evt_buf)

1050

if (!iommu->evt_buf)

1051

return -ENOMEM;

1051

return -ENOMEM;

1052

1053

iommu->int_enabled = false;

1053

iommu->int_enabled = false;

1054

1055

init_iommu_from_pci(iommu);

1055

init_iommu_from_pci(iommu);

1056

init_iommu_from_acpi(iommu, h);

1056

init_iommu_from_acpi(iommu, h);

1057

init_iommu_devices(iommu);

1057

init_iommu_devices(iommu);

1058

1059

if (iommu_feature(iommu, FEATURE_PPR)) {

1059

if (iommu_feature(iommu, FEATURE_PPR)) {

1060

iommu->ppr_log = alloc_ppr_log(iommu);

1060

iommu->ppr_log = alloc_ppr_log(iommu);

1061

if (!iommu->ppr_log)

1061

if (!iommu->ppr_log)

1062

return -ENOMEM;

1062

return -ENOMEM;

1063

}

1063

}

1064

1065

if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))

1065

if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))

1066

amd_iommu_np_cache = true;

1066

amd_iommu_np_cache = true;

1067

1068

return pci_enable_device(iommu->dev);

1068

return pci_enable_device(iommu->dev);

1069

}

1069

}

1070

1071

/*

1071

/*

1072

* Iterates over all IOMMU entries in the ACPI table, allocates the

1072

* Iterates over all IOMMU entries in the ACPI table, allocates the

1073

* IOMMU structure and initializes it with init_iommu_one()

1073

* IOMMU structure and initializes it with init_iommu_one()

1074

*/

1074

*/

1075

static int __init init_iommu_all(struct acpi_table_header *table)

1075

static int __init init_iommu_all(struct acpi_table_header *table)

1076

{

1076

{

1077

u8 *p = (u8 *)table, *end = (u8 *)table;

1077

u8 *p = (u8 *)table, *end = (u8 *)table;

1078

struct ivhd_header *h;

1078

struct ivhd_header *h;

1079

struct amd_iommu *iommu;

1079

struct amd_iommu *iommu;

1080

int ret;

1080

int ret;

1081

1082

end += table->length;

1082

end += table->length;

1083

p += IVRS_HEADER_LENGTH;

1083

p += IVRS_HEADER_LENGTH;

1084

1085

while (p < end) {

1085

while (p < end) {

1086

h = (struct ivhd_header *)p;

1086

h = (struct ivhd_header *)p;

1087

switch (*p) {

1087

switch (*p) {

1088

case ACPI_IVHD_TYPE:

1088

case ACPI_IVHD_TYPE:

1089

1090

DUMP_printk("device: %02x:%02x.%01x cap: %04x "

1090

DUMP_printk("device: %02x:%02x.%01x cap: %04x "

1091

"seg: %d flags: %01x info %04x\n",

1091

"seg: %d flags: %01x info %04x\n",

1092

PCI_BUS(h->devid), PCI_SLOT(h->devid),

1092

PCI_BUS(h->devid), PCI_SLOT(h->devid),

1093

PCI_FUNC(h->devid), h->cap_ptr,

1093

PCI_FUNC(h->devid), h->cap_ptr,

1094

h->pci_seg, h->flags, h->info);

1094

h->pci_seg, h->flags, h->info);

1095

DUMP_printk(" mmio-addr: %016llx\n",

1095

DUMP_printk(" mmio-addr: %016llx\n",

1096

h->mmio_phys);

1096

h->mmio_phys);

1097

1098

iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);

1098

iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);

1099

if (iommu == NULL) {

1099

if (iommu == NULL) {

1100

amd_iommu_init_err = -ENOMEM;

1100

amd_iommu_init_err = -ENOMEM;

1101

return 0;

1101

return 0;

1102

}

1102

}

1103

1104

ret = init_iommu_one(iommu, h);

1104

ret = init_iommu_one(iommu, h);

1105

if (ret) {

1105

if (ret) {

1106

amd_iommu_init_err = ret;

1106

amd_iommu_init_err = ret;

1107

return 0;

1107

return 0;

1108

}

1108

}

1109

break;

1109

break;

1110

default:

1110

default:

1111

break;

1111

break;

1112

}

1112

}

1113

p += h->length;

1113

p += h->length;

1114

1115

}

1115

}

1116

WARN_ON(p != end);

1116

WARN_ON(p != end);

1117

1118

return 0;

1118

return 0;

1119

}

1119

}

1120

1121

/****************************************************************************

1121

/****************************************************************************

1122

*

1122

*

1123

* The following functions initialize the MSI interrupts for all IOMMUs

1123

* The following functions initialize the MSI interrupts for all IOMMUs

1124

* in the system. Its a bit challenging because there could be multiple

1124

* in the system. Its a bit challenging because there could be multiple

1125

* IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per

1125

* IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per

1126

* pci_dev.

1126

* pci_dev.

1127

*

1127

*

1128

****************************************************************************/

1128

****************************************************************************/

1129

1130

static int iommu_setup_msi(struct amd_iommu *iommu)

1130

static int iommu_setup_msi(struct amd_iommu *iommu)

1131

{

1131

{

1132

int r;

1132

int r;

1133

1134

if (pci_enable_msi(iommu->dev))

1134

if (pci_enable_msi(iommu->dev))

1135

return 1;

1135

return 1;

1136

1137

r = request_threaded_irq(iommu->dev->irq,

1137

r = request_threaded_irq(iommu->dev->irq,

1138

amd_iommu_int_handler,

1138

amd_iommu_int_handler,

1139

amd_iommu_int_thread,

1139

amd_iommu_int_thread,

1140

0, "AMD-Vi",

1140

0, "AMD-Vi",

1141

iommu->dev);

1141

iommu->dev);

1142

1143

if (r) {

1143

if (r) {

1144

pci_disable_msi(iommu->dev);

1144

pci_disable_msi(iommu->dev);

1145

return 1;

1145

return 1;

1146

}

1146

}

1147

1148

iommu->int_enabled = true;

1148

iommu->int_enabled = true;

1149

iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);

1149

iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);

1150

1151

if (iommu->ppr_log != NULL)

1151

if (iommu->ppr_log != NULL)

1152

iommu_feature_enable(iommu, CONTROL_PPFINT_EN);

1152

iommu_feature_enable(iommu, CONTROL_PPFINT_EN);

1153

1154

return 0;

1154

return 0;

1155

}

1155

}

1156

1157

static int iommu_init_msi(struct amd_iommu *iommu)

1157

static int iommu_init_msi(struct amd_iommu *iommu)

1158

{

1158

{

1159

if (iommu->int_enabled)

1159

if (iommu->int_enabled)

1160

return 0;

1160

return 0;

1161

1162

if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))

1162

if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))

1163

return iommu_setup_msi(iommu);

1163

return iommu_setup_msi(iommu);

1164

1165

return 1;

1165

return 1;

1166

}

1166

}

1167

1168

/****************************************************************************

1168

/****************************************************************************

1169

*

1169

*

1170

* The next functions belong to the third pass of parsing the ACPI

1170

* The next functions belong to the third pass of parsing the ACPI

1171

* table. In this last pass the memory mapping requirements are

1171

* table. In this last pass the memory mapping requirements are

1172

* gathered (like exclusion and unity mapping reanges).

1172

* gathered (like exclusion and unity mapping reanges).

1173

*

1173

*

1174

****************************************************************************/

1174

****************************************************************************/

1175

1176

static void __init free_unity_maps(void)

1176

static void __init free_unity_maps(void)

1177

{

1177

{

1178

struct unity_map_entry *entry, *next;

1178

struct unity_map_entry *entry, *next;

1179

1180

list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {

1180

list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {

1181

list_del(&entry->list);

1181

list_del(&entry->list);

1182

kfree(entry);

1182

kfree(entry);

1183

}

1183

}

1184

}

1184

}

1185

1186

/* called when we find an exclusion range definition in ACPI */

1186

/* called when we find an exclusion range definition in ACPI */

1187

static int __init init_exclusion_range(struct ivmd_header *m)

1187

static int __init init_exclusion_range(struct ivmd_header *m)

1188

{

1188

{

1189

int i;

1189

int i;

1190

1191

switch (m->type) {

1191

switch (m->type) {

1192

case ACPI_IVMD_TYPE:

1192

case ACPI_IVMD_TYPE:

1193

set_device_exclusion_range(m->devid, m);

1193

set_device_exclusion_range(m->devid, m);

1194

break;

1194

break;

1195

case ACPI_IVMD_TYPE_ALL:

1195

case ACPI_IVMD_TYPE_ALL:

1196

for (i = 0; i <= amd_iommu_last_bdf; ++i)

1196

for (i = 0; i <= amd_iommu_last_bdf; ++i)

1197

set_device_exclusion_range(i, m);

1197

set_device_exclusion_range(i, m);

1198

break;

1198

break;

1199

case ACPI_IVMD_TYPE_RANGE:

1199

case ACPI_IVMD_TYPE_RANGE:

1200

for (i = m->devid; i <= m->aux; ++i)

1200

for (i = m->devid; i <= m->aux; ++i)

1201

set_device_exclusion_range(i, m);

1201

set_device_exclusion_range(i, m);

1202

break;

1202

break;

1203

default:

1203

default:

1204

break;

1204

break;

1205

}

1205

}

1206

1207

return 0;

1207

return 0;

1208

}

1208

}

1209

1210

/* called for unity map ACPI definition */

1210

/* called for unity map ACPI definition */

1211

static int __init init_unity_map_range(struct ivmd_header *m)

1211

static int __init init_unity_map_range(struct ivmd_header *m)

1212

{

1212

{

1213

struct unity_map_entry *e = 0;

1213

struct unity_map_entry *e = 0;

1214

char *s;

1214

char *s;

1215

1216

e = kzalloc(sizeof(*e), GFP_KERNEL);

1216

e = kzalloc(sizeof(*e), GFP_KERNEL);

1217

if (e == NULL)

1217

if (e == NULL)

1218

return -ENOMEM;

1218

return -ENOMEM;

1219

1220

switch (m->type) {

1220

switch (m->type) {

1221

default:

1221

default:

1222

kfree(e);

1222

kfree(e);

1223

return 0;

1223

return 0;

1224

case ACPI_IVMD_TYPE:

1224

case ACPI_IVMD_TYPE:

1225

s = "IVMD_TYPEi\t\t\t";

1225

s = "IVMD_TYPEi\t\t\t";

1226

e->devid_start = e->devid_end = m->devid;

1226

e->devid_start = e->devid_end = m->devid;

1227

break;

1227

break;

1228

case ACPI_IVMD_TYPE_ALL:

1228

case ACPI_IVMD_TYPE_ALL:

1229

s = "IVMD_TYPE_ALL\t\t";

1229

s = "IVMD_TYPE_ALL\t\t";

1230

e->devid_start = 0;

1230

e->devid_start = 0;

1231

e->devid_end = amd_iommu_last_bdf;

1231

e->devid_end = amd_iommu_last_bdf;

1232

break;

1232

break;

1233

case ACPI_IVMD_TYPE_RANGE:

1233

case ACPI_IVMD_TYPE_RANGE:

1234

s = "IVMD_TYPE_RANGE\t\t";

1234

s = "IVMD_TYPE_RANGE\t\t";

1235

e->devid_start = m->devid;

1235

e->devid_start = m->devid;

1236

e->devid_end = m->aux;

1236

e->devid_end = m->aux;

1237

break;

1237

break;

1238

}

1238

}

1239

e->address_start = PAGE_ALIGN(m->range_start);

1239

e->address_start = PAGE_ALIGN(m->range_start);

1240

e->address_end = e->address_start + PAGE_ALIGN(m->range_length);

1240

e->address_end = e->address_start + PAGE_ALIGN(m->range_length);

1241

e->prot = m->flags >> 1;

1241

e->prot = m->flags >> 1;

1242

1243

DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"

1243

DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"

1244

" range_start: %016llx range_end: %016llx flags: %x\n", s,

1244

" range_start: %016llx range_end: %016llx flags: %x\n", s,

1245

PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),

1245

PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),

1246

PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),

1246

PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),

1247

PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),

1247

PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),

1248

e->address_start, e->address_end, m->flags);

1248

e->address_start, e->address_end, m->flags);

1249

1250

list_add_tail(&e->list, &amd_iommu_unity_map);

1250

list_add_tail(&e->list, &amd_iommu_unity_map);

1251

1252

return 0;

1252

return 0;

1253

}

1253

}

1254

1255

/* iterates over all memory definitions we find in the ACPI table */

1255

/* iterates over all memory definitions we find in the ACPI table */

1256

static int __init init_memory_definitions(struct acpi_table_header *table)

1256

static int __init init_memory_definitions(struct acpi_table_header *table)

1257

{

1257

{

1258

u8 *p = (u8 *)table, *end = (u8 *)table;

1258

u8 *p = (u8 *)table, *end = (u8 *)table;

1259

struct ivmd_header *m;

1259

struct ivmd_header *m;

1260

1261

end += table->length;

1261

end += table->length;

1262

p += IVRS_HEADER_LENGTH;

1262

p += IVRS_HEADER_LENGTH;

1263

1264

while (p < end) {

1264

while (p < end) {

1265

m = (struct ivmd_header *)p;

1265

m = (struct ivmd_header *)p;

1266

if (m->flags & IVMD_FLAG_EXCL_RANGE)

1266

if (m->flags & IVMD_FLAG_EXCL_RANGE)

1267

init_exclusion_range(m);

1267

init_exclusion_range(m);

1268

else if (m->flags & IVMD_FLAG_UNITY_MAP)

1268

else if (m->flags & IVMD_FLAG_UNITY_MAP)

1269

init_unity_map_range(m);

1269

init_unity_map_range(m);

1270

1271

p += m->length;

1271

p += m->length;

1272

}

1272

}

1273

1274

return 0;

1274

return 0;

1275

}

1275

}

1276

1277

/*

1277

/*

1278

* Init the device table to not allow DMA access for devices and

1278

* Init the device table to not allow DMA access for devices and

1279

* suppress all page faults

1279

* suppress all page faults

1280

*/

1280

*/

1281

static void init_device_table(void)

1281

static void init_device_table(void)

1282

{

1282

{

1283

u32 devid;

1283

u32 devid;

1284

1285

for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {

1285

for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {

1286

set_dev_entry_bit(devid, DEV_ENTRY_VALID);

1286

set_dev_entry_bit(devid, DEV_ENTRY_VALID);

1287

set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);

1287

set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);

1288

}

1288

}

1289

}

1289

}

1290

1291

static void iommu_init_flags(struct amd_iommu *iommu)

1291

static void iommu_init_flags(struct amd_iommu *iommu)

1292

{

1292

{

1293

iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?

1293

iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?

1294

iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :

1294

iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :

1295

iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);

1295

iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);

1296

1297

iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?

1297

iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?

1298

iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :

1298

iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :

1299

iommu_feature_disable(iommu, CONTROL_PASSPW_EN);

1299

iommu_feature_disable(iommu, CONTROL_PASSPW_EN);

1300

1301

iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?

1301

iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?

1302

iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :

1302

iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :

1303

iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);

1303

iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);

1304

1305

iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?

1305

iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?

1306

iommu_feature_enable(iommu, CONTROL_ISOC_EN) :

1306

iommu_feature_enable(iommu, CONTROL_ISOC_EN) :

1307

iommu_feature_disable(iommu, CONTROL_ISOC_EN);

1307

iommu_feature_disable(iommu, CONTROL_ISOC_EN);

1308

1309

/*

1309

/*

1310

* make IOMMU memory accesses cache coherent

1310

* make IOMMU memory accesses cache coherent

1311

*/

1311

*/

1312

iommu_feature_enable(iommu, CONTROL_COHERENT_EN);

1312

iommu_feature_enable(iommu, CONTROL_COHERENT_EN);

1313

1314

/* Set IOTLB invalidation timeout to 1s */

1314

/* Set IOTLB invalidation timeout to 1s */

1315

iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);

1315

iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);

1316

}

1316

}

1317

1318

static void iommu_apply_resume_quirks(struct amd_iommu *iommu)

1318

static void iommu_apply_resume_quirks(struct amd_iommu *iommu)

1319

{

1319

{

1320

int i, j;

1320

int i, j;

1321

u32 ioc_feature_control;

1321

u32 ioc_feature_control;

1322

struct pci_dev *pdev = NULL;

1322

struct pci_dev *pdev = NULL;

1323

1324

/* RD890 BIOSes may not have completely reconfigured the iommu */

1324

/* RD890 BIOSes may not have completely reconfigured the iommu */

1325

if (!is_rd890_iommu(iommu->dev))

1325

if (!is_rd890_iommu(iommu->dev))

1326

return;

1326

return;

1327

1328

/*

1328

/*

1329

* First, we need to ensure that the iommu is enabled. This is

1329

* First, we need to ensure that the iommu is enabled. This is

1330

* controlled by a register in the northbridge

1330

* controlled by a register in the northbridge

1331

*/

1331

*/

1332

pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0));

1332

pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0));

1333

1334

if (!pdev)

1334

if (!pdev)

1335

return;

1335

return;

1336

1337

/* Select Northbridge indirect register 0x75 and enable writing */

1337

/* Select Northbridge indirect register 0x75 and enable writing */

1338

pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));

1338

pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));

1339

pci_read_config_dword(pdev, 0x64, &ioc_feature_control);

1339

pci_read_config_dword(pdev, 0x64, &ioc_feature_control);

1340

1341

/* Enable the iommu */

1341

/* Enable the iommu */

1342

if (!(ioc_feature_control & 0x1))

1342

if (!(ioc_feature_control & 0x1))

1343

pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);

1343

pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);

1344

1345

pci_dev_put(pdev);

1345

pci_dev_put(pdev);

1346

1347

/* Restore the iommu BAR */

1347

/* Restore the iommu BAR */

1348

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

1348

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

1349

iommu->stored_addr_lo);

1349

iommu->stored_addr_lo);

1350

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,

1350

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,

1351

iommu->stored_addr_hi);

1351

iommu->stored_addr_hi);

1352

1353

/* Restore the l1 indirect regs for each of the 6 l1s */

1353

/* Restore the l1 indirect regs for each of the 6 l1s */

1354

for (i = 0; i < 6; i++)

1354

for (i = 0; i < 6; i++)

1355

for (j = 0; j < 0x12; j++)

1355

for (j = 0; j < 0x12; j++)

1356

iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);

1356

iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);

1357

1358

/* Restore the l2 indirect regs */

1358

/* Restore the l2 indirect regs */

1359

for (i = 0; i < 0x83; i++)

1359

for (i = 0; i < 0x83; i++)

1360

iommu_write_l2(iommu, i, iommu->stored_l2[i]);

1360

iommu_write_l2(iommu, i, iommu->stored_l2[i]);

1361

1362

/* Lock PCI setup registers */

1362

/* Lock PCI setup registers */

1363

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

1363

pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,

1364

iommu->stored_addr_lo | 1);

1364

iommu->stored_addr_lo | 1);

1365

}

1365

}

1366

1367

/*

1367

/*

1368

* This function finally enables all IOMMUs found in the system after

1368

* This function finally enables all IOMMUs found in the system after

1369

* they have been initialized

1369

* they have been initialized

1370

*/

1370

*/

1371

static void enable_iommus(void)

1371

static void enable_iommus(void)

1372

{

1372

{

1373

struct amd_iommu *iommu;

1373

struct amd_iommu *iommu;

1374

1375

for_each_iommu(iommu) {

1375

for_each_iommu(iommu) {

1376

iommu_disable(iommu);

1376

iommu_disable(iommu);

1377

iommu_init_flags(iommu);

1377

iommu_init_flags(iommu);

1378

iommu_set_device_table(iommu);

1378

iommu_set_device_table(iommu);

1379

iommu_enable_command_buffer(iommu);

1379

iommu_enable_command_buffer(iommu);

1380

iommu_enable_event_buffer(iommu);

1380

iommu_enable_event_buffer(iommu);

1381

iommu_enable_ppr_log(iommu);

1381

iommu_enable_ppr_log(iommu);

1382

iommu_enable_gt(iommu);

1382

iommu_enable_gt(iommu);

1383

iommu_set_exclusion_range(iommu);

1383

iommu_set_exclusion_range(iommu);

1384

iommu_init_msi(iommu);

1384

iommu_init_msi(iommu);

1385

iommu_enable(iommu);

1385

iommu_enable(iommu);

1386

iommu_flush_all_caches(iommu);

1386

iommu_flush_all_caches(iommu);

1387

}

1387

}

1388

}

1388

}

1389

1390

static void disable_iommus(void)

1390

static void disable_iommus(void)

1391

{

1391

{

1392

struct amd_iommu *iommu;

1392

struct amd_iommu *iommu;

1393

1394

for_each_iommu(iommu)

1394

for_each_iommu(iommu)

1395

iommu_disable(iommu);

1395

iommu_disable(iommu);

1396

}

1396

}

1397

1398

/*

1398

/*

1399

* Suspend/Resume support

1399

* Suspend/Resume support

1400

* disable suspend until real resume implemented

1400

* disable suspend until real resume implemented

1401

*/

1401

*/

1402

1403

static void amd_iommu_resume(void)

1403

static void amd_iommu_resume(void)

1404

{

1404

{

1405

struct amd_iommu *iommu;

1405

struct amd_iommu *iommu;

1406

1407

for_each_iommu(iommu)

1407

for_each_iommu(iommu)

1408

iommu_apply_resume_quirks(iommu);

1408

iommu_apply_resume_quirks(iommu);

1409

1410

/* re-load the hardware */

1410

/* re-load the hardware */

1411

enable_iommus();

1411

enable_iommus();

1412

}

1412

}

1413

1414

static int amd_iommu_suspend(void)

1414

static int amd_iommu_suspend(void)

1415

{

1415

{

1416

/* disable IOMMUs to go out of the way for BIOS */

1416

/* disable IOMMUs to go out of the way for BIOS */

1417

disable_iommus();

1417

disable_iommus();

1418

1419

return 0;

1419

return 0;

1420

}

1420

}

1421

1422

static struct syscore_ops amd_iommu_syscore_ops = {

1422

static struct syscore_ops amd_iommu_syscore_ops = {

1423

.suspend = amd_iommu_suspend,

1423

.suspend = amd_iommu_suspend,

1424

.resume = amd_iommu_resume,

1424

.resume = amd_iommu_resume,

1425

};

1425

};

1426

1427

/*

1427

/*

1428

* This is the core init function for AMD IOMMU hardware in the system.

1428

* This is the core init function for AMD IOMMU hardware in the system.

1429

* This function is called from the generic x86 DMA layer initialization

1429

* This function is called from the generic x86 DMA layer initialization

1430

* code.

1430

* code.

1431

*

1431

*

1432

* This function basically parses the ACPI table for AMD IOMMU (IVRS)

1432

* This function basically parses the ACPI table for AMD IOMMU (IVRS)

1433

* three times:

1433

* three times:

1434

*

1434

*

1435

* 1 pass) Find the highest PCI device id the driver has to handle.

1435

* 1 pass) Find the highest PCI device id the driver has to handle.

1436

* Upon this information the size of the data structures is

1436

* Upon this information the size of the data structures is

1437

* determined that needs to be allocated.

1437

* determined that needs to be allocated.

1438

*

1438

*

1439

* 2 pass) Initialize the data structures just allocated with the

1439

* 2 pass) Initialize the data structures just allocated with the

1440

* information in the ACPI table about available AMD IOMMUs

1440

* information in the ACPI table about available AMD IOMMUs

1441

* in the system. It also maps the PCI devices in the

1441

* in the system. It also maps the PCI devices in the

1442

* system to specific IOMMUs

1442

* system to specific IOMMUs

1443

*

1443

*

1444

* 3 pass) After the basic data structures are allocated and

1444

* 3 pass) After the basic data structures are allocated and

1445

* initialized we update them with information about memory

1445

* initialized we update them with information about memory

1446

* remapping requirements parsed out of the ACPI table in

1446

* remapping requirements parsed out of the ACPI table in

1447

* this last pass.

1447

* this last pass.

1448

*

1448

*

1449

* After that the hardware is initialized and ready to go. In the last

1449

* After that the hardware is initialized and ready to go. In the last

1450

* step we do some Linux specific things like registering the driver in

1450

* step we do some Linux specific things like registering the driver in

1451

* the dma_ops interface and initializing the suspend/resume support

1451

* the dma_ops interface and initializing the suspend/resume support

1452

* functions. Finally it prints some information about AMD IOMMUs and

1452

* functions. Finally it prints some information about AMD IOMMUs and

1453

* the driver state and enables the hardware.

1453

* the driver state and enables the hardware.

1454

*/

1454

*/

1455

static int __init amd_iommu_init(void)

1455

static int __init amd_iommu_init(void)

1456

{

1456

{

1457

int i, ret = 0;

1457

int i, ret = 0;

1458

1459

/*

1459

/*

1460

* First parse ACPI tables to find the largest Bus/Dev/Func

1460

* First parse ACPI tables to find the largest Bus/Dev/Func

1461

* we need to handle. Upon this information the shared data

1461

* we need to handle. Upon this information the shared data

1462

* structures for the IOMMUs in the system will be allocated

1462

* structures for the IOMMUs in the system will be allocated

1463

*/

1463

*/

1464

if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0)

1464

if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0)

1465

return -ENODEV;

1465

return -ENODEV;

1466

1467

ret = amd_iommu_init_err;

1467

ret = amd_iommu_init_err;

1468

if (ret)

1468

if (ret)

1469

goto out;

1469

goto out;

1470

1471

dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);

1471

dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);

1472

alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);

1472

alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);

1473

rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);

1473

rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);

1474

1475

ret = -ENOMEM;

1475

ret = -ENOMEM;

1476

1477

/* Device table - directly used by all IOMMUs */

1477

/* Device table - directly used by all IOMMUs */

1478

amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

1478

amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,

1479

get_order(dev_table_size));

1479

get_order(dev_table_size));

1480

if (amd_iommu_dev_table == NULL)

1480

if (amd_iommu_dev_table == NULL)

1481

goto out;

1481

goto out;

1482

1483

/*

1483

/*

1484

* Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the

1484

* Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the

1485

* IOMMU see for that device

1485

* IOMMU see for that device

1486

*/

1486

*/

1487

amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,

1487

amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,

1488

get_order(alias_table_size));

1488

get_order(alias_table_size));

1489

if (amd_iommu_alias_table == NULL)

1489

if (amd_iommu_alias_table == NULL)

1490

goto free;

1490

goto free;

1491

1492

/* IOMMU rlookup table - find the IOMMU for a specific device */

1492

/* IOMMU rlookup table - find the IOMMU for a specific device */

1493

amd_iommu_rlookup_table = (void *)__get_free_pages(

1493

amd_iommu_rlookup_table = (void *)__get_free_pages(

1494

GFP_KERNEL | __GFP_ZERO,

1494

GFP_KERNEL | __GFP_ZERO,

1495

get_order(rlookup_table_size));

1495

get_order(rlookup_table_size));

1496

if (amd_iommu_rlookup_table == NULL)

1496

if (amd_iommu_rlookup_table == NULL)

1497

goto free;

1497

goto free;

1498

1499

amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(

1499

amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(

1500

GFP_KERNEL | __GFP_ZERO,

1500

GFP_KERNEL | __GFP_ZERO,

1501

get_order(MAX_DOMAIN_ID/8));

1501

get_order(MAX_DOMAIN_ID/8));

1502

if (amd_iommu_pd_alloc_bitmap == NULL)

1502

if (amd_iommu_pd_alloc_bitmap == NULL)

1503

goto free;

1503

goto free;

1504

1505

/* init the device table */

1505

/* init the device table */

1506

init_device_table();

1506

init_device_table();

1507

1508

/*

1508

/*

1509

* let all alias entries point to itself

1509

* let all alias entries point to itself

1510

*/

1510

*/

1511

for (i = 0; i <= amd_iommu_last_bdf; ++i)

1511

for (i = 0; i <= amd_iommu_last_bdf; ++i)

1512

amd_iommu_alias_table[i] = i;

1512

amd_iommu_alias_table[i] = i;

1513

1514

/*

1514

/*

1515

* never allocate domain 0 because its used as the non-allocated and

1515

* never allocate domain 0 because its used as the non-allocated and

1516

* error value placeholder

1516

* error value placeholder

1517

*/

1517

*/

1518

amd_iommu_pd_alloc_bitmap[0] = 1;

1518

amd_iommu_pd_alloc_bitmap[0] = 1;

1519

1520

spin_lock_init(&amd_iommu_pd_lock);

1520

spin_lock_init(&amd_iommu_pd_lock);

1521

1522

/*

1522

/*

1523

* now the data structures are allocated and basically initialized

1523

* now the data structures are allocated and basically initialized

1524

* start the real acpi table scan

1524

* start the real acpi table scan

1525

*/

1525

*/

1526

ret = -ENODEV;

1526

ret = -ENODEV;

1527

if (acpi_table_parse("IVRS", init_iommu_all) != 0)

1527

if (acpi_table_parse("IVRS", init_iommu_all) != 0)

1528

goto free;

1528

goto free;

1529

1530

if (amd_iommu_init_err) {

1530

if (amd_iommu_init_err) {

1531

ret = amd_iommu_init_err;

1531

ret = amd_iommu_init_err;

1532

goto free;

1532

goto free;

1533

}

1533

}

1534

1535

if (acpi_table_parse("IVRS", init_memory_definitions) != 0)

1535

if (acpi_table_parse("IVRS", init_memory_definitions) != 0)

1536

goto free;

1536

goto free;

1537

1538

if (amd_iommu_init_err) {

1538

if (amd_iommu_init_err) {

1539

ret = amd_iommu_init_err;

1539

ret = amd_iommu_init_err;

1540

goto free;

1540

goto free;

1541

}

1541

}

1542

1543

ret = amd_iommu_init_devices();

1543

ret = amd_iommu_init_devices();

1544

if (ret)

1544

if (ret)

1545

goto free;

1545

goto free;

1546

1547

enable_iommus();

1547

enable_iommus();

1548

1549

if (iommu_pass_through)

1549

if (iommu_pass_through)

1550

ret = amd_iommu_init_passthrough();

1550

ret = amd_iommu_init_passthrough();

1551

else

1551

else

1552

ret = amd_iommu_init_dma_ops();

1552

ret = amd_iommu_init_dma_ops();

1553

1554

if (ret)

1554

if (ret)

1555

goto free_disable;

1555

goto free_disable;

1556

1557

amd_iommu_init_api();

1557

amd_iommu_init_api();

1558

1559

amd_iommu_init_notifier();

1559

amd_iommu_init_notifier();

1560

1561

register_syscore_ops(&amd_iommu_syscore_ops);

1561

register_syscore_ops(&amd_iommu_syscore_ops);

1562

1563

if (iommu_pass_through)

1563

if (iommu_pass_through)

1564

goto out;

1564

goto out;

1565

1566

if (amd_iommu_unmap_flush)

1566

if (amd_iommu_unmap_flush)

1567

printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n");

1567

printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n");

1568

else

1568

else

1569

printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n");

1569

printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n");

1570

1571

x86_platform.iommu_shutdown = disable_iommus;

1571

x86_platform.iommu_shutdown = disable_iommus;

1572

out:

1572

out:

1573

return ret;

1573

return ret;

1574

1575

free_disable:

1575

free_disable:

1576

disable_iommus();

1576

disable_iommus();

1577

1578

free:

1578

free:

1579

amd_iommu_uninit_devices();

1579

amd_iommu_uninit_devices();

1580

1581

free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,

1581

free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,

1582

get_order(MAX_DOMAIN_ID/8));

1582

get_order(MAX_DOMAIN_ID/8));

1583

1584

free_pages((unsigned long)amd_iommu_rlookup_table,

1584

free_pages((unsigned long)amd_iommu_rlookup_table,

1585

get_order(rlookup_table_size));

1585

get_order(rlookup_table_size));

1586

1587

free_pages((unsigned long)amd_iommu_alias_table,

1587

free_pages((unsigned long)amd_iommu_alias_table,

1588

get_order(alias_table_size));

1588

get_order(alias_table_size));

1589

1590

free_pages((unsigned long)amd_iommu_dev_table,

1590

free_pages((unsigned long)amd_iommu_dev_table,

1591

get_order(dev_table_size));

1591

get_order(dev_table_size));

1592

1593

free_iommu_all();

1593

free_iommu_all();

1594

1595

free_unity_maps();

1595

free_unity_maps();

1596

1597

#ifdef CONFIG_GART_IOMMU

1597

#ifdef CONFIG_GART_IOMMU

1598

/*

1598

/*

1599

* We failed to initialize the AMD IOMMU - try fallback to GART

1599

* We failed to initialize the AMD IOMMU - try fallback to GART

1600

* if possible.

1600

* if possible.

1601

*/

1601

*/

1602

gart_iommu_init();

1602

gart_iommu_init();

1603

1604

#endif

1604

#endif

1605

1606

goto out;

1606

goto out;

1607

}

1607

}

1608

1609

/****************************************************************************

1609

/****************************************************************************

1610

*

1610

*

1611

* Early detect code. This code runs at IOMMU detection time in the DMA

1611

* Early detect code. This code runs at IOMMU detection time in the DMA

1612

* layer. It just looks if there is an IVRS ACPI table to detect AMD

1612

* layer. It just looks if there is an IVRS ACPI table to detect AMD

1613

* IOMMUs

1613

* IOMMUs

1614

*

1614

*

1615

****************************************************************************/

1615

****************************************************************************/

1616

static int __init early_amd_iommu_detect(struct acpi_table_header *table)

1616

static int __init early_amd_iommu_detect(struct acpi_table_header *table)

1617

{

1617

{

1618

return 0;

1618

return 0;

1619

}

1619

}

1620

1621

int __init amd_iommu_detect(void)

1621

int __init amd_iommu_detect(void)

1622

{

1622

{

1623

if (no_iommu || (iommu_detected && !gart_iommu_aperture))

1623

if (no_iommu || (iommu_detected && !gart_iommu_aperture))

1624

return -ENODEV;

1624

return -ENODEV;

1625

1626

if (amd_iommu_disabled)

1626

if (amd_iommu_disabled)

1627

return -ENODEV;

1627

return -ENODEV;

1628

1629

if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {

1629

if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {

1630

iommu_detected = 1;

1630

iommu_detected = 1;

1631

amd_iommu_detected = 1;

1631

amd_iommu_detected = 1;

1632

x86_init.iommu.iommu_init = amd_iommu_init;

1632

x86_init.iommu.iommu_init = amd_iommu_init;

1633

1634

/* Make sure ACS will be enabled */

1634

/* Make sure ACS will be enabled */

1635

pci_request_acs();

1635

pci_request_acs();

1636

return 1;

1636

return 1;

1637

}

1637

}

1638

return -ENODEV;

1638

return -ENODEV;

1639

}

1639

}

1640

1641

/****************************************************************************

1641

/****************************************************************************

1642

*

1642

*

1643

* Parsing functions for the AMD IOMMU specific kernel command line

1643

* Parsing functions for the AMD IOMMU specific kernel command line

1644

* options.

1644

* options.

1645

*

1645

*

1646

****************************************************************************/

1646

****************************************************************************/

1647

1648

static int __init parse_amd_iommu_dump(char *str)

1648

static int __init parse_amd_iommu_dump(char *str)

1649

{

1649

{

1650

amd_iommu_dump = true;

1650

amd_iommu_dump = true;

1651

1652

return 1;

1652

return 1;

1653

}

1653

}

1654

1655

static int __init parse_amd_iommu_options(char *str)

1655

static int __init parse_amd_iommu_options(char *str)

1656

{

1656

{

1657

for (; *str; ++str) {

1657

for (; *str; ++str) {

1658

if (strncmp(str, "fullflush", 9) == 0)

1658

if (strncmp(str, "fullflush", 9) == 0)

1659

amd_iommu_unmap_flush = true;

1659

amd_iommu_unmap_flush = true;

1660

if (strncmp(str, "off", 3) == 0)

1660

if (strncmp(str, "off", 3) == 0)

1661

amd_iommu_disabled = true;

1661

amd_iommu_disabled = true;

1662

if (strncmp(str, "force_isolation", 15) == 0)

1662

if (strncmp(str, "force_isolation", 15) == 0)

1663

amd_iommu_force_isolation = true;

1663

amd_iommu_force_isolation = true;

1664

}

1664

}

1665

1666

return 1;

1666

return 1;

1667

}

1667

}

1668

1669

__setup("amd_iommu_dump", parse_amd_iommu_dump);

1669

__setup("amd_iommu_dump", parse_amd_iommu_dump);

1670

__setup("amd_iommu=", parse_amd_iommu_options);

1670

__setup("amd_iommu=", parse_amd_iommu_options);

1671

1672

IOMMU_INIT_FINISH(amd_iommu_detect,

1672

IOMMU_INIT_FINISH(amd_iommu_detect,

1673

gart_iommu_hole_init,

1673

gart_iommu_hole_init,

1674

0,

1674

0,

1675

0);

1675

0);

1676

1677

bool amd_iommu_v2_supported(void)

1677

bool amd_iommu_v2_supported(void)

1678

{

1678

{

1679

return amd_iommu_v2_present;

1679

return amd_iommu_v2_present;

1680

}

1680

}

1681

EXPORT_SYMBOL(amd_iommu_v2_supported);

1681

EXPORT_SYMBOL(amd_iommu_v2_supported);

1682

GITLAB

x86/amd: iommu_set_device_table() must not be __init