Eric Lee / linux-smarc-t335x-v3.2

1

/*

1

/*

2

3

*

3

*

4

* This program is free software; you can redistribute it and/or modify it

4

* This program is free software; you can redistribute it and/or modify it

5

* under the terms and conditions of the GNU General Public License,

5

* under the terms and conditions of the GNU General Public License,

6

* version 2, as published by the Free Software Foundation.

6

* version 2, as published by the Free Software Foundation.

7

*

7

*

8

* This program is distributed in the hope it will be useful, but WITHOUT

8

* This program is distributed in the hope it will be useful, but WITHOUT

9

* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

9

* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

10

* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for

10

* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for

11

* more details.

11

* more details.

12

*

12

*

13

* You should have received a copy of the GNU General Public License along with

13

* You should have received a copy of the GNU General Public License along with

14

* this program; if not, write to the Free Software Foundation, Inc., 59 Temple

14

* this program; if not, write to the Free Software Foundation, Inc., 59 Temple

15

* Place - Suite 330, Boston, MA 02111-1307 USA.

15

* Place - Suite 330, Boston, MA 02111-1307 USA.

16

*

16

*

17

18

* Author: Ashok Raj <ashok.raj@intel.com>

18

* Author: Ashok Raj <ashok.raj@intel.com>

19

* Author: Shaohua Li <shaohua.li@intel.com>

19

* Author: Shaohua Li <shaohua.li@intel.com>

20

* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>

20

* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>

21

* Author: Fenghua Yu <fenghua.yu@intel.com>

21

* Author: Fenghua Yu <fenghua.yu@intel.com>

22

*/

22

*/

23

24

#include <linux/init.h>

24

#include <linux/init.h>

25

#include <linux/bitmap.h>

25

#include <linux/bitmap.h>

26

#include <linux/debugfs.h>

26

#include <linux/debugfs.h>

27

#include <linux/slab.h>

27

#include <linux/slab.h>

28

#include <linux/irq.h>

28

#include <linux/irq.h>

29

#include <linux/interrupt.h>

29

#include <linux/interrupt.h>

30

#include <linux/spinlock.h>

30

#include <linux/spinlock.h>

31

#include <linux/pci.h>

31

#include <linux/pci.h>

32

#include <linux/dmar.h>

32

#include <linux/dmar.h>

33

#include <linux/dma-mapping.h>

33

#include <linux/dma-mapping.h>

34

#include <linux/mempool.h>

34

#include <linux/mempool.h>

35

#include <linux/timer.h>

35

#include <linux/timer.h>

36

#include <linux/iova.h>

36

#include <linux/iova.h>

37

#include <linux/iommu.h>

37

#include <linux/iommu.h>

38

#include <linux/intel-iommu.h>

38

#include <linux/intel-iommu.h>

39

#include <linux/syscore_ops.h>

39

#include <linux/syscore_ops.h>

40

#include <linux/tboot.h>

40

#include <linux/tboot.h>

41

#include <linux/dmi.h>

41

#include <linux/dmi.h>

42

#include <linux/pci-ats.h>

42

#include <linux/pci-ats.h>

43

#include <asm/cacheflush.h>

43

#include <asm/cacheflush.h>

44

#include <asm/iommu.h>

44

#include <asm/iommu.h>

45

46

#define ROOT_SIZE VTD_PAGE_SIZE

46

#define ROOT_SIZE VTD_PAGE_SIZE

47

#define CONTEXT_SIZE VTD_PAGE_SIZE

47

#define CONTEXT_SIZE VTD_PAGE_SIZE

48

49

#define IS_BRIDGE_HOST_DEVICE(pdev) \

49

#define IS_BRIDGE_HOST_DEVICE(pdev) \

50

((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)

50

((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)

51

#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)

51

#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)

52

#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)

52

#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)

53

#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)

53

#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)

54

55

#define IOAPIC_RANGE_START (0xfee00000)

55

#define IOAPIC_RANGE_START (0xfee00000)

56

#define IOAPIC_RANGE_END (0xfeefffff)

56

#define IOAPIC_RANGE_END (0xfeefffff)

57

#define IOVA_START_ADDR (0x1000)

57

#define IOVA_START_ADDR (0x1000)

58

59

#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48

59

#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48

60

61

#define MAX_AGAW_WIDTH 64

61

#define MAX_AGAW_WIDTH 64

62

63

#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)

63

#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)

64

#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)

64

#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)

65

66

/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR

66

/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR

67

to match. That way, we can use 'unsigned long' for PFNs with impunity. */

67

to match. That way, we can use 'unsigned long' for PFNs with impunity. */

68

#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \

68

#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \

69

__DOMAIN_MAX_PFN(gaw), (unsigned long)-1))

69

__DOMAIN_MAX_PFN(gaw), (unsigned long)-1))

70

#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)

70

#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)

71

72

#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)

72

#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)

73

#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))

73

#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))

74

#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))

74

#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))

75

76

/* page table handling */

76

/* page table handling */

77

#define LEVEL_STRIDE (9)

77

#define LEVEL_STRIDE (9)

78

#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)

78

#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)

79

80

static inline int agaw_to_level(int agaw)

80

static inline int agaw_to_level(int agaw)

81

{

81

{

82

return agaw + 2;

82

return agaw + 2;

83

}

83

}

84

85

static inline int agaw_to_width(int agaw)

85

static inline int agaw_to_width(int agaw)

86

{

86

{

87

return 30 + agaw * LEVEL_STRIDE;

87

return 30 + agaw * LEVEL_STRIDE;

88

}

88

}

89

90

static inline int width_to_agaw(int width)

90

static inline int width_to_agaw(int width)

91

{

91

{

92

return (width - 30) / LEVEL_STRIDE;

92

return (width - 30) / LEVEL_STRIDE;

93

}

93

}

94

95

static inline unsigned int level_to_offset_bits(int level)

95

static inline unsigned int level_to_offset_bits(int level)

96

{

96

{

97

return (level - 1) * LEVEL_STRIDE;

97

return (level - 1) * LEVEL_STRIDE;

98

}

98

}

99

100

static inline int pfn_level_offset(unsigned long pfn, int level)

100

static inline int pfn_level_offset(unsigned long pfn, int level)

101

{

101

{

102

return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;

102

return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;

103

}

103

}

104

105

static inline unsigned long level_mask(int level)

105

static inline unsigned long level_mask(int level)

106

{

106

{

107

return -1UL << level_to_offset_bits(level);

107

return -1UL << level_to_offset_bits(level);

108

}

108

}

109

110

static inline unsigned long level_size(int level)

110

static inline unsigned long level_size(int level)

111

{

111

{

112

return 1UL << level_to_offset_bits(level);

112

return 1UL << level_to_offset_bits(level);

113

}

113

}

114

115

static inline unsigned long align_to_level(unsigned long pfn, int level)

115

static inline unsigned long align_to_level(unsigned long pfn, int level)

116

{

116

{

117

return (pfn + level_size(level) - 1) & level_mask(level);

117

return (pfn + level_size(level) - 1) & level_mask(level);

118

}

118

}

119

120

static inline unsigned long lvl_to_nr_pages(unsigned int lvl)

120

static inline unsigned long lvl_to_nr_pages(unsigned int lvl)

121

{

121

{

122

return 1 << ((lvl - 1) * LEVEL_STRIDE);

122

return 1 << ((lvl - 1) * LEVEL_STRIDE);

123

}

123

}

124

125

/* VT-d pages must always be _smaller_ than MM pages. Otherwise things

125

/* VT-d pages must always be _smaller_ than MM pages. Otherwise things

126

are never going to work. */

126

are never going to work. */

127

static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)

127

static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)

128

{

128

{

129

return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);

129

return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);

130

}

130

}

131

132

static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)

132

static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)

133

{

133

{

134

return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);

134

return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);

135

}

135

}

136

static inline unsigned long page_to_dma_pfn(struct page *pg)

136

static inline unsigned long page_to_dma_pfn(struct page *pg)

137

{

137

{

138

return mm_to_dma_pfn(page_to_pfn(pg));

138

return mm_to_dma_pfn(page_to_pfn(pg));

139

}

139

}

140

static inline unsigned long virt_to_dma_pfn(void *p)

140

static inline unsigned long virt_to_dma_pfn(void *p)

141

{

141

{

142

return page_to_dma_pfn(virt_to_page(p));

142

return page_to_dma_pfn(virt_to_page(p));

143

}

143

}

144

145

/* global iommu list, set NULL for ignored DMAR units */

145

/* global iommu list, set NULL for ignored DMAR units */

146

static struct intel_iommu **g_iommus;

146

static struct intel_iommu **g_iommus;

147

148

static void __init check_tylersburg_isoch(void);

148

static void __init check_tylersburg_isoch(void);

149

static int rwbf_quirk;

149

static int rwbf_quirk;

150

151

/*

151

/*

152

* set to 1 to panic kernel if can't successfully enable VT-d

152

* set to 1 to panic kernel if can't successfully enable VT-d

153

* (used when kernel is launched w/ TXT)

153

* (used when kernel is launched w/ TXT)

154

*/

154

*/

155

static int force_on = 0;

155

static int force_on = 0;

156

157

/*

157

/*

158

* 0: Present

158

* 0: Present

159

* 1-11: Reserved

159

* 1-11: Reserved

160

* 12-63: Context Ptr (12 - (haw-1))

160

* 12-63: Context Ptr (12 - (haw-1))

161

* 64-127: Reserved

161

* 64-127: Reserved

162

*/

162

*/

163

struct root_entry {

163

struct root_entry {

164

u64 val;

164

u64 val;

165

u64 rsvd1;

165

u64 rsvd1;

166

};

166

};

167

#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))

167

#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))

168

static inline bool root_present(struct root_entry *root)

168

static inline bool root_present(struct root_entry *root)

169

{

169

{

170

return (root->val & 1);

170

return (root->val & 1);

171

}

171

}

172

static inline void set_root_present(struct root_entry *root)

172

static inline void set_root_present(struct root_entry *root)

173

{

173

{

174

root->val |= 1;

174

root->val |= 1;

175

}

175

}

176

static inline void set_root_value(struct root_entry *root, unsigned long value)

176

static inline void set_root_value(struct root_entry *root, unsigned long value)

177

{

177

{

178

root->val |= value & VTD_PAGE_MASK;

178

root->val |= value & VTD_PAGE_MASK;

179

}

179

}

180

181

static inline struct context_entry *

181

static inline struct context_entry *

182

get_context_addr_from_root(struct root_entry *root)

182

get_context_addr_from_root(struct root_entry *root)

183

{

183

{

184

return (struct context_entry *)

184

return (struct context_entry *)

185

(root_present(root)?phys_to_virt(

185

(root_present(root)?phys_to_virt(

186

root->val & VTD_PAGE_MASK) :

186

root->val & VTD_PAGE_MASK) :

187

NULL);

187

NULL);

188

}

188

}

189

190

/*

190

/*

191

* low 64 bits:

191

* low 64 bits:

192

* 0: present

192

* 0: present

193

* 1: fault processing disable

193

* 1: fault processing disable

194

* 2-3: translation type

194

* 2-3: translation type

195

* 12-63: address space root

195

* 12-63: address space root

196

* high 64 bits:

196

* high 64 bits:

197

* 0-2: address width

197

* 0-2: address width

198

* 3-6: aval

198

* 3-6: aval

199

* 8-23: domain id

199

* 8-23: domain id

200

*/

200

*/

201

struct context_entry {

201

struct context_entry {

202

u64 lo;

202

u64 lo;

203

u64 hi;

203

u64 hi;

204

};

204

};

205

206

static inline bool context_present(struct context_entry *context)

206

static inline bool context_present(struct context_entry *context)

207

{

207

{

208

return (context->lo & 1);

208

return (context->lo & 1);

209

}

209

}

210

static inline void context_set_present(struct context_entry *context)

210

static inline void context_set_present(struct context_entry *context)

211

{

211

{

212

context->lo |= 1;

212

context->lo |= 1;

213

}

213

}

214

215

static inline void context_set_fault_enable(struct context_entry *context)

215

static inline void context_set_fault_enable(struct context_entry *context)

216

{

216

{

217

context->lo &= (((u64)-1) << 2) | 1;

217

context->lo &= (((u64)-1) << 2) | 1;

218

}

218

}

219

220

static inline void context_set_translation_type(struct context_entry *context,

220

static inline void context_set_translation_type(struct context_entry *context,

221

unsigned long value)

221

unsigned long value)

222

{

222

{

223

context->lo &= (((u64)-1) << 4) | 3;

223

context->lo &= (((u64)-1) << 4) | 3;

224

context->lo |= (value & 3) << 2;

224

context->lo |= (value & 3) << 2;

225

}

225

}

226

227

static inline void context_set_address_root(struct context_entry *context,

227

static inline void context_set_address_root(struct context_entry *context,

228

unsigned long value)

228

unsigned long value)

229

{

229

{

230

context->lo |= value & VTD_PAGE_MASK;

230

context->lo |= value & VTD_PAGE_MASK;

231

}

231

}

232

233

static inline void context_set_address_width(struct context_entry *context,

233

static inline void context_set_address_width(struct context_entry *context,

234

unsigned long value)

234

unsigned long value)

235

{

235

{

236

context->hi |= value & 7;

236

context->hi |= value & 7;

237

}

237

}

238

239

static inline void context_set_domain_id(struct context_entry *context,

239

static inline void context_set_domain_id(struct context_entry *context,

240

unsigned long value)

240

unsigned long value)

241

{

241

{

242

context->hi |= (value & ((1 << 16) - 1)) << 8;

242

context->hi |= (value & ((1 << 16) - 1)) << 8;

243

}

243

}

244

245

static inline void context_clear_entry(struct context_entry *context)

245

static inline void context_clear_entry(struct context_entry *context)

246

{

246

{

247

context->lo = 0;

247

context->lo = 0;

248

context->hi = 0;

248

context->hi = 0;

249

}

249

}

250

251

/*

251

/*

252

* 0: readable

252

* 0: readable

253

* 1: writable

253

* 1: writable

254

* 2-6: reserved

254

* 2-6: reserved

255

* 7: super page

255

* 7: super page

256

* 8-10: available

256

* 8-10: available

257

* 11: snoop behavior

257

* 11: snoop behavior

258

* 12-63: Host physcial address

258

* 12-63: Host physcial address

259

*/

259

*/

260

struct dma_pte {

260

struct dma_pte {

261

u64 val;

261

u64 val;

262

};

262

};

263

264

static inline void dma_clear_pte(struct dma_pte *pte)

264

static inline void dma_clear_pte(struct dma_pte *pte)

265

{

265

{

266

pte->val = 0;

266

pte->val = 0;

267

}

267

}

268

269

static inline void dma_set_pte_readable(struct dma_pte *pte)

269

static inline void dma_set_pte_readable(struct dma_pte *pte)

270

{

270

{

271

pte->val |= DMA_PTE_READ;

271

pte->val |= DMA_PTE_READ;

272

}

272

}

273

274

static inline void dma_set_pte_writable(struct dma_pte *pte)

274

static inline void dma_set_pte_writable(struct dma_pte *pte)

275

{

275

{

276

pte->val |= DMA_PTE_WRITE;

276

pte->val |= DMA_PTE_WRITE;

277

}

277

}

278

279

static inline void dma_set_pte_snp(struct dma_pte *pte)

279

static inline void dma_set_pte_snp(struct dma_pte *pte)

280

{

280

{

281

pte->val |= DMA_PTE_SNP;

281

pte->val |= DMA_PTE_SNP;

282

}

282

}

283

284

static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)

284

static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)

285

{

285

{

286

pte->val = (pte->val & ~3) | (prot & 3);

286

pte->val = (pte->val & ~3) | (prot & 3);

287

}

287

}

288

289

static inline u64 dma_pte_addr(struct dma_pte *pte)

289

static inline u64 dma_pte_addr(struct dma_pte *pte)

290

{

290

{

291

#ifdef CONFIG_64BIT

291

#ifdef CONFIG_64BIT

292

return pte->val & VTD_PAGE_MASK;

292

return pte->val & VTD_PAGE_MASK;

293

#else

293

#else

294

/* Must have a full atomic 64-bit read */

294

/* Must have a full atomic 64-bit read */

295

return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;

295

return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;

296

#endif

296

#endif

297

}

297

}

298

299

static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)

299

static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)

300

{

300

{

301

pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;

301

pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;

302

}

302

}

303

304

static inline bool dma_pte_present(struct dma_pte *pte)

304

static inline bool dma_pte_present(struct dma_pte *pte)

305

{

305

{

306

return (pte->val & 3) != 0;

306

return (pte->val & 3) != 0;

307

}

307

}

308

309

static inline bool dma_pte_superpage(struct dma_pte *pte)

310

{

311

return (pte->val & (1 << 7));

312

}

313

309

static inline int first_pte_in_page(struct dma_pte *pte)

314

static inline int first_pte_in_page(struct dma_pte *pte)

310

{

315

{

311

return !((unsigned long)pte & ~VTD_PAGE_MASK);

316

return !((unsigned long)pte & ~VTD_PAGE_MASK);

312

}

317

}

313

318

314

/*

319

/*

315

* This domain is a statically identity mapping domain.

320

* This domain is a statically identity mapping domain.

316

* 1. This domain creats a static 1:1 mapping to all usable memory.

321

* 1. This domain creats a static 1:1 mapping to all usable memory.

317

* 2. It maps to each iommu if successful.

322

* 2. It maps to each iommu if successful.

318

* 3. Each iommu mapps to this domain if successful.

323

* 3. Each iommu mapps to this domain if successful.

319

*/

324

*/

320

static struct dmar_domain *si_domain;

325

static struct dmar_domain *si_domain;

321

static int hw_pass_through = 1;

326

static int hw_pass_through = 1;

322

327

323

/* devices under the same p2p bridge are owned in one domain */

328

/* devices under the same p2p bridge are owned in one domain */

324

#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)

329

#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)

325

330

326

/* domain represents a virtual machine, more than one devices

331

/* domain represents a virtual machine, more than one devices

327

* across iommus may be owned in one domain, e.g. kvm guest.

332

* across iommus may be owned in one domain, e.g. kvm guest.

328

*/

333

*/

329

#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)

334

#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)

330

335

331

/* si_domain contains mulitple devices */

336

/* si_domain contains mulitple devices */

332

#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)

337

#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)

333

338

334

struct dmar_domain {

339

struct dmar_domain {

335

int id; /* domain id */

340

int id; /* domain id */

336

int nid; /* node id */

341

int nid; /* node id */

337

unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/

342

unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/

338

343

339

struct list_head devices; /* all devices' list */

344

struct list_head devices; /* all devices' list */

340

struct iova_domain iovad; /* iova's that belong to this domain */

345

struct iova_domain iovad; /* iova's that belong to this domain */

341

346

342

struct dma_pte *pgd; /* virtual address */

347

struct dma_pte *pgd; /* virtual address */

343

int gaw; /* max guest address width */

348

int gaw; /* max guest address width */

344

349

345

/* adjusted guest address width, 0 is level 2 30-bit */

350

/* adjusted guest address width, 0 is level 2 30-bit */

346

int agaw;

351

int agaw;

347

352

348

int flags; /* flags to find out type of domain */

353

int flags; /* flags to find out type of domain */

349

354

350

int iommu_coherency;/* indicate coherency of iommu access */

355

int iommu_coherency;/* indicate coherency of iommu access */

351

int iommu_snooping; /* indicate snooping control feature*/

356

int iommu_snooping; /* indicate snooping control feature*/

352

int iommu_count; /* reference count of iommu */

357

int iommu_count; /* reference count of iommu */

353

int iommu_superpage;/* Level of superpages supported:

358

int iommu_superpage;/* Level of superpages supported:

354

0 == 4KiB (no superpages), 1 == 2MiB,

359

0 == 4KiB (no superpages), 1 == 2MiB,

355

2 == 1GiB, 3 == 512GiB, 4 == 1TiB */

360

2 == 1GiB, 3 == 512GiB, 4 == 1TiB */

356

spinlock_t iommu_lock; /* protect iommu set in domain */

361

spinlock_t iommu_lock; /* protect iommu set in domain */

357

u64 max_addr; /* maximum mapped address */

362

u64 max_addr; /* maximum mapped address */

358

};

363

};

359

364

360

/* PCI domain-device relationship */

365

/* PCI domain-device relationship */

361

struct device_domain_info {

366

struct device_domain_info {

362

struct list_head link; /* link to domain siblings */

367

struct list_head link; /* link to domain siblings */

363

struct list_head global; /* link to global list */

368

struct list_head global; /* link to global list */

364

int segment; /* PCI domain */

369

int segment; /* PCI domain */

365

u8 bus; /* PCI bus number */

370

u8 bus; /* PCI bus number */

366

u8 devfn; /* PCI devfn number */

371

u8 devfn; /* PCI devfn number */

367

struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */

372

struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */

368

struct intel_iommu *iommu; /* IOMMU used by this device */

373

struct intel_iommu *iommu; /* IOMMU used by this device */

369

struct dmar_domain *domain; /* pointer to domain */

374

struct dmar_domain *domain; /* pointer to domain */

370

};

375

};

371

376

372

static void flush_unmaps_timeout(unsigned long data);

377

static void flush_unmaps_timeout(unsigned long data);

373

378

374

DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);

379

DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);

375

380

376

#define HIGH_WATER_MARK 250

381

#define HIGH_WATER_MARK 250

377

struct deferred_flush_tables {

382

struct deferred_flush_tables {

378

int next;

383

int next;

379

struct iova *iova[HIGH_WATER_MARK];

384

struct iova *iova[HIGH_WATER_MARK];

380

struct dmar_domain *domain[HIGH_WATER_MARK];

385

struct dmar_domain *domain[HIGH_WATER_MARK];

381

};

386

};

382

387

383

static struct deferred_flush_tables *deferred_flush;

388

static struct deferred_flush_tables *deferred_flush;

384

389

385

/* bitmap for indexing intel_iommus */

390

/* bitmap for indexing intel_iommus */

386

static int g_num_of_iommus;

391

static int g_num_of_iommus;

387

392

388

static DEFINE_SPINLOCK(async_umap_flush_lock);

393

static DEFINE_SPINLOCK(async_umap_flush_lock);

389

static LIST_HEAD(unmaps_to_do);

394

static LIST_HEAD(unmaps_to_do);

390

395

391

static int timer_on;

396

static int timer_on;

392

static long list_size;

397

static long list_size;

393

398

394

static void domain_remove_dev_info(struct dmar_domain *domain);

399

static void domain_remove_dev_info(struct dmar_domain *domain);

395

400

396

#ifdef CONFIG_DMAR_DEFAULT_ON

401

#ifdef CONFIG_DMAR_DEFAULT_ON

397

int dmar_disabled = 0;

402

int dmar_disabled = 0;

398

#else

403

#else

399

int dmar_disabled = 1;

404

int dmar_disabled = 1;

400

#endif /*CONFIG_DMAR_DEFAULT_ON*/

405

#endif /*CONFIG_DMAR_DEFAULT_ON*/

401

406

402

static int dmar_map_gfx = 1;

407

static int dmar_map_gfx = 1;

403

static int dmar_forcedac;

408

static int dmar_forcedac;

404

static int intel_iommu_strict;

409

static int intel_iommu_strict;

405

static int intel_iommu_superpage = 1;

410

static int intel_iommu_superpage = 1;

406

411

407

int intel_iommu_gfx_mapped;

412

int intel_iommu_gfx_mapped;

408

EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);

413

EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);

409

414

410

#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))

415

#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))

411

static DEFINE_SPINLOCK(device_domain_lock);

416

static DEFINE_SPINLOCK(device_domain_lock);

412

static LIST_HEAD(device_domain_list);

417

static LIST_HEAD(device_domain_list);

413

418

414

static struct iommu_ops intel_iommu_ops;

419

static struct iommu_ops intel_iommu_ops;

415

420

416

static int __init intel_iommu_setup(char *str)

421

static int __init intel_iommu_setup(char *str)

417

{

422

{

418

if (!str)

423

if (!str)

419

return -EINVAL;

424

return -EINVAL;

420

while (*str) {

425

while (*str) {

421

if (!strncmp(str, "on", 2)) {

426

if (!strncmp(str, "on", 2)) {

422

dmar_disabled = 0;

427

dmar_disabled = 0;

423

printk(KERN_INFO "Intel-IOMMU: enabled\n");

428

printk(KERN_INFO "Intel-IOMMU: enabled\n");

424

} else if (!strncmp(str, "off", 3)) {

429

} else if (!strncmp(str, "off", 3)) {

425

dmar_disabled = 1;

430

dmar_disabled = 1;

426

printk(KERN_INFO "Intel-IOMMU: disabled\n");

431

printk(KERN_INFO "Intel-IOMMU: disabled\n");

427

} else if (!strncmp(str, "igfx_off", 8)) {

432

} else if (!strncmp(str, "igfx_off", 8)) {

428

dmar_map_gfx = 0;

433

dmar_map_gfx = 0;

429

printk(KERN_INFO

434

printk(KERN_INFO

430

"Intel-IOMMU: disable GFX device mapping\n");

435

"Intel-IOMMU: disable GFX device mapping\n");

431

} else if (!strncmp(str, "forcedac", 8)) {

436

} else if (!strncmp(str, "forcedac", 8)) {

432

printk(KERN_INFO

437

printk(KERN_INFO

433

"Intel-IOMMU: Forcing DAC for PCI devices\n");

438

"Intel-IOMMU: Forcing DAC for PCI devices\n");

434

dmar_forcedac = 1;

439

dmar_forcedac = 1;

435

} else if (!strncmp(str, "strict", 6)) {

440

} else if (!strncmp(str, "strict", 6)) {

436

printk(KERN_INFO

441

printk(KERN_INFO

437

"Intel-IOMMU: disable batched IOTLB flush\n");

442

"Intel-IOMMU: disable batched IOTLB flush\n");

438

intel_iommu_strict = 1;

443

intel_iommu_strict = 1;

439

} else if (!strncmp(str, "sp_off", 6)) {

444

} else if (!strncmp(str, "sp_off", 6)) {

440

printk(KERN_INFO

445

printk(KERN_INFO

441

"Intel-IOMMU: disable supported super page\n");

446

"Intel-IOMMU: disable supported super page\n");

442

intel_iommu_superpage = 0;

447

intel_iommu_superpage = 0;

443

}

448

}

444

449

445

str += strcspn(str, ",");

450

str += strcspn(str, ",");

446

while (*str == ',')

451

while (*str == ',')

447

str++;

452

str++;

448

}

453

}

449

return 0;

454

return 0;

450

}

455

}

451

__setup("intel_iommu=", intel_iommu_setup);

456

__setup("intel_iommu=", intel_iommu_setup);

452

457

453

static struct kmem_cache *iommu_domain_cache;

458

static struct kmem_cache *iommu_domain_cache;

454

static struct kmem_cache *iommu_devinfo_cache;

459

static struct kmem_cache *iommu_devinfo_cache;

455

static struct kmem_cache *iommu_iova_cache;

460

static struct kmem_cache *iommu_iova_cache;

456

461

457

static inline void *alloc_pgtable_page(int node)

462

static inline void *alloc_pgtable_page(int node)

458

{

463

{

459

struct page *page;

464

struct page *page;

460

void *vaddr = NULL;

465

void *vaddr = NULL;

461

466

462

page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);

467

page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);

463

if (page)

468

if (page)

464

vaddr = page_address(page);

469

vaddr = page_address(page);

465

return vaddr;

470

return vaddr;

466

}

471

}

467

472

468

static inline void free_pgtable_page(void *vaddr)

473

static inline void free_pgtable_page(void *vaddr)

469

{

474

{

470

free_page((unsigned long)vaddr);

475

free_page((unsigned long)vaddr);

471

}

476

}

472

477

473

static inline void *alloc_domain_mem(void)

478

static inline void *alloc_domain_mem(void)

474

{

479

{

475

return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);

480

return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);

476

}

481

}

477

482

478

static void free_domain_mem(void *vaddr)

483

static void free_domain_mem(void *vaddr)

479

{

484

{

480

kmem_cache_free(iommu_domain_cache, vaddr);

485

kmem_cache_free(iommu_domain_cache, vaddr);

481

}

486

}

482

487

483

static inline void * alloc_devinfo_mem(void)

488

static inline void * alloc_devinfo_mem(void)

484

{

489

{

485

return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);

490

return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);

486

}

491

}

487

492

488

static inline void free_devinfo_mem(void *vaddr)

493

static inline void free_devinfo_mem(void *vaddr)

489

{

494

{

490

kmem_cache_free(iommu_devinfo_cache, vaddr);

495

kmem_cache_free(iommu_devinfo_cache, vaddr);

491

}

496

}

492

497

493

struct iova *alloc_iova_mem(void)

498

struct iova *alloc_iova_mem(void)

494

{

499

{

495

return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);

500

return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);

496

}

501

}

497

502

498

void free_iova_mem(struct iova *iova)

503

void free_iova_mem(struct iova *iova)

499

{

504

{

500

kmem_cache_free(iommu_iova_cache, iova);

505

kmem_cache_free(iommu_iova_cache, iova);

501

}

506

}

502

507

503

508

504

static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)

509

static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)

505

{

510

{

506

unsigned long sagaw;

511

unsigned long sagaw;

507

int agaw = -1;

512

int agaw = -1;

508

513

509

sagaw = cap_sagaw(iommu->cap);

514

sagaw = cap_sagaw(iommu->cap);

510

for (agaw = width_to_agaw(max_gaw);

515

for (agaw = width_to_agaw(max_gaw);

511

agaw >= 0; agaw--) {

516

agaw >= 0; agaw--) {

512

if (test_bit(agaw, &sagaw))

517

if (test_bit(agaw, &sagaw))

513

break;

518

break;

514

}

519

}

515

520

516

return agaw;

521

return agaw;

517

}

522

}

518

523

519

/*

524

/*

520

* Calculate max SAGAW for each iommu.

525

* Calculate max SAGAW for each iommu.

521

*/

526

*/

522

int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

527

int iommu_calculate_max_sagaw(struct intel_iommu *iommu)

523

{

528

{

524

return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);

529

return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);

525

}

530

}

526

531

527

/*

532

/*

528

* calculate agaw for each iommu.

533

* calculate agaw for each iommu.

529

* "SAGAW" may be different across iommus, use a default agaw, and

534

* "SAGAW" may be different across iommus, use a default agaw, and

530

* get a supported less agaw for iommus that don't support the default agaw.

535

* get a supported less agaw for iommus that don't support the default agaw.

531

*/

536

*/

532

int iommu_calculate_agaw(struct intel_iommu *iommu)

537

int iommu_calculate_agaw(struct intel_iommu *iommu)

533

{

538

{

534

return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);

539

return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);

535

}

540

}

536

541

537

/* This functionin only returns single iommu in a domain */

542

/* This functionin only returns single iommu in a domain */

538

static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)

543

static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)

539

{

544

{

540

int iommu_id;

545

int iommu_id;

541

546

542

/* si_domain and vm domain should not get here. */

547

/* si_domain and vm domain should not get here. */

543

BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);

548

BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);

544

BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);

549

BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);

545

550

546

iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);

551

iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);

547

if (iommu_id < 0 || iommu_id >= g_num_of_iommus)

552

if (iommu_id < 0 || iommu_id >= g_num_of_iommus)

548

return NULL;

553

return NULL;

549

554

550

return g_iommus[iommu_id];

555

return g_iommus[iommu_id];

551

}

556

}

552

557

553

static void domain_update_iommu_coherency(struct dmar_domain *domain)

558

static void domain_update_iommu_coherency(struct dmar_domain *domain)

554

{

559

{

555

int i;

560

int i;

556

561

557

domain->iommu_coherency = 1;

562

domain->iommu_coherency = 1;

558

563

559

for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {

564

for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {

560

if (!ecap_coherent(g_iommus[i]->ecap)) {

565

if (!ecap_coherent(g_iommus[i]->ecap)) {

561

domain->iommu_coherency = 0;

566

domain->iommu_coherency = 0;

562

break;

567

break;

563

}

568

}

564

}

569

}

565

}

570

}

566

571

567

static void domain_update_iommu_snooping(struct dmar_domain *domain)

572

static void domain_update_iommu_snooping(struct dmar_domain *domain)

568

{

573

{

569

int i;

574

int i;

570

575

571

domain->iommu_snooping = 1;

576

domain->iommu_snooping = 1;

572

577

573

for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {

578

for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {

574

if (!ecap_sc_support(g_iommus[i]->ecap)) {

579

if (!ecap_sc_support(g_iommus[i]->ecap)) {

575

domain->iommu_snooping = 0;

580

domain->iommu_snooping = 0;

576

break;

581

break;

577

}

582

}

578

}

583

}

579

}

584

}

580

585

581

static void domain_update_iommu_superpage(struct dmar_domain *domain)

586

static void domain_update_iommu_superpage(struct dmar_domain *domain)

582

{

587

{

583

struct dmar_drhd_unit *drhd;

588

struct dmar_drhd_unit *drhd;

584

struct intel_iommu *iommu = NULL;

589

struct intel_iommu *iommu = NULL;

585

int mask = 0xf;

590

int mask = 0xf;

586

591

587

if (!intel_iommu_superpage) {

592

if (!intel_iommu_superpage) {

588

domain->iommu_superpage = 0;

593

domain->iommu_superpage = 0;

589

return;

594

return;

590

}

595

}

591

596

592

/* set iommu_superpage to the smallest common denominator */

597

/* set iommu_superpage to the smallest common denominator */

593

for_each_active_iommu(iommu, drhd) {

598

for_each_active_iommu(iommu, drhd) {

594

mask &= cap_super_page_val(iommu->cap);

599

mask &= cap_super_page_val(iommu->cap);

595

if (!mask) {

600

if (!mask) {

596

break;

601

break;

597

}

602

}

598

}

603

}

599

domain->iommu_superpage = fls(mask);

604

domain->iommu_superpage = fls(mask);

600

}

605

}

601

606

602

/* Some capabilities may be different across iommus */

607

/* Some capabilities may be different across iommus */

603

static void domain_update_iommu_cap(struct dmar_domain *domain)

608

static void domain_update_iommu_cap(struct dmar_domain *domain)

604

{

609

{

605

domain_update_iommu_coherency(domain);

610

domain_update_iommu_coherency(domain);

606

domain_update_iommu_snooping(domain);

611

domain_update_iommu_snooping(domain);

607

domain_update_iommu_superpage(domain);

612

domain_update_iommu_superpage(domain);

608

}

613

}

609

614

610

static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)

615

static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)

611

{

616

{

612

struct dmar_drhd_unit *drhd = NULL;

617

struct dmar_drhd_unit *drhd = NULL;

613

int i;

618

int i;

614

619

615

for_each_drhd_unit(drhd) {

620

for_each_drhd_unit(drhd) {

616

if (drhd->ignored)

621

if (drhd->ignored)

617

continue;

622

continue;

618

if (segment != drhd->segment)

623

if (segment != drhd->segment)

619

continue;

624

continue;

620

625

621

for (i = 0; i < drhd->devices_cnt; i++) {

626

for (i = 0; i < drhd->devices_cnt; i++) {

622

if (drhd->devices[i] &&

627

if (drhd->devices[i] &&

623

drhd->devices[i]->bus->number == bus &&

628

drhd->devices[i]->bus->number == bus &&

624

drhd->devices[i]->devfn == devfn)

629

drhd->devices[i]->devfn == devfn)

625

return drhd->iommu;

630

return drhd->iommu;

626

if (drhd->devices[i] &&

631

if (drhd->devices[i] &&

627

drhd->devices[i]->subordinate &&

632

drhd->devices[i]->subordinate &&

628

drhd->devices[i]->subordinate->number <= bus &&

633

drhd->devices[i]->subordinate->number <= bus &&

629

drhd->devices[i]->subordinate->subordinate >= bus)

634

drhd->devices[i]->subordinate->subordinate >= bus)

630

return drhd->iommu;

635

return drhd->iommu;

631

}

636

}

632

637

633

if (drhd->include_all)

638

if (drhd->include_all)

634

return drhd->iommu;

639

return drhd->iommu;

635

}

640

}

636

641

637

return NULL;

642

return NULL;

638

}

643

}

639

644

640

static void domain_flush_cache(struct dmar_domain *domain,

645

static void domain_flush_cache(struct dmar_domain *domain,

641

void *addr, int size)

646

void *addr, int size)

642

{

647

{

643

if (!domain->iommu_coherency)

648

if (!domain->iommu_coherency)

644

clflush_cache_range(addr, size);

649

clflush_cache_range(addr, size);

645

}

650

}

646

651

647

/* Gets context entry for a given bus and devfn */

652

/* Gets context entry for a given bus and devfn */

648

static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,

653

static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,

649

u8 bus, u8 devfn)

654

u8 bus, u8 devfn)

650

{

655

{

651

struct root_entry *root;

656

struct root_entry *root;

652

struct context_entry *context;

657

struct context_entry *context;

653

unsigned long phy_addr;

658

unsigned long phy_addr;

654

unsigned long flags;

659

unsigned long flags;

655

660

656

spin_lock_irqsave(&iommu->lock, flags);

661

spin_lock_irqsave(&iommu->lock, flags);

657

root = &iommu->root_entry[bus];

662

root = &iommu->root_entry[bus];

658

context = get_context_addr_from_root(root);

663

context = get_context_addr_from_root(root);

659

if (!context) {

664

if (!context) {

660

context = (struct context_entry *)

665

context = (struct context_entry *)

661

alloc_pgtable_page(iommu->node);

666

alloc_pgtable_page(iommu->node);

662

if (!context) {

667

if (!context) {

663

spin_unlock_irqrestore(&iommu->lock, flags);

668

spin_unlock_irqrestore(&iommu->lock, flags);

664

return NULL;

669

return NULL;

665

}

670

}

666

__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);

671

__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);

667

phy_addr = virt_to_phys((void *)context);

672

phy_addr = virt_to_phys((void *)context);

668

set_root_value(root, phy_addr);

673

set_root_value(root, phy_addr);

669

set_root_present(root);

674

set_root_present(root);

670

__iommu_flush_cache(iommu, root, sizeof(*root));

675

__iommu_flush_cache(iommu, root, sizeof(*root));

671

}

676

}

672

spin_unlock_irqrestore(&iommu->lock, flags);

677

spin_unlock_irqrestore(&iommu->lock, flags);

673

return &context[devfn];

678

return &context[devfn];

674

}

679

}

675

680

676

static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)

681

static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)

677

{

682

{

678

struct root_entry *root;

683

struct root_entry *root;

679

struct context_entry *context;

684

struct context_entry *context;

680

int ret;

685

int ret;

681

unsigned long flags;

686

unsigned long flags;

682

687

683

spin_lock_irqsave(&iommu->lock, flags);

688

spin_lock_irqsave(&iommu->lock, flags);

684

root = &iommu->root_entry[bus];

689

root = &iommu->root_entry[bus];

685

context = get_context_addr_from_root(root);

690

context = get_context_addr_from_root(root);

686

if (!context) {

691

if (!context) {

687

ret = 0;

692

ret = 0;

688

goto out;

693

goto out;

689

}

694

}

690

ret = context_present(&context[devfn]);

695

ret = context_present(&context[devfn]);

691

out:

696

out:

692

spin_unlock_irqrestore(&iommu->lock, flags);

697

spin_unlock_irqrestore(&iommu->lock, flags);

693

return ret;

698

return ret;

694

}

699

}

695

700

696

static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)

701

static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)

697

{

702

{

698

struct root_entry *root;

703

struct root_entry *root;

699

struct context_entry *context;

704

struct context_entry *context;

700

unsigned long flags;

705

unsigned long flags;

701

706

702

spin_lock_irqsave(&iommu->lock, flags);

707

spin_lock_irqsave(&iommu->lock, flags);

703

root = &iommu->root_entry[bus];

708

root = &iommu->root_entry[bus];

704

context = get_context_addr_from_root(root);

709

context = get_context_addr_from_root(root);

705

if (context) {

710

if (context) {

706

context_clear_entry(&context[devfn]);

711

context_clear_entry(&context[devfn]);

707

__iommu_flush_cache(iommu, &context[devfn], \

712

__iommu_flush_cache(iommu, &context[devfn], \

708

sizeof(*context));

713

sizeof(*context));

709

}

714

}

710

spin_unlock_irqrestore(&iommu->lock, flags);

715

spin_unlock_irqrestore(&iommu->lock, flags);

711

}

716

}

712

717

713

static void free_context_table(struct intel_iommu *iommu)

718

static void free_context_table(struct intel_iommu *iommu)

714

{

719

{

715

struct root_entry *root;

720

struct root_entry *root;

716

int i;

721

int i;

717

unsigned long flags;

722

unsigned long flags;

718

struct context_entry *context;

723

struct context_entry *context;

719

724

720

spin_lock_irqsave(&iommu->lock, flags);

725

spin_lock_irqsave(&iommu->lock, flags);

721

if (!iommu->root_entry) {

726

if (!iommu->root_entry) {

722

goto out;

727

goto out;

723

}

728

}

724

for (i = 0; i < ROOT_ENTRY_NR; i++) {

729

for (i = 0; i < ROOT_ENTRY_NR; i++) {

725

root = &iommu->root_entry[i];

730

root = &iommu->root_entry[i];

726

context = get_context_addr_from_root(root);

731

context = get_context_addr_from_root(root);

727

if (context)

732

if (context)

728

free_pgtable_page(context);

733

free_pgtable_page(context);

729

}

734

}

730

free_pgtable_page(iommu->root_entry);

735

free_pgtable_page(iommu->root_entry);

731

iommu->root_entry = NULL;

736

iommu->root_entry = NULL;

732

out:

737

out:

733

spin_unlock_irqrestore(&iommu->lock, flags);

738

spin_unlock_irqrestore(&iommu->lock, flags);

734

}

739

}

735

740

736

static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,

741

static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,

737

unsigned long pfn, int large_level)

742

unsigned long pfn, int target_level)

738

{

743

{

739

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

744

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

740

struct dma_pte *parent, *pte = NULL;

745

struct dma_pte *parent, *pte = NULL;

741

int level = agaw_to_level(domain->agaw);

746

int level = agaw_to_level(domain->agaw);

742

int offset, target_level;

747

int offset;

743

748

744

BUG_ON(!domain->pgd);

749

BUG_ON(!domain->pgd);

745

BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);

750

BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);

746

parent = domain->pgd;

751

parent = domain->pgd;

747

752

748

/* Search pte */

749

if (!large_level)

750

target_level = 1;

751

else

752

target_level = large_level;

753

754

while (level > 0) {

753

while (level > 0) {

755

void *tmp_page;

754

void *tmp_page;

756

755

757

offset = pfn_level_offset(pfn, level);

756

offset = pfn_level_offset(pfn, level);

758

pte = &parent[offset];

757

pte = &parent[offset];

759

if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))

758

if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))

760

break;

759

break;

761

if (level == target_level)

760

if (level == target_level)

762

break;

761

break;

763

762

764

if (!dma_pte_present(pte)) {

763

if (!dma_pte_present(pte)) {

765

uint64_t pteval;

764

uint64_t pteval;

766

765

767

tmp_page = alloc_pgtable_page(domain->nid);

766

tmp_page = alloc_pgtable_page(domain->nid);

768

767

769

if (!tmp_page)

768

if (!tmp_page)

770

return NULL;

769

return NULL;

771

770

772

domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);

771

domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);

773

pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;

772

pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;

774

if (cmpxchg64(&pte->val, 0ULL, pteval)) {

773

if (cmpxchg64(&pte->val, 0ULL, pteval)) {

775

/* Someone else set it while we were thinking; use theirs. */

774

/* Someone else set it while we were thinking; use theirs. */

776

free_pgtable_page(tmp_page);

775

free_pgtable_page(tmp_page);

777

} else {

776

} else {

778

dma_pte_addr(pte);

777

dma_pte_addr(pte);

779

domain_flush_cache(domain, pte, sizeof(*pte));

778

domain_flush_cache(domain, pte, sizeof(*pte));

780

}

779

}

781

}

780

}

782

parent = phys_to_virt(dma_pte_addr(pte));

781

parent = phys_to_virt(dma_pte_addr(pte));

783

level--;

782

level--;

784

}

783

}

785

784

786

return pte;

785

return pte;

787

}

786

}

788

787

789

788

790

/* return address's pte at specific level */

789

/* return address's pte at specific level */

791

static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,

790

static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,

792

unsigned long pfn,

791

unsigned long pfn,

793

int level, int *large_page)

792

int level, int *large_page)

794

{

793

{

795

struct dma_pte *parent, *pte = NULL;

794

struct dma_pte *parent, *pte = NULL;

796

int total = agaw_to_level(domain->agaw);

795

int total = agaw_to_level(domain->agaw);

797

int offset;

796

int offset;

798

797

799

parent = domain->pgd;

798

parent = domain->pgd;

800

while (level <= total) {

799

while (level <= total) {

801

offset = pfn_level_offset(pfn, total);

800

offset = pfn_level_offset(pfn, total);

802

pte = &parent[offset];

801

pte = &parent[offset];

803

if (level == total)

802

if (level == total)

804

return pte;

803

return pte;

805

804

806

if (!dma_pte_present(pte)) {

805

if (!dma_pte_present(pte)) {

807

*large_page = total;

806

*large_page = total;

808

break;

807

break;

809

}

808

}

810

809

811

if (pte->val & DMA_PTE_LARGE_PAGE) {

810

if (pte->val & DMA_PTE_LARGE_PAGE) {

812

*large_page = total;

811

*large_page = total;

813

return pte;

812

return pte;

814

}

813

}

815

814

816

parent = phys_to_virt(dma_pte_addr(pte));

815

parent = phys_to_virt(dma_pte_addr(pte));

817

total--;

816

total--;

818

}

817

}

819

return NULL;

818

return NULL;

820

}

819

}

821

820

822

/* clear last level pte, a tlb flush should be followed */

821

/* clear last level pte, a tlb flush should be followed */

823

static int dma_pte_clear_range(struct dmar_domain *domain,

822

static int dma_pte_clear_range(struct dmar_domain *domain,

824

unsigned long start_pfn,

823

unsigned long start_pfn,

825

unsigned long last_pfn)

824

unsigned long last_pfn)

826

{

825

{

827

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

826

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

828

unsigned int large_page = 1;

827

unsigned int large_page = 1;

829

struct dma_pte *first_pte, *pte;

828

struct dma_pte *first_pte, *pte;

830

int order;

829

int order;

831

830

832

BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);

831

BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);

833

BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);

832

BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);

834

BUG_ON(start_pfn > last_pfn);

833

BUG_ON(start_pfn > last_pfn);

835

834

836

/* we don't need lock here; nobody else touches the iova range */

835

/* we don't need lock here; nobody else touches the iova range */

837

do {

836

do {

838

large_page = 1;

837

large_page = 1;

839

first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);

838

first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);

840

if (!pte) {

839

if (!pte) {

841

start_pfn = align_to_level(start_pfn + 1, large_page + 1);

840

start_pfn = align_to_level(start_pfn + 1, large_page + 1);

842

continue;

841

continue;

843

}

842

}

844

do {

843

do {

845

dma_clear_pte(pte);

844

dma_clear_pte(pte);

846

start_pfn += lvl_to_nr_pages(large_page);

845

start_pfn += lvl_to_nr_pages(large_page);

847

pte++;

846

pte++;

848

} while (start_pfn <= last_pfn && !first_pte_in_page(pte));

847

} while (start_pfn <= last_pfn && !first_pte_in_page(pte));

849

848

850

domain_flush_cache(domain, first_pte,

849

domain_flush_cache(domain, first_pte,

851

(void *)pte - (void *)first_pte);

850

(void *)pte - (void *)first_pte);

852

851

853

} while (start_pfn && start_pfn <= last_pfn);

852

} while (start_pfn && start_pfn <= last_pfn);

854

853

855

order = (large_page - 1) * 9;

854

order = (large_page - 1) * 9;

856

return order;

855

return order;

857

}

856

}

858

857

859

/* free page table pages. last level pte should already be cleared */

858

/* free page table pages. last level pte should already be cleared */

860

static void dma_pte_free_pagetable(struct dmar_domain *domain,

859

static void dma_pte_free_pagetable(struct dmar_domain *domain,

861

unsigned long start_pfn,

860

unsigned long start_pfn,

862

unsigned long last_pfn)

861

unsigned long last_pfn)

863

{

862

{

864

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

863

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

865

struct dma_pte *first_pte, *pte;

864

struct dma_pte *first_pte, *pte;

866

int total = agaw_to_level(domain->agaw);

865

int total = agaw_to_level(domain->agaw);

867

int level;

866

int level;

868

unsigned long tmp;

867

unsigned long tmp;

869

int large_page = 2;

868

int large_page = 2;

870

869

871

BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);

870

BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);

872

BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);

871

BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);

873

BUG_ON(start_pfn > last_pfn);

872

BUG_ON(start_pfn > last_pfn);

874

873

875

/* We don't need lock here; nobody else touches the iova range */

874

/* We don't need lock here; nobody else touches the iova range */

876

level = 2;

875

level = 2;

877

while (level <= total) {

876

while (level <= total) {

878

tmp = align_to_level(start_pfn, level);

877

tmp = align_to_level(start_pfn, level);

879

878

880

/* If we can't even clear one PTE at this level, we're done */

879

/* If we can't even clear one PTE at this level, we're done */

881

if (tmp + level_size(level) - 1 > last_pfn)

880

if (tmp + level_size(level) - 1 > last_pfn)

882

return;

881

return;

883

882

884

do {

883

do {

885

large_page = level;

884

large_page = level;

886

first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);

885

first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);

887

if (large_page > level)

886

if (large_page > level)

888

level = large_page + 1;

887

level = large_page + 1;

889

if (!pte) {

888

if (!pte) {

890

tmp = align_to_level(tmp + 1, level + 1);

889

tmp = align_to_level(tmp + 1, level + 1);

891

continue;

890

continue;

892

}

891

}

893

do {

892

do {

894

if (dma_pte_present(pte)) {

893

if (dma_pte_present(pte)) {

895

free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));

894

free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));

896

dma_clear_pte(pte);

895

dma_clear_pte(pte);

897

}

896

}

898

pte++;

897

pte++;

899

tmp += level_size(level);

898

tmp += level_size(level);

900

} while (!first_pte_in_page(pte) &&

899

} while (!first_pte_in_page(pte) &&

901

tmp + level_size(level) - 1 <= last_pfn);

900

tmp + level_size(level) - 1 <= last_pfn);

902

901

903

domain_flush_cache(domain, first_pte,

902

domain_flush_cache(domain, first_pte,

904

(void *)pte - (void *)first_pte);

903

(void *)pte - (void *)first_pte);

905

904

906

} while (tmp && tmp + level_size(level) - 1 <= last_pfn);

905

} while (tmp && tmp + level_size(level) - 1 <= last_pfn);

907

level++;

906

level++;

908

}

907

}

909

/* free pgd */

908

/* free pgd */

910

if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {

909

if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {

911

free_pgtable_page(domain->pgd);

910

free_pgtable_page(domain->pgd);

912

domain->pgd = NULL;

911

domain->pgd = NULL;

913

}

912

}

914

}

913

}

915

914

916

/* iommu handling */

915

/* iommu handling */

917

static int iommu_alloc_root_entry(struct intel_iommu *iommu)

916

static int iommu_alloc_root_entry(struct intel_iommu *iommu)

918

{

917

{

919

struct root_entry *root;

918

struct root_entry *root;

920

unsigned long flags;

919

unsigned long flags;

921

920

922

root = (struct root_entry *)alloc_pgtable_page(iommu->node);

921

root = (struct root_entry *)alloc_pgtable_page(iommu->node);

923

if (!root)

922

if (!root)

924

return -ENOMEM;

923

return -ENOMEM;

925

924

926

__iommu_flush_cache(iommu, root, ROOT_SIZE);

925

__iommu_flush_cache(iommu, root, ROOT_SIZE);

927

926

928

spin_lock_irqsave(&iommu->lock, flags);

927

spin_lock_irqsave(&iommu->lock, flags);

929

iommu->root_entry = root;

928

iommu->root_entry = root;

930

spin_unlock_irqrestore(&iommu->lock, flags);

929

spin_unlock_irqrestore(&iommu->lock, flags);

931

930

932

return 0;

931

return 0;

933

}

932

}

934

933

935

static void iommu_set_root_entry(struct intel_iommu *iommu)

934

static void iommu_set_root_entry(struct intel_iommu *iommu)

936

{

935

{

937

void *addr;

936

void *addr;

938

u32 sts;

937

u32 sts;

939

unsigned long flag;

938

unsigned long flag;

940

939

941

addr = iommu->root_entry;

940

addr = iommu->root_entry;

942

941

943

spin_lock_irqsave(&iommu->register_lock, flag);

942

spin_lock_irqsave(&iommu->register_lock, flag);

944

dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));

943

dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));

945

944

946

writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

945

writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);

947

946

948

/* Make sure hardware complete it */

947

/* Make sure hardware complete it */

949

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

948

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

950

readl, (sts & DMA_GSTS_RTPS), sts);

949

readl, (sts & DMA_GSTS_RTPS), sts);

951

950

952

spin_unlock_irqrestore(&iommu->register_lock, flag);

951

spin_unlock_irqrestore(&iommu->register_lock, flag);

953

}

952

}

954

953

955

static void iommu_flush_write_buffer(struct intel_iommu *iommu)

954

static void iommu_flush_write_buffer(struct intel_iommu *iommu)

956

{

955

{

957

u32 val;

956

u32 val;

958

unsigned long flag;

957

unsigned long flag;

959

958

960

if (!rwbf_quirk && !cap_rwbf(iommu->cap))

959

if (!rwbf_quirk && !cap_rwbf(iommu->cap))

961

return;

960

return;

962

961

963

spin_lock_irqsave(&iommu->register_lock, flag);

962

spin_lock_irqsave(&iommu->register_lock, flag);

964

writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);

963

writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);

965

964

966

/* Make sure hardware complete it */

965

/* Make sure hardware complete it */

967

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

966

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

968

readl, (!(val & DMA_GSTS_WBFS)), val);

967

readl, (!(val & DMA_GSTS_WBFS)), val);

969

968

970

spin_unlock_irqrestore(&iommu->register_lock, flag);

969

spin_unlock_irqrestore(&iommu->register_lock, flag);

971

}

970

}

972

971

973

/* return value determine if we need a write buffer flush */

972

/* return value determine if we need a write buffer flush */

974

static void __iommu_flush_context(struct intel_iommu *iommu,

973

static void __iommu_flush_context(struct intel_iommu *iommu,

975

u16 did, u16 source_id, u8 function_mask,

974

u16 did, u16 source_id, u8 function_mask,

976

u64 type)

975

u64 type)

977

{

976

{

978

u64 val = 0;

977

u64 val = 0;

979

unsigned long flag;

978

unsigned long flag;

980

979

981

switch (type) {

980

switch (type) {

982

case DMA_CCMD_GLOBAL_INVL:

981

case DMA_CCMD_GLOBAL_INVL:

983

val = DMA_CCMD_GLOBAL_INVL;

982

val = DMA_CCMD_GLOBAL_INVL;

984

break;

983

break;

985

case DMA_CCMD_DOMAIN_INVL:

984

case DMA_CCMD_DOMAIN_INVL:

986

val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);

985

val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);

987

break;

986

break;

988

case DMA_CCMD_DEVICE_INVL:

987

case DMA_CCMD_DEVICE_INVL:

989

val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)

988

val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)

990

| DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);

989

| DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);

991

break;

990

break;

992

default:

991

default:

993

BUG();

992

BUG();

994

}

993

}

995

val |= DMA_CCMD_ICC;

994

val |= DMA_CCMD_ICC;

996

995

997

spin_lock_irqsave(&iommu->register_lock, flag);

996

spin_lock_irqsave(&iommu->register_lock, flag);

998

dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);

997

dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);

999

998

1000

/* Make sure hardware complete it */

999

/* Make sure hardware complete it */

1001

IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,

1000

IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,

1002

dmar_readq, (!(val & DMA_CCMD_ICC)), val);

1001

dmar_readq, (!(val & DMA_CCMD_ICC)), val);

1003

1002

1004

spin_unlock_irqrestore(&iommu->register_lock, flag);

1003

spin_unlock_irqrestore(&iommu->register_lock, flag);

1005

}

1004

}

1006

1005

1007

/* return value determine if we need a write buffer flush */

1006

/* return value determine if we need a write buffer flush */

1008

static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,

1007

static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,

1009

u64 addr, unsigned int size_order, u64 type)

1008

u64 addr, unsigned int size_order, u64 type)

1010

{

1009

{

1011

int tlb_offset = ecap_iotlb_offset(iommu->ecap);

1010

int tlb_offset = ecap_iotlb_offset(iommu->ecap);

1012

u64 val = 0, val_iva = 0;

1011

u64 val = 0, val_iva = 0;

1013

unsigned long flag;

1012

unsigned long flag;

1014

1013

1015

switch (type) {

1014

switch (type) {

1016

case DMA_TLB_GLOBAL_FLUSH:

1015

case DMA_TLB_GLOBAL_FLUSH:

1017

/* global flush doesn't need set IVA_REG */

1016

/* global flush doesn't need set IVA_REG */

1018

val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;

1017

val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;

1019

break;

1018

break;

1020

case DMA_TLB_DSI_FLUSH:

1019

case DMA_TLB_DSI_FLUSH:

1021

val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);

1020

val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);

1022

break;

1021

break;

1023

case DMA_TLB_PSI_FLUSH:

1022

case DMA_TLB_PSI_FLUSH:

1024

val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);

1023

val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);

1025

/* Note: always flush non-leaf currently */

1024

/* Note: always flush non-leaf currently */

1026

val_iva = size_order | addr;

1025

val_iva = size_order | addr;

1027

break;

1026

break;

1028

default:

1027

default:

1029

BUG();

1028

BUG();

1030

}

1029

}

1031

/* Note: set drain read/write */

1030

/* Note: set drain read/write */

1032

#if 0

1031

#if 0

1033

/*

1032

/*

1034

* This is probably to be super secure.. Looks like we can

1033

* This is probably to be super secure.. Looks like we can

1035

* ignore it without any impact.

1034

* ignore it without any impact.

1036

*/

1035

*/

1037

if (cap_read_drain(iommu->cap))

1036

if (cap_read_drain(iommu->cap))

1038

val |= DMA_TLB_READ_DRAIN;

1037

val |= DMA_TLB_READ_DRAIN;

1039

#endif

1038

#endif

1040

if (cap_write_drain(iommu->cap))

1039

if (cap_write_drain(iommu->cap))

1041

val |= DMA_TLB_WRITE_DRAIN;

1040

val |= DMA_TLB_WRITE_DRAIN;

1042

1041

1043

spin_lock_irqsave(&iommu->register_lock, flag);

1042

spin_lock_irqsave(&iommu->register_lock, flag);

1044

/* Note: Only uses first TLB reg currently */

1043

/* Note: Only uses first TLB reg currently */

1045

if (val_iva)

1044

if (val_iva)

1046

dmar_writeq(iommu->reg + tlb_offset, val_iva);

1045

dmar_writeq(iommu->reg + tlb_offset, val_iva);

1047

dmar_writeq(iommu->reg + tlb_offset + 8, val);

1046

dmar_writeq(iommu->reg + tlb_offset + 8, val);

1048

1047

1049

/* Make sure hardware complete it */

1048

/* Make sure hardware complete it */

1050

IOMMU_WAIT_OP(iommu, tlb_offset + 8,

1049

IOMMU_WAIT_OP(iommu, tlb_offset + 8,

1051

dmar_readq, (!(val & DMA_TLB_IVT)), val);

1050

dmar_readq, (!(val & DMA_TLB_IVT)), val);

1052

1051

1053

spin_unlock_irqrestore(&iommu->register_lock, flag);

1052

spin_unlock_irqrestore(&iommu->register_lock, flag);

1054

1053

1055

/* check IOTLB invalidation granularity */

1054

/* check IOTLB invalidation granularity */

1056

if (DMA_TLB_IAIG(val) == 0)

1055

if (DMA_TLB_IAIG(val) == 0)

1057

printk(KERN_ERR"IOMMU: flush IOTLB failed\n");

1056

printk(KERN_ERR"IOMMU: flush IOTLB failed\n");

1058

if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))

1057

if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))

1059

pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",

1058

pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",

1060

(unsigned long long)DMA_TLB_IIRG(type),

1059

(unsigned long long)DMA_TLB_IIRG(type),

1061

(unsigned long long)DMA_TLB_IAIG(val));

1060

(unsigned long long)DMA_TLB_IAIG(val));

1062

}

1061

}

1063

1062

1064

static struct device_domain_info *iommu_support_dev_iotlb(

1063

static struct device_domain_info *iommu_support_dev_iotlb(

1065

struct dmar_domain *domain, int segment, u8 bus, u8 devfn)

1064

struct dmar_domain *domain, int segment, u8 bus, u8 devfn)

1066

{

1065

{

1067

int found = 0;

1066

int found = 0;

1068

unsigned long flags;

1067

unsigned long flags;

1069

struct device_domain_info *info;

1068

struct device_domain_info *info;

1070

struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);

1069

struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);

1071

1070

1072

if (!ecap_dev_iotlb_support(iommu->ecap))

1071

if (!ecap_dev_iotlb_support(iommu->ecap))

1073

return NULL;

1072

return NULL;

1074

1073

1075

if (!iommu->qi)

1074

if (!iommu->qi)

1076

return NULL;

1075

return NULL;

1077

1076

1078

spin_lock_irqsave(&device_domain_lock, flags);

1077

spin_lock_irqsave(&device_domain_lock, flags);

1079

list_for_each_entry(info, &domain->devices, link)

1078

list_for_each_entry(info, &domain->devices, link)

1080

if (info->bus == bus && info->devfn == devfn) {

1079

if (info->bus == bus && info->devfn == devfn) {

1081

found = 1;

1080

found = 1;

1082

break;

1081

break;

1083

}

1082

}

1084

spin_unlock_irqrestore(&device_domain_lock, flags);

1083

spin_unlock_irqrestore(&device_domain_lock, flags);

1085

1084

1086

if (!found || !info->dev)

1085

if (!found || !info->dev)

1087

return NULL;

1086

return NULL;

1088

1087

1089

if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))

1088

if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))

1090

return NULL;

1089

return NULL;

1091

1090

1092

if (!dmar_find_matched_atsr_unit(info->dev))

1091

if (!dmar_find_matched_atsr_unit(info->dev))

1093

return NULL;

1092

return NULL;

1094

1093

1095

info->iommu = iommu;

1094

info->iommu = iommu;

1096

1095

1097

return info;

1096

return info;

1098

}

1097

}

1099

1098

1100

static void iommu_enable_dev_iotlb(struct device_domain_info *info)

1099

static void iommu_enable_dev_iotlb(struct device_domain_info *info)

1101

{

1100

{

1102

if (!info)

1101

if (!info)

1103

return;

1102

return;

1104

1103

1105

pci_enable_ats(info->dev, VTD_PAGE_SHIFT);

1104

pci_enable_ats(info->dev, VTD_PAGE_SHIFT);

1106

}

1105

}

1107

1106

1108

static void iommu_disable_dev_iotlb(struct device_domain_info *info)

1107

static void iommu_disable_dev_iotlb(struct device_domain_info *info)

1109

{

1108

{

1110

if (!info->dev || !pci_ats_enabled(info->dev))

1109

if (!info->dev || !pci_ats_enabled(info->dev))

1111

return;

1110

return;

1112

1111

1113

pci_disable_ats(info->dev);

1112

pci_disable_ats(info->dev);

1114

}

1113

}

1115

1114

1116

static void iommu_flush_dev_iotlb(struct dmar_domain *domain,

1115

static void iommu_flush_dev_iotlb(struct dmar_domain *domain,

1117

u64 addr, unsigned mask)

1116

u64 addr, unsigned mask)

1118

{

1117

{

1119

u16 sid, qdep;

1118

u16 sid, qdep;

1120

unsigned long flags;

1119

unsigned long flags;

1121

struct device_domain_info *info;

1120

struct device_domain_info *info;

1122

1121

1123

spin_lock_irqsave(&device_domain_lock, flags);

1122

spin_lock_irqsave(&device_domain_lock, flags);

1124

list_for_each_entry(info, &domain->devices, link) {

1123

list_for_each_entry(info, &domain->devices, link) {

1125

if (!info->dev || !pci_ats_enabled(info->dev))

1124

if (!info->dev || !pci_ats_enabled(info->dev))

1126

continue;

1125

continue;

1127

1126

1128

sid = info->bus << 8 | info->devfn;

1127

sid = info->bus << 8 | info->devfn;

1129

qdep = pci_ats_queue_depth(info->dev);

1128

qdep = pci_ats_queue_depth(info->dev);

1130

qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);

1129

qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);

1131

}

1130

}

1132

spin_unlock_irqrestore(&device_domain_lock, flags);

1131

spin_unlock_irqrestore(&device_domain_lock, flags);

1133

}

1132

}

1134

1133

1135

static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,

1134

static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,

1136

unsigned long pfn, unsigned int pages, int map)

1135

unsigned long pfn, unsigned int pages, int map)

1137

{

1136

{

1138

unsigned int mask = ilog2(__roundup_pow_of_two(pages));

1137

unsigned int mask = ilog2(__roundup_pow_of_two(pages));

1139

uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;

1138

uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;

1140

1139

1141

BUG_ON(pages == 0);

1140

BUG_ON(pages == 0);

1142

1141

1143

/*

1142

/*

1144

* Fallback to domain selective flush if no PSI support or the size is

1143

* Fallback to domain selective flush if no PSI support or the size is

1145

* too big.

1144

* too big.

1146

* PSI requires page size to be 2 ^ x, and the base address is naturally

1145

* PSI requires page size to be 2 ^ x, and the base address is naturally

1147

* aligned to the size

1146

* aligned to the size

1148

*/

1147

*/

1149

if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))

1148

if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))

1150

iommu->flush.flush_iotlb(iommu, did, 0, 0,

1149

iommu->flush.flush_iotlb(iommu, did, 0, 0,

1151

DMA_TLB_DSI_FLUSH);

1150

DMA_TLB_DSI_FLUSH);

1152

else

1151

else

1153

iommu->flush.flush_iotlb(iommu, did, addr, mask,

1152

iommu->flush.flush_iotlb(iommu, did, addr, mask,

1154

DMA_TLB_PSI_FLUSH);

1153

DMA_TLB_PSI_FLUSH);

1155

1154

1156

/*

1155

/*

1157

* In caching mode, changes of pages from non-present to present require

1156

* In caching mode, changes of pages from non-present to present require

1158

* flush. However, device IOTLB doesn't need to be flushed in this case.

1157

* flush. However, device IOTLB doesn't need to be flushed in this case.

1159

*/

1158

*/

1160

if (!cap_caching_mode(iommu->cap) || !map)

1159

if (!cap_caching_mode(iommu->cap) || !map)

1161

iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);

1160

iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);

1162

}

1161

}

1163

1162

1164

static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)

1163

static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)

1165

{

1164

{

1166

u32 pmen;

1165

u32 pmen;

1167

unsigned long flags;

1166

unsigned long flags;

1168

1167

1169

spin_lock_irqsave(&iommu->register_lock, flags);

1168

spin_lock_irqsave(&iommu->register_lock, flags);

1170

pmen = readl(iommu->reg + DMAR_PMEN_REG);

1169

pmen = readl(iommu->reg + DMAR_PMEN_REG);

1171

pmen &= ~DMA_PMEN_EPM;

1170

pmen &= ~DMA_PMEN_EPM;

1172

writel(pmen, iommu->reg + DMAR_PMEN_REG);

1171

writel(pmen, iommu->reg + DMAR_PMEN_REG);

1173

1172

1174

/* wait for the protected region status bit to clear */

1173

/* wait for the protected region status bit to clear */

1175

IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,

1174

IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,

1176

readl, !(pmen & DMA_PMEN_PRS), pmen);

1175

readl, !(pmen & DMA_PMEN_PRS), pmen);

1177

1176

1178

spin_unlock_irqrestore(&iommu->register_lock, flags);

1177

spin_unlock_irqrestore(&iommu->register_lock, flags);

1179

}

1178

}

1180

1179

1181

static int iommu_enable_translation(struct intel_iommu *iommu)

1180

static int iommu_enable_translation(struct intel_iommu *iommu)

1182

{

1181

{

1183

u32 sts;

1182

u32 sts;

1184

unsigned long flags;

1183

unsigned long flags;

1185

1184

1186

spin_lock_irqsave(&iommu->register_lock, flags);

1185

spin_lock_irqsave(&iommu->register_lock, flags);

1187

iommu->gcmd |= DMA_GCMD_TE;

1186

iommu->gcmd |= DMA_GCMD_TE;

1188

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

1187

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

1189

1188

1190

/* Make sure hardware complete it */

1189

/* Make sure hardware complete it */

1191

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

1190

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

1192

readl, (sts & DMA_GSTS_TES), sts);

1191

readl, (sts & DMA_GSTS_TES), sts);

1193

1192

1194

spin_unlock_irqrestore(&iommu->register_lock, flags);

1193

spin_unlock_irqrestore(&iommu->register_lock, flags);

1195

return 0;

1194

return 0;

1196

}

1195

}

1197

1196

1198

static int iommu_disable_translation(struct intel_iommu *iommu)

1197

static int iommu_disable_translation(struct intel_iommu *iommu)

1199

{

1198

{

1200

u32 sts;

1199

u32 sts;

1201

unsigned long flag;

1200

unsigned long flag;

1202

1201

1203

spin_lock_irqsave(&iommu->register_lock, flag);

1202

spin_lock_irqsave(&iommu->register_lock, flag);

1204

iommu->gcmd &= ~DMA_GCMD_TE;

1203

iommu->gcmd &= ~DMA_GCMD_TE;

1205

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

1204

writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

1206

1205

1207

/* Make sure hardware complete it */

1206

/* Make sure hardware complete it */

1208

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

1207

IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,

1209

readl, (!(sts & DMA_GSTS_TES)), sts);

1208

readl, (!(sts & DMA_GSTS_TES)), sts);

1210

1209

1211

spin_unlock_irqrestore(&iommu->register_lock, flag);

1210

spin_unlock_irqrestore(&iommu->register_lock, flag);

1212

return 0;

1211

return 0;

1213

}

1212

}

1214

1213

1215

1214

1216

static int iommu_init_domains(struct intel_iommu *iommu)

1215

static int iommu_init_domains(struct intel_iommu *iommu)

1217

{

1216

{

1218

unsigned long ndomains;

1217

unsigned long ndomains;

1219

unsigned long nlongs;

1218

unsigned long nlongs;

1220

1219

1221

ndomains = cap_ndoms(iommu->cap);

1220

ndomains = cap_ndoms(iommu->cap);

1222

pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,

1221

pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,

1223

ndomains);

1222

ndomains);

1224

nlongs = BITS_TO_LONGS(ndomains);

1223

nlongs = BITS_TO_LONGS(ndomains);

1225

1224

1226

spin_lock_init(&iommu->lock);

1225

spin_lock_init(&iommu->lock);

1227

1226

1228

/* TBD: there might be 64K domains,

1227

/* TBD: there might be 64K domains,

1229

* consider other allocation for future chip

1228

* consider other allocation for future chip

1230

*/

1229

*/

1231

iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);

1230

iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);

1232

if (!iommu->domain_ids) {

1231

if (!iommu->domain_ids) {

1233

printk(KERN_ERR "Allocating domain id array failed\n");

1232

printk(KERN_ERR "Allocating domain id array failed\n");

1234

return -ENOMEM;

1233

return -ENOMEM;

1235

}

1234

}

1236

iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),

1235

iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),

1237

GFP_KERNEL);

1236

GFP_KERNEL);

1238

if (!iommu->domains) {

1237

if (!iommu->domains) {

1239

printk(KERN_ERR "Allocating domain array failed\n");

1238

printk(KERN_ERR "Allocating domain array failed\n");

1240

return -ENOMEM;

1239

return -ENOMEM;

1241

}

1240

}

1242

1241

1243

/*

1242

/*

1244

* if Caching mode is set, then invalid translations are tagged

1243

* if Caching mode is set, then invalid translations are tagged

1245

* with domainid 0. Hence we need to pre-allocate it.

1244

* with domainid 0. Hence we need to pre-allocate it.

1246

*/

1245

*/

1247

if (cap_caching_mode(iommu->cap))

1246

if (cap_caching_mode(iommu->cap))

1248

set_bit(0, iommu->domain_ids);

1247

set_bit(0, iommu->domain_ids);

1249

return 0;

1248

return 0;

1250

}

1249

}

1251

1250

1252

1251

1253

static void domain_exit(struct dmar_domain *domain);

1252

static void domain_exit(struct dmar_domain *domain);

1254

static void vm_domain_exit(struct dmar_domain *domain);

1253

static void vm_domain_exit(struct dmar_domain *domain);

1255

1254

1256

void free_dmar_iommu(struct intel_iommu *iommu)

1255

void free_dmar_iommu(struct intel_iommu *iommu)

1257

{

1256

{

1258

struct dmar_domain *domain;

1257

struct dmar_domain *domain;

1259

int i;

1258

int i;

1260

unsigned long flags;

1259

unsigned long flags;

1261

1260

1262

if ((iommu->domains) && (iommu->domain_ids)) {

1261

if ((iommu->domains) && (iommu->domain_ids)) {

1263

for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {

1262

for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {

1264

domain = iommu->domains[i];

1263

domain = iommu->domains[i];

1265

clear_bit(i, iommu->domain_ids);

1264

clear_bit(i, iommu->domain_ids);

1266

1265

1267

spin_lock_irqsave(&domain->iommu_lock, flags);

1266

spin_lock_irqsave(&domain->iommu_lock, flags);

1268

if (--domain->iommu_count == 0) {

1267

if (--domain->iommu_count == 0) {

1269

if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)

1268

if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)

1270

vm_domain_exit(domain);

1269

vm_domain_exit(domain);

1271

else

1270

else

1272

domain_exit(domain);

1271

domain_exit(domain);

1273

}

1272

}

1274

spin_unlock_irqrestore(&domain->iommu_lock, flags);

1273

spin_unlock_irqrestore(&domain->iommu_lock, flags);

1275

}

1274

}

1276

}

1275

}

1277

1276

1278

if (iommu->gcmd & DMA_GCMD_TE)

1277

if (iommu->gcmd & DMA_GCMD_TE)

1279

iommu_disable_translation(iommu);

1278

iommu_disable_translation(iommu);

1280

1279

1281

if (iommu->irq) {

1280

if (iommu->irq) {

1282

irq_set_handler_data(iommu->irq, NULL);

1281

irq_set_handler_data(iommu->irq, NULL);

1283

/* This will mask the irq */

1282

/* This will mask the irq */

1284

free_irq(iommu->irq, iommu);

1283

free_irq(iommu->irq, iommu);

1285

destroy_irq(iommu->irq);

1284

destroy_irq(iommu->irq);

1286

}

1285

}

1287

1286

1288

kfree(iommu->domains);

1287

kfree(iommu->domains);

1289

kfree(iommu->domain_ids);

1288

kfree(iommu->domain_ids);

1290

1289

1291

g_iommus[iommu->seq_id] = NULL;

1290

g_iommus[iommu->seq_id] = NULL;

1292

1291

1293

/* if all iommus are freed, free g_iommus */

1292

/* if all iommus are freed, free g_iommus */

1294

for (i = 0; i < g_num_of_iommus; i++) {

1293

for (i = 0; i < g_num_of_iommus; i++) {

1295

if (g_iommus[i])

1294

if (g_iommus[i])

1296

break;

1295

break;

1297

}

1296

}

1298

1297

1299

if (i == g_num_of_iommus)

1298

if (i == g_num_of_iommus)

1300

kfree(g_iommus);

1299

kfree(g_iommus);

1301

1300

1302

/* free context mapping */

1301

/* free context mapping */

1303

free_context_table(iommu);

1302

free_context_table(iommu);

1304

}

1303

}

1305

1304

1306

static struct dmar_domain *alloc_domain(void)

1305

static struct dmar_domain *alloc_domain(void)

1307

{

1306

{

1308

struct dmar_domain *domain;

1307

struct dmar_domain *domain;

1309

1308

1310

domain = alloc_domain_mem();

1309

domain = alloc_domain_mem();

1311

if (!domain)

1310

if (!domain)

1312

return NULL;

1311

return NULL;

1313

1312

1314

domain->nid = -1;

1313

domain->nid = -1;

1315

memset(&domain->iommu_bmp, 0, sizeof(unsigned long));

1314

memset(&domain->iommu_bmp, 0, sizeof(unsigned long));

1316

domain->flags = 0;

1315

domain->flags = 0;

1317

1316

1318

return domain;

1317

return domain;

1319

}

1318

}

1320

1319

1321

static int iommu_attach_domain(struct dmar_domain *domain,

1320

static int iommu_attach_domain(struct dmar_domain *domain,

1322

struct intel_iommu *iommu)

1321

struct intel_iommu *iommu)

1323

{

1322

{

1324

int num;

1323

int num;

1325

unsigned long ndomains;

1324

unsigned long ndomains;

1326

unsigned long flags;

1325

unsigned long flags;

1327

1326

1328

ndomains = cap_ndoms(iommu->cap);

1327

ndomains = cap_ndoms(iommu->cap);

1329

1328

1330

spin_lock_irqsave(&iommu->lock, flags);

1329

spin_lock_irqsave(&iommu->lock, flags);

1331

1330

1332

num = find_first_zero_bit(iommu->domain_ids, ndomains);

1331

num = find_first_zero_bit(iommu->domain_ids, ndomains);

1333

if (num >= ndomains) {

1332

if (num >= ndomains) {

1334

spin_unlock_irqrestore(&iommu->lock, flags);

1333

spin_unlock_irqrestore(&iommu->lock, flags);

1335

printk(KERN_ERR "IOMMU: no free domain ids\n");

1334

printk(KERN_ERR "IOMMU: no free domain ids\n");

1336

return -ENOMEM;

1335

return -ENOMEM;

1337

}

1336

}

1338

1337

1339

domain->id = num;

1338

domain->id = num;

1340

set_bit(num, iommu->domain_ids);

1339

set_bit(num, iommu->domain_ids);

1341

set_bit(iommu->seq_id, &domain->iommu_bmp);

1340

set_bit(iommu->seq_id, &domain->iommu_bmp);

1342

iommu->domains[num] = domain;

1341

iommu->domains[num] = domain;

1343

spin_unlock_irqrestore(&iommu->lock, flags);

1342

spin_unlock_irqrestore(&iommu->lock, flags);

1344

1343

1345

return 0;

1344

return 0;

1346

}

1345

}

1347

1346

1348

static void iommu_detach_domain(struct dmar_domain *domain,

1347

static void iommu_detach_domain(struct dmar_domain *domain,

1349

struct intel_iommu *iommu)

1348

struct intel_iommu *iommu)

1350

{

1349

{

1351

unsigned long flags;

1350

unsigned long flags;

1352

int num, ndomains;

1351

int num, ndomains;

1353

int found = 0;

1352

int found = 0;

1354

1353

1355

spin_lock_irqsave(&iommu->lock, flags);

1354

spin_lock_irqsave(&iommu->lock, flags);

1356

ndomains = cap_ndoms(iommu->cap);

1355

ndomains = cap_ndoms(iommu->cap);

1357

for_each_set_bit(num, iommu->domain_ids, ndomains) {

1356

for_each_set_bit(num, iommu->domain_ids, ndomains) {

1358

if (iommu->domains[num] == domain) {

1357

if (iommu->domains[num] == domain) {

1359

found = 1;

1358

found = 1;

1360

break;

1359

break;

1361

}

1360

}

1362

}

1361

}

1363

1362

1364

if (found) {

1363

if (found) {

1365

clear_bit(num, iommu->domain_ids);

1364

clear_bit(num, iommu->domain_ids);

1366

clear_bit(iommu->seq_id, &domain->iommu_bmp);

1365

clear_bit(iommu->seq_id, &domain->iommu_bmp);

1367

iommu->domains[num] = NULL;

1366

iommu->domains[num] = NULL;

1368

}

1367

}

1369

spin_unlock_irqrestore(&iommu->lock, flags);

1368

spin_unlock_irqrestore(&iommu->lock, flags);

1370

}

1369

}

1371

1370

1372

static struct iova_domain reserved_iova_list;

1371

static struct iova_domain reserved_iova_list;

1373

static struct lock_class_key reserved_rbtree_key;

1372

static struct lock_class_key reserved_rbtree_key;

1374

1373

1375

static int dmar_init_reserved_ranges(void)

1374

static int dmar_init_reserved_ranges(void)

1376

{

1375

{

1377

struct pci_dev *pdev = NULL;

1376

struct pci_dev *pdev = NULL;

1378

struct iova *iova;

1377

struct iova *iova;

1379

int i;

1378

int i;

1380

1379

1381

init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);

1380

init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);

1382

1381

1383

lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,

1382

lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,

1384

&reserved_rbtree_key);

1383

&reserved_rbtree_key);

1385

1384

1386

/* IOAPIC ranges shouldn't be accessed by DMA */

1385

/* IOAPIC ranges shouldn't be accessed by DMA */

1387

iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),

1386

iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),

1388

IOVA_PFN(IOAPIC_RANGE_END));

1387

IOVA_PFN(IOAPIC_RANGE_END));

1389

if (!iova) {

1388

if (!iova) {

1390

printk(KERN_ERR "Reserve IOAPIC range failed\n");

1389

printk(KERN_ERR "Reserve IOAPIC range failed\n");

1391

return -ENODEV;

1390

return -ENODEV;

1392

}

1391

}

1393

1392

1394

/* Reserve all PCI MMIO to avoid peer-to-peer access */

1393

/* Reserve all PCI MMIO to avoid peer-to-peer access */

1395

for_each_pci_dev(pdev) {

1394

for_each_pci_dev(pdev) {

1396

struct resource *r;

1395

struct resource *r;

1397

1396

1398

for (i = 0; i < PCI_NUM_RESOURCES; i++) {

1397

for (i = 0; i < PCI_NUM_RESOURCES; i++) {

1399

r = &pdev->resource[i];

1398

r = &pdev->resource[i];

1400

if (!r->flags || !(r->flags & IORESOURCE_MEM))

1399

if (!r->flags || !(r->flags & IORESOURCE_MEM))

1401

continue;

1400

continue;

1402

iova = reserve_iova(&reserved_iova_list,

1401

iova = reserve_iova(&reserved_iova_list,

1403

IOVA_PFN(r->start),

1402

IOVA_PFN(r->start),

1404

IOVA_PFN(r->end));

1403

IOVA_PFN(r->end));

1405

if (!iova) {

1404

if (!iova) {

1406

printk(KERN_ERR "Reserve iova failed\n");

1405

printk(KERN_ERR "Reserve iova failed\n");

1407

return -ENODEV;

1406

return -ENODEV;

1408

}

1407

}

1409

}

1408

}

1410

}

1409

}

1411

return 0;

1410

return 0;

1412

}

1411

}

1413

1412

1414

static void domain_reserve_special_ranges(struct dmar_domain *domain)

1413

static void domain_reserve_special_ranges(struct dmar_domain *domain)

1415

{

1414

{

1416

copy_reserved_iova(&reserved_iova_list, &domain->iovad);

1415

copy_reserved_iova(&reserved_iova_list, &domain->iovad);

1417

}

1416

}

1418

1417

1419

static inline int guestwidth_to_adjustwidth(int gaw)

1418

static inline int guestwidth_to_adjustwidth(int gaw)

1420

{

1419

{

1421

int agaw;

1420

int agaw;

1422

int r = (gaw - 12) % 9;

1421

int r = (gaw - 12) % 9;

1423

1422

1424

if (r == 0)

1423

if (r == 0)

1425

agaw = gaw;

1424

agaw = gaw;

1426

else

1425

else

1427

agaw = gaw + 9 - r;

1426

agaw = gaw + 9 - r;

1428

if (agaw > 64)

1427

if (agaw > 64)

1429

agaw = 64;

1428

agaw = 64;

1430

return agaw;

1429

return agaw;

1431

}

1430

}

1432

1431

1433

static int domain_init(struct dmar_domain *domain, int guest_width)

1432

static int domain_init(struct dmar_domain *domain, int guest_width)

1434

{

1433

{

1435

struct intel_iommu *iommu;

1434

struct intel_iommu *iommu;

1436

int adjust_width, agaw;

1435

int adjust_width, agaw;

1437

unsigned long sagaw;

1436

unsigned long sagaw;

1438

1437

1439

init_iova_domain(&domain->iovad, DMA_32BIT_PFN);

1438

init_iova_domain(&domain->iovad, DMA_32BIT_PFN);

1440

spin_lock_init(&domain->iommu_lock);

1439

spin_lock_init(&domain->iommu_lock);

1441

1440

1442

domain_reserve_special_ranges(domain);

1441

domain_reserve_special_ranges(domain);

1443

1442

1444

/* calculate AGAW */

1443

/* calculate AGAW */

1445

iommu = domain_get_iommu(domain);

1444

iommu = domain_get_iommu(domain);

1446

if (guest_width > cap_mgaw(iommu->cap))

1445

if (guest_width > cap_mgaw(iommu->cap))

1447

guest_width = cap_mgaw(iommu->cap);

1446

guest_width = cap_mgaw(iommu->cap);

1448

domain->gaw = guest_width;

1447

domain->gaw = guest_width;

1449

adjust_width = guestwidth_to_adjustwidth(guest_width);

1448

adjust_width = guestwidth_to_adjustwidth(guest_width);

1450

agaw = width_to_agaw(adjust_width);

1449

agaw = width_to_agaw(adjust_width);

1451

sagaw = cap_sagaw(iommu->cap);

1450

sagaw = cap_sagaw(iommu->cap);

1452

if (!test_bit(agaw, &sagaw)) {

1451

if (!test_bit(agaw, &sagaw)) {

1453

/* hardware doesn't support it, choose a bigger one */

1452

/* hardware doesn't support it, choose a bigger one */

1454

pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);

1453

pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);

1455

agaw = find_next_bit(&sagaw, 5, agaw);

1454

agaw = find_next_bit(&sagaw, 5, agaw);

1456

if (agaw >= 5)

1455

if (agaw >= 5)

1457

return -ENODEV;

1456

return -ENODEV;

1458

}

1457

}

1459

domain->agaw = agaw;

1458

domain->agaw = agaw;

1460

INIT_LIST_HEAD(&domain->devices);

1459

INIT_LIST_HEAD(&domain->devices);

1461

1460

1462

if (ecap_coherent(iommu->ecap))

1461

if (ecap_coherent(iommu->ecap))

1463

domain->iommu_coherency = 1;

1462

domain->iommu_coherency = 1;

1464

else

1463

else

1465

domain->iommu_coherency = 0;

1464

domain->iommu_coherency = 0;

1466

1465

1467

if (ecap_sc_support(iommu->ecap))

1466

if (ecap_sc_support(iommu->ecap))

1468

domain->iommu_snooping = 1;

1467

domain->iommu_snooping = 1;

1469

else

1468

else

1470

domain->iommu_snooping = 0;

1469

domain->iommu_snooping = 0;

1471

1470

1472

domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));

1471

domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));

1473

domain->iommu_count = 1;

1472

domain->iommu_count = 1;

1474

domain->nid = iommu->node;

1473

domain->nid = iommu->node;

1475

1474

1476

/* always allocate the top pgd */

1475

/* always allocate the top pgd */

1477

domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);

1476

domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);

1478

if (!domain->pgd)

1477

if (!domain->pgd)

1479

return -ENOMEM;

1478

return -ENOMEM;

1480

__iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);

1479

__iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);

1481

return 0;

1480

return 0;

1482

}

1481

}

1483

1482

1484

static void domain_exit(struct dmar_domain *domain)

1483

static void domain_exit(struct dmar_domain *domain)

1485

{

1484

{

1486

struct dmar_drhd_unit *drhd;

1485

struct dmar_drhd_unit *drhd;

1487

struct intel_iommu *iommu;

1486

struct intel_iommu *iommu;

1488

1487

1489

/* Domain 0 is reserved, so dont process it */

1488

/* Domain 0 is reserved, so dont process it */

1490

if (!domain)

1489

if (!domain)

1491

return;

1490

return;

1492

1491

1493

/* Flush any lazy unmaps that may reference this domain */

1492

/* Flush any lazy unmaps that may reference this domain */

1494

if (!intel_iommu_strict)

1493

if (!intel_iommu_strict)

1495

flush_unmaps_timeout(0);

1494

flush_unmaps_timeout(0);

1496

1495

1497

domain_remove_dev_info(domain);

1496

domain_remove_dev_info(domain);

1498

/* destroy iovas */

1497

/* destroy iovas */

1499

put_iova_domain(&domain->iovad);

1498

put_iova_domain(&domain->iovad);

1500

1499

1501

/* clear ptes */

1500

/* clear ptes */

1502

dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

1501

dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

1503

1502

1504

/* free page tables */

1503

/* free page tables */

1505

dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

1504

dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

1506

1505

1507

for_each_active_iommu(iommu, drhd)

1506

for_each_active_iommu(iommu, drhd)

1508

if (test_bit(iommu->seq_id, &domain->iommu_bmp))

1507

if (test_bit(iommu->seq_id, &domain->iommu_bmp))

1509

iommu_detach_domain(domain, iommu);

1508

iommu_detach_domain(domain, iommu);

1510

1509

1511

free_domain_mem(domain);

1510

free_domain_mem(domain);

1512

}

1511

}

1513

1512

1514

static int domain_context_mapping_one(struct dmar_domain *domain, int segment,

1513

static int domain_context_mapping_one(struct dmar_domain *domain, int segment,

1515

u8 bus, u8 devfn, int translation)

1514

u8 bus, u8 devfn, int translation)

1516

{

1515

{

1517

struct context_entry *context;

1516

struct context_entry *context;

1518

unsigned long flags;

1517

unsigned long flags;

1519

struct intel_iommu *iommu;

1518

struct intel_iommu *iommu;

1520

struct dma_pte *pgd;

1519

struct dma_pte *pgd;

1521

unsigned long num;

1520

unsigned long num;

1522

unsigned long ndomains;

1521

unsigned long ndomains;

1523

int id;

1522

int id;

1524

int agaw;

1523

int agaw;

1525

struct device_domain_info *info = NULL;

1524

struct device_domain_info *info = NULL;

1526

1525

1527

pr_debug("Set context mapping for %02x:%02x.%d\n",

1526

pr_debug("Set context mapping for %02x:%02x.%d\n",

1528

bus, PCI_SLOT(devfn), PCI_FUNC(devfn));

1527

bus, PCI_SLOT(devfn), PCI_FUNC(devfn));

1529

1528

1530

BUG_ON(!domain->pgd);

1529

BUG_ON(!domain->pgd);

1531

BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&

1530

BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&

1532

translation != CONTEXT_TT_MULTI_LEVEL);

1531

translation != CONTEXT_TT_MULTI_LEVEL);

1533

1532

1534

iommu = device_to_iommu(segment, bus, devfn);

1533

iommu = device_to_iommu(segment, bus, devfn);

1535

if (!iommu)

1534

if (!iommu)

1536

return -ENODEV;

1535

return -ENODEV;

1537

1536

1538

context = device_to_context_entry(iommu, bus, devfn);

1537

context = device_to_context_entry(iommu, bus, devfn);

1539

if (!context)

1538

if (!context)

1540

return -ENOMEM;

1539

return -ENOMEM;

1541

spin_lock_irqsave(&iommu->lock, flags);

1540

spin_lock_irqsave(&iommu->lock, flags);

1542

if (context_present(context)) {

1541

if (context_present(context)) {

1543

spin_unlock_irqrestore(&iommu->lock, flags);

1542

spin_unlock_irqrestore(&iommu->lock, flags);

1544

return 0;

1543

return 0;

1545

}

1544

}

1546

1545

1547

id = domain->id;

1546

id = domain->id;

1548

pgd = domain->pgd;

1547

pgd = domain->pgd;

1549

1548

1550

if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||

1549

if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||

1551

domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {

1550

domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {

1552

int found = 0;

1551

int found = 0;

1553

1552

1554

/* find an available domain id for this device in iommu */

1553

/* find an available domain id for this device in iommu */

1555

ndomains = cap_ndoms(iommu->cap);

1554

ndomains = cap_ndoms(iommu->cap);

1556

for_each_set_bit(num, iommu->domain_ids, ndomains) {

1555

for_each_set_bit(num, iommu->domain_ids, ndomains) {

1557

if (iommu->domains[num] == domain) {

1556

if (iommu->domains[num] == domain) {

1558

id = num;

1557

id = num;

1559

found = 1;

1558

found = 1;

1560

break;

1559

break;

1561

}

1560

}

1562

}

1561

}

1563

1562

1564

if (found == 0) {

1563

if (found == 0) {

1565

num = find_first_zero_bit(iommu->domain_ids, ndomains);

1564

num = find_first_zero_bit(iommu->domain_ids, ndomains);

1566

if (num >= ndomains) {

1565

if (num >= ndomains) {

1567

spin_unlock_irqrestore(&iommu->lock, flags);

1566

spin_unlock_irqrestore(&iommu->lock, flags);

1568

printk(KERN_ERR "IOMMU: no free domain ids\n");

1567

printk(KERN_ERR "IOMMU: no free domain ids\n");

1569

return -EFAULT;

1568

return -EFAULT;

1570

}

1569

}

1571

1570

1572

set_bit(num, iommu->domain_ids);

1571

set_bit(num, iommu->domain_ids);

1573

iommu->domains[num] = domain;

1572

iommu->domains[num] = domain;

1574

id = num;

1573

id = num;

1575

}

1574

}

1576

1575

1577

/* Skip top levels of page tables for

1576

/* Skip top levels of page tables for

1578

* iommu which has less agaw than default.

1577

* iommu which has less agaw than default.

1579

* Unnecessary for PT mode.

1578

* Unnecessary for PT mode.

1580

*/

1579

*/

1581

if (translation != CONTEXT_TT_PASS_THROUGH) {

1580

if (translation != CONTEXT_TT_PASS_THROUGH) {

1582

for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {

1581

for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {

1583

pgd = phys_to_virt(dma_pte_addr(pgd));

1582

pgd = phys_to_virt(dma_pte_addr(pgd));

1584

if (!dma_pte_present(pgd)) {

1583

if (!dma_pte_present(pgd)) {

1585

spin_unlock_irqrestore(&iommu->lock, flags);

1584

spin_unlock_irqrestore(&iommu->lock, flags);

1586

return -ENOMEM;

1585

return -ENOMEM;

1587

}

1586

}

1588

}

1587

}

1589

}

1588

}

1590

}

1589

}

1591

1590

1592

context_set_domain_id(context, id);

1591

context_set_domain_id(context, id);

1593

1592

1594

if (translation != CONTEXT_TT_PASS_THROUGH) {

1593

if (translation != CONTEXT_TT_PASS_THROUGH) {

1595

info = iommu_support_dev_iotlb(domain, segment, bus, devfn);

1594

info = iommu_support_dev_iotlb(domain, segment, bus, devfn);

1596

translation = info ? CONTEXT_TT_DEV_IOTLB :

1595

translation = info ? CONTEXT_TT_DEV_IOTLB :

1597

CONTEXT_TT_MULTI_LEVEL;

1596

CONTEXT_TT_MULTI_LEVEL;

1598

}

1597

}

1599

/*

1598

/*

1600

* In pass through mode, AW must be programmed to indicate the largest

1599

* In pass through mode, AW must be programmed to indicate the largest

1601

* AGAW value supported by hardware. And ASR is ignored by hardware.

1600

* AGAW value supported by hardware. And ASR is ignored by hardware.

1602

*/

1601

*/

1603

if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))

1602

if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))

1604

context_set_address_width(context, iommu->msagaw);

1603

context_set_address_width(context, iommu->msagaw);

1605

else {

1604

else {

1606

context_set_address_root(context, virt_to_phys(pgd));

1605

context_set_address_root(context, virt_to_phys(pgd));

1607

context_set_address_width(context, iommu->agaw);

1606

context_set_address_width(context, iommu->agaw);

1608

}

1607

}

1609

1608

1610

context_set_translation_type(context, translation);

1609

context_set_translation_type(context, translation);

1611

context_set_fault_enable(context);

1610

context_set_fault_enable(context);

1612

context_set_present(context);

1611

context_set_present(context);

1613

domain_flush_cache(domain, context, sizeof(*context));

1612

domain_flush_cache(domain, context, sizeof(*context));

1614

1613

1615

/*

1614

/*

1616

* It's a non-present to present mapping. If hardware doesn't cache

1615

* It's a non-present to present mapping. If hardware doesn't cache

1617

* non-present entry we only need to flush the write-buffer. If the

1616

* non-present entry we only need to flush the write-buffer. If the

1618

* _does_ cache non-present entries, then it does so in the special

1617

* _does_ cache non-present entries, then it does so in the special

1619

* domain #0, which we have to flush:

1618

* domain #0, which we have to flush:

1620

*/

1619

*/

1621

if (cap_caching_mode(iommu->cap)) {

1620

if (cap_caching_mode(iommu->cap)) {

1622

iommu->flush.flush_context(iommu, 0,

1621

iommu->flush.flush_context(iommu, 0,

1623

(((u16)bus) << 8) | devfn,

1622

(((u16)bus) << 8) | devfn,

1624

DMA_CCMD_MASK_NOBIT,

1623

DMA_CCMD_MASK_NOBIT,

1625

DMA_CCMD_DEVICE_INVL);

1624

DMA_CCMD_DEVICE_INVL);

1626

iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);

1625

iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);

1627

} else {

1626

} else {

1628

iommu_flush_write_buffer(iommu);

1627

iommu_flush_write_buffer(iommu);

1629

}

1628

}

1630

iommu_enable_dev_iotlb(info);

1629

iommu_enable_dev_iotlb(info);

1631

spin_unlock_irqrestore(&iommu->lock, flags);

1630

spin_unlock_irqrestore(&iommu->lock, flags);

1632

1631

1633

spin_lock_irqsave(&domain->iommu_lock, flags);

1632

spin_lock_irqsave(&domain->iommu_lock, flags);

1634

if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {

1633

if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {

1635

domain->iommu_count++;

1634

domain->iommu_count++;

1636

if (domain->iommu_count == 1)

1635

if (domain->iommu_count == 1)

1637

domain->nid = iommu->node;

1636

domain->nid = iommu->node;

1638

domain_update_iommu_cap(domain);

1637

domain_update_iommu_cap(domain);

1639

}

1638

}

1640

spin_unlock_irqrestore(&domain->iommu_lock, flags);

1639

spin_unlock_irqrestore(&domain->iommu_lock, flags);

1641

return 0;

1640

return 0;

1642

}

1641

}

1643

1642

1644

static int

1643

static int

1645

domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,

1644

domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,

1646

int translation)

1645

int translation)

1647

{

1646

{

1648

int ret;

1647

int ret;

1649

struct pci_dev *tmp, *parent;

1648

struct pci_dev *tmp, *parent;

1650

1649

1651

ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),

1650

ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),

1652

pdev->bus->number, pdev->devfn,

1651

pdev->bus->number, pdev->devfn,

1653

translation);

1652

translation);

1654

if (ret)

1653

if (ret)

1655

return ret;

1654

return ret;

1656

1655

1657

/* dependent device mapping */

1656

/* dependent device mapping */

1658

tmp = pci_find_upstream_pcie_bridge(pdev);

1657

tmp = pci_find_upstream_pcie_bridge(pdev);

1659

if (!tmp)

1658

if (!tmp)

1660

return 0;

1659

return 0;

1661

/* Secondary interface's bus number and devfn 0 */

1660

/* Secondary interface's bus number and devfn 0 */

1662

parent = pdev->bus->self;

1661

parent = pdev->bus->self;

1663

while (parent != tmp) {

1662

while (parent != tmp) {

1664

ret = domain_context_mapping_one(domain,

1663

ret = domain_context_mapping_one(domain,

1665

pci_domain_nr(parent->bus),

1664

pci_domain_nr(parent->bus),

1666

parent->bus->number,

1665

parent->bus->number,

1667

parent->devfn, translation);

1666

parent->devfn, translation);

1668

if (ret)

1667

if (ret)

1669

return ret;

1668

return ret;

1670

parent = parent->bus->self;

1669

parent = parent->bus->self;

1671

}

1670

}

1672

if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */

1671

if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */

1673

return domain_context_mapping_one(domain,

1672

return domain_context_mapping_one(domain,

1674

pci_domain_nr(tmp->subordinate),

1673

pci_domain_nr(tmp->subordinate),

1675

tmp->subordinate->number, 0,

1674

tmp->subordinate->number, 0,

1676

translation);

1675

translation);

1677

else /* this is a legacy PCI bridge */

1676

else /* this is a legacy PCI bridge */

1678

return domain_context_mapping_one(domain,

1677

return domain_context_mapping_one(domain,

1679

pci_domain_nr(tmp->bus),

1678

pci_domain_nr(tmp->bus),

1680

tmp->bus->number,

1679

tmp->bus->number,

1681

tmp->devfn,

1680

tmp->devfn,

1682

translation);

1681

translation);

1683

}

1682

}

1684

1683

1685

static int domain_context_mapped(struct pci_dev *pdev)

1684

static int domain_context_mapped(struct pci_dev *pdev)

1686

{

1685

{

1687

int ret;

1686

int ret;

1688

struct pci_dev *tmp, *parent;

1687

struct pci_dev *tmp, *parent;

1689

struct intel_iommu *iommu;

1688

struct intel_iommu *iommu;

1690

1689

1691

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

1690

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

1692

pdev->devfn);

1691

pdev->devfn);

1693

if (!iommu)

1692

if (!iommu)

1694

return -ENODEV;

1693

return -ENODEV;

1695

1694

1696

ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);

1695

ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);

1697

if (!ret)

1696

if (!ret)

1698

return ret;

1697

return ret;

1699

/* dependent device mapping */

1698

/* dependent device mapping */

1700

tmp = pci_find_upstream_pcie_bridge(pdev);

1699

tmp = pci_find_upstream_pcie_bridge(pdev);

1701

if (!tmp)

1700

if (!tmp)

1702

return ret;

1701

return ret;

1703

/* Secondary interface's bus number and devfn 0 */

1702

/* Secondary interface's bus number and devfn 0 */

1704

parent = pdev->bus->self;

1703

parent = pdev->bus->self;

1705

while (parent != tmp) {

1704

while (parent != tmp) {

1706

ret = device_context_mapped(iommu, parent->bus->number,

1705

ret = device_context_mapped(iommu, parent->bus->number,

1707

parent->devfn);

1706

parent->devfn);

1708

if (!ret)

1707

if (!ret)

1709

return ret;

1708

return ret;

1710

parent = parent->bus->self;

1709

parent = parent->bus->self;

1711

}

1710

}

1712

if (pci_is_pcie(tmp))

1711

if (pci_is_pcie(tmp))

1713

return device_context_mapped(iommu, tmp->subordinate->number,

1712

return device_context_mapped(iommu, tmp->subordinate->number,

1714

0);

1713

0);

1715

else

1714

else

1716

return device_context_mapped(iommu, tmp->bus->number,

1715

return device_context_mapped(iommu, tmp->bus->number,

1717

tmp->devfn);

1716

tmp->devfn);

1718

}

1717

}

1719

1718

1720

/* Returns a number of VTD pages, but aligned to MM page size */

1719

/* Returns a number of VTD pages, but aligned to MM page size */

1721

static inline unsigned long aligned_nrpages(unsigned long host_addr,

1720

static inline unsigned long aligned_nrpages(unsigned long host_addr,

1722

size_t size)

1721

size_t size)

1723

{

1722

{

1724

host_addr &= ~PAGE_MASK;

1723

host_addr &= ~PAGE_MASK;

1725

return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;

1724

return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;

1726

}

1725

}

1727

1726

1728

/* Return largest possible superpage level for a given mapping */

1727

/* Return largest possible superpage level for a given mapping */

1729

static inline int hardware_largepage_caps(struct dmar_domain *domain,

1728

static inline int hardware_largepage_caps(struct dmar_domain *domain,

1730

unsigned long iov_pfn,

1729

unsigned long iov_pfn,

1731

unsigned long phy_pfn,

1730

unsigned long phy_pfn,

1732

unsigned long pages)

1731

unsigned long pages)

1733

{

1732

{

1734

int support, level = 1;

1733

int support, level = 1;

1735

unsigned long pfnmerge;

1734

unsigned long pfnmerge;

1736

1735

1737

support = domain->iommu_superpage;

1736

support = domain->iommu_superpage;

1738

1737

1739

/* To use a large page, the virtual *and* physical addresses

1738

/* To use a large page, the virtual *and* physical addresses

1740

must be aligned to 2MiB/1GiB/etc. Lower bits set in either

1739

must be aligned to 2MiB/1GiB/etc. Lower bits set in either

1741

of them will mean we have to use smaller pages. So just

1740

of them will mean we have to use smaller pages. So just

1742

merge them and check both at once. */

1741

merge them and check both at once. */

1743

pfnmerge = iov_pfn | phy_pfn;

1742

pfnmerge = iov_pfn | phy_pfn;

1744

1743

1745

while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {

1744

while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {

1746

pages >>= VTD_STRIDE_SHIFT;

1745

pages >>= VTD_STRIDE_SHIFT;

1747

if (!pages)

1746

if (!pages)

1748

break;

1747

break;

1749

pfnmerge >>= VTD_STRIDE_SHIFT;

1748

pfnmerge >>= VTD_STRIDE_SHIFT;

1750

level++;

1749

level++;

1751

support--;

1750

support--;

1752

}

1751

}

1753

return level;

1752

return level;

1754

}

1753

}

1755

1754

1756

static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1755

static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1757

struct scatterlist *sg, unsigned long phys_pfn,

1756

struct scatterlist *sg, unsigned long phys_pfn,

1758

unsigned long nr_pages, int prot)

1757

unsigned long nr_pages, int prot)

1759

{

1758

{

1760

struct dma_pte *first_pte = NULL, *pte = NULL;

1759

struct dma_pte *first_pte = NULL, *pte = NULL;

1761

phys_addr_t uninitialized_var(pteval);

1760

phys_addr_t uninitialized_var(pteval);

1762

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

1761

int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;

1763

unsigned long sg_res;

1762

unsigned long sg_res;

1764

unsigned int largepage_lvl = 0;

1763

unsigned int largepage_lvl = 0;

1765

unsigned long lvl_pages = 0;

1764

unsigned long lvl_pages = 0;

1766

1765

1767

BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);

1766

BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);

1768

1767

1769

if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)

1768

if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)

1770

return -EINVAL;

1769

return -EINVAL;

1771

1770

1772

prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;

1771

prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;

1773

1772

1774

if (sg)

1773

if (sg)

1775

sg_res = 0;

1774

sg_res = 0;

1776

else {

1775

else {

1777

sg_res = nr_pages + 1;

1776

sg_res = nr_pages + 1;

1778

pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;

1777

pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;

1779

}

1778

}

1780

1779

1781

while (nr_pages > 0) {

1780

while (nr_pages > 0) {

1782

uint64_t tmp;

1781

uint64_t tmp;

1783

1782

1784

if (!sg_res) {

1783

if (!sg_res) {

1785

sg_res = aligned_nrpages(sg->offset, sg->length);

1784

sg_res = aligned_nrpages(sg->offset, sg->length);

1786

sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;

1785

sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;

1787

sg->dma_length = sg->length;

1786

sg->dma_length = sg->length;

1788

pteval = page_to_phys(sg_page(sg)) | prot;

1787

pteval = page_to_phys(sg_page(sg)) | prot;

1789

phys_pfn = pteval >> VTD_PAGE_SHIFT;

1788

phys_pfn = pteval >> VTD_PAGE_SHIFT;

1790

}

1789

}

1791

1790

1792

if (!pte) {

1791

if (!pte) {

1793

largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);

1792

largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);

1794

1793

1795

first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);

1794

first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);

1796

if (!pte)

1795

if (!pte)

1797

return -ENOMEM;

1796

return -ENOMEM;

1798

/* It is large page*/

1797

/* It is large page*/

1799

if (largepage_lvl > 1)

1798

if (largepage_lvl > 1)

1800

pteval |= DMA_PTE_LARGE_PAGE;

1799

pteval |= DMA_PTE_LARGE_PAGE;

1801

else

1800

else

1802

pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;

1801

pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;

1803

1802

1804

}

1803

}

1805

/* We don't need lock here, nobody else

1804

/* We don't need lock here, nobody else

1806

* touches the iova range

1805

* touches the iova range

1807

*/

1806

*/

1808

tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);

1807

tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);

1809

if (tmp) {

1808

if (tmp) {

1810

static int dumps = 5;

1809

static int dumps = 5;

1811

printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",

1810

printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",

1812

iov_pfn, tmp, (unsigned long long)pteval);

1811

iov_pfn, tmp, (unsigned long long)pteval);

1813

if (dumps) {

1812

if (dumps) {

1814

dumps--;

1813

dumps--;

1815

debug_dma_dump_mappings(NULL);

1814

debug_dma_dump_mappings(NULL);

1816

}

1815

}

1817

WARN_ON(1);

1816

WARN_ON(1);

1818

}

1817

}

1819

1818

1820

lvl_pages = lvl_to_nr_pages(largepage_lvl);

1819

lvl_pages = lvl_to_nr_pages(largepage_lvl);

1821

1820

1822

BUG_ON(nr_pages < lvl_pages);

1821

BUG_ON(nr_pages < lvl_pages);

1823

BUG_ON(sg_res < lvl_pages);

1822

BUG_ON(sg_res < lvl_pages);

1824

1823

1825

nr_pages -= lvl_pages;

1824

nr_pages -= lvl_pages;

1826

iov_pfn += lvl_pages;

1825

iov_pfn += lvl_pages;

1827

phys_pfn += lvl_pages;

1826

phys_pfn += lvl_pages;

1828

pteval += lvl_pages * VTD_PAGE_SIZE;

1827

pteval += lvl_pages * VTD_PAGE_SIZE;

1829

sg_res -= lvl_pages;

1828

sg_res -= lvl_pages;

1830

1829

1831

/* If the next PTE would be the first in a new page, then we

1830

/* If the next PTE would be the first in a new page, then we

1832

need to flush the cache on the entries we've just written.

1831

need to flush the cache on the entries we've just written.

1833

And then we'll need to recalculate 'pte', so clear it and

1832

And then we'll need to recalculate 'pte', so clear it and

1834

let it get set again in the if (!pte) block above.

1833

let it get set again in the if (!pte) block above.

1835

1834

1836

If we're done (!nr_pages) we need to flush the cache too.

1835

If we're done (!nr_pages) we need to flush the cache too.

1837

1836

1838

Also if we've been setting superpages, we may need to

1837

Also if we've been setting superpages, we may need to

1839

recalculate 'pte' and switch back to smaller pages for the

1838

recalculate 'pte' and switch back to smaller pages for the

1840

end of the mapping, if the trailing size is not enough to

1839

end of the mapping, if the trailing size is not enough to

1841

use another superpage (i.e. sg_res < lvl_pages). */

1840

use another superpage (i.e. sg_res < lvl_pages). */

1842

pte++;

1841

pte++;

1843

if (!nr_pages || first_pte_in_page(pte) ||

1842

if (!nr_pages || first_pte_in_page(pte) ||

1844

(largepage_lvl > 1 && sg_res < lvl_pages)) {

1843

(largepage_lvl > 1 && sg_res < lvl_pages)) {

1845

domain_flush_cache(domain, first_pte,

1844

domain_flush_cache(domain, first_pte,

1846

(void *)pte - (void *)first_pte);

1845

(void *)pte - (void *)first_pte);

1847

pte = NULL;

1846

pte = NULL;

1848

}

1847

}

1849

1848

1850

if (!sg_res && nr_pages)

1849

if (!sg_res && nr_pages)

1851

sg = sg_next(sg);

1850

sg = sg_next(sg);

1852

}

1851

}

1853

return 0;

1852

return 0;

1854

}

1853

}

1855

1854

1856

static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1855

static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1857

struct scatterlist *sg, unsigned long nr_pages,

1856

struct scatterlist *sg, unsigned long nr_pages,

1858

int prot)

1857

int prot)

1859

{

1858

{

1860

return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);

1859

return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);

1861

}

1860

}

1862

1861

1863

static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1862

static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,

1864

unsigned long phys_pfn, unsigned long nr_pages,

1863

unsigned long phys_pfn, unsigned long nr_pages,

1865

int prot)

1864

int prot)

1866

{

1865

{

1867

return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);

1866

return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);

1868

}

1867

}

1869

1868

1870

static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)

1869

static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)

1871

{

1870

{

1872

if (!iommu)

1871

if (!iommu)

1873

return;

1872

return;

1874

1873

1875

clear_context_table(iommu, bus, devfn);

1874

clear_context_table(iommu, bus, devfn);

1876

iommu->flush.flush_context(iommu, 0, 0, 0,

1875

iommu->flush.flush_context(iommu, 0, 0, 0,

1877

DMA_CCMD_GLOBAL_INVL);

1876

DMA_CCMD_GLOBAL_INVL);

1878

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

1877

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

1879

}

1878

}

1880

1879

1881

static void domain_remove_dev_info(struct dmar_domain *domain)

1880

static void domain_remove_dev_info(struct dmar_domain *domain)

1882

{

1881

{

1883

struct device_domain_info *info;

1882

struct device_domain_info *info;

1884

unsigned long flags;

1883

unsigned long flags;

1885

struct intel_iommu *iommu;

1884

struct intel_iommu *iommu;

1886

1885

1887

spin_lock_irqsave(&device_domain_lock, flags);

1886

spin_lock_irqsave(&device_domain_lock, flags);

1888

while (!list_empty(&domain->devices)) {

1887

while (!list_empty(&domain->devices)) {

1889

info = list_entry(domain->devices.next,

1888

info = list_entry(domain->devices.next,

1890

struct device_domain_info, link);

1889

struct device_domain_info, link);

1891

list_del(&info->link);

1890

list_del(&info->link);

1892

list_del(&info->global);

1891

list_del(&info->global);

1893

if (info->dev)

1892

if (info->dev)

1894

info->dev->dev.archdata.iommu = NULL;

1893

info->dev->dev.archdata.iommu = NULL;

1895

spin_unlock_irqrestore(&device_domain_lock, flags);

1894

spin_unlock_irqrestore(&device_domain_lock, flags);

1896

1895

1897

iommu_disable_dev_iotlb(info);

1896

iommu_disable_dev_iotlb(info);

1898

iommu = device_to_iommu(info->segment, info->bus, info->devfn);

1897

iommu = device_to_iommu(info->segment, info->bus, info->devfn);

1899

iommu_detach_dev(iommu, info->bus, info->devfn);

1898

iommu_detach_dev(iommu, info->bus, info->devfn);

1900

free_devinfo_mem(info);

1899

free_devinfo_mem(info);

1901

1900

1902

spin_lock_irqsave(&device_domain_lock, flags);

1901

spin_lock_irqsave(&device_domain_lock, flags);

1903

}

1902

}

1904

spin_unlock_irqrestore(&device_domain_lock, flags);

1903

spin_unlock_irqrestore(&device_domain_lock, flags);

1905

}

1904

}

1906

1905

1907

/*

1906

/*

1908

* find_domain

1907

* find_domain

1909

* Note: we use struct pci_dev->dev.archdata.iommu stores the info

1908

* Note: we use struct pci_dev->dev.archdata.iommu stores the info

1910

*/

1909

*/

1911

static struct dmar_domain *

1910

static struct dmar_domain *

1912

find_domain(struct pci_dev *pdev)

1911

find_domain(struct pci_dev *pdev)

1913

{

1912

{

1914

struct device_domain_info *info;

1913

struct device_domain_info *info;

1915

1914

1916

/* No lock here, assumes no domain exit in normal case */

1915

/* No lock here, assumes no domain exit in normal case */

1917

info = pdev->dev.archdata.iommu;

1916

info = pdev->dev.archdata.iommu;

1918

if (info)

1917

if (info)

1919

return info->domain;

1918

return info->domain;

1920

return NULL;

1919

return NULL;

1921

}

1920

}

1922

1921

1923

/* domain is initialized */

1922

/* domain is initialized */

1924

static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)

1923

static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)

1925

{

1924

{

1926

struct dmar_domain *domain, *found = NULL;

1925

struct dmar_domain *domain, *found = NULL;

1927

struct intel_iommu *iommu;

1926

struct intel_iommu *iommu;

1928

struct dmar_drhd_unit *drhd;

1927

struct dmar_drhd_unit *drhd;

1929

struct device_domain_info *info, *tmp;

1928

struct device_domain_info *info, *tmp;

1930

struct pci_dev *dev_tmp;

1929

struct pci_dev *dev_tmp;

1931

unsigned long flags;

1930

unsigned long flags;

1932

int bus = 0, devfn = 0;

1931

int bus = 0, devfn = 0;

1933

int segment;

1932

int segment;

1934

int ret;

1933

int ret;

1935

1934

1936

domain = find_domain(pdev);

1935

domain = find_domain(pdev);

1937

if (domain)

1936

if (domain)

1938

return domain;

1937

return domain;

1939

1938

1940

segment = pci_domain_nr(pdev->bus);

1939

segment = pci_domain_nr(pdev->bus);

1941

1940

1942

dev_tmp = pci_find_upstream_pcie_bridge(pdev);

1941

dev_tmp = pci_find_upstream_pcie_bridge(pdev);

1943

if (dev_tmp) {

1942

if (dev_tmp) {

1944

if (pci_is_pcie(dev_tmp)) {

1943

if (pci_is_pcie(dev_tmp)) {

1945

bus = dev_tmp->subordinate->number;

1944

bus = dev_tmp->subordinate->number;

1946

devfn = 0;

1945

devfn = 0;

1947

} else {

1946

} else {

1948

bus = dev_tmp->bus->number;

1947

bus = dev_tmp->bus->number;

1949

devfn = dev_tmp->devfn;

1948

devfn = dev_tmp->devfn;

1950

}

1949

}

1951

spin_lock_irqsave(&device_domain_lock, flags);

1950

spin_lock_irqsave(&device_domain_lock, flags);

1952

list_for_each_entry(info, &device_domain_list, global) {

1951

list_for_each_entry(info, &device_domain_list, global) {

1953

if (info->segment == segment &&

1952

if (info->segment == segment &&

1954

info->bus == bus && info->devfn == devfn) {

1953

info->bus == bus && info->devfn == devfn) {

1955

found = info->domain;

1954

found = info->domain;

1956

break;

1955

break;

1957

}

1956

}

1958

}

1957

}

1959

spin_unlock_irqrestore(&device_domain_lock, flags);

1958

spin_unlock_irqrestore(&device_domain_lock, flags);

1960

/* pcie-pci bridge already has a domain, uses it */

1959

/* pcie-pci bridge already has a domain, uses it */

1961

if (found) {

1960

if (found) {

1962

domain = found;

1961

domain = found;

1963

goto found_domain;

1962

goto found_domain;

1964

}

1963

}

1965

}

1964

}

1966

1965

1967

domain = alloc_domain();

1966

domain = alloc_domain();

1968

if (!domain)

1967

if (!domain)

1969

goto error;

1968

goto error;

1970

1969

1971

/* Allocate new domain for the device */

1970

/* Allocate new domain for the device */

1972

drhd = dmar_find_matched_drhd_unit(pdev);

1971

drhd = dmar_find_matched_drhd_unit(pdev);

1973

if (!drhd) {

1972

if (!drhd) {

1974

printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",

1973

printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",

1975

pci_name(pdev));

1974

pci_name(pdev));

1976

return NULL;

1975

return NULL;

1977

}

1976

}

1978

iommu = drhd->iommu;

1977

iommu = drhd->iommu;

1979

1978

1980

ret = iommu_attach_domain(domain, iommu);

1979

ret = iommu_attach_domain(domain, iommu);

1981

if (ret) {

1980

if (ret) {

1982

free_domain_mem(domain);

1981

free_domain_mem(domain);

1983

goto error;

1982

goto error;

1984

}

1983

}

1985

1984

1986

if (domain_init(domain, gaw)) {

1985

if (domain_init(domain, gaw)) {

1987

domain_exit(domain);

1986

domain_exit(domain);

1988

goto error;

1987

goto error;

1989

}

1988

}

1990

1989

1991

/* register pcie-to-pci device */

1990

/* register pcie-to-pci device */

1992

if (dev_tmp) {

1991

if (dev_tmp) {

1993

info = alloc_devinfo_mem();

1992

info = alloc_devinfo_mem();

1994

if (!info) {

1993

if (!info) {

1995

domain_exit(domain);

1994

domain_exit(domain);

1996

goto error;

1995

goto error;

1997

}

1996

}

1998

info->segment = segment;

1997

info->segment = segment;

1999

info->bus = bus;

1998

info->bus = bus;

2000

info->devfn = devfn;

1999

info->devfn = devfn;

2001

info->dev = NULL;

2000

info->dev = NULL;

2002

info->domain = domain;

2001

info->domain = domain;

2003

/* This domain is shared by devices under p2p bridge */

2002

/* This domain is shared by devices under p2p bridge */

2004

domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;

2003

domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;

2005

2004

2006

/* pcie-to-pci bridge already has a domain, uses it */

2005

/* pcie-to-pci bridge already has a domain, uses it */

2007

found = NULL;

2006

found = NULL;

2008

spin_lock_irqsave(&device_domain_lock, flags);

2007

spin_lock_irqsave(&device_domain_lock, flags);

2009

list_for_each_entry(tmp, &device_domain_list, global) {

2008

list_for_each_entry(tmp, &device_domain_list, global) {

2010

if (tmp->segment == segment &&

2009

if (tmp->segment == segment &&

2011

tmp->bus == bus && tmp->devfn == devfn) {

2010

tmp->bus == bus && tmp->devfn == devfn) {

2012

found = tmp->domain;

2011

found = tmp->domain;

2013

break;

2012

break;

2014

}

2013

}

2015

}

2014

}

2016

if (found) {

2015

if (found) {

2017

spin_unlock_irqrestore(&device_domain_lock, flags);

2016

spin_unlock_irqrestore(&device_domain_lock, flags);

2018

free_devinfo_mem(info);

2017

free_devinfo_mem(info);

2019

domain_exit(domain);

2018

domain_exit(domain);

2020

domain = found;

2019

domain = found;

2021

} else {

2020

} else {

2022

list_add(&info->link, &domain->devices);

2021

list_add(&info->link, &domain->devices);

2023

list_add(&info->global, &device_domain_list);

2022

list_add(&info->global, &device_domain_list);

2024

spin_unlock_irqrestore(&device_domain_lock, flags);

2023

spin_unlock_irqrestore(&device_domain_lock, flags);

2025

}

2024

}

2026

}

2025

}

2027

2026

2028

found_domain:

2027

found_domain:

2029

info = alloc_devinfo_mem();

2028

info = alloc_devinfo_mem();

2030

if (!info)

2029

if (!info)

2031

goto error;

2030

goto error;

2032

info->segment = segment;

2031

info->segment = segment;

2033

info->bus = pdev->bus->number;

2032

info->bus = pdev->bus->number;

2034

info->devfn = pdev->devfn;

2033

info->devfn = pdev->devfn;

2035

info->dev = pdev;

2034

info->dev = pdev;

2036

info->domain = domain;

2035

info->domain = domain;

2037

spin_lock_irqsave(&device_domain_lock, flags);

2036

spin_lock_irqsave(&device_domain_lock, flags);

2038

/* somebody is fast */

2037

/* somebody is fast */

2039

found = find_domain(pdev);

2038

found = find_domain(pdev);

2040

if (found != NULL) {

2039

if (found != NULL) {

2041

spin_unlock_irqrestore(&device_domain_lock, flags);

2040

spin_unlock_irqrestore(&device_domain_lock, flags);

2042

if (found != domain) {

2041

if (found != domain) {

2043

domain_exit(domain);

2042

domain_exit(domain);

2044

domain = found;

2043

domain = found;

2045

}

2044

}

2046

free_devinfo_mem(info);

2045

free_devinfo_mem(info);

2047

return domain;

2046

return domain;

2048

}

2047

}

2049

list_add(&info->link, &domain->devices);

2048

list_add(&info->link, &domain->devices);

2050

list_add(&info->global, &device_domain_list);

2049

list_add(&info->global, &device_domain_list);

2051

pdev->dev.archdata.iommu = info;

2050

pdev->dev.archdata.iommu = info;

2052

spin_unlock_irqrestore(&device_domain_lock, flags);

2051

spin_unlock_irqrestore(&device_domain_lock, flags);

2053

return domain;

2052

return domain;

2054

error:

2053

error:

2055

/* recheck it here, maybe others set it */

2054

/* recheck it here, maybe others set it */

2056

return find_domain(pdev);

2055

return find_domain(pdev);

2057

}

2056

}

2058

2057

2059

static int iommu_identity_mapping;

2058

static int iommu_identity_mapping;

2060

#define IDENTMAP_ALL 1

2059

#define IDENTMAP_ALL 1

2061

#define IDENTMAP_GFX 2

2060

#define IDENTMAP_GFX 2

2062

#define IDENTMAP_AZALIA 4

2061

#define IDENTMAP_AZALIA 4

2063

2062

2064

static int iommu_domain_identity_map(struct dmar_domain *domain,

2063

static int iommu_domain_identity_map(struct dmar_domain *domain,

2065

unsigned long long start,

2064

unsigned long long start,

2066

unsigned long long end)

2065

unsigned long long end)

2067

{

2066

{

2068

unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;

2067

unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;

2069

unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;

2068

unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;

2070

2069

2071

if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),

2070

if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),

2072

dma_to_mm_pfn(last_vpfn))) {

2071

dma_to_mm_pfn(last_vpfn))) {

2073

printk(KERN_ERR "IOMMU: reserve iova failed\n");

2072

printk(KERN_ERR "IOMMU: reserve iova failed\n");

2074

return -ENOMEM;

2073

return -ENOMEM;

2075

}

2074

}

2076

2075

2077

pr_debug("Mapping reserved region %llx-%llx for domain %d\n",

2076

pr_debug("Mapping reserved region %llx-%llx for domain %d\n",

2078

start, end, domain->id);

2077

start, end, domain->id);

2079

/*

2078

/*

2080

* RMRR range might have overlap with physical memory range,

2079

* RMRR range might have overlap with physical memory range,

2081

* clear it first

2080

* clear it first

2082

*/

2081

*/

2083

dma_pte_clear_range(domain, first_vpfn, last_vpfn);

2082

dma_pte_clear_range(domain, first_vpfn, last_vpfn);

2084

2083

2085

return domain_pfn_mapping(domain, first_vpfn, first_vpfn,

2084

return domain_pfn_mapping(domain, first_vpfn, first_vpfn,

2086

last_vpfn - first_vpfn + 1,

2085

last_vpfn - first_vpfn + 1,

2087

DMA_PTE_READ|DMA_PTE_WRITE);

2086

DMA_PTE_READ|DMA_PTE_WRITE);

2088

}

2087

}

2089

2088

2090

static int iommu_prepare_identity_map(struct pci_dev *pdev,

2089

static int iommu_prepare_identity_map(struct pci_dev *pdev,

2091

unsigned long long start,

2090

unsigned long long start,

2092

unsigned long long end)

2091

unsigned long long end)

2093

{

2092

{

2094

struct dmar_domain *domain;

2093

struct dmar_domain *domain;

2095

int ret;

2094

int ret;

2096

2095

2097

domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);

2096

domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);

2098

if (!domain)

2097

if (!domain)

2099

return -ENOMEM;

2098

return -ENOMEM;

2100

2099

2101

/* For _hardware_ passthrough, don't bother. But for software

2100

/* For _hardware_ passthrough, don't bother. But for software

2102

passthrough, we do it anyway -- it may indicate a memory

2101

passthrough, we do it anyway -- it may indicate a memory

2103

range which is reserved in E820, so which didn't get set

2102

range which is reserved in E820, so which didn't get set

2104

up to start with in si_domain */

2103

up to start with in si_domain */

2105

if (domain == si_domain && hw_pass_through) {

2104

if (domain == si_domain && hw_pass_through) {

2106

printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",

2105

printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",

2107

pci_name(pdev), start, end);

2106

pci_name(pdev), start, end);

2108

return 0;

2107

return 0;

2109

}

2108

}

2110

2109

2111

printk(KERN_INFO

2110

printk(KERN_INFO

2112

"IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",

2111

"IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",

2113

pci_name(pdev), start, end);

2112

pci_name(pdev), start, end);

2114

2113

2115

if (end < start) {

2114

if (end < start) {

2116

WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"

2115

WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"

2117

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

2116

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

2118

dmi_get_system_info(DMI_BIOS_VENDOR),

2117

dmi_get_system_info(DMI_BIOS_VENDOR),

2119

dmi_get_system_info(DMI_BIOS_VERSION),

2118

dmi_get_system_info(DMI_BIOS_VERSION),

2120

dmi_get_system_info(DMI_PRODUCT_VERSION));

2119

dmi_get_system_info(DMI_PRODUCT_VERSION));

2121

ret = -EIO;

2120

ret = -EIO;

2122

goto error;

2121

goto error;

2123

}

2122

}

2124

2123

2125

if (end >> agaw_to_width(domain->agaw)) {

2124

if (end >> agaw_to_width(domain->agaw)) {

2126

WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"

2125

WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"

2127

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

2126

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

2128

agaw_to_width(domain->agaw),

2127

agaw_to_width(domain->agaw),

2129

dmi_get_system_info(DMI_BIOS_VENDOR),

2128

dmi_get_system_info(DMI_BIOS_VENDOR),

2130

dmi_get_system_info(DMI_BIOS_VERSION),

2129

dmi_get_system_info(DMI_BIOS_VERSION),

2131

dmi_get_system_info(DMI_PRODUCT_VERSION));

2130

dmi_get_system_info(DMI_PRODUCT_VERSION));

2132

ret = -EIO;

2131

ret = -EIO;

2133

goto error;

2132

goto error;

2134

}

2133

}

2135

2134

2136

ret = iommu_domain_identity_map(domain, start, end);

2135

ret = iommu_domain_identity_map(domain, start, end);

2137

if (ret)

2136

if (ret)

2138

goto error;

2137

goto error;

2139

2138

2140

/* context entry init */

2139

/* context entry init */

2141

ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);

2140

ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);

2142

if (ret)

2141

if (ret)

2143

goto error;

2142

goto error;

2144

2143

2145

return 0;

2144

return 0;

2146

2145

2147

error:

2146

error:

2148

domain_exit(domain);

2147

domain_exit(domain);

2149

return ret;

2148

return ret;

2150

}

2149

}

2151

2150

2152

static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,

2151

static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,

2153

struct pci_dev *pdev)

2152

struct pci_dev *pdev)

2154

{

2153

{

2155

if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)

2154

if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)

2156

return 0;

2155

return 0;

2157

return iommu_prepare_identity_map(pdev, rmrr->base_address,

2156

return iommu_prepare_identity_map(pdev, rmrr->base_address,

2158

rmrr->end_address);

2157

rmrr->end_address);

2159

}

2158

}

2160

2159

2161

#ifdef CONFIG_DMAR_FLOPPY_WA

2160

#ifdef CONFIG_DMAR_FLOPPY_WA

2162

static inline void iommu_prepare_isa(void)

2161

static inline void iommu_prepare_isa(void)

2163

{

2162

{

2164

struct pci_dev *pdev;

2163

struct pci_dev *pdev;

2165

int ret;

2164

int ret;

2166

2165

2167

pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);

2166

pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);

2168

if (!pdev)

2167

if (!pdev)

2169

return;

2168

return;

2170

2169

2171

printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");

2170

printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");

2172

ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);

2171

ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);

2173

2172

2174

if (ret)

2173

if (ret)

2175

printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "

2174

printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "

2176

"floppy might not work\n");

2175

"floppy might not work\n");

2177

2176

2178

}

2177

}

2179

#else

2178

#else

2180

static inline void iommu_prepare_isa(void)

2179

static inline void iommu_prepare_isa(void)

2181

{

2180

{

2182

return;

2181

return;

2183

}

2182

}

2184

#endif /* !CONFIG_DMAR_FLPY_WA */

2183

#endif /* !CONFIG_DMAR_FLPY_WA */

2185

2184

2186

static int md_domain_init(struct dmar_domain *domain, int guest_width);

2185

static int md_domain_init(struct dmar_domain *domain, int guest_width);

2187

2186

2188

static int __init si_domain_work_fn(unsigned long start_pfn,

2187

static int __init si_domain_work_fn(unsigned long start_pfn,

2189

unsigned long end_pfn, void *datax)

2188

unsigned long end_pfn, void *datax)

2190

{

2189

{

2191

int *ret = datax;

2190

int *ret = datax;

2192

2191

2193

*ret = iommu_domain_identity_map(si_domain,

2192

*ret = iommu_domain_identity_map(si_domain,

2194

(uint64_t)start_pfn << PAGE_SHIFT,

2193

(uint64_t)start_pfn << PAGE_SHIFT,

2195

(uint64_t)end_pfn << PAGE_SHIFT);

2194

(uint64_t)end_pfn << PAGE_SHIFT);

2196

return *ret;

2195

return *ret;

2197

2196

2198

}

2197

}

2199

2198

2200

static int __init si_domain_init(int hw)

2199

static int __init si_domain_init(int hw)

2201

{

2200

{

2202

struct dmar_drhd_unit *drhd;

2201

struct dmar_drhd_unit *drhd;

2203

struct intel_iommu *iommu;

2202

struct intel_iommu *iommu;

2204

int nid, ret = 0;

2203

int nid, ret = 0;

2205

2204

2206

si_domain = alloc_domain();

2205

si_domain = alloc_domain();

2207

if (!si_domain)

2206

if (!si_domain)

2208

return -EFAULT;

2207

return -EFAULT;

2209

2208

2210

pr_debug("Identity mapping domain is domain %d\n", si_domain->id);

2209

pr_debug("Identity mapping domain is domain %d\n", si_domain->id);

2211

2210

2212

for_each_active_iommu(iommu, drhd) {

2211

for_each_active_iommu(iommu, drhd) {

2213

ret = iommu_attach_domain(si_domain, iommu);

2212

ret = iommu_attach_domain(si_domain, iommu);

2214

if (ret) {

2213

if (ret) {

2215

domain_exit(si_domain);

2214

domain_exit(si_domain);

2216

return -EFAULT;

2215

return -EFAULT;

2217

}

2216

}

2218

}

2217

}

2219

2218

2220

if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {

2219

if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {

2221

domain_exit(si_domain);

2220

domain_exit(si_domain);

2222

return -EFAULT;

2221

return -EFAULT;

2223

}

2222

}

2224

2223

2225

si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;

2224

si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;

2226

2225

2227

if (hw)

2226

if (hw)

2228

return 0;

2227

return 0;

2229

2228

2230

for_each_online_node(nid) {

2229

for_each_online_node(nid) {

2231

work_with_active_regions(nid, si_domain_work_fn, &ret);

2230

work_with_active_regions(nid, si_domain_work_fn, &ret);

2232

if (ret)

2231

if (ret)

2233

return ret;

2232

return ret;

2234

}

2233

}

2235

2234

2236

return 0;

2235

return 0;

2237

}

2236

}

2238

2237

2239

static void domain_remove_one_dev_info(struct dmar_domain *domain,

2238

static void domain_remove_one_dev_info(struct dmar_domain *domain,

2240

struct pci_dev *pdev);

2239

struct pci_dev *pdev);

2241

static int identity_mapping(struct pci_dev *pdev)

2240

static int identity_mapping(struct pci_dev *pdev)

2242

{

2241

{

2243

struct device_domain_info *info;

2242

struct device_domain_info *info;

2244

2243

2245

if (likely(!iommu_identity_mapping))

2244

if (likely(!iommu_identity_mapping))

2246

return 0;

2245

return 0;

2247

2246

2248

info = pdev->dev.archdata.iommu;

2247

info = pdev->dev.archdata.iommu;

2249

if (info && info != DUMMY_DEVICE_DOMAIN_INFO)

2248

if (info && info != DUMMY_DEVICE_DOMAIN_INFO)

2250

return (info->domain == si_domain);

2249

return (info->domain == si_domain);

2251

2250

2252

return 0;

2251

return 0;

2253

}

2252

}

2254

2253

2255

static int domain_add_dev_info(struct dmar_domain *domain,

2254

static int domain_add_dev_info(struct dmar_domain *domain,

2256

struct pci_dev *pdev,

2255

struct pci_dev *pdev,

2257

int translation)

2256

int translation)

2258

{

2257

{

2259

struct device_domain_info *info;

2258

struct device_domain_info *info;

2260

unsigned long flags;

2259

unsigned long flags;

2261

int ret;

2260

int ret;

2262

2261

2263

info = alloc_devinfo_mem();

2262

info = alloc_devinfo_mem();

2264

if (!info)

2263

if (!info)

2265

return -ENOMEM;

2264

return -ENOMEM;

2266

2265

2267

ret = domain_context_mapping(domain, pdev, translation);

2266

ret = domain_context_mapping(domain, pdev, translation);

2268

if (ret) {

2267

if (ret) {

2269

free_devinfo_mem(info);

2268

free_devinfo_mem(info);

2270

return ret;

2269

return ret;

2271

}

2270

}

2272

2271

2273

info->segment = pci_domain_nr(pdev->bus);

2272

info->segment = pci_domain_nr(pdev->bus);

2274

info->bus = pdev->bus->number;

2273

info->bus = pdev->bus->number;

2275

info->devfn = pdev->devfn;

2274

info->devfn = pdev->devfn;

2276

info->dev = pdev;

2275

info->dev = pdev;

2277

info->domain = domain;

2276

info->domain = domain;

2278

2277

2279

spin_lock_irqsave(&device_domain_lock, flags);

2278

spin_lock_irqsave(&device_domain_lock, flags);

2280

list_add(&info->link, &domain->devices);

2279

list_add(&info->link, &domain->devices);

2281

list_add(&info->global, &device_domain_list);

2280

list_add(&info->global, &device_domain_list);

2282

pdev->dev.archdata.iommu = info;

2281

pdev->dev.archdata.iommu = info;

2283

spin_unlock_irqrestore(&device_domain_lock, flags);

2282

spin_unlock_irqrestore(&device_domain_lock, flags);

2284

2283

2285

return 0;

2284

return 0;

2286

}

2285

}

2287

2286

2288

static int iommu_should_identity_map(struct pci_dev *pdev, int startup)

2287

static int iommu_should_identity_map(struct pci_dev *pdev, int startup)

2289

{

2288

{

2290

if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))

2289

if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))

2291

return 1;

2290

return 1;

2292

2291

2293

if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))

2292

if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))

2294

return 1;

2293

return 1;

2295

2294

2296

if (!(iommu_identity_mapping & IDENTMAP_ALL))

2295

if (!(iommu_identity_mapping & IDENTMAP_ALL))

2297

return 0;

2296

return 0;

2298

2297

2299

/*

2298

/*

2300

* We want to start off with all devices in the 1:1 domain, and

2299

* We want to start off with all devices in the 1:1 domain, and

2301

* take them out later if we find they can't access all of memory.

2300

* take them out later if we find they can't access all of memory.

2302

*

2301

*

2303

* However, we can't do this for PCI devices behind bridges,

2302

* However, we can't do this for PCI devices behind bridges,

2304

* because all PCI devices behind the same bridge will end up

2303

* because all PCI devices behind the same bridge will end up

2305

* with the same source-id on their transactions.

2304

* with the same source-id on their transactions.

2306

*

2305

*

2307

* Practically speaking, we can't change things around for these

2306

* Practically speaking, we can't change things around for these

2308

* devices at run-time, because we can't be sure there'll be no

2307

* devices at run-time, because we can't be sure there'll be no

2309

* DMA transactions in flight for any of their siblings.

2308

* DMA transactions in flight for any of their siblings.

2310

*

2309

*

2311

* So PCI devices (unless they're on the root bus) as well as

2310

* So PCI devices (unless they're on the root bus) as well as

2312

* their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of

2311

* their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of

2313

* the 1:1 domain, just in _case_ one of their siblings turns out

2312

* the 1:1 domain, just in _case_ one of their siblings turns out

2314

* not to be able to map all of memory.

2313

* not to be able to map all of memory.

2315

*/

2314

*/

2316

if (!pci_is_pcie(pdev)) {

2315

if (!pci_is_pcie(pdev)) {

2317

if (!pci_is_root_bus(pdev->bus))

2316

if (!pci_is_root_bus(pdev->bus))

2318

return 0;

2317

return 0;

2319

if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)

2318

if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)

2320

return 0;

2319

return 0;

2321

} else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)

2320

} else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)

2322

return 0;

2321

return 0;

2323

2322

2324

/*

2323

/*

2325

* At boot time, we don't yet know if devices will be 64-bit capable.

2324

* At boot time, we don't yet know if devices will be 64-bit capable.

2326

* Assume that they will -- if they turn out not to be, then we can

2325

* Assume that they will -- if they turn out not to be, then we can

2327

* take them out of the 1:1 domain later.

2326

* take them out of the 1:1 domain later.

2328

*/

2327

*/

2329

if (!startup) {

2328

if (!startup) {

2330

/*

2329

/*

2331

* If the device's dma_mask is less than the system's memory

2330

* If the device's dma_mask is less than the system's memory

2332

* size then this is not a candidate for identity mapping.

2331

* size then this is not a candidate for identity mapping.

2333

*/

2332

*/

2334

u64 dma_mask = pdev->dma_mask;

2333

u64 dma_mask = pdev->dma_mask;

2335

2334

2336

if (pdev->dev.coherent_dma_mask &&

2335

if (pdev->dev.coherent_dma_mask &&

2337

pdev->dev.coherent_dma_mask < dma_mask)

2336

pdev->dev.coherent_dma_mask < dma_mask)

2338

dma_mask = pdev->dev.coherent_dma_mask;

2337

dma_mask = pdev->dev.coherent_dma_mask;

2339

2338

2340

return dma_mask >= dma_get_required_mask(&pdev->dev);

2339

return dma_mask >= dma_get_required_mask(&pdev->dev);

2341

}

2340

}

2342

2341

2343

return 1;

2342

return 1;

2344

}

2343

}

2345

2344

2346

static int __init iommu_prepare_static_identity_mapping(int hw)

2345

static int __init iommu_prepare_static_identity_mapping(int hw)

2347

{

2346

{

2348

struct pci_dev *pdev = NULL;

2347

struct pci_dev *pdev = NULL;

2349

int ret;

2348

int ret;

2350

2349

2351

ret = si_domain_init(hw);

2350

ret = si_domain_init(hw);

2352

if (ret)

2351

if (ret)

2353

return -EFAULT;

2352

return -EFAULT;

2354

2353

2355

for_each_pci_dev(pdev) {

2354

for_each_pci_dev(pdev) {

2356

/* Skip Host/PCI Bridge devices */

2355

/* Skip Host/PCI Bridge devices */

2357

if (IS_BRIDGE_HOST_DEVICE(pdev))

2356

if (IS_BRIDGE_HOST_DEVICE(pdev))

2358

continue;

2357

continue;

2359

if (iommu_should_identity_map(pdev, 1)) {

2358

if (iommu_should_identity_map(pdev, 1)) {

2360

printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",

2359

printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",

2361

hw ? "hardware" : "software", pci_name(pdev));

2360

hw ? "hardware" : "software", pci_name(pdev));

2362

2361

2363

ret = domain_add_dev_info(si_domain, pdev,

2362

ret = domain_add_dev_info(si_domain, pdev,

2364

hw ? CONTEXT_TT_PASS_THROUGH :

2363

hw ? CONTEXT_TT_PASS_THROUGH :

2365

CONTEXT_TT_MULTI_LEVEL);

2364

CONTEXT_TT_MULTI_LEVEL);

2366

if (ret)

2365

if (ret)

2367

return ret;

2366

return ret;

2368

}

2367

}

2369

}

2368

}

2370

2369

2371

return 0;

2370

return 0;

2372

}

2371

}

2373

2372

2374

static int __init init_dmars(void)

2373

static int __init init_dmars(void)

2375

{

2374

{

2376

struct dmar_drhd_unit *drhd;

2375

struct dmar_drhd_unit *drhd;

2377

struct dmar_rmrr_unit *rmrr;

2376

struct dmar_rmrr_unit *rmrr;

2378

struct pci_dev *pdev;

2377

struct pci_dev *pdev;

2379

struct intel_iommu *iommu;

2378

struct intel_iommu *iommu;

2380

int i, ret;

2379

int i, ret;

2381

2380

2382

/*

2381

/*

2383

* for each drhd

2382

* for each drhd

2384

* allocate root

2383

* allocate root

2385

* initialize and program root entry to not present

2384

* initialize and program root entry to not present

2386

* endfor

2385

* endfor

2387

*/

2386

*/

2388

for_each_drhd_unit(drhd) {

2387

for_each_drhd_unit(drhd) {

2389

g_num_of_iommus++;

2388

g_num_of_iommus++;

2390

/*

2389

/*

2391

* lock not needed as this is only incremented in the single

2390

* lock not needed as this is only incremented in the single

2392

* threaded kernel __init code path all other access are read

2391

* threaded kernel __init code path all other access are read

2393

* only

2392

* only

2394

*/

2393

*/

2395

}

2394

}

2396

2395

2397

g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),

2396

g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),

2398

GFP_KERNEL);

2397

GFP_KERNEL);

2399

if (!g_iommus) {

2398

if (!g_iommus) {

2400

printk(KERN_ERR "Allocating global iommu array failed\n");

2399

printk(KERN_ERR "Allocating global iommu array failed\n");

2401

ret = -ENOMEM;

2400

ret = -ENOMEM;

2402

goto error;

2401

goto error;

2403

}

2402

}

2404

2403

2405

deferred_flush = kzalloc(g_num_of_iommus *

2404

deferred_flush = kzalloc(g_num_of_iommus *

2406

sizeof(struct deferred_flush_tables), GFP_KERNEL);

2405

sizeof(struct deferred_flush_tables), GFP_KERNEL);

2407

if (!deferred_flush) {

2406

if (!deferred_flush) {

2408

ret = -ENOMEM;

2407

ret = -ENOMEM;

2409

goto error;

2408

goto error;

2410

}

2409

}

2411

2410

2412

for_each_drhd_unit(drhd) {

2411

for_each_drhd_unit(drhd) {

2413

if (drhd->ignored)

2412

if (drhd->ignored)

2414

continue;

2413

continue;

2415

2414

2416

iommu = drhd->iommu;

2415

iommu = drhd->iommu;

2417

g_iommus[iommu->seq_id] = iommu;

2416

g_iommus[iommu->seq_id] = iommu;

2418

2417

2419

ret = iommu_init_domains(iommu);

2418

ret = iommu_init_domains(iommu);

2420

if (ret)

2419

if (ret)

2421

goto error;

2420

goto error;

2422

2421

2423

/*

2422

/*

2424

* TBD:

2423

* TBD:

2425

* we could share the same root & context tables

2424

* we could share the same root & context tables

2426

* among all IOMMU's. Need to Split it later.

2425

* among all IOMMU's. Need to Split it later.

2427

*/

2426

*/

2428

ret = iommu_alloc_root_entry(iommu);

2427

ret = iommu_alloc_root_entry(iommu);

2429

if (ret) {

2428

if (ret) {

2430

printk(KERN_ERR "IOMMU: allocate root entry failed\n");

2429

printk(KERN_ERR "IOMMU: allocate root entry failed\n");

2431

goto error;

2430

goto error;

2432

}

2431

}

2433

if (!ecap_pass_through(iommu->ecap))

2432

if (!ecap_pass_through(iommu->ecap))

2434

hw_pass_through = 0;

2433

hw_pass_through = 0;

2435

}

2434

}

2436

2435

2437

/*

2436

/*

2438

* Start from the sane iommu hardware state.

2437

* Start from the sane iommu hardware state.

2439

*/

2438

*/

2440

for_each_drhd_unit(drhd) {

2439

for_each_drhd_unit(drhd) {

2441

if (drhd->ignored)

2440

if (drhd->ignored)

2442

continue;

2441

continue;

2443

2442

2444

iommu = drhd->iommu;

2443

iommu = drhd->iommu;

2445

2444

2446

/*

2445

/*

2447

* If the queued invalidation is already initialized by us

2446

* If the queued invalidation is already initialized by us

2448

* (for example, while enabling interrupt-remapping) then

2447

* (for example, while enabling interrupt-remapping) then

2449

* we got the things already rolling from a sane state.

2448

* we got the things already rolling from a sane state.

2450

*/

2449

*/

2451

if (iommu->qi)

2450

if (iommu->qi)

2452

continue;

2451

continue;

2453

2452

2454

/*

2453

/*

2455

* Clear any previous faults.

2454

* Clear any previous faults.

2456

*/

2455

*/

2457

dmar_fault(-1, iommu);

2456

dmar_fault(-1, iommu);

2458

/*

2457

/*

2459

* Disable queued invalidation if supported and already enabled

2458

* Disable queued invalidation if supported and already enabled

2460

* before OS handover.

2459

* before OS handover.

2461

*/

2460

*/

2462

dmar_disable_qi(iommu);

2461

dmar_disable_qi(iommu);

2463

}

2462

}

2464

2463

2465

for_each_drhd_unit(drhd) {

2464

for_each_drhd_unit(drhd) {

2466

if (drhd->ignored)

2465

if (drhd->ignored)

2467

continue;

2466

continue;

2468

2467

2469

iommu = drhd->iommu;

2468

iommu = drhd->iommu;

2470

2469

2471

if (dmar_enable_qi(iommu)) {

2470

if (dmar_enable_qi(iommu)) {

2472

/*

2471

/*

2473

* Queued Invalidate not enabled, use Register Based

2472

* Queued Invalidate not enabled, use Register Based

2474

* Invalidate

2473

* Invalidate

2475

*/

2474

*/

2476

iommu->flush.flush_context = __iommu_flush_context;

2475

iommu->flush.flush_context = __iommu_flush_context;

2477

iommu->flush.flush_iotlb = __iommu_flush_iotlb;

2476

iommu->flush.flush_iotlb = __iommu_flush_iotlb;

2478

printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "

2477

printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "

2479

"invalidation\n",

2478

"invalidation\n",

2480

iommu->seq_id,

2479

iommu->seq_id,

2481

(unsigned long long)drhd->reg_base_addr);

2480

(unsigned long long)drhd->reg_base_addr);

2482

} else {

2481

} else {

2483

iommu->flush.flush_context = qi_flush_context;

2482

iommu->flush.flush_context = qi_flush_context;

2484

iommu->flush.flush_iotlb = qi_flush_iotlb;

2483

iommu->flush.flush_iotlb = qi_flush_iotlb;

2485

printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "

2484

printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "

2486

"invalidation\n",

2485

"invalidation\n",

2487

iommu->seq_id,

2486

iommu->seq_id,

2488

(unsigned long long)drhd->reg_base_addr);

2487

(unsigned long long)drhd->reg_base_addr);

2489

}

2488

}

2490

}

2489

}

2491

2490

2492

if (iommu_pass_through)

2491

if (iommu_pass_through)

2493

iommu_identity_mapping |= IDENTMAP_ALL;

2492

iommu_identity_mapping |= IDENTMAP_ALL;

2494

2493

2495

#ifdef CONFIG_DMAR_BROKEN_GFX_WA

2494

#ifdef CONFIG_DMAR_BROKEN_GFX_WA

2496

iommu_identity_mapping |= IDENTMAP_GFX;

2495

iommu_identity_mapping |= IDENTMAP_GFX;

2497

#endif

2496

#endif

2498

2497

2499

check_tylersburg_isoch();

2498

check_tylersburg_isoch();

2500

2499

2501

/*

2500

/*

2502

* If pass through is not set or not enabled, setup context entries for

2501

* If pass through is not set or not enabled, setup context entries for

2503

* identity mappings for rmrr, gfx, and isa and may fall back to static

2502

* identity mappings for rmrr, gfx, and isa and may fall back to static

2504

* identity mapping if iommu_identity_mapping is set.

2503

* identity mapping if iommu_identity_mapping is set.

2505

*/

2504

*/

2506

if (iommu_identity_mapping) {

2505

if (iommu_identity_mapping) {

2507

ret = iommu_prepare_static_identity_mapping(hw_pass_through);

2506

ret = iommu_prepare_static_identity_mapping(hw_pass_through);

2508

if (ret) {

2507

if (ret) {

2509

printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");

2508

printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");

2510

goto error;

2509

goto error;

2511

}

2510

}

2512

}

2511

}

2513

/*

2512

/*

2514

* For each rmrr

2513

* For each rmrr

2515

* for each dev attached to rmrr

2514

* for each dev attached to rmrr

2516

* do

2515

* do

2517

* locate drhd for dev, alloc domain for dev

2516

* locate drhd for dev, alloc domain for dev

2518

* allocate free domain

2517

* allocate free domain

2519

* allocate page table entries for rmrr

2518

* allocate page table entries for rmrr

2520

* if context not allocated for bus

2519

* if context not allocated for bus

2521

* allocate and init context

2520

* allocate and init context

2522

* set present in root table for this bus

2521

* set present in root table for this bus

2523

* init context with domain, translation etc

2522

* init context with domain, translation etc

2524

* endfor

2523

* endfor

2525

* endfor

2524

* endfor

2526

*/

2525

*/

2527

printk(KERN_INFO "IOMMU: Setting RMRR:\n");

2526

printk(KERN_INFO "IOMMU: Setting RMRR:\n");

2528

for_each_rmrr_units(rmrr) {

2527

for_each_rmrr_units(rmrr) {

2529

for (i = 0; i < rmrr->devices_cnt; i++) {

2528

for (i = 0; i < rmrr->devices_cnt; i++) {

2530

pdev = rmrr->devices[i];

2529

pdev = rmrr->devices[i];

2531

/*

2530

/*

2532

* some BIOS lists non-exist devices in DMAR

2531

* some BIOS lists non-exist devices in DMAR

2533

* table.

2532

* table.

2534

*/

2533

*/

2535

if (!pdev)

2534

if (!pdev)

2536

continue;

2535

continue;

2537

ret = iommu_prepare_rmrr_dev(rmrr, pdev);

2536

ret = iommu_prepare_rmrr_dev(rmrr, pdev);

2538

if (ret)

2537

if (ret)

2539

printk(KERN_ERR

2538

printk(KERN_ERR

2540

"IOMMU: mapping reserved region failed\n");

2539

"IOMMU: mapping reserved region failed\n");

2541

}

2540

}

2542

}

2541

}

2543

2542

2544

iommu_prepare_isa();

2543

iommu_prepare_isa();

2545

2544

2546

/*

2545

/*

2547

* for each drhd

2546

* for each drhd

2548

* enable fault log

2547

* enable fault log

2549

* global invalidate context cache

2548

* global invalidate context cache

2550

* global invalidate iotlb

2549

* global invalidate iotlb

2551

* enable translation

2550

* enable translation

2552

*/

2551

*/

2553

for_each_drhd_unit(drhd) {

2552

for_each_drhd_unit(drhd) {

2554

if (drhd->ignored) {

2553

if (drhd->ignored) {

2555

/*

2554

/*

2556

* we always have to disable PMRs or DMA may fail on

2555

* we always have to disable PMRs or DMA may fail on

2557

* this device

2556

* this device

2558

*/

2557

*/

2559

if (force_on)

2558

if (force_on)

2560

iommu_disable_protect_mem_regions(drhd->iommu);

2559

iommu_disable_protect_mem_regions(drhd->iommu);

2561

continue;

2560

continue;

2562

}

2561

}

2563

iommu = drhd->iommu;

2562

iommu = drhd->iommu;

2564

2563

2565

iommu_flush_write_buffer(iommu);

2564

iommu_flush_write_buffer(iommu);

2566

2565

2567

ret = dmar_set_interrupt(iommu);

2566

ret = dmar_set_interrupt(iommu);

2568

if (ret)

2567

if (ret)

2569

goto error;

2568

goto error;

2570

2569

2571

iommu_set_root_entry(iommu);

2570

iommu_set_root_entry(iommu);

2572

2571

2573

iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);

2572

iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);

2574

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

2573

iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);

2575

2574

2576

ret = iommu_enable_translation(iommu);

2575

ret = iommu_enable_translation(iommu);

2577

if (ret)

2576

if (ret)

2578

goto error;

2577

goto error;

2579

2578

2580

iommu_disable_protect_mem_regions(iommu);

2579

iommu_disable_protect_mem_regions(iommu);

2581

}

2580

}

2582

2581

2583

return 0;

2582

return 0;

2584

error:

2583

error:

2585

for_each_drhd_unit(drhd) {

2584

for_each_drhd_unit(drhd) {

2586

if (drhd->ignored)

2585

if (drhd->ignored)

2587

continue;

2586

continue;

2588

iommu = drhd->iommu;

2587

iommu = drhd->iommu;

2589

free_iommu(iommu);

2588

free_iommu(iommu);

2590

}

2589

}

2591

kfree(g_iommus);

2590

kfree(g_iommus);

2592

return ret;

2591

return ret;

2593

}

2592

}

2594

2593

2595

/* This takes a number of _MM_ pages, not VTD pages */

2594

/* This takes a number of _MM_ pages, not VTD pages */

2596

static struct iova *intel_alloc_iova(struct device *dev,

2595

static struct iova *intel_alloc_iova(struct device *dev,

2597

struct dmar_domain *domain,

2596

struct dmar_domain *domain,

2598

unsigned long nrpages, uint64_t dma_mask)

2597

unsigned long nrpages, uint64_t dma_mask)

2599

{

2598

{

2600

struct pci_dev *pdev = to_pci_dev(dev);

2599

struct pci_dev *pdev = to_pci_dev(dev);

2601

struct iova *iova = NULL;

2600

struct iova *iova = NULL;

2602

2601

2603

/* Restrict dma_mask to the width that the iommu can handle */

2602

/* Restrict dma_mask to the width that the iommu can handle */

2604

dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);

2603

dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);

2605

2604

2606

if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {

2605

if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {

2607

/*

2606

/*

2608

* First try to allocate an io virtual address in

2607

* First try to allocate an io virtual address in

2609

* DMA_BIT_MASK(32) and if that fails then try allocating

2608

* DMA_BIT_MASK(32) and if that fails then try allocating

2610

* from higher range

2609

* from higher range

2611

*/

2610

*/

2612

iova = alloc_iova(&domain->iovad, nrpages,

2611

iova = alloc_iova(&domain->iovad, nrpages,

2613

IOVA_PFN(DMA_BIT_MASK(32)), 1);

2612

IOVA_PFN(DMA_BIT_MASK(32)), 1);

2614

if (iova)

2613

if (iova)

2615

return iova;

2614

return iova;

2616

}

2615

}

2617

iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);

2616

iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);

2618

if (unlikely(!iova)) {

2617

if (unlikely(!iova)) {

2619

printk(KERN_ERR "Allocating %ld-page iova for %s failed",

2618

printk(KERN_ERR "Allocating %ld-page iova for %s failed",

2620

nrpages, pci_name(pdev));

2619

nrpages, pci_name(pdev));

2621

return NULL;

2620

return NULL;

2622

}

2621

}

2623

2622

2624

return iova;

2623

return iova;

2625

}

2624

}

2626

2625

2627

static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)

2626

static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)

2628

{

2627

{

2629

struct dmar_domain *domain;

2628

struct dmar_domain *domain;

2630

int ret;

2629

int ret;

2631

2630

2632

domain = get_domain_for_dev(pdev,

2631

domain = get_domain_for_dev(pdev,

2633

DEFAULT_DOMAIN_ADDRESS_WIDTH);

2632

DEFAULT_DOMAIN_ADDRESS_WIDTH);

2634

if (!domain) {

2633

if (!domain) {

2635

printk(KERN_ERR

2634

printk(KERN_ERR

2636

"Allocating domain for %s failed", pci_name(pdev));

2635

"Allocating domain for %s failed", pci_name(pdev));

2637

return NULL;

2636

return NULL;

2638

}

2637

}

2639

2638

2640

/* make sure context mapping is ok */

2639

/* make sure context mapping is ok */

2641

if (unlikely(!domain_context_mapped(pdev))) {

2640

if (unlikely(!domain_context_mapped(pdev))) {

2642

ret = domain_context_mapping(domain, pdev,

2641

ret = domain_context_mapping(domain, pdev,

2643

CONTEXT_TT_MULTI_LEVEL);

2642

CONTEXT_TT_MULTI_LEVEL);

2644

if (ret) {

2643

if (ret) {

2645

printk(KERN_ERR

2644

printk(KERN_ERR

2646

"Domain context map for %s failed",

2645

"Domain context map for %s failed",

2647

pci_name(pdev));

2646

pci_name(pdev));

2648

return NULL;

2647

return NULL;

2649

}

2648

}

2650

}

2649

}

2651

2650

2652

return domain;

2651

return domain;

2653

}

2652

}

2654

2653

2655

static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)

2654

static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)

2656

{

2655

{

2657

struct device_domain_info *info;

2656

struct device_domain_info *info;

2658

2657

2659

/* No lock here, assumes no domain exit in normal case */

2658

/* No lock here, assumes no domain exit in normal case */

2660

info = dev->dev.archdata.iommu;

2659

info = dev->dev.archdata.iommu;

2661

if (likely(info))

2660

if (likely(info))

2662

return info->domain;

2661

return info->domain;

2663

2662

2664

return __get_valid_domain_for_dev(dev);

2663

return __get_valid_domain_for_dev(dev);

2665

}

2664

}

2666

2665

2667

static int iommu_dummy(struct pci_dev *pdev)

2666

static int iommu_dummy(struct pci_dev *pdev)

2668

{

2667

{

2669

return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;

2668

return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;

2670

}

2669

}

2671

2670

2672

/* Check if the pdev needs to go through non-identity map and unmap process.*/

2671

/* Check if the pdev needs to go through non-identity map and unmap process.*/

2673

static int iommu_no_mapping(struct device *dev)

2672

static int iommu_no_mapping(struct device *dev)

2674

{

2673

{

2675

struct pci_dev *pdev;

2674

struct pci_dev *pdev;

2676

int found;

2675

int found;

2677

2676

2678

if (unlikely(dev->bus != &pci_bus_type))

2677

if (unlikely(dev->bus != &pci_bus_type))

2679

return 1;

2678

return 1;

2680

2679

2681

pdev = to_pci_dev(dev);

2680

pdev = to_pci_dev(dev);

2682

if (iommu_dummy(pdev))

2681

if (iommu_dummy(pdev))

2683

return 1;

2682

return 1;

2684

2683

2685

if (!iommu_identity_mapping)

2684

if (!iommu_identity_mapping)

2686

return 0;

2685

return 0;

2687

2686

2688

found = identity_mapping(pdev);

2687

found = identity_mapping(pdev);

2689

if (found) {

2688

if (found) {

2690

if (iommu_should_identity_map(pdev, 0))

2689

if (iommu_should_identity_map(pdev, 0))

2691

return 1;

2690

return 1;

2692

else {

2691

else {

2693

/*

2692

/*

2694

* 32 bit DMA is removed from si_domain and fall back

2693

* 32 bit DMA is removed from si_domain and fall back

2695

* to non-identity mapping.

2694

* to non-identity mapping.

2696

*/

2695

*/

2697

domain_remove_one_dev_info(si_domain, pdev);

2696

domain_remove_one_dev_info(si_domain, pdev);

2698

printk(KERN_INFO "32bit %s uses non-identity mapping\n",

2697

printk(KERN_INFO "32bit %s uses non-identity mapping\n",

2699

pci_name(pdev));

2698

pci_name(pdev));

2700

return 0;

2699

return 0;

2701

}

2700

}

2702

} else {

2701

} else {

2703

/*

2702

/*

2704

* In case of a detached 64 bit DMA device from vm, the device

2703

* In case of a detached 64 bit DMA device from vm, the device

2705

* is put into si_domain for identity mapping.

2704

* is put into si_domain for identity mapping.

2706

*/

2705

*/

2707

if (iommu_should_identity_map(pdev, 0)) {

2706

if (iommu_should_identity_map(pdev, 0)) {

2708

int ret;

2707

int ret;

2709

ret = domain_add_dev_info(si_domain, pdev,

2708

ret = domain_add_dev_info(si_domain, pdev,

2710

hw_pass_through ?

2709

hw_pass_through ?

2711

CONTEXT_TT_PASS_THROUGH :

2710

CONTEXT_TT_PASS_THROUGH :

2712

CONTEXT_TT_MULTI_LEVEL);

2711

CONTEXT_TT_MULTI_LEVEL);

2713

if (!ret) {

2712

if (!ret) {

2714

printk(KERN_INFO "64bit %s uses identity mapping\n",

2713

printk(KERN_INFO "64bit %s uses identity mapping\n",

2715

pci_name(pdev));

2714

pci_name(pdev));

2716

return 1;

2715

return 1;

2717

}

2716

}

2718

}

2717

}

2719

}

2718

}

2720

2719

2721

return 0;

2720

return 0;

2722

}

2721

}

2723

2722

2724

static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,

2723

static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,

2725

size_t size, int dir, u64 dma_mask)

2724

size_t size, int dir, u64 dma_mask)

2726

{

2725

{

2727

struct pci_dev *pdev = to_pci_dev(hwdev);

2726

struct pci_dev *pdev = to_pci_dev(hwdev);

2728

struct dmar_domain *domain;

2727

struct dmar_domain *domain;

2729

phys_addr_t start_paddr;

2728

phys_addr_t start_paddr;

2730

struct iova *iova;

2729

struct iova *iova;

2731

int prot = 0;

2730

int prot = 0;

2732

int ret;

2731

int ret;

2733

struct intel_iommu *iommu;

2732

struct intel_iommu *iommu;

2734

unsigned long paddr_pfn = paddr >> PAGE_SHIFT;

2733

unsigned long paddr_pfn = paddr >> PAGE_SHIFT;

2735

2734

2736

BUG_ON(dir == DMA_NONE);

2735

BUG_ON(dir == DMA_NONE);

2737

2736

2738

if (iommu_no_mapping(hwdev))

2737

if (iommu_no_mapping(hwdev))

2739

return paddr;

2738

return paddr;

2740

2739

2741

domain = get_valid_domain_for_dev(pdev);

2740

domain = get_valid_domain_for_dev(pdev);

2742

if (!domain)

2741

if (!domain)

2743

return 0;

2742

return 0;

2744

2743

2745

iommu = domain_get_iommu(domain);

2744

iommu = domain_get_iommu(domain);

2746

size = aligned_nrpages(paddr, size);

2745

size = aligned_nrpages(paddr, size);

2747

2746

2748

iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);

2747

iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);

2749

if (!iova)

2748

if (!iova)

2750

goto error;

2749

goto error;

2751

2750

2752

/*

2751

/*

2753

* Check if DMAR supports zero-length reads on write only

2752

* Check if DMAR supports zero-length reads on write only

2754

* mappings..

2753

* mappings..

2755

*/

2754

*/

2756

if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \

2755

if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \

2757

!cap_zlr(iommu->cap))

2756

!cap_zlr(iommu->cap))

2758

prot |= DMA_PTE_READ;

2757

prot |= DMA_PTE_READ;

2759

if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)

2758

if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)

2760

prot |= DMA_PTE_WRITE;

2759

prot |= DMA_PTE_WRITE;

2761

/*

2760

/*

2762

* paddr - (paddr + size) might be partial page, we should map the whole

2761

* paddr - (paddr + size) might be partial page, we should map the whole

2763

* page. Note: if two part of one page are separately mapped, we

2762

* page. Note: if two part of one page are separately mapped, we

2764

* might have two guest_addr mapping to the same host paddr, but this

2763

* might have two guest_addr mapping to the same host paddr, but this

2765

* is not a big problem

2764

* is not a big problem

2766

*/

2765

*/

2767

ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),

2766

ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),

2768

mm_to_dma_pfn(paddr_pfn), size, prot);

2767

mm_to_dma_pfn(paddr_pfn), size, prot);

2769

if (ret)

2768

if (ret)

2770

goto error;

2769

goto error;

2771

2770

2772

/* it's a non-present to present mapping. Only flush if caching mode */

2771

/* it's a non-present to present mapping. Only flush if caching mode */

2773

if (cap_caching_mode(iommu->cap))

2772

if (cap_caching_mode(iommu->cap))

2774

iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);

2773

iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);

2775

else

2774

else

2776

iommu_flush_write_buffer(iommu);

2775

iommu_flush_write_buffer(iommu);

2777

2776

2778

start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;

2777

start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;

2779

start_paddr += paddr & ~PAGE_MASK;

2778

start_paddr += paddr & ~PAGE_MASK;

2780

return start_paddr;

2779

return start_paddr;

2781

2780

2782

error:

2781

error:

2783

if (iova)

2782

if (iova)

2784

__free_iova(&domain->iovad, iova);

2783

__free_iova(&domain->iovad, iova);

2785

printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",

2784

printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",

2786

pci_name(pdev), size, (unsigned long long)paddr, dir);

2785

pci_name(pdev), size, (unsigned long long)paddr, dir);

2787

return 0;

2786

return 0;

2788

}

2787

}

2789

2788

2790

static dma_addr_t intel_map_page(struct device *dev, struct page *page,

2789

static dma_addr_t intel_map_page(struct device *dev, struct page *page,

2791

unsigned long offset, size_t size,

2790

unsigned long offset, size_t size,

2792

enum dma_data_direction dir,

2791

enum dma_data_direction dir,

2793

struct dma_attrs *attrs)

2792

struct dma_attrs *attrs)

2794

{

2793

{

2795

return __intel_map_single(dev, page_to_phys(page) + offset, size,

2794

return __intel_map_single(dev, page_to_phys(page) + offset, size,

2796

dir, to_pci_dev(dev)->dma_mask);

2795

dir, to_pci_dev(dev)->dma_mask);

2797

}

2796

}

2798

2797

2799

static void flush_unmaps(void)

2798

static void flush_unmaps(void)

2800

{

2799

{

2801

int i, j;

2800

int i, j;

2802

2801

2803

timer_on = 0;

2802

timer_on = 0;

2804

2803

2805

/* just flush them all */

2804

/* just flush them all */

2806

for (i = 0; i < g_num_of_iommus; i++) {

2805

for (i = 0; i < g_num_of_iommus; i++) {

2807

struct intel_iommu *iommu = g_iommus[i];

2806

struct intel_iommu *iommu = g_iommus[i];

2808

if (!iommu)

2807

if (!iommu)

2809

continue;

2808

continue;

2810

2809

2811

if (!deferred_flush[i].next)

2810

if (!deferred_flush[i].next)

2812

continue;

2811

continue;

2813

2812

2814

/* In caching mode, global flushes turn emulation expensive */

2813

/* In caching mode, global flushes turn emulation expensive */

2815

if (!cap_caching_mode(iommu->cap))

2814

if (!cap_caching_mode(iommu->cap))

2816

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

2815

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

2817

DMA_TLB_GLOBAL_FLUSH);

2816

DMA_TLB_GLOBAL_FLUSH);

2818

for (j = 0; j < deferred_flush[i].next; j++) {

2817

for (j = 0; j < deferred_flush[i].next; j++) {

2819

unsigned long mask;

2818

unsigned long mask;

2820

struct iova *iova = deferred_flush[i].iova[j];

2819

struct iova *iova = deferred_flush[i].iova[j];

2821

struct dmar_domain *domain = deferred_flush[i].domain[j];

2820

struct dmar_domain *domain = deferred_flush[i].domain[j];

2822

2821

2823

/* On real hardware multiple invalidations are expensive */

2822

/* On real hardware multiple invalidations are expensive */

2824

if (cap_caching_mode(iommu->cap))

2823

if (cap_caching_mode(iommu->cap))

2825

iommu_flush_iotlb_psi(iommu, domain->id,

2824

iommu_flush_iotlb_psi(iommu, domain->id,

2826

iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);

2825

iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);

2827

else {

2826

else {

2828

mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));

2827

mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));

2829

iommu_flush_dev_iotlb(deferred_flush[i].domain[j],

2828

iommu_flush_dev_iotlb(deferred_flush[i].domain[j],

2830

(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);

2829

(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);

2831

}

2830

}

2832

__free_iova(&deferred_flush[i].domain[j]->iovad, iova);

2831

__free_iova(&deferred_flush[i].domain[j]->iovad, iova);

2833

}

2832

}

2834

deferred_flush[i].next = 0;

2833

deferred_flush[i].next = 0;

2835

}

2834

}

2836

2835

2837

list_size = 0;

2836

list_size = 0;

2838

}

2837

}

2839

2838

2840

static void flush_unmaps_timeout(unsigned long data)

2839

static void flush_unmaps_timeout(unsigned long data)

2841

{

2840

{

2842

unsigned long flags;

2841

unsigned long flags;

2843

2842

2844

spin_lock_irqsave(&async_umap_flush_lock, flags);

2843

spin_lock_irqsave(&async_umap_flush_lock, flags);

2845

flush_unmaps();

2844

flush_unmaps();

2846

spin_unlock_irqrestore(&async_umap_flush_lock, flags);

2845

spin_unlock_irqrestore(&async_umap_flush_lock, flags);

2847

}

2846

}

2848

2847

2849

static void add_unmap(struct dmar_domain *dom, struct iova *iova)

2848

static void add_unmap(struct dmar_domain *dom, struct iova *iova)

2850

{

2849

{

2851

unsigned long flags;

2850

unsigned long flags;

2852

int next, iommu_id;

2851

int next, iommu_id;

2853

struct intel_iommu *iommu;

2852

struct intel_iommu *iommu;

2854

2853

2855

spin_lock_irqsave(&async_umap_flush_lock, flags);

2854

spin_lock_irqsave(&async_umap_flush_lock, flags);

2856

if (list_size == HIGH_WATER_MARK)

2855

if (list_size == HIGH_WATER_MARK)

2857

flush_unmaps();

2856

flush_unmaps();

2858

2857

2859

iommu = domain_get_iommu(dom);

2858

iommu = domain_get_iommu(dom);

2860

iommu_id = iommu->seq_id;

2859

iommu_id = iommu->seq_id;

2861

2860

2862

next = deferred_flush[iommu_id].next;

2861

next = deferred_flush[iommu_id].next;

2863

deferred_flush[iommu_id].domain[next] = dom;

2862

deferred_flush[iommu_id].domain[next] = dom;

2864

deferred_flush[iommu_id].iova[next] = iova;

2863

deferred_flush[iommu_id].iova[next] = iova;

2865

deferred_flush[iommu_id].next++;

2864

deferred_flush[iommu_id].next++;

2866

2865

2867

if (!timer_on) {

2866

if (!timer_on) {

2868

mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));

2867

mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));

2869

timer_on = 1;

2868

timer_on = 1;

2870

}

2869

}

2871

list_size++;

2870

list_size++;

2872

spin_unlock_irqrestore(&async_umap_flush_lock, flags);

2871

spin_unlock_irqrestore(&async_umap_flush_lock, flags);

2873

}

2872

}

2874

2873

2875

static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,

2874

static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,

2876

size_t size, enum dma_data_direction dir,

2875

size_t size, enum dma_data_direction dir,

2877

struct dma_attrs *attrs)

2876

struct dma_attrs *attrs)

2878

{

2877

{

2879

struct pci_dev *pdev = to_pci_dev(dev);

2878

struct pci_dev *pdev = to_pci_dev(dev);

2880

struct dmar_domain *domain;

2879

struct dmar_domain *domain;

2881

unsigned long start_pfn, last_pfn;

2880

unsigned long start_pfn, last_pfn;

2882

struct iova *iova;

2881

struct iova *iova;

2883

struct intel_iommu *iommu;

2882

struct intel_iommu *iommu;

2884

2883

2885

if (iommu_no_mapping(dev))

2884

if (iommu_no_mapping(dev))

2886

return;

2885

return;

2887

2886

2888

domain = find_domain(pdev);

2887

domain = find_domain(pdev);

2889

BUG_ON(!domain);

2888

BUG_ON(!domain);

2890

2889

2891

iommu = domain_get_iommu(domain);

2890

iommu = domain_get_iommu(domain);

2892

2891

2893

iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));

2892

iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));

2894

if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",

2893

if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",

2895

(unsigned long long)dev_addr))

2894

(unsigned long long)dev_addr))

2896

return;

2895

return;

2897

2896

2898

start_pfn = mm_to_dma_pfn(iova->pfn_lo);

2897

start_pfn = mm_to_dma_pfn(iova->pfn_lo);

2899

last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;

2898

last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;

2900

2899

2901

pr_debug("Device %s unmapping: pfn %lx-%lx\n",

2900

pr_debug("Device %s unmapping: pfn %lx-%lx\n",

2902

pci_name(pdev), start_pfn, last_pfn);

2901

pci_name(pdev), start_pfn, last_pfn);

2903

2902

2904

/* clear the whole page */

2903

/* clear the whole page */

2905

dma_pte_clear_range(domain, start_pfn, last_pfn);

2904

dma_pte_clear_range(domain, start_pfn, last_pfn);

2906

2905

2907

/* free page tables */

2906

/* free page tables */

2908

dma_pte_free_pagetable(domain, start_pfn, last_pfn);

2907

dma_pte_free_pagetable(domain, start_pfn, last_pfn);

2909

2908

2910

if (intel_iommu_strict) {

2909

if (intel_iommu_strict) {

2911

iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,

2910

iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,

2912

last_pfn - start_pfn + 1, 0);

2911

last_pfn - start_pfn + 1, 0);

2913

/* free iova */

2912

/* free iova */

2914

__free_iova(&domain->iovad, iova);

2913

__free_iova(&domain->iovad, iova);

2915

} else {

2914

} else {

2916

add_unmap(domain, iova);

2915

add_unmap(domain, iova);

2917

/*

2916

/*

2918

* queue up the release of the unmap to save the 1/6th of the

2917

* queue up the release of the unmap to save the 1/6th of the

2919

* cpu used up by the iotlb flush operation...

2918

* cpu used up by the iotlb flush operation...

2920

*/

2919

*/

2921

}

2920

}

2922

}

2921

}

2923

2922

2924

static void *intel_alloc_coherent(struct device *hwdev, size_t size,

2923

static void *intel_alloc_coherent(struct device *hwdev, size_t size,

2925

dma_addr_t *dma_handle, gfp_t flags)

2924

dma_addr_t *dma_handle, gfp_t flags)

2926

{

2925

{

2927

void *vaddr;

2926

void *vaddr;

2928

int order;

2927

int order;

2929

2928

2930

size = PAGE_ALIGN(size);

2929

size = PAGE_ALIGN(size);

2931

order = get_order(size);

2930

order = get_order(size);

2932

2931

2933

if (!iommu_no_mapping(hwdev))

2932

if (!iommu_no_mapping(hwdev))

2934

flags &= ~(GFP_DMA | GFP_DMA32);

2933

flags &= ~(GFP_DMA | GFP_DMA32);

2935

else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {

2934

else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {

2936

if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))

2935

if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))

2937

flags |= GFP_DMA;

2936

flags |= GFP_DMA;

2938

else

2937

else

2939

flags |= GFP_DMA32;

2938

flags |= GFP_DMA32;

2940

}

2939

}

2941

2940

2942

vaddr = (void *)__get_free_pages(flags, order);

2941

vaddr = (void *)__get_free_pages(flags, order);

2943

if (!vaddr)

2942

if (!vaddr)

2944

return NULL;

2943

return NULL;

2945

memset(vaddr, 0, size);

2944

memset(vaddr, 0, size);

2946

2945

2947

*dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,

2946

*dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,

2948

DMA_BIDIRECTIONAL,

2947

DMA_BIDIRECTIONAL,

2949

hwdev->coherent_dma_mask);

2948

hwdev->coherent_dma_mask);

2950

if (*dma_handle)

2949

if (*dma_handle)

2951

return vaddr;

2950

return vaddr;

2952

free_pages((unsigned long)vaddr, order);

2951

free_pages((unsigned long)vaddr, order);

2953

return NULL;

2952

return NULL;

2954

}

2953

}

2955

2954

2956

static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,

2955

static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,

2957

dma_addr_t dma_handle)

2956

dma_addr_t dma_handle)

2958

{

2957

{

2959

int order;

2958

int order;

2960

2959

2961

size = PAGE_ALIGN(size);

2960

size = PAGE_ALIGN(size);

2962

order = get_order(size);

2961

order = get_order(size);

2963

2962

2964

intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);

2963

intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);

2965

free_pages((unsigned long)vaddr, order);

2964

free_pages((unsigned long)vaddr, order);

2966

}

2965

}

2967

2966

2968

static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,

2967

static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,

2969

int nelems, enum dma_data_direction dir,

2968

int nelems, enum dma_data_direction dir,

2970

struct dma_attrs *attrs)

2969

struct dma_attrs *attrs)

2971

{

2970

{

2972

struct pci_dev *pdev = to_pci_dev(hwdev);

2971

struct pci_dev *pdev = to_pci_dev(hwdev);

2973

struct dmar_domain *domain;

2972

struct dmar_domain *domain;

2974

unsigned long start_pfn, last_pfn;

2973

unsigned long start_pfn, last_pfn;

2975

struct iova *iova;

2974

struct iova *iova;

2976

struct intel_iommu *iommu;

2975

struct intel_iommu *iommu;

2977

2976

2978

if (iommu_no_mapping(hwdev))

2977

if (iommu_no_mapping(hwdev))

2979

return;

2978

return;

2980

2979

2981

domain = find_domain(pdev);

2980

domain = find_domain(pdev);

2982

BUG_ON(!domain);

2981

BUG_ON(!domain);

2983

2982

2984

iommu = domain_get_iommu(domain);

2983

iommu = domain_get_iommu(domain);

2985

2984

2986

iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));

2985

iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));

2987

if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",

2986

if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",

2988

(unsigned long long)sglist[0].dma_address))

2987

(unsigned long long)sglist[0].dma_address))

2989

return;

2988

return;

2990

2989

2991

start_pfn = mm_to_dma_pfn(iova->pfn_lo);

2990

start_pfn = mm_to_dma_pfn(iova->pfn_lo);

2992

last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;

2991

last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;

2993

2992

2994

/* clear the whole page */

2993

/* clear the whole page */

2995

dma_pte_clear_range(domain, start_pfn, last_pfn);

2994

dma_pte_clear_range(domain, start_pfn, last_pfn);

2996

2995

2997

/* free page tables */

2996

/* free page tables */

2998

dma_pte_free_pagetable(domain, start_pfn, last_pfn);

2997

dma_pte_free_pagetable(domain, start_pfn, last_pfn);

2999

2998

3000

if (intel_iommu_strict) {

2999

if (intel_iommu_strict) {

3001

iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,

3000

iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,

3002

last_pfn - start_pfn + 1, 0);

3001

last_pfn - start_pfn + 1, 0);

3003

/* free iova */

3002

/* free iova */

3004

__free_iova(&domain->iovad, iova);

3003

__free_iova(&domain->iovad, iova);

3005

} else {

3004

} else {

3006

add_unmap(domain, iova);

3005

add_unmap(domain, iova);

3007

/*

3006

/*

3008

* queue up the release of the unmap to save the 1/6th of the

3007

* queue up the release of the unmap to save the 1/6th of the

3009

* cpu used up by the iotlb flush operation...

3008

* cpu used up by the iotlb flush operation...

3010

*/

3009

*/

3011

}

3010

}

3012

}

3011

}

3013

3012

3014

static int intel_nontranslate_map_sg(struct device *hddev,

3013

static int intel_nontranslate_map_sg(struct device *hddev,

3015

struct scatterlist *sglist, int nelems, int dir)

3014

struct scatterlist *sglist, int nelems, int dir)

3016

{

3015

{

3017

int i;

3016

int i;

3018

struct scatterlist *sg;

3017

struct scatterlist *sg;

3019

3018

3020

for_each_sg(sglist, sg, nelems, i) {

3019

for_each_sg(sglist, sg, nelems, i) {

3021

BUG_ON(!sg_page(sg));

3020

BUG_ON(!sg_page(sg));

3022

sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;

3021

sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;

3023

sg->dma_length = sg->length;

3022

sg->dma_length = sg->length;

3024

}

3023

}

3025

return nelems;

3024

return nelems;

3026

}

3025

}

3027

3026

3028

static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,

3027

static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,

3029

enum dma_data_direction dir, struct dma_attrs *attrs)

3028

enum dma_data_direction dir, struct dma_attrs *attrs)

3030

{

3029

{

3031

int i;

3030

int i;

3032

struct pci_dev *pdev = to_pci_dev(hwdev);

3031

struct pci_dev *pdev = to_pci_dev(hwdev);

3033

struct dmar_domain *domain;

3032

struct dmar_domain *domain;

3034

size_t size = 0;

3033

size_t size = 0;

3035

int prot = 0;

3034

int prot = 0;

3036

struct iova *iova = NULL;

3035

struct iova *iova = NULL;

3037

int ret;

3036

int ret;

3038

struct scatterlist *sg;

3037

struct scatterlist *sg;

3039

unsigned long start_vpfn;

3038

unsigned long start_vpfn;

3040

struct intel_iommu *iommu;

3039

struct intel_iommu *iommu;

3041

3040

3042

BUG_ON(dir == DMA_NONE);

3041

BUG_ON(dir == DMA_NONE);

3043

if (iommu_no_mapping(hwdev))

3042

if (iommu_no_mapping(hwdev))

3044

return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);

3043

return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);

3045

3044

3046

domain = get_valid_domain_for_dev(pdev);

3045

domain = get_valid_domain_for_dev(pdev);

3047

if (!domain)

3046

if (!domain)

3048

return 0;

3047

return 0;

3049

3048

3050

iommu = domain_get_iommu(domain);

3049

iommu = domain_get_iommu(domain);

3051

3050

3052

for_each_sg(sglist, sg, nelems, i)

3051

for_each_sg(sglist, sg, nelems, i)

3053

size += aligned_nrpages(sg->offset, sg->length);

3052

size += aligned_nrpages(sg->offset, sg->length);

3054

3053

3055

iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),

3054

iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),

3056

pdev->dma_mask);

3055

pdev->dma_mask);

3057

if (!iova) {

3056

if (!iova) {

3058

sglist->dma_length = 0;

3057

sglist->dma_length = 0;

3059

return 0;

3058

return 0;

3060

}

3059

}

3061

3060

3062

/*

3061

/*

3063

* Check if DMAR supports zero-length reads on write only

3062

* Check if DMAR supports zero-length reads on write only

3064

* mappings..

3063

* mappings..

3065

*/

3064

*/

3066

if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \

3065

if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \

3067

!cap_zlr(iommu->cap))

3066

!cap_zlr(iommu->cap))

3068

prot |= DMA_PTE_READ;

3067

prot |= DMA_PTE_READ;

3069

if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)

3068

if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)

3070

prot |= DMA_PTE_WRITE;

3069

prot |= DMA_PTE_WRITE;

3071

3070

3072

start_vpfn = mm_to_dma_pfn(iova->pfn_lo);

3071

start_vpfn = mm_to_dma_pfn(iova->pfn_lo);

3073

3072

3074

ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);

3073

ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);

3075

if (unlikely(ret)) {

3074

if (unlikely(ret)) {

3076

/* clear the page */

3075

/* clear the page */

3077

dma_pte_clear_range(domain, start_vpfn,

3076

dma_pte_clear_range(domain, start_vpfn,

3078

start_vpfn + size - 1);

3077

start_vpfn + size - 1);

3079

/* free page tables */

3078

/* free page tables */

3080

dma_pte_free_pagetable(domain, start_vpfn,

3079

dma_pte_free_pagetable(domain, start_vpfn,

3081

start_vpfn + size - 1);

3080

start_vpfn + size - 1);

3082

/* free iova */

3081

/* free iova */

3083

__free_iova(&domain->iovad, iova);

3082

__free_iova(&domain->iovad, iova);

3084

return 0;

3083

return 0;

3085

}

3084

}

3086

3085

3087

/* it's a non-present to present mapping. Only flush if caching mode */

3086

/* it's a non-present to present mapping. Only flush if caching mode */

3088

if (cap_caching_mode(iommu->cap))

3087

if (cap_caching_mode(iommu->cap))

3089

iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);

3088

iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);

3090

else

3089

else

3091

iommu_flush_write_buffer(iommu);

3090

iommu_flush_write_buffer(iommu);

3092

3091

3093

return nelems;

3092

return nelems;

3094

}

3093

}

3095

3094

3096

static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)

3095

static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)

3097

{

3096

{

3098

return !dma_addr;

3097

return !dma_addr;

3099

}

3098

}

3100

3099

3101

struct dma_map_ops intel_dma_ops = {

3100

struct dma_map_ops intel_dma_ops = {

3102

.alloc_coherent = intel_alloc_coherent,

3101

.alloc_coherent = intel_alloc_coherent,

3103

.free_coherent = intel_free_coherent,

3102

.free_coherent = intel_free_coherent,

3104

.map_sg = intel_map_sg,

3103

.map_sg = intel_map_sg,

3105

.unmap_sg = intel_unmap_sg,

3104

.unmap_sg = intel_unmap_sg,

3106

.map_page = intel_map_page,

3105

.map_page = intel_map_page,

3107

.unmap_page = intel_unmap_page,

3106

.unmap_page = intel_unmap_page,

3108

.mapping_error = intel_mapping_error,

3107

.mapping_error = intel_mapping_error,

3109

};

3108

};

3110

3109

3111

static inline int iommu_domain_cache_init(void)

3110

static inline int iommu_domain_cache_init(void)

3112

{

3111

{

3113

int ret = 0;

3112

int ret = 0;

3114

3113

3115

iommu_domain_cache = kmem_cache_create("iommu_domain",

3114

iommu_domain_cache = kmem_cache_create("iommu_domain",

3116

sizeof(struct dmar_domain),

3115

sizeof(struct dmar_domain),

3117

0,

3116

0,

3118

SLAB_HWCACHE_ALIGN,

3117

SLAB_HWCACHE_ALIGN,

3119

3118

3120

NULL);

3119

NULL);

3121

if (!iommu_domain_cache) {

3120

if (!iommu_domain_cache) {

3122

printk(KERN_ERR "Couldn't create iommu_domain cache\n");

3121

printk(KERN_ERR "Couldn't create iommu_domain cache\n");

3123

ret = -ENOMEM;

3122

ret = -ENOMEM;

3124

}

3123

}

3125

3124

3126

return ret;

3125

return ret;

3127

}

3126

}

3128

3127

3129

static inline int iommu_devinfo_cache_init(void)

3128

static inline int iommu_devinfo_cache_init(void)

3130

{

3129

{

3131

int ret = 0;

3130

int ret = 0;

3132

3131

3133

iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",

3132

iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",

3134

sizeof(struct device_domain_info),

3133

sizeof(struct device_domain_info),

3135

0,

3134

0,

3136

SLAB_HWCACHE_ALIGN,

3135

SLAB_HWCACHE_ALIGN,

3137

NULL);

3136

NULL);

3138

if (!iommu_devinfo_cache) {

3137

if (!iommu_devinfo_cache) {

3139

printk(KERN_ERR "Couldn't create devinfo cache\n");

3138

printk(KERN_ERR "Couldn't create devinfo cache\n");

3140

ret = -ENOMEM;

3139

ret = -ENOMEM;

3141

}

3140

}

3142

3141

3143

return ret;

3142

return ret;

3144

}

3143

}

3145

3144

3146

static inline int iommu_iova_cache_init(void)

3145

static inline int iommu_iova_cache_init(void)

3147

{

3146

{

3148

int ret = 0;

3147

int ret = 0;

3149

3148

3150

iommu_iova_cache = kmem_cache_create("iommu_iova",

3149

iommu_iova_cache = kmem_cache_create("iommu_iova",

3151

sizeof(struct iova),

3150

sizeof(struct iova),

3152

0,

3151

0,

3153

SLAB_HWCACHE_ALIGN,

3152

SLAB_HWCACHE_ALIGN,

3154

NULL);

3153

NULL);

3155

if (!iommu_iova_cache) {

3154

if (!iommu_iova_cache) {

3156

printk(KERN_ERR "Couldn't create iova cache\n");

3155

printk(KERN_ERR "Couldn't create iova cache\n");

3157

ret = -ENOMEM;

3156

ret = -ENOMEM;

3158

}

3157

}

3159

3158

3160

return ret;

3159

return ret;

3161

}

3160

}

3162

3161

3163

static int __init iommu_init_mempool(void)

3162

static int __init iommu_init_mempool(void)

3164

{

3163

{

3165

int ret;

3164

int ret;

3166

ret = iommu_iova_cache_init();

3165

ret = iommu_iova_cache_init();

3167

if (ret)

3166

if (ret)

3168

return ret;

3167

return ret;

3169

3168

3170

ret = iommu_domain_cache_init();

3169

ret = iommu_domain_cache_init();

3171

if (ret)

3170

if (ret)

3172

goto domain_error;

3171

goto domain_error;

3173

3172

3174

ret = iommu_devinfo_cache_init();

3173

ret = iommu_devinfo_cache_init();

3175

if (!ret)

3174

if (!ret)

3176

return ret;

3175

return ret;

3177

3176

3178

kmem_cache_destroy(iommu_domain_cache);

3177

kmem_cache_destroy(iommu_domain_cache);

3179

domain_error:

3178

domain_error:

3180

kmem_cache_destroy(iommu_iova_cache);

3179

kmem_cache_destroy(iommu_iova_cache);

3181

3180

3182

return -ENOMEM;

3181

return -ENOMEM;

3183

}

3182

}

3184

3183

3185

static void __init iommu_exit_mempool(void)

3184

static void __init iommu_exit_mempool(void)

3186

{

3185

{

3187

kmem_cache_destroy(iommu_devinfo_cache);

3186

kmem_cache_destroy(iommu_devinfo_cache);

3188

kmem_cache_destroy(iommu_domain_cache);

3187

kmem_cache_destroy(iommu_domain_cache);

3189

kmem_cache_destroy(iommu_iova_cache);

3188

kmem_cache_destroy(iommu_iova_cache);

3190

3189

3191

}

3190

}

3192

3191

3193

static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)

3192

static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)

3194

{

3193

{

3195

struct dmar_drhd_unit *drhd;

3194

struct dmar_drhd_unit *drhd;

3196

u32 vtbar;

3195

u32 vtbar;

3197

int rc;

3196

int rc;

3198

3197

3199

/* We know that this device on this chipset has its own IOMMU.

3198

/* We know that this device on this chipset has its own IOMMU.

3200

* If we find it under a different IOMMU, then the BIOS is lying

3199

* If we find it under a different IOMMU, then the BIOS is lying

3201

* to us. Hope that the IOMMU for this device is actually

3200

* to us. Hope that the IOMMU for this device is actually

3202

* disabled, and it needs no translation...

3201

* disabled, and it needs no translation...

3203

*/

3202

*/

3204

rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);

3203

rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);

3205

if (rc) {

3204

if (rc) {

3206

/* "can't" happen */

3205

/* "can't" happen */

3207

dev_info(&pdev->dev, "failed to run vt-d quirk\n");

3206

dev_info(&pdev->dev, "failed to run vt-d quirk\n");

3208

return;

3207

return;

3209

}

3208

}

3210

vtbar &= 0xffff0000;

3209

vtbar &= 0xffff0000;

3211

3210

3212

/* we know that the this iommu should be at offset 0xa000 from vtbar */

3211

/* we know that the this iommu should be at offset 0xa000 from vtbar */

3213

drhd = dmar_find_matched_drhd_unit(pdev);

3212

drhd = dmar_find_matched_drhd_unit(pdev);

3214

if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,

3213

if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,

3215

TAINT_FIRMWARE_WORKAROUND,

3214

TAINT_FIRMWARE_WORKAROUND,

3216

"BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))

3215

"BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))

3217

pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;

3216

pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;

3218

}

3217

}

3219

DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);

3218

DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);

3220

3219

3221

static void __init init_no_remapping_devices(void)

3220

static void __init init_no_remapping_devices(void)

3222

{

3221

{

3223

struct dmar_drhd_unit *drhd;

3222

struct dmar_drhd_unit *drhd;

3224

3223

3225

for_each_drhd_unit(drhd) {

3224

for_each_drhd_unit(drhd) {

3226

if (!drhd->include_all) {

3225

if (!drhd->include_all) {

3227

int i;

3226

int i;

3228

for (i = 0; i < drhd->devices_cnt; i++)

3227

for (i = 0; i < drhd->devices_cnt; i++)

3229

if (drhd->devices[i] != NULL)

3228

if (drhd->devices[i] != NULL)

3230

break;

3229

break;

3231

/* ignore DMAR unit if no pci devices exist */

3230

/* ignore DMAR unit if no pci devices exist */

3232

if (i == drhd->devices_cnt)

3231

if (i == drhd->devices_cnt)

3233

drhd->ignored = 1;

3232

drhd->ignored = 1;

3234

}

3233

}

3235

}

3234

}

3236

3235

3237

for_each_drhd_unit(drhd) {

3236

for_each_drhd_unit(drhd) {

3238

int i;

3237

int i;

3239

if (drhd->ignored || drhd->include_all)

3238

if (drhd->ignored || drhd->include_all)

3240

continue;

3239

continue;

3241

3240

3242

for (i = 0; i < drhd->devices_cnt; i++)

3241

for (i = 0; i < drhd->devices_cnt; i++)

3243

if (drhd->devices[i] &&

3242

if (drhd->devices[i] &&

3244

!IS_GFX_DEVICE(drhd->devices[i]))

3243

!IS_GFX_DEVICE(drhd->devices[i]))

3245

break;

3244

break;

3246

3245

3247

if (i < drhd->devices_cnt)

3246

if (i < drhd->devices_cnt)

3248

continue;

3247

continue;

3249

3248

3250

/* This IOMMU has *only* gfx devices. Either bypass it or

3249

/* This IOMMU has *only* gfx devices. Either bypass it or

3251

set the gfx_mapped flag, as appropriate */

3250

set the gfx_mapped flag, as appropriate */

3252

if (dmar_map_gfx) {

3251

if (dmar_map_gfx) {

3253

intel_iommu_gfx_mapped = 1;

3252

intel_iommu_gfx_mapped = 1;

3254

} else {

3253

} else {

3255

drhd->ignored = 1;

3254

drhd->ignored = 1;

3256

for (i = 0; i < drhd->devices_cnt; i++) {

3255

for (i = 0; i < drhd->devices_cnt; i++) {

3257

if (!drhd->devices[i])

3256

if (!drhd->devices[i])

3258

continue;

3257

continue;

3259

drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;

3258

drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;

3260

}

3259

}

3261

}

3260

}

3262

}

3261

}

3263

}

3262

}

3264

3263

3265

#ifdef CONFIG_SUSPEND

3264

#ifdef CONFIG_SUSPEND

3266

static int init_iommu_hw(void)

3265

static int init_iommu_hw(void)

3267

{

3266

{

3268

struct dmar_drhd_unit *drhd;

3267

struct dmar_drhd_unit *drhd;

3269

struct intel_iommu *iommu = NULL;

3268

struct intel_iommu *iommu = NULL;

3270

3269

3271

for_each_active_iommu(iommu, drhd)

3270

for_each_active_iommu(iommu, drhd)

3272

if (iommu->qi)

3271

if (iommu->qi)

3273

dmar_reenable_qi(iommu);

3272

dmar_reenable_qi(iommu);

3274

3273

3275

for_each_iommu(iommu, drhd) {

3274

for_each_iommu(iommu, drhd) {

3276

if (drhd->ignored) {

3275

if (drhd->ignored) {

3277

/*

3276

/*

3278

* we always have to disable PMRs or DMA may fail on

3277

* we always have to disable PMRs or DMA may fail on

3279

* this device

3278

* this device

3280

*/

3279

*/

3281

if (force_on)

3280

if (force_on)

3282

iommu_disable_protect_mem_regions(iommu);

3281

iommu_disable_protect_mem_regions(iommu);

3283

continue;

3282

continue;

3284

}

3283

}

3285

3284

3286

iommu_flush_write_buffer(iommu);

3285

iommu_flush_write_buffer(iommu);

3287

3286

3288

iommu_set_root_entry(iommu);

3287

iommu_set_root_entry(iommu);

3289

3288

3290

iommu->flush.flush_context(iommu, 0, 0, 0,

3289

iommu->flush.flush_context(iommu, 0, 0, 0,

3291

DMA_CCMD_GLOBAL_INVL);

3290

DMA_CCMD_GLOBAL_INVL);

3292

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

3291

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

3293

DMA_TLB_GLOBAL_FLUSH);

3292

DMA_TLB_GLOBAL_FLUSH);

3294

if (iommu_enable_translation(iommu))

3293

if (iommu_enable_translation(iommu))

3295

return 1;

3294

return 1;

3296

iommu_disable_protect_mem_regions(iommu);

3295

iommu_disable_protect_mem_regions(iommu);

3297

}

3296

}

3298

3297

3299

return 0;

3298

return 0;

3300

}

3299

}

3301

3300

3302

static void iommu_flush_all(void)

3301

static void iommu_flush_all(void)

3303

{

3302

{

3304

struct dmar_drhd_unit *drhd;

3303

struct dmar_drhd_unit *drhd;

3305

struct intel_iommu *iommu;

3304

struct intel_iommu *iommu;

3306

3305

3307

for_each_active_iommu(iommu, drhd) {

3306

for_each_active_iommu(iommu, drhd) {

3308

iommu->flush.flush_context(iommu, 0, 0, 0,

3307

iommu->flush.flush_context(iommu, 0, 0, 0,

3309

DMA_CCMD_GLOBAL_INVL);

3308

DMA_CCMD_GLOBAL_INVL);

3310

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

3309

iommu->flush.flush_iotlb(iommu, 0, 0, 0,

3311

DMA_TLB_GLOBAL_FLUSH);

3310

DMA_TLB_GLOBAL_FLUSH);

3312

}

3311

}

3313

}

3312

}

3314

3313

3315

static int iommu_suspend(void)

3314

static int iommu_suspend(void)

3316

{

3315

{

3317

struct dmar_drhd_unit *drhd;

3316

struct dmar_drhd_unit *drhd;

3318

struct intel_iommu *iommu = NULL;

3317

struct intel_iommu *iommu = NULL;

3319

unsigned long flag;

3318

unsigned long flag;

3320

3319

3321

for_each_active_iommu(iommu, drhd) {

3320

for_each_active_iommu(iommu, drhd) {

3322

iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,

3321

iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,

3323

GFP_ATOMIC);

3322

GFP_ATOMIC);

3324

if (!iommu->iommu_state)

3323

if (!iommu->iommu_state)

3325

goto nomem;

3324

goto nomem;

3326

}

3325

}

3327

3326

3328

iommu_flush_all();

3327

iommu_flush_all();

3329

3328

3330

for_each_active_iommu(iommu, drhd) {

3329

for_each_active_iommu(iommu, drhd) {

3331

iommu_disable_translation(iommu);

3330

iommu_disable_translation(iommu);

3332

3331

3333

spin_lock_irqsave(&iommu->register_lock, flag);

3332

spin_lock_irqsave(&iommu->register_lock, flag);

3334

3333

3335

iommu->iommu_state[SR_DMAR_FECTL_REG] =

3334

iommu->iommu_state[SR_DMAR_FECTL_REG] =

3336

readl(iommu->reg + DMAR_FECTL_REG);

3335

readl(iommu->reg + DMAR_FECTL_REG);

3337

iommu->iommu_state[SR_DMAR_FEDATA_REG] =

3336

iommu->iommu_state[SR_DMAR_FEDATA_REG] =

3338

readl(iommu->reg + DMAR_FEDATA_REG);

3337

readl(iommu->reg + DMAR_FEDATA_REG);

3339

iommu->iommu_state[SR_DMAR_FEADDR_REG] =

3338

iommu->iommu_state[SR_DMAR_FEADDR_REG] =

3340

readl(iommu->reg + DMAR_FEADDR_REG);

3339

readl(iommu->reg + DMAR_FEADDR_REG);

3341

iommu->iommu_state[SR_DMAR_FEUADDR_REG] =

3340

iommu->iommu_state[SR_DMAR_FEUADDR_REG] =

3342

readl(iommu->reg + DMAR_FEUADDR_REG);

3341

readl(iommu->reg + DMAR_FEUADDR_REG);

3343

3342

3344

spin_unlock_irqrestore(&iommu->register_lock, flag);

3343

spin_unlock_irqrestore(&iommu->register_lock, flag);

3345

}

3344

}

3346

return 0;

3345

return 0;

3347

3346

3348

nomem:

3347

nomem:

3349

for_each_active_iommu(iommu, drhd)

3348

for_each_active_iommu(iommu, drhd)

3350

kfree(iommu->iommu_state);

3349

kfree(iommu->iommu_state);

3351

3350

3352

return -ENOMEM;

3351

return -ENOMEM;

3353

}

3352

}

3354

3353

3355

static void iommu_resume(void)

3354

static void iommu_resume(void)

3356

{

3355

{

3357

struct dmar_drhd_unit *drhd;

3356

struct dmar_drhd_unit *drhd;

3358

struct intel_iommu *iommu = NULL;

3357

struct intel_iommu *iommu = NULL;

3359

unsigned long flag;

3358

unsigned long flag;

3360

3359

3361

if (init_iommu_hw()) {

3360

if (init_iommu_hw()) {

3362

if (force_on)

3361

if (force_on)

3363

panic("tboot: IOMMU setup failed, DMAR can not resume!\n");

3362

panic("tboot: IOMMU setup failed, DMAR can not resume!\n");

3364

else

3363

else

3365

WARN(1, "IOMMU setup failed, DMAR can not resume!\n");

3364

WARN(1, "IOMMU setup failed, DMAR can not resume!\n");

3366

return;

3365

return;

3367

}

3366

}

3368

3367

3369

for_each_active_iommu(iommu, drhd) {

3368

for_each_active_iommu(iommu, drhd) {

3370

3369

3371

spin_lock_irqsave(&iommu->register_lock, flag);

3370

spin_lock_irqsave(&iommu->register_lock, flag);

3372

3371

3373

writel(iommu->iommu_state[SR_DMAR_FECTL_REG],

3372

writel(iommu->iommu_state[SR_DMAR_FECTL_REG],

3374

iommu->reg + DMAR_FECTL_REG);

3373

iommu->reg + DMAR_FECTL_REG);

3375

writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],

3374

writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],

3376

iommu->reg + DMAR_FEDATA_REG);

3375

iommu->reg + DMAR_FEDATA_REG);

3377

writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],

3376

writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],

3378

iommu->reg + DMAR_FEADDR_REG);

3377

iommu->reg + DMAR_FEADDR_REG);

3379

writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],

3378

writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],

3380

iommu->reg + DMAR_FEUADDR_REG);

3379

iommu->reg + DMAR_FEUADDR_REG);

3381

3380

3382

spin_unlock_irqrestore(&iommu->register_lock, flag);

3381

spin_unlock_irqrestore(&iommu->register_lock, flag);

3383

}

3382

}

3384

3383

3385

for_each_active_iommu(iommu, drhd)

3384

for_each_active_iommu(iommu, drhd)

3386

kfree(iommu->iommu_state);

3385

kfree(iommu->iommu_state);

3387

}

3386

}

3388

3387

3389

static struct syscore_ops iommu_syscore_ops = {

3388

static struct syscore_ops iommu_syscore_ops = {

3390

.resume = iommu_resume,

3389

.resume = iommu_resume,

3391

.suspend = iommu_suspend,

3390

.suspend = iommu_suspend,

3392

};

3391

};

3393

3392

3394

static void __init init_iommu_pm_ops(void)

3393

static void __init init_iommu_pm_ops(void)

3395

{

3394

{

3396

register_syscore_ops(&iommu_syscore_ops);

3395

register_syscore_ops(&iommu_syscore_ops);

3397

}

3396

}

3398

3397

3399

#else

3398

#else

3400

static inline void init_iommu_pm_ops(void) {}

3399

static inline void init_iommu_pm_ops(void) {}

3401

#endif /* CONFIG_PM */

3400

#endif /* CONFIG_PM */

3402

3401

3403

/*

3402

/*

3404

* Here we only respond to action of unbound device from driver.

3403

* Here we only respond to action of unbound device from driver.

3405

*

3404

*

3406

* Added device is not attached to its DMAR domain here yet. That will happen

3405

* Added device is not attached to its DMAR domain here yet. That will happen

3407

* when mapping the device to iova.

3406

* when mapping the device to iova.

3408

*/

3407

*/

3409

static int device_notifier(struct notifier_block *nb,

3408

static int device_notifier(struct notifier_block *nb,

3410

unsigned long action, void *data)

3409

unsigned long action, void *data)

3411

{

3410

{

3412

struct device *dev = data;

3411

struct device *dev = data;

3413

struct pci_dev *pdev = to_pci_dev(dev);

3412

struct pci_dev *pdev = to_pci_dev(dev);

3414

struct dmar_domain *domain;

3413

struct dmar_domain *domain;

3415

3414

3416

if (iommu_no_mapping(dev))

3415

if (iommu_no_mapping(dev))

3417

return 0;

3416

return 0;

3418

3417

3419

domain = find_domain(pdev);

3418

domain = find_domain(pdev);

3420

if (!domain)

3419

if (!domain)

3421

return 0;

3420

return 0;

3422

3421

3423

if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {

3422

if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {

3424

domain_remove_one_dev_info(domain, pdev);

3423

domain_remove_one_dev_info(domain, pdev);

3425

3424

3426

if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&

3425

if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&

3427

!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&

3426

!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&

3428

list_empty(&domain->devices))

3427

list_empty(&domain->devices))

3429

domain_exit(domain);

3428

domain_exit(domain);

3430

}

3429

}

3431

3430

3432

return 0;

3431

return 0;

3433

}

3432

}

3434

3433

3435

static struct notifier_block device_nb = {

3434

static struct notifier_block device_nb = {

3436

.notifier_call = device_notifier,

3435

.notifier_call = device_notifier,

3437

};

3436

};

3438

3437

3439

int __init intel_iommu_init(void)

3438

int __init intel_iommu_init(void)

3440

{

3439

{

3441

int ret = 0;

3440

int ret = 0;

3442

3441

3443

/* VT-d is required for a TXT/tboot launch, so enforce that */

3442

/* VT-d is required for a TXT/tboot launch, so enforce that */

3444

force_on = tboot_force_iommu();

3443

force_on = tboot_force_iommu();

3445

3444

3446

if (dmar_table_init()) {

3445

if (dmar_table_init()) {

3447

if (force_on)

3446

if (force_on)

3448

panic("tboot: Failed to initialize DMAR table\n");

3447

panic("tboot: Failed to initialize DMAR table\n");

3449

return -ENODEV;

3448

return -ENODEV;

3450

}

3449

}

3451

3450

3452

if (dmar_dev_scope_init()) {

3451

if (dmar_dev_scope_init()) {

3453

if (force_on)

3452

if (force_on)

3454

panic("tboot: Failed to initialize DMAR device scope\n");

3453

panic("tboot: Failed to initialize DMAR device scope\n");

3455

return -ENODEV;

3454

return -ENODEV;

3456

}

3455

}

3457

3456

3458

/*

3457

/*

3459

* Check the need for DMA-remapping initialization now.

3458

* Check the need for DMA-remapping initialization now.

3460

* Above initialization will also be used by Interrupt-remapping.

3459

* Above initialization will also be used by Interrupt-remapping.

3461

*/

3460

*/

3462

if (no_iommu || dmar_disabled)

3461

if (no_iommu || dmar_disabled)

3463

return -ENODEV;

3462

return -ENODEV;

3464

3463

3465

if (iommu_init_mempool()) {

3464

if (iommu_init_mempool()) {

3466

if (force_on)

3465

if (force_on)

3467

panic("tboot: Failed to initialize iommu memory\n");

3466

panic("tboot: Failed to initialize iommu memory\n");

3468

return -ENODEV;

3467

return -ENODEV;

3469

}

3468

}

3470

3469

3471

if (dmar_init_reserved_ranges()) {

3470

if (dmar_init_reserved_ranges()) {

3472

if (force_on)

3471

if (force_on)

3473

panic("tboot: Failed to reserve iommu ranges\n");

3472

panic("tboot: Failed to reserve iommu ranges\n");

3474

return -ENODEV;

3473

return -ENODEV;

3475

}

3474

}

3476

3475

3477

init_no_remapping_devices();

3476

init_no_remapping_devices();

3478

3477

3479

ret = init_dmars();

3478

ret = init_dmars();

3480

if (ret) {

3479

if (ret) {

3481

if (force_on)

3480

if (force_on)

3482

panic("tboot: Failed to initialize DMARs\n");

3481

panic("tboot: Failed to initialize DMARs\n");

3483

printk(KERN_ERR "IOMMU: dmar init failed\n");

3482

printk(KERN_ERR "IOMMU: dmar init failed\n");

3484

put_iova_domain(&reserved_iova_list);

3483

put_iova_domain(&reserved_iova_list);

3485

iommu_exit_mempool();

3484

iommu_exit_mempool();

3486

return ret;

3485

return ret;

3487

}

3486

}

3488

printk(KERN_INFO

3487

printk(KERN_INFO

3489

"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");

3488

"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");

3490

3489

3491

init_timer(&unmap_timer);

3490

init_timer(&unmap_timer);

3492

#ifdef CONFIG_SWIOTLB

3491

#ifdef CONFIG_SWIOTLB

3493

swiotlb = 0;

3492

swiotlb = 0;

3494

#endif

3493

#endif

3495

dma_ops = &intel_dma_ops;

3494

dma_ops = &intel_dma_ops;

3496

3495

3497

init_iommu_pm_ops();

3496

init_iommu_pm_ops();

3498

3497

3499

register_iommu(&intel_iommu_ops);

3498

register_iommu(&intel_iommu_ops);

3500

3499

3501

bus_register_notifier(&pci_bus_type, &device_nb);

3500

bus_register_notifier(&pci_bus_type, &device_nb);

3502

3501

3503

return 0;

3502

return 0;

3504

}

3503

}

3505

3504

3506

static void iommu_detach_dependent_devices(struct intel_iommu *iommu,

3505

static void iommu_detach_dependent_devices(struct intel_iommu *iommu,

3507

struct pci_dev *pdev)

3506

struct pci_dev *pdev)

3508

{

3507

{

3509

struct pci_dev *tmp, *parent;

3508

struct pci_dev *tmp, *parent;

3510

3509

3511

if (!iommu || !pdev)

3510

if (!iommu || !pdev)

3512

return;

3511

return;

3513

3512

3514

/* dependent device detach */

3513

/* dependent device detach */

3515

tmp = pci_find_upstream_pcie_bridge(pdev);

3514

tmp = pci_find_upstream_pcie_bridge(pdev);

3516

/* Secondary interface's bus number and devfn 0 */

3515

/* Secondary interface's bus number and devfn 0 */

3517

if (tmp) {

3516

if (tmp) {

3518

parent = pdev->bus->self;

3517

parent = pdev->bus->self;

3519

while (parent != tmp) {

3518

while (parent != tmp) {

3520

iommu_detach_dev(iommu, parent->bus->number,

3519

iommu_detach_dev(iommu, parent->bus->number,

3521

parent->devfn);

3520

parent->devfn);

3522

parent = parent->bus->self;

3521

parent = parent->bus->self;

3523

}

3522

}

3524

if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */

3523

if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */

3525

iommu_detach_dev(iommu,

3524

iommu_detach_dev(iommu,

3526

tmp->subordinate->number, 0);

3525

tmp->subordinate->number, 0);

3527

else /* this is a legacy PCI bridge */

3526

else /* this is a legacy PCI bridge */

3528

iommu_detach_dev(iommu, tmp->bus->number,

3527

iommu_detach_dev(iommu, tmp->bus->number,

3529

tmp->devfn);

3528

tmp->devfn);

3530

}

3529

}

3531

}

3530

}

3532

3531

3533

static void domain_remove_one_dev_info(struct dmar_domain *domain,

3532

static void domain_remove_one_dev_info(struct dmar_domain *domain,

3534

struct pci_dev *pdev)

3533

struct pci_dev *pdev)

3535

{

3534

{

3536

struct device_domain_info *info;

3535

struct device_domain_info *info;

3537

struct intel_iommu *iommu;

3536

struct intel_iommu *iommu;

3538

unsigned long flags;

3537

unsigned long flags;

3539

int found = 0;

3538

int found = 0;

3540

struct list_head *entry, *tmp;

3539

struct list_head *entry, *tmp;

3541

3540

3542

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

3541

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

3543

pdev->devfn);

3542

pdev->devfn);

3544

if (!iommu)

3543

if (!iommu)

3545

return;

3544

return;

3546

3545

3547

spin_lock_irqsave(&device_domain_lock, flags);

3546

spin_lock_irqsave(&device_domain_lock, flags);

3548

list_for_each_safe(entry, tmp, &domain->devices) {

3547

list_for_each_safe(entry, tmp, &domain->devices) {

3549

info = list_entry(entry, struct device_domain_info, link);

3548

info = list_entry(entry, struct device_domain_info, link);

3550

if (info->segment == pci_domain_nr(pdev->bus) &&

3549

if (info->segment == pci_domain_nr(pdev->bus) &&

3551

info->bus == pdev->bus->number &&

3550

info->bus == pdev->bus->number &&

3552

info->devfn == pdev->devfn) {

3551

info->devfn == pdev->devfn) {

3553

list_del(&info->link);

3552

list_del(&info->link);

3554

list_del(&info->global);

3553

list_del(&info->global);

3555

if (info->dev)

3554

if (info->dev)

3556

info->dev->dev.archdata.iommu = NULL;

3555

info->dev->dev.archdata.iommu = NULL;

3557

spin_unlock_irqrestore(&device_domain_lock, flags);

3556

spin_unlock_irqrestore(&device_domain_lock, flags);

3558

3557

3559

iommu_disable_dev_iotlb(info);

3558

iommu_disable_dev_iotlb(info);

3560

iommu_detach_dev(iommu, info->bus, info->devfn);

3559

iommu_detach_dev(iommu, info->bus, info->devfn);

3561

iommu_detach_dependent_devices(iommu, pdev);

3560

iommu_detach_dependent_devices(iommu, pdev);

3562

free_devinfo_mem(info);

3561

free_devinfo_mem(info);

3563

3562

3564

spin_lock_irqsave(&device_domain_lock, flags);

3563

spin_lock_irqsave(&device_domain_lock, flags);

3565

3564

3566

if (found)

3565

if (found)

3567

break;

3566

break;

3568

else

3567

else

3569

continue;

3568

continue;

3570

}

3569

}

3571

3570

3572

/* if there is no other devices under the same iommu

3571

/* if there is no other devices under the same iommu

3573

* owned by this domain, clear this iommu in iommu_bmp

3572

* owned by this domain, clear this iommu in iommu_bmp

3574

* update iommu count and coherency

3573

* update iommu count and coherency

3575

*/

3574

*/

3576

if (iommu == device_to_iommu(info->segment, info->bus,

3575

if (iommu == device_to_iommu(info->segment, info->bus,

3577

info->devfn))

3576

info->devfn))

3578

found = 1;

3577

found = 1;

3579

}

3578

}

3580

3579

3581

spin_unlock_irqrestore(&device_domain_lock, flags);

3580

spin_unlock_irqrestore(&device_domain_lock, flags);

3582

3581

3583

if (found == 0) {

3582

if (found == 0) {

3584

unsigned long tmp_flags;

3583

unsigned long tmp_flags;

3585

spin_lock_irqsave(&domain->iommu_lock, tmp_flags);

3584

spin_lock_irqsave(&domain->iommu_lock, tmp_flags);

3586

clear_bit(iommu->seq_id, &domain->iommu_bmp);

3585

clear_bit(iommu->seq_id, &domain->iommu_bmp);

3587

domain->iommu_count--;

3586

domain->iommu_count--;

3588

domain_update_iommu_cap(domain);

3587

domain_update_iommu_cap(domain);

3589

spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);

3588

spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);

3590

3589

3591

if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&

3590

if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&

3592

!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {

3591

!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {

3593

spin_lock_irqsave(&iommu->lock, tmp_flags);

3592

spin_lock_irqsave(&iommu->lock, tmp_flags);

3594

clear_bit(domain->id, iommu->domain_ids);

3593

clear_bit(domain->id, iommu->domain_ids);

3595

iommu->domains[domain->id] = NULL;

3594

iommu->domains[domain->id] = NULL;

3596

spin_unlock_irqrestore(&iommu->lock, tmp_flags);

3595

spin_unlock_irqrestore(&iommu->lock, tmp_flags);

3597

}

3596

}

3598

}

3597

}

3599

}

3598

}

3600

3599

3601

static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)

3600

static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)

3602

{

3601

{

3603

struct device_domain_info *info;

3602

struct device_domain_info *info;

3604

struct intel_iommu *iommu;

3603

struct intel_iommu *iommu;

3605

unsigned long flags1, flags2;

3604

unsigned long flags1, flags2;

3606

3605

3607

spin_lock_irqsave(&device_domain_lock, flags1);

3606

spin_lock_irqsave(&device_domain_lock, flags1);

3608

while (!list_empty(&domain->devices)) {

3607

while (!list_empty(&domain->devices)) {

3609

info = list_entry(domain->devices.next,

3608

info = list_entry(domain->devices.next,

3610

struct device_domain_info, link);

3609

struct device_domain_info, link);

3611

list_del(&info->link);

3610

list_del(&info->link);

3612

list_del(&info->global);

3611

list_del(&info->global);

3613

if (info->dev)

3612

if (info->dev)

3614

info->dev->dev.archdata.iommu = NULL;

3613

info->dev->dev.archdata.iommu = NULL;

3615

3614

3616

spin_unlock_irqrestore(&device_domain_lock, flags1);

3615

spin_unlock_irqrestore(&device_domain_lock, flags1);

3617

3616

3618

iommu_disable_dev_iotlb(info);

3617

iommu_disable_dev_iotlb(info);

3619

iommu = device_to_iommu(info->segment, info->bus, info->devfn);

3618

iommu = device_to_iommu(info->segment, info->bus, info->devfn);

3620

iommu_detach_dev(iommu, info->bus, info->devfn);

3619

iommu_detach_dev(iommu, info->bus, info->devfn);

3621

iommu_detach_dependent_devices(iommu, info->dev);

3620

iommu_detach_dependent_devices(iommu, info->dev);

3622

3621

3623

/* clear this iommu in iommu_bmp, update iommu count

3622

/* clear this iommu in iommu_bmp, update iommu count

3624

* and capabilities

3623

* and capabilities

3625

*/

3624

*/

3626

spin_lock_irqsave(&domain->iommu_lock, flags2);

3625

spin_lock_irqsave(&domain->iommu_lock, flags2);

3627

if (test_and_clear_bit(iommu->seq_id,

3626

if (test_and_clear_bit(iommu->seq_id,

3628

&domain->iommu_bmp)) {

3627

&domain->iommu_bmp)) {

3629

domain->iommu_count--;

3628

domain->iommu_count--;

3630

domain_update_iommu_cap(domain);

3629

domain_update_iommu_cap(domain);

3631

}

3630

}

3632

spin_unlock_irqrestore(&domain->iommu_lock, flags2);

3631

spin_unlock_irqrestore(&domain->iommu_lock, flags2);

3633

3632

3634

free_devinfo_mem(info);

3633

free_devinfo_mem(info);

3635

spin_lock_irqsave(&device_domain_lock, flags1);

3634

spin_lock_irqsave(&device_domain_lock, flags1);

3636

}

3635

}

3637

spin_unlock_irqrestore(&device_domain_lock, flags1);

3636

spin_unlock_irqrestore(&device_domain_lock, flags1);

3638

}

3637

}

3639

3638

3640

/* domain id for virtual machine, it won't be set in context */

3639

/* domain id for virtual machine, it won't be set in context */

3641

static unsigned long vm_domid;

3640

static unsigned long vm_domid;

3642

3641

3643

static struct dmar_domain *iommu_alloc_vm_domain(void)

3642

static struct dmar_domain *iommu_alloc_vm_domain(void)

3644

{

3643

{

3645

struct dmar_domain *domain;

3644

struct dmar_domain *domain;

3646

3645

3647

domain = alloc_domain_mem();

3646

domain = alloc_domain_mem();

3648

if (!domain)

3647

if (!domain)

3649

return NULL;

3648

return NULL;

3650

3649

3651

domain->id = vm_domid++;

3650

domain->id = vm_domid++;

3652

domain->nid = -1;

3651

domain->nid = -1;

3653

memset(&domain->iommu_bmp, 0, sizeof(unsigned long));

3652

memset(&domain->iommu_bmp, 0, sizeof(unsigned long));

3654

domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;

3653

domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;

3655

3654

3656

return domain;

3655

return domain;

3657

}

3656

}

3658

3657

3659

static int md_domain_init(struct dmar_domain *domain, int guest_width)

3658

static int md_domain_init(struct dmar_domain *domain, int guest_width)

3660

{

3659

{

3661

int adjust_width;

3660

int adjust_width;

3662

3661

3663

init_iova_domain(&domain->iovad, DMA_32BIT_PFN);

3662

init_iova_domain(&domain->iovad, DMA_32BIT_PFN);

3664

spin_lock_init(&domain->iommu_lock);

3663

spin_lock_init(&domain->iommu_lock);

3665

3664

3666

domain_reserve_special_ranges(domain);

3665

domain_reserve_special_ranges(domain);

3667

3666

3668

/* calculate AGAW */

3667

/* calculate AGAW */

3669

domain->gaw = guest_width;

3668

domain->gaw = guest_width;

3670

adjust_width = guestwidth_to_adjustwidth(guest_width);

3669

adjust_width = guestwidth_to_adjustwidth(guest_width);

3671

domain->agaw = width_to_agaw(adjust_width);

3670

domain->agaw = width_to_agaw(adjust_width);

3672

3671

3673

INIT_LIST_HEAD(&domain->devices);

3672

INIT_LIST_HEAD(&domain->devices);

3674

3673

3675

domain->iommu_count = 0;

3674

domain->iommu_count = 0;

3676

domain->iommu_coherency = 0;

3675

domain->iommu_coherency = 0;

3677

domain->iommu_snooping = 0;

3676

domain->iommu_snooping = 0;

3678

domain->iommu_superpage = 0;

3677

domain->iommu_superpage = 0;

3679

domain->max_addr = 0;

3678

domain->max_addr = 0;

3680

domain->nid = -1;

3679

domain->nid = -1;

3681

3680

3682

/* always allocate the top pgd */

3681

/* always allocate the top pgd */

3683

domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);

3682

domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);

3684

if (!domain->pgd)

3683

if (!domain->pgd)

3685

return -ENOMEM;

3684

return -ENOMEM;

3686

domain_flush_cache(domain, domain->pgd, PAGE_SIZE);

3685

domain_flush_cache(domain, domain->pgd, PAGE_SIZE);

3687

return 0;

3686

return 0;

3688

}

3687

}

3689

3688

3690

static void iommu_free_vm_domain(struct dmar_domain *domain)

3689

static void iommu_free_vm_domain(struct dmar_domain *domain)

3691

{

3690

{

3692

unsigned long flags;

3691

unsigned long flags;

3693

struct dmar_drhd_unit *drhd;

3692

struct dmar_drhd_unit *drhd;

3694

struct intel_iommu *iommu;

3693

struct intel_iommu *iommu;

3695

unsigned long i;

3694

unsigned long i;

3696

unsigned long ndomains;

3695

unsigned long ndomains;

3697

3696

3698

for_each_drhd_unit(drhd) {

3697

for_each_drhd_unit(drhd) {

3699

if (drhd->ignored)

3698

if (drhd->ignored)

3700

continue;

3699

continue;

3701

iommu = drhd->iommu;

3700

iommu = drhd->iommu;

3702

3701

3703

ndomains = cap_ndoms(iommu->cap);

3702

ndomains = cap_ndoms(iommu->cap);

3704

for_each_set_bit(i, iommu->domain_ids, ndomains) {

3703

for_each_set_bit(i, iommu->domain_ids, ndomains) {

3705

if (iommu->domains[i] == domain) {

3704

if (iommu->domains[i] == domain) {

3706

spin_lock_irqsave(&iommu->lock, flags);

3705

spin_lock_irqsave(&iommu->lock, flags);

3707

clear_bit(i, iommu->domain_ids);

3706

clear_bit(i, iommu->domain_ids);

3708

iommu->domains[i] = NULL;

3707

iommu->domains[i] = NULL;

3709

spin_unlock_irqrestore(&iommu->lock, flags);

3708

spin_unlock_irqrestore(&iommu->lock, flags);

3710

break;

3709

break;

3711

}

3710

}

3712

}

3711

}

3713

}

3712

}

3714

}

3713

}

3715

3714

3716

static void vm_domain_exit(struct dmar_domain *domain)

3715

static void vm_domain_exit(struct dmar_domain *domain)

3717

{

3716

{

3718

/* Domain 0 is reserved, so dont process it */

3717

/* Domain 0 is reserved, so dont process it */

3719

if (!domain)

3718

if (!domain)

3720

return;

3719

return;

3721

3720

3722

vm_domain_remove_all_dev_info(domain);

3721

vm_domain_remove_all_dev_info(domain);

3723

/* destroy iovas */

3722

/* destroy iovas */

3724

put_iova_domain(&domain->iovad);

3723

put_iova_domain(&domain->iovad);

3725

3724

3726

/* clear ptes */

3725

/* clear ptes */

3727

dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

3726

dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

3728

3727

3729

/* free page tables */

3728

/* free page tables */

3730

dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

3729

dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));

3731

3730

3732

iommu_free_vm_domain(domain);

3731

iommu_free_vm_domain(domain);

3733

free_domain_mem(domain);

3732

free_domain_mem(domain);

3734

}

3733

}

3735

3734

3736

static int intel_iommu_domain_init(struct iommu_domain *domain)

3735

static int intel_iommu_domain_init(struct iommu_domain *domain)

3737

{

3736

{

3738

struct dmar_domain *dmar_domain;

3737

struct dmar_domain *dmar_domain;

3739

3738

3740

dmar_domain = iommu_alloc_vm_domain();

3739

dmar_domain = iommu_alloc_vm_domain();

3741

if (!dmar_domain) {

3740

if (!dmar_domain) {

3742

printk(KERN_ERR

3741

printk(KERN_ERR

3743

"intel_iommu_domain_init: dmar_domain == NULL\n");

3742

"intel_iommu_domain_init: dmar_domain == NULL\n");

3744

return -ENOMEM;

3743

return -ENOMEM;

3745

}

3744

}

3746

if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {

3745

if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {

3747

printk(KERN_ERR

3746

printk(KERN_ERR

3748

"intel_iommu_domain_init() failed\n");

3747

"intel_iommu_domain_init() failed\n");

3749

vm_domain_exit(dmar_domain);

3748

vm_domain_exit(dmar_domain);

3750

return -ENOMEM;

3749

return -ENOMEM;

3751

}

3750

}

3752

domain_update_iommu_cap(dmar_domain);

3751

domain_update_iommu_cap(dmar_domain);

3753

domain->priv = dmar_domain;

3752

domain->priv = dmar_domain;

3754

3753

3755

return 0;

3754

return 0;

3756

}

3755

}

3757

3756

3758

static void intel_iommu_domain_destroy(struct iommu_domain *domain)

3757

static void intel_iommu_domain_destroy(struct iommu_domain *domain)

3759

{

3758

{

3760

struct dmar_domain *dmar_domain = domain->priv;

3759

struct dmar_domain *dmar_domain = domain->priv;

3761

3760

3762

domain->priv = NULL;

3761

domain->priv = NULL;

3763

vm_domain_exit(dmar_domain);

3762

vm_domain_exit(dmar_domain);

3764

}

3763

}

3765

3764

3766

static int intel_iommu_attach_device(struct iommu_domain *domain,

3765

static int intel_iommu_attach_device(struct iommu_domain *domain,

3767

struct device *dev)

3766

struct device *dev)

3768

{

3767

{

3769

struct dmar_domain *dmar_domain = domain->priv;

3768

struct dmar_domain *dmar_domain = domain->priv;

3770

struct pci_dev *pdev = to_pci_dev(dev);

3769

struct pci_dev *pdev = to_pci_dev(dev);

3771

struct intel_iommu *iommu;

3770

struct intel_iommu *iommu;

3772

int addr_width;

3771

int addr_width;

3773

3772

3774

/* normally pdev is not mapped */

3773

/* normally pdev is not mapped */

3775

if (unlikely(domain_context_mapped(pdev))) {

3774

if (unlikely(domain_context_mapped(pdev))) {

3776

struct dmar_domain *old_domain;

3775

struct dmar_domain *old_domain;

3777

3776

3778

old_domain = find_domain(pdev);

3777

old_domain = find_domain(pdev);

3779

if (old_domain) {

3778

if (old_domain) {

3780

if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||

3779

if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||

3781

dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)

3780

dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)

3782

domain_remove_one_dev_info(old_domain, pdev);

3781

domain_remove_one_dev_info(old_domain, pdev);

3783

else

3782

else

3784

domain_remove_dev_info(old_domain);

3783

domain_remove_dev_info(old_domain);

3785

}

3784

}

3786

}

3785

}

3787

3786

3788

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

3787

iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,

3789

pdev->devfn);

3788

pdev->devfn);

3790

if (!iommu)

3789

if (!iommu)

3791

return -ENODEV;

3790

return -ENODEV;

3792

3791

3793

/* check if this iommu agaw is sufficient for max mapped address */

3792

/* check if this iommu agaw is sufficient for max mapped address */

3794

addr_width = agaw_to_width(iommu->agaw);

3793

addr_width = agaw_to_width(iommu->agaw);

3795

if (addr_width > cap_mgaw(iommu->cap))

3794

if (addr_width > cap_mgaw(iommu->cap))

3796

addr_width = cap_mgaw(iommu->cap);

3795

addr_width = cap_mgaw(iommu->cap);

3797

3796

3798

if (dmar_domain->max_addr > (1LL << addr_width)) {

3797

if (dmar_domain->max_addr > (1LL << addr_width)) {

3799

printk(KERN_ERR "%s: iommu width (%d) is not "

3798

printk(KERN_ERR "%s: iommu width (%d) is not "

3800

"sufficient for the mapped address (%llx)\n",

3799

"sufficient for the mapped address (%llx)\n",

3801

__func__, addr_width, dmar_domain->max_addr);

3800

__func__, addr_width, dmar_domain->max_addr);

3802

return -EFAULT;

3801

return -EFAULT;

3803

}

3802

}

3804

dmar_domain->gaw = addr_width;

3803

dmar_domain->gaw = addr_width;

3805

3804

3806

/*

3805

/*

3807

* Knock out extra levels of page tables if necessary

3806

* Knock out extra levels of page tables if necessary

3808

*/

3807

*/

3809

while (iommu->agaw < dmar_domain->agaw) {

3808

while (iommu->agaw < dmar_domain->agaw) {

3810

struct dma_pte *pte;

3809

struct dma_pte *pte;

3811

3810

3812

pte = dmar_domain->pgd;

3811

pte = dmar_domain->pgd;

3813

if (dma_pte_present(pte)) {

3812

if (dma_pte_present(pte)) {

3814

dmar_domain->pgd = (struct dma_pte *)

3813

dmar_domain->pgd = (struct dma_pte *)

3815

phys_to_virt(dma_pte_addr(pte));

3814

phys_to_virt(dma_pte_addr(pte));

3816

free_pgtable_page(pte);

3815

free_pgtable_page(pte);

3817

}

3816

}

3818

dmar_domain->agaw--;

3817

dmar_domain->agaw--;

3819

}

3818

}

3820

3819

3821

return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);

3820

return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);

3822

}

3821

}

3823

3822

3824

static void intel_iommu_detach_device(struct iommu_domain *domain,

3823

static void intel_iommu_detach_device(struct iommu_domain *domain,

3825

struct device *dev)

3824

struct device *dev)

3826

{

3825

{

3827

struct dmar_domain *dmar_domain = domain->priv;

3826

struct dmar_domain *dmar_domain = domain->priv;

3828

struct pci_dev *pdev = to_pci_dev(dev);

3827

struct pci_dev *pdev = to_pci_dev(dev);

3829

3828

3830

domain_remove_one_dev_info(dmar_domain, pdev);

3829

domain_remove_one_dev_info(dmar_domain, pdev);

3831

}

3830

}

3832

3831

3833

static int intel_iommu_map(struct iommu_domain *domain,

3832

static int intel_iommu_map(struct iommu_domain *domain,

3834

unsigned long iova, phys_addr_t hpa,

3833

unsigned long iova, phys_addr_t hpa,

3835

int gfp_order, int iommu_prot)

3834

int gfp_order, int iommu_prot)

3836

{

3835

{

3837

struct dmar_domain *dmar_domain = domain->priv;

3836

struct dmar_domain *dmar_domain = domain->priv;

3838

u64 max_addr;

3837

u64 max_addr;

3839

int prot = 0;

3838

int prot = 0;

3840

size_t size;

3839

size_t size;

3841

int ret;

3840

int ret;

3842

3841

3843

if (iommu_prot & IOMMU_READ)

3842

if (iommu_prot & IOMMU_READ)

3844

prot |= DMA_PTE_READ;

3843

prot |= DMA_PTE_READ;

3845

if (iommu_prot & IOMMU_WRITE)

3844

if (iommu_prot & IOMMU_WRITE)

3846

prot |= DMA_PTE_WRITE;

3845

prot |= DMA_PTE_WRITE;

3847

if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)

3846

if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)

3848

prot |= DMA_PTE_SNP;

3847

prot |= DMA_PTE_SNP;

3849

3848

3850

size = PAGE_SIZE << gfp_order;

3849

size = PAGE_SIZE << gfp_order;

3851

max_addr = iova + size;

3850

max_addr = iova + size;

3852

if (dmar_domain->max_addr < max_addr) {

3851

if (dmar_domain->max_addr < max_addr) {

3853

u64 end;

3852

u64 end;

3854

3853

3855

/* check if minimum agaw is sufficient for mapped address */

3854

/* check if minimum agaw is sufficient for mapped address */

3856

end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;

3855

end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;

3857

if (end < max_addr) {

3856

if (end < max_addr) {

3858

printk(KERN_ERR "%s: iommu width (%d) is not "

3857

printk(KERN_ERR "%s: iommu width (%d) is not "

3859

"sufficient for the mapped address (%llx)\n",

3858

"sufficient for the mapped address (%llx)\n",

3860

__func__, dmar_domain->gaw, max_addr);

3859

__func__, dmar_domain->gaw, max_addr);

3861

return -EFAULT;

3860

return -EFAULT;

3862

}

3861

}

3863

dmar_domain->max_addr = max_addr;

3862

dmar_domain->max_addr = max_addr;

3864

}

3863

}

3865

/* Round up size to next multiple of PAGE_SIZE, if it and

3864

/* Round up size to next multiple of PAGE_SIZE, if it and

3866

the low bits of hpa would take us onto the next page */

3865

the low bits of hpa would take us onto the next page */

3867

size = aligned_nrpages(hpa, size);

3866

size = aligned_nrpages(hpa, size);

3868

ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,

3867

ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,

3869

hpa >> VTD_PAGE_SHIFT, size, prot);

3868

hpa >> VTD_PAGE_SHIFT, size, prot);

3870

return ret;

3869

return ret;

3871

}

3870

}

3872

3871

3873

static int intel_iommu_unmap(struct iommu_domain *domain,

3872

static int intel_iommu_unmap(struct iommu_domain *domain,

3874

unsigned long iova, int gfp_order)

3873

unsigned long iova, int gfp_order)

3875

{

3874

{

3876

struct dmar_domain *dmar_domain = domain->priv;

3875

struct dmar_domain *dmar_domain = domain->priv;

3877

size_t size = PAGE_SIZE << gfp_order;

3876

size_t size = PAGE_SIZE << gfp_order;

3878

int order;

3877

int order;

3879

3878

3880

order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,

3879

order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,

3881

(iova + size - 1) >> VTD_PAGE_SHIFT);

3880

(iova + size - 1) >> VTD_PAGE_SHIFT);

3882

3881

3883

if (dmar_domain->max_addr == iova + size)

3882

if (dmar_domain->max_addr == iova + size)

3884

dmar_domain->max_addr = iova;

3883

dmar_domain->max_addr = iova;

3885

3884

3886

return order;

3885

return order;

3887

}

3886

}

3888

3887

3889

static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,

3888

static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,

3890

unsigned long iova)

3889

unsigned long iova)

3891

{

3890

{

3892

struct dmar_domain *dmar_domain = domain->priv;

3891

struct dmar_domain *dmar_domain = domain->priv;

3893

struct dma_pte *pte;

3892

struct dma_pte *pte;

3894

u64 phys = 0;

3893

u64 phys = 0;

3895

3894

3896

pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);

3895

pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);

3897

if (pte)

3896

if (pte)

3898

phys = dma_pte_addr(pte);

3897

phys = dma_pte_addr(pte);

3899

3898

3900

return phys;

3899

return phys;

3901

}

3900

}

3902

3901

3903

static int intel_iommu_domain_has_cap(struct iommu_domain *domain,

3902

static int intel_iommu_domain_has_cap(struct iommu_domain *domain,

3904

unsigned long cap)

3903

unsigned long cap)

3905

{

3904

{

3906

struct dmar_domain *dmar_domain = domain->priv;

3905

struct dmar_domain *dmar_domain = domain->priv;

3907

3906

3908

if (cap == IOMMU_CAP_CACHE_COHERENCY)

3907

if (cap == IOMMU_CAP_CACHE_COHERENCY)

3909

return dmar_domain->iommu_snooping;

3908

return dmar_domain->iommu_snooping;

3910

if (cap == IOMMU_CAP_INTR_REMAP)

3909

if (cap == IOMMU_CAP_INTR_REMAP)

3911

return intr_remapping_enabled;

3910

return intr_remapping_enabled;

3912

3911

3913

return 0;

3912

return 0;

3914

}

3913

}

3915

3914

3916

static struct iommu_ops intel_iommu_ops = {

3915

static struct iommu_ops intel_iommu_ops = {

3917

.domain_init = intel_iommu_domain_init,

3916

.domain_init = intel_iommu_domain_init,

3918

.domain_destroy = intel_iommu_domain_destroy,

3917

.domain_destroy = intel_iommu_domain_destroy,

3919

.attach_dev = intel_iommu_attach_device,

3918

.attach_dev = intel_iommu_attach_device,

3920

.detach_dev = intel_iommu_detach_device,

3919

.detach_dev = intel_iommu_detach_device,

3921

.map = intel_iommu_map,

3920

.map = intel_iommu_map,

3922

.unmap = intel_iommu_unmap,

3921

.unmap = intel_iommu_unmap,

3923

.iova_to_phys = intel_iommu_iova_to_phys,

3922

.iova_to_phys = intel_iommu_iova_to_phys,

3924

.domain_has_cap = intel_iommu_domain_has_cap,

3923

.domain_has_cap = intel_iommu_domain_has_cap,

3925

};

3924

};

3926

3925

3927

static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)

3926

static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)

3928

{

3927

{

3929

/*

3928

/*

3930

* Mobile 4 Series Chipset neglects to set RWBF capability,

3929

* Mobile 4 Series Chipset neglects to set RWBF capability,

3931

* but needs it:

3930

* but needs it:

3932

*/

3931

*/

3933

printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");

3932

printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");

3934

rwbf_quirk = 1;

3933

rwbf_quirk = 1;

3935

3934

3936

/* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */

3935

/* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */

3937

if (dev->revision == 0x07) {

3936

if (dev->revision == 0x07) {

3938

printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");

3937

printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");

3939

dmar_map_gfx = 0;

3938

dmar_map_gfx = 0;

3940

}

3939

}

3941

}

3940

}

3942

3941

3943

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);

3942

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);

3944

3943

3945

#define GGC 0x52

3944

#define GGC 0x52

3946

#define GGC_MEMORY_SIZE_MASK (0xf << 8)

3945

#define GGC_MEMORY_SIZE_MASK (0xf << 8)

3947

#define GGC_MEMORY_SIZE_NONE (0x0 << 8)

3946

#define GGC_MEMORY_SIZE_NONE (0x0 << 8)

3948

#define GGC_MEMORY_SIZE_1M (0x1 << 8)

3947

#define GGC_MEMORY_SIZE_1M (0x1 << 8)

3949

#define GGC_MEMORY_SIZE_2M (0x3 << 8)

3948

#define GGC_MEMORY_SIZE_2M (0x3 << 8)

3950

#define GGC_MEMORY_VT_ENABLED (0x8 << 8)

3949

#define GGC_MEMORY_VT_ENABLED (0x8 << 8)

3951

#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)

3950

#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)

3952

#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)

3951

#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)

3953

#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)

3952

#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)

3954

3953

3955

static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)

3954

static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)

3956

{

3955

{

3957

unsigned short ggc;

3956

unsigned short ggc;

3958

3957

3959

if (pci_read_config_word(dev, GGC, &ggc))

3958

if (pci_read_config_word(dev, GGC, &ggc))

3960

return;

3959

return;

3961

3960

3962

if (!(ggc & GGC_MEMORY_VT_ENABLED)) {

3961

if (!(ggc & GGC_MEMORY_VT_ENABLED)) {

3963

printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");

3962

printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");

3964

dmar_map_gfx = 0;

3963

dmar_map_gfx = 0;

3965

} else if (dmar_map_gfx) {

3964

} else if (dmar_map_gfx) {

3966

/* we have to ensure the gfx device is idle before we flush */

3965

/* we have to ensure the gfx device is idle before we flush */

3967

printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");

3966

printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");

3968

intel_iommu_strict = 1;

3967

intel_iommu_strict = 1;

3969

}

3968

}

3970

}

3969

}

3971

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);

3970

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);

3972

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);

3971

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);

3973

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);

3972

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);

3974

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);

3973

DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);

3975

3974

3976

/* On Tylersburg chipsets, some BIOSes have been known to enable the

3975

/* On Tylersburg chipsets, some BIOSes have been known to enable the

3977

ISOCH DMAR unit for the Azalia sound device, but not give it any

3976

ISOCH DMAR unit for the Azalia sound device, but not give it any

3978

TLB entries, which causes it to deadlock. Check for that. We do

3977

TLB entries, which causes it to deadlock. Check for that. We do

3979

this in a function called from init_dmars(), instead of in a PCI

3978

this in a function called from init_dmars(), instead of in a PCI

3980

quirk, because we don't want to print the obnoxious "BIOS broken"

3979

quirk, because we don't want to print the obnoxious "BIOS broken"

3981

message if VT-d is actually disabled.

3980

message if VT-d is actually disabled.

3982

*/

3981

*/

3983

static void __init check_tylersburg_isoch(void)

3982

static void __init check_tylersburg_isoch(void)

3984

{

3983

{

3985

struct pci_dev *pdev;

3984

struct pci_dev *pdev;

3986

uint32_t vtisochctrl;

3985

uint32_t vtisochctrl;

3987

3986

3988

/* If there's no Azalia in the system anyway, forget it. */

3987

/* If there's no Azalia in the system anyway, forget it. */

3989

pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);

3988

pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);

3990

if (!pdev)

3989

if (!pdev)

3991

return;

3990

return;

3992

pci_dev_put(pdev);

3991

pci_dev_put(pdev);

3993

3992

3994

/* System Management Registers. Might be hidden, in which case

3993

/* System Management Registers. Might be hidden, in which case

3995

we can't do the sanity check. But that's OK, because the

3994

we can't do the sanity check. But that's OK, because the

3996

known-broken BIOSes _don't_ actually hide it, so far. */

3995

known-broken BIOSes _don't_ actually hide it, so far. */

3997

pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);

3996

pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);

3998

if (!pdev)

3997

if (!pdev)

3999

return;

3998

return;

4000

3999

4001

if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {

4000

if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {

4002

pci_dev_put(pdev);

4001

pci_dev_put(pdev);

4003

return;

4002

return;

4004

}

4003

}

4005

4004

4006

pci_dev_put(pdev);

4005

pci_dev_put(pdev);

4007

4006

4008

/* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */

4007

/* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */

4009

if (vtisochctrl & 1)

4008

if (vtisochctrl & 1)

4010

return;

4009

return;

4011

4010

4012

/* Drop all bits other than the number of TLB entries */

4011

/* Drop all bits other than the number of TLB entries */

4013

vtisochctrl &= 0x1c;

4012

vtisochctrl &= 0x1c;

4014

4013

4015

/* If we have the recommended number of TLB entries (16), fine. */

4014

/* If we have the recommended number of TLB entries (16), fine. */

4016

if (vtisochctrl == 0x10)

4015

if (vtisochctrl == 0x10)

4017

return;

4016

return;

4018

4017

4019

/* Zero TLB entries? You get to ride the short bus to school. */

4018

/* Zero TLB entries? You get to ride the short bus to school. */

4020

if (!vtisochctrl) {

4019

if (!vtisochctrl) {

4021

WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"

4020

WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"

4022

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

4021

"BIOS vendor: %s; Ver: %s; Product Version: %s\n",

4023

dmi_get_system_info(DMI_BIOS_VENDOR),

4022

dmi_get_system_info(DMI_BIOS_VENDOR),

4024

dmi_get_system_info(DMI_BIOS_VERSION),

4023

dmi_get_system_info(DMI_BIOS_VERSION),

4025

dmi_get_system_info(DMI_PRODUCT_VERSION));

4024

dmi_get_system_info(DMI_PRODUCT_VERSION));

4026

iommu_identity_mapping |= IDENTMAP_AZALIA;

4025

iommu_identity_mapping |= IDENTMAP_AZALIA;

4027

return;

4026

return;

4028

}

4027

}

GITLAB

Eric Lee / linux-smarc-t335x-v3.2

intel-iommu: fix superpage support in pfn_to_dma_pte()

 /*
  * Copyright (c) 2006, Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  * Place - Suite 330, Boston, MA 02111-1307 USA.
  *
  * Copyright (C) 2006-2008 Intel Corporation
  * Author: Ashok Raj <ashok.raj@intel.com>
  * Author: Shaohua Li <shaohua.li@intel.com>
  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  * Author: Fenghua Yu <fenghua.yu@intel.com>
  */
 #include <linux/init.h>
 #include <linux/bitmap.h>
 #include <linux/debugfs.h>
 #include <linux/slab.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/spinlock.h>
 #include <linux/pci.h>
 #include <linux/dmar.h>
 #include <linux/dma-mapping.h>
 #include <linux/mempool.h>
 #include <linux/timer.h>
 #include <linux/iova.h>
 #include <linux/iommu.h>
 #include <linux/intel-iommu.h>
 #include <linux/syscore_ops.h>
 #include <linux/tboot.h>
 #include <linux/dmi.h>
 #include <linux/pci-ats.h>
 #include <asm/cacheflush.h>
 #include <asm/iommu.h>
 #define ROOT_SIZE		VTD_PAGE_SIZE
 #define CONTEXT_SIZE		VTD_PAGE_SIZE
 #define IS_BRIDGE_HOST_DEVICE(pdev) \
 			    ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
 #define IOAPIC_RANGE_START	(0xfee00000)
 #define IOAPIC_RANGE_END	(0xfeefffff)
 #define IOVA_START_ADDR		(0x1000)
 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
 #define MAX_AGAW_WIDTH 64
 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
 #define DOMAIN_MAX_PFN(gaw)	((unsigned long) min_t(uint64_t, \
 				__DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
 #define DOMAIN_MAX_ADDR(gaw)	(((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
 #define IOVA_PFN(addr)		((addr) >> PAGE_SHIFT)
 #define DMA_32BIT_PFN		IOVA_PFN(DMA_BIT_MASK(32))
 #define DMA_64BIT_PFN		IOVA_PFN(DMA_BIT_MASK(64))
 /* page table handling */
 #define LEVEL_STRIDE		(9)
 #define LEVEL_MASK		(((u64)1 << LEVEL_STRIDE) - 1)
 static inline int agaw_to_level(int agaw)
 {
 	return agaw + 2;
 }
 static inline int agaw_to_width(int agaw)
 {
 	return 30 + agaw * LEVEL_STRIDE;
 }
 static inline int width_to_agaw(int width)
 {
 	return (width - 30) / LEVEL_STRIDE;
 }
 static inline unsigned int level_to_offset_bits(int level)
 {
 	return (level - 1) * LEVEL_STRIDE;
 }
 static inline int pfn_level_offset(unsigned long pfn, int level)
 {
 	return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 }
 static inline unsigned long level_mask(int level)
 {
 	return -1UL << level_to_offset_bits(level);
 }
 static inline unsigned long level_size(int level)
 {
 	return 1UL << level_to_offset_bits(level);
 }
 static inline unsigned long align_to_level(unsigned long pfn, int level)
 {
 	return (pfn + level_size(level) - 1) & level_mask(level);
 }
 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 {
 	return  1 << ((lvl - 1) * LEVEL_STRIDE);
 }
 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
    are never going to work. */
 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 {
 	return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 }
 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 {
 	return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 }
 static inline unsigned long page_to_dma_pfn(struct page *pg)
 {
 	return mm_to_dma_pfn(page_to_pfn(pg));
 }
 static inline unsigned long virt_to_dma_pfn(void *p)
 {
 	return page_to_dma_pfn(virt_to_page(p));
 }
 /* global iommu list, set NULL for ignored DMAR units */
 static struct intel_iommu **g_iommus;
 static void __init check_tylersburg_isoch(void);
 static int rwbf_quirk;
 /*
  * set to 1 to panic kernel if can't successfully enable VT-d
  * (used when kernel is launched w/ TXT)
  */
 static int force_on = 0;
 /*
  * 0: Present
  * 1-11: Reserved
  * 12-63: Context Ptr (12 - (haw-1))
  * 64-127: Reserved
  */
 struct root_entry {
 	u64	val;
 	u64	rsvd1;
 };
 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 static inline bool root_present(struct root_entry *root)
 {
 	return (root->val & 1);
 }
 static inline void set_root_present(struct root_entry *root)
 {
 	root->val |= 1;
 }
 static inline void set_root_value(struct root_entry *root, unsigned long value)
 {
 	root->val |= value & VTD_PAGE_MASK;
 }
 static inline struct context_entry *
 get_context_addr_from_root(struct root_entry *root)
 {
 	return (struct context_entry *)
 		(root_present(root)?phys_to_virt(
 		root->val & VTD_PAGE_MASK) :
 		NULL);
 }
 /*
  * low 64 bits:
  * 0: present
  * 1: fault processing disable
  * 2-3: translation type
  * 12-63: address space root
  * high 64 bits:
  * 0-2: address width
  * 3-6: aval
  * 8-23: domain id
  */
 struct context_entry {
 	u64 lo;
 	u64 hi;
 };
 static inline bool context_present(struct context_entry *context)
 {
 	return (context->lo & 1);
 }
 static inline void context_set_present(struct context_entry *context)
 {
 	context->lo |= 1;
 }
 static inline void context_set_fault_enable(struct context_entry *context)
 {
 	context->lo &= (((u64)-1) << 2) | 1;
 }
 static inline void context_set_translation_type(struct context_entry *context,
 						unsigned long value)
 {
 	context->lo &= (((u64)-1) << 4) | 3;
 	context->lo |= (value & 3) << 2;
 }
 static inline void context_set_address_root(struct context_entry *context,
 					    unsigned long value)
 {
 	context->lo |= value & VTD_PAGE_MASK;
 }
 static inline void context_set_address_width(struct context_entry *context,
 					     unsigned long value)
 {
 	context->hi |= value & 7;
 }
 static inline void context_set_domain_id(struct context_entry *context,
 					 unsigned long value)
 {
 	context->hi |= (value & ((1 << 16) - 1)) << 8;
 }
 static inline void context_clear_entry(struct context_entry *context)
 {
 	context->lo = 0;
 	context->hi = 0;
 }
 /*
  * 0: readable
  * 1: writable
  * 2-6: reserved
  * 7: super page
  * 8-10: available
  * 11: snoop behavior
  * 12-63: Host physcial address
  */
 struct dma_pte {
 	u64 val;
 };
 static inline void dma_clear_pte(struct dma_pte *pte)
 {
 	pte->val = 0;
 }
 static inline void dma_set_pte_readable(struct dma_pte *pte)
 {
 	pte->val |= DMA_PTE_READ;
 }
 static inline void dma_set_pte_writable(struct dma_pte *pte)
 {
 	pte->val |= DMA_PTE_WRITE;
 }
 static inline void dma_set_pte_snp(struct dma_pte *pte)
 {
 	pte->val |= DMA_PTE_SNP;
 }
 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 {
 	pte->val = (pte->val & ~3) | (prot & 3);
 }
 static inline u64 dma_pte_addr(struct dma_pte *pte)
 {
 #ifdef CONFIG_64BIT
 	return pte->val & VTD_PAGE_MASK;
 #else
 	/* Must have a full atomic 64-bit read */
 	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 #endif
 }
 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
 {
 	pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
 }
 static inline bool dma_pte_present(struct dma_pte *pte)
 {
 	return (pte->val & 3) != 0;
 }
+static inline bool dma_pte_superpage(struct dma_pte *pte)
+{
+	return (pte->val & (1 << 7));
+}
 static inline int first_pte_in_page(struct dma_pte *pte)
 {
 	return !((unsigned long)pte & ~VTD_PAGE_MASK);
 }
 /*
  * This domain is a statically identity mapping domain.
  *	1. This domain creats a static 1:1 mapping to all usable memory.
  * 	2. It maps to each iommu if successful.
  *	3. Each iommu mapps to this domain if successful.
  */
 static struct dmar_domain *si_domain;
 static int hw_pass_through = 1;
 /* devices under the same p2p bridge are owned in one domain */
 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 /* domain represents a virtual machine, more than one devices
  * across iommus may be owned in one domain, e.g. kvm guest.
  */
 #define DOMAIN_FLAG_VIRTUAL_MACHINE	(1 << 1)
 /* si_domain contains mulitple devices */
 #define DOMAIN_FLAG_STATIC_IDENTITY	(1 << 2)
 struct dmar_domain {
 	int	id;			/* domain id */
 	int	nid;			/* node id */
 	unsigned long iommu_bmp;	/* bitmap of iommus this domain uses*/
 	struct list_head devices; 	/* all devices' list */
 	struct iova_domain iovad;	/* iova's that belong to this domain */
 	struct dma_pte	*pgd;		/* virtual address */
 	int		gaw;		/* max guest address width */
 	/* adjusted guest address width, 0 is level 2 30-bit */
 	int		agaw;
 	int		flags;		/* flags to find out type of domain */
 	int		iommu_coherency;/* indicate coherency of iommu access */
 	int		iommu_snooping; /* indicate snooping control feature*/
 	int		iommu_count;	/* reference count of iommu */
 	int		iommu_superpage;/* Level of superpages supported:
 					   0 == 4KiB (no superpages), 1 == 2MiB,
 					   2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
 	spinlock_t	iommu_lock;	/* protect iommu set in domain */
 	u64		max_addr;	/* maximum mapped address */
 };
 /* PCI domain-device relationship */
 struct device_domain_info {
 	struct list_head link;	/* link to domain siblings */
 	struct list_head global; /* link to global list */
 	int segment;		/* PCI domain */
 	u8 bus;			/* PCI bus number */
 	u8 devfn;		/* PCI devfn number */
 	struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
 	struct intel_iommu *iommu; /* IOMMU used by this device */
 	struct dmar_domain *domain; /* pointer to domain */
 };
 static void flush_unmaps_timeout(unsigned long data);
 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 #define HIGH_WATER_MARK 250
 struct deferred_flush_tables {
 	int next;
 	struct iova *iova[HIGH_WATER_MARK];
 	struct dmar_domain *domain[HIGH_WATER_MARK];
 };
 static struct deferred_flush_tables *deferred_flush;
 /* bitmap for indexing intel_iommus */
 static int g_num_of_iommus;
 static DEFINE_SPINLOCK(async_umap_flush_lock);
 static LIST_HEAD(unmaps_to_do);
 static int timer_on;
 static long list_size;
 static void domain_remove_dev_info(struct dmar_domain *domain);
 #ifdef CONFIG_DMAR_DEFAULT_ON
 int dmar_disabled = 0;
 #else
 int dmar_disabled = 1;
 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 static int dmar_map_gfx = 1;
 static int dmar_forcedac;
 static int intel_iommu_strict;
 static int intel_iommu_superpage = 1;
 int intel_iommu_gfx_mapped;
 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 static DEFINE_SPINLOCK(device_domain_lock);
 static LIST_HEAD(device_domain_list);
 static struct iommu_ops intel_iommu_ops;
 static int __init intel_iommu_setup(char *str)
 {
 	if (!str)
 		return -EINVAL;
 	while (*str) {
 		if (!strncmp(str, "on", 2)) {
 			dmar_disabled = 0;
 			printk(KERN_INFO "Intel-IOMMU: enabled\n");
 		} else if (!strncmp(str, "off", 3)) {
 			dmar_disabled = 1;
 			printk(KERN_INFO "Intel-IOMMU: disabled\n");
 		} else if (!strncmp(str, "igfx_off", 8)) {
 			dmar_map_gfx = 0;
 			printk(KERN_INFO
 				"Intel-IOMMU: disable GFX device mapping\n");
 		} else if (!strncmp(str, "forcedac", 8)) {
 			printk(KERN_INFO
 				"Intel-IOMMU: Forcing DAC for PCI devices\n");
 			dmar_forcedac = 1;
 		} else if (!strncmp(str, "strict", 6)) {
 			printk(KERN_INFO
 				"Intel-IOMMU: disable batched IOTLB flush\n");
 			intel_iommu_strict = 1;
 		} else if (!strncmp(str, "sp_off", 6)) {
 			printk(KERN_INFO
 				"Intel-IOMMU: disable supported super page\n");
 			intel_iommu_superpage = 0;
 		}
 		str += strcspn(str, ",");
 		while (*str == ',')
 			str++;
 	}
 	return 0;
 }
 __setup("intel_iommu=", intel_iommu_setup);
 static struct kmem_cache *iommu_domain_cache;
 static struct kmem_cache *iommu_devinfo_cache;
 static struct kmem_cache *iommu_iova_cache;
 static inline void *alloc_pgtable_page(int node)
 {
 	struct page *page;
 	void *vaddr = NULL;
 	page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 	if (page)
 		vaddr = page_address(page);
 	return vaddr;
 }
 static inline void free_pgtable_page(void *vaddr)
 {
 	free_page((unsigned long)vaddr);
 }
 static inline void *alloc_domain_mem(void)
 {
 	return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 }
 static void free_domain_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_domain_cache, vaddr);
 }
 static inline void * alloc_devinfo_mem(void)
 {
 	return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 }
 static inline void free_devinfo_mem(void *vaddr)
 {
 	kmem_cache_free(iommu_devinfo_cache, vaddr);
 }
 struct iova *alloc_iova_mem(void)
 {
 	return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
 }
 void free_iova_mem(struct iova *iova)
 {
 	kmem_cache_free(iommu_iova_cache, iova);
 }
 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 {
 	unsigned long sagaw;
 	int agaw = -1;
 	sagaw = cap_sagaw(iommu->cap);
 	for (agaw = width_to_agaw(max_gaw);
 	     agaw >= 0; agaw--) {
 		if (test_bit(agaw, &sagaw))
 			break;
 	}
 	return agaw;
 }
 /*
  * Calculate max SAGAW for each iommu.
  */
 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 {
 	return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 }
 /*
  * calculate agaw for each iommu.
  * "SAGAW" may be different across iommus, use a default agaw, and
  * get a supported less agaw for iommus that don't support the default agaw.
  */
 int iommu_calculate_agaw(struct intel_iommu *iommu)
 {
 	return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 }
 /* This functionin only returns single iommu in a domain */
 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 {
 	int iommu_id;
 	/* si_domain and vm domain should not get here. */
 	BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 	BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
 	iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 	if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 		return NULL;
 	return g_iommus[iommu_id];
 }
 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 {
 	int i;
 	domain->iommu_coherency = 1;
 	for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 		if (!ecap_coherent(g_iommus[i]->ecap)) {
 			domain->iommu_coherency = 0;
 			break;
 		}
 	}
 }
 static void domain_update_iommu_snooping(struct dmar_domain *domain)
 {
 	int i;
 	domain->iommu_snooping = 1;
 	for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
 		if (!ecap_sc_support(g_iommus[i]->ecap)) {
 			domain->iommu_snooping = 0;
 			break;
 		}
 	}
 }
 static void domain_update_iommu_superpage(struct dmar_domain *domain)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu = NULL;
 	int mask = 0xf;
 	if (!intel_iommu_superpage) {
 		domain->iommu_superpage = 0;
 		return;
 	}
 	/* set iommu_superpage to the smallest common denominator */
 	for_each_active_iommu(iommu, drhd) {
 		mask &= cap_super_page_val(iommu->cap);
 		if (!mask) {
 			break;
 		}
 	}
 	domain->iommu_superpage = fls(mask);
 }
 /* Some capabilities may be different across iommus */
 static void domain_update_iommu_cap(struct dmar_domain *domain)
 {
 	domain_update_iommu_coherency(domain);
 	domain_update_iommu_snooping(domain);
 	domain_update_iommu_superpage(domain);
 }
 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
 {
 	struct dmar_drhd_unit *drhd = NULL;
 	int i;
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		if (segment != drhd->segment)
 			continue;
 		for (i = 0; i < drhd->devices_cnt; i++) {
 			if (drhd->devices[i] &&
 			    drhd->devices[i]->bus->number == bus &&
 			    drhd->devices[i]->devfn == devfn)
 				return drhd->iommu;
 			if (drhd->devices[i] &&
 			    drhd->devices[i]->subordinate &&
 			    drhd->devices[i]->subordinate->number <= bus &&
 			    drhd->devices[i]->subordinate->subordinate >= bus)
 				return drhd->iommu;
 		}
 		if (drhd->include_all)
 			return drhd->iommu;
 	}
 	return NULL;
 }
 static void domain_flush_cache(struct dmar_domain *domain,
 			       void *addr, int size)
 {
 	if (!domain->iommu_coherency)
 		clflush_cache_range(addr, size);
 }
 /* Gets context entry for a given bus and devfn */
 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 		u8 bus, u8 devfn)
 {
 	struct root_entry *root;
 	struct context_entry *context;
 	unsigned long phy_addr;
 	unsigned long flags;
 	spin_lock_irqsave(&iommu->lock, flags);
 	root = &iommu->root_entry[bus];
 	context = get_context_addr_from_root(root);
 	if (!context) {
 		context = (struct context_entry *)
 				alloc_pgtable_page(iommu->node);
 		if (!context) {
 			spin_unlock_irqrestore(&iommu->lock, flags);
 			return NULL;
 		}
 		__iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 		phy_addr = virt_to_phys((void *)context);
 		set_root_value(root, phy_addr);
 		set_root_present(root);
 		__iommu_flush_cache(iommu, root, sizeof(*root));
 	}
 	spin_unlock_irqrestore(&iommu->lock, flags);
 	return &context[devfn];
 }
 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 {
 	struct root_entry *root;
 	struct context_entry *context;
 	int ret;
 	unsigned long flags;
 	spin_lock_irqsave(&iommu->lock, flags);
 	root = &iommu->root_entry[bus];
 	context = get_context_addr_from_root(root);
 	if (!context) {
 		ret = 0;
 		goto out;
 	}
 	ret = context_present(&context[devfn]);
 out:
 	spin_unlock_irqrestore(&iommu->lock, flags);
 	return ret;
 }
 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 {
 	struct root_entry *root;
 	struct context_entry *context;
 	unsigned long flags;
 	spin_lock_irqsave(&iommu->lock, flags);
 	root = &iommu->root_entry[bus];
 	context = get_context_addr_from_root(root);
 	if (context) {
 		context_clear_entry(&context[devfn]);
 		__iommu_flush_cache(iommu, &context[devfn], \
 			sizeof(*context));
 	}
 	spin_unlock_irqrestore(&iommu->lock, flags);
 }
 static void free_context_table(struct intel_iommu *iommu)
 {
 	struct root_entry *root;
 	int i;
 	unsigned long flags;
 	struct context_entry *context;
 	spin_lock_irqsave(&iommu->lock, flags);
 	if (!iommu->root_entry) {
 		goto out;
 	}
 	for (i = 0; i < ROOT_ENTRY_NR; i++) {
 		root = &iommu->root_entry[i];
 		context = get_context_addr_from_root(root);
 		if (context)
 			free_pgtable_page(context);
 	}
 	free_pgtable_page(iommu->root_entry);
 	iommu->root_entry = NULL;
 out:
 	spin_unlock_irqrestore(&iommu->lock, flags);
 }
 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
-				      unsigned long pfn, int large_level)
+				      unsigned long pfn, int target_level)
 {
 	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 	struct dma_pte *parent, *pte = NULL;
 	int level = agaw_to_level(domain->agaw);
-	int offset, target_level;
+	int offset;
 	BUG_ON(!domain->pgd);
 	BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
 	parent = domain->pgd;
-	/* Search pte */
-	if (!large_level)
-		target_level = 1;
-	else
-		target_level = large_level;
 	while (level > 0) {
 		void *tmp_page;
 		offset = pfn_level_offset(pfn, level);
 		pte = &parent[offset];
-		if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
+		if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 			break;
 		if (level == target_level)
 			break;
 		if (!dma_pte_present(pte)) {
 			uint64_t pteval;
 			tmp_page = alloc_pgtable_page(domain->nid);
 			if (!tmp_page)
 				return NULL;
 			domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 			pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 			if (cmpxchg64(&pte->val, 0ULL, pteval)) {
 				/* Someone else set it while we were thinking; use theirs. */
 				free_pgtable_page(tmp_page);
 			} else {
 				dma_pte_addr(pte);
 				domain_flush_cache(domain, pte, sizeof(*pte));
 			}
 		}
 		parent = phys_to_virt(dma_pte_addr(pte));
 		level--;
 	}
 	return pte;
 }
 /* return address's pte at specific level */
 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
 					 unsigned long pfn,
 					 int level, int *large_page)
 {
 	struct dma_pte *parent, *pte = NULL;
 	int total = agaw_to_level(domain->agaw);
 	int offset;
 	parent = domain->pgd;
 	while (level <= total) {
 		offset = pfn_level_offset(pfn, total);
 		pte = &parent[offset];
 		if (level == total)
 			return pte;
 		if (!dma_pte_present(pte)) {
 			*large_page = total;
 			break;
 		}
 		if (pte->val & DMA_PTE_LARGE_PAGE) {
 			*large_page = total;
 			return pte;
 		}
 		parent = phys_to_virt(dma_pte_addr(pte));
 		total--;
 	}
 	return NULL;
 }
 /* clear last level pte, a tlb flush should be followed */
 static int dma_pte_clear_range(struct dmar_domain *domain,
 				unsigned long start_pfn,
 				unsigned long last_pfn)
 {
 	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 	unsigned int large_page = 1;
 	struct dma_pte *first_pte, *pte;
 	int order;
 	BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 	BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 	BUG_ON(start_pfn > last_pfn);
 	/* we don't need lock here; nobody else touches the iova range */
 	do {
 		large_page = 1;
 		first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
 		if (!pte) {
 			start_pfn = align_to_level(start_pfn + 1, large_page + 1);
 			continue;
 		}
 		do {
 			dma_clear_pte(pte);
 			start_pfn += lvl_to_nr_pages(large_page);
 			pte++;
 		} while (start_pfn <= last_pfn && !first_pte_in_page(pte));
 		domain_flush_cache(domain, first_pte,
 				   (void *)pte - (void *)first_pte);
 	} while (start_pfn && start_pfn <= last_pfn);
 	order = (large_page - 1) * 9;
 	return order;
 }
 /* free page table pages. last level pte should already be cleared */
 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 				   unsigned long start_pfn,
 				   unsigned long last_pfn)
 {
 	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 	struct dma_pte *first_pte, *pte;
 	int total = agaw_to_level(domain->agaw);
 	int level;
 	unsigned long tmp;
 	int large_page = 2;
 	BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
 	BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
 	BUG_ON(start_pfn > last_pfn);
 	/* We don't need lock here; nobody else touches the iova range */
 	level = 2;
 	while (level <= total) {
 		tmp = align_to_level(start_pfn, level);
 		/* If we can't even clear one PTE at this level, we're done */
 		if (tmp + level_size(level) - 1 > last_pfn)
 			return;
 		do {
 			large_page = level;
 			first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
 			if (large_page > level)
 				level = large_page + 1;
 			if (!pte) {
 				tmp = align_to_level(tmp + 1, level + 1);
 				continue;
 			}
 			do {
 				if (dma_pte_present(pte)) {
 					free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
 					dma_clear_pte(pte);
 				}
 				pte++;
 				tmp += level_size(level);
 			} while (!first_pte_in_page(pte) &&
 				 tmp + level_size(level) - 1 <= last_pfn);
 			domain_flush_cache(domain, first_pte,
 					   (void *)pte - (void *)first_pte);
 		} while (tmp && tmp + level_size(level) - 1 <= last_pfn);
 		level++;
 	}
 	/* free pgd */
 	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
 		free_pgtable_page(domain->pgd);
 		domain->pgd = NULL;
 	}
 }
 /* iommu handling */
 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 {
 	struct root_entry *root;
 	unsigned long flags;
 	root = (struct root_entry *)alloc_pgtable_page(iommu->node);
 	if (!root)
 		return -ENOMEM;
 	__iommu_flush_cache(iommu, root, ROOT_SIZE);
 	spin_lock_irqsave(&iommu->lock, flags);
 	iommu->root_entry = root;
 	spin_unlock_irqrestore(&iommu->lock, flags);
 	return 0;
 }
 static void iommu_set_root_entry(struct intel_iommu *iommu)
 {
 	void *addr;
 	u32 sts;
 	unsigned long flag;
 	addr = iommu->root_entry;
 	spin_lock_irqsave(&iommu->register_lock, flag);
 	dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 	writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 		      readl, (sts & DMA_GSTS_RTPS), sts);
 	spin_unlock_irqrestore(&iommu->register_lock, flag);
 }
 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 {
 	u32 val;
 	unsigned long flag;
 	if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 		return;
 	spin_lock_irqsave(&iommu->register_lock, flag);
 	writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 		      readl, (!(val & DMA_GSTS_WBFS)), val);
 	spin_unlock_irqrestore(&iommu->register_lock, flag);
 }
 /* return value determine if we need a write buffer flush */
 static void __iommu_flush_context(struct intel_iommu *iommu,
 				  u16 did, u16 source_id, u8 function_mask,
 				  u64 type)
 {
 	u64 val = 0;
 	unsigned long flag;
 	switch (type) {
 	case DMA_CCMD_GLOBAL_INVL:
 		val = DMA_CCMD_GLOBAL_INVL;
 		break;
 	case DMA_CCMD_DOMAIN_INVL:
 		val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 		break;
 	case DMA_CCMD_DEVICE_INVL:
 		val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 			| DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 		break;
 	default:
 		BUG();
 	}
 	val |= DMA_CCMD_ICC;
 	spin_lock_irqsave(&iommu->register_lock, flag);
 	dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 		dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 	spin_unlock_irqrestore(&iommu->register_lock, flag);
 }
 /* return value determine if we need a write buffer flush */
 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 				u64 addr, unsigned int size_order, u64 type)
 {
 	int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 	u64 val = 0, val_iva = 0;
 	unsigned long flag;
 	switch (type) {
 	case DMA_TLB_GLOBAL_FLUSH:
 		/* global flush doesn't need set IVA_REG */
 		val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 		break;
 	case DMA_TLB_DSI_FLUSH:
 		val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 		break;
 	case DMA_TLB_PSI_FLUSH:
 		val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 		/* Note: always flush non-leaf currently */
 		val_iva = size_order | addr;
 		break;
 	default:
 		BUG();
 	}
 	/* Note: set drain read/write */
 #if 0
 	/*
 	 * This is probably to be super secure.. Looks like we can
 	 * ignore it without any impact.
 	 */
 	if (cap_read_drain(iommu->cap))
 		val |= DMA_TLB_READ_DRAIN;
 #endif
 	if (cap_write_drain(iommu->cap))
 		val |= DMA_TLB_WRITE_DRAIN;
 	spin_lock_irqsave(&iommu->register_lock, flag);
 	/* Note: Only uses first TLB reg currently */
 	if (val_iva)
 		dmar_writeq(iommu->reg + tlb_offset, val_iva);
 	dmar_writeq(iommu->reg + tlb_offset + 8, val);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 		dmar_readq, (!(val & DMA_TLB_IVT)), val);
 	spin_unlock_irqrestore(&iommu->register_lock, flag);
 	/* check IOTLB invalidation granularity */
 	if (DMA_TLB_IAIG(val) == 0)
 		printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 	if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 		pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 			(unsigned long long)DMA_TLB_IIRG(type),
 			(unsigned long long)DMA_TLB_IAIG(val));
 }
 static struct device_domain_info *iommu_support_dev_iotlb(
 	struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
 {
 	int found = 0;
 	unsigned long flags;
 	struct device_domain_info *info;
 	struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
 	if (!ecap_dev_iotlb_support(iommu->ecap))
 		return NULL;
 	if (!iommu->qi)
 		return NULL;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	list_for_each_entry(info, &domain->devices, link)
 		if (info->bus == bus && info->devfn == devfn) {
 			found = 1;
 			break;
 		}
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 	if (!found || !info->dev)
 		return NULL;
 	if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
 		return NULL;
 	if (!dmar_find_matched_atsr_unit(info->dev))
 		return NULL;
 	info->iommu = iommu;
 	return info;
 }
 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
 {
 	if (!info)
 		return;
 	pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
 }
 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
 {
 	if (!info->dev || !pci_ats_enabled(info->dev))
 		return;
 	pci_disable_ats(info->dev);
 }
 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
 				  u64 addr, unsigned mask)
 {
 	u16 sid, qdep;
 	unsigned long flags;
 	struct device_domain_info *info;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	list_for_each_entry(info, &domain->devices, link) {
 		if (!info->dev || !pci_ats_enabled(info->dev))
 			continue;
 		sid = info->bus << 8 | info->devfn;
 		qdep = pci_ats_queue_depth(info->dev);
 		qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
 	}
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 }
 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 				  unsigned long pfn, unsigned int pages, int map)
 {
 	unsigned int mask = ilog2(__roundup_pow_of_two(pages));
 	uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
 	BUG_ON(pages == 0);
 	/*
 	 * Fallback to domain selective flush if no PSI support or the size is
 	 * too big.
 	 * PSI requires page size to be 2 ^ x, and the base address is naturally
 	 * aligned to the size
 	 */
 	if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
 		iommu->flush.flush_iotlb(iommu, did, 0, 0,
 						DMA_TLB_DSI_FLUSH);
 	else
 		iommu->flush.flush_iotlb(iommu, did, addr, mask,
 						DMA_TLB_PSI_FLUSH);
 	/*
 	 * In caching mode, changes of pages from non-present to present require
 	 * flush. However, device IOTLB doesn't need to be flushed in this case.
 	 */
 	if (!cap_caching_mode(iommu->cap) || !map)
 		iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
 }
 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 {
 	u32 pmen;
 	unsigned long flags;
 	spin_lock_irqsave(&iommu->register_lock, flags);
 	pmen = readl(iommu->reg + DMAR_PMEN_REG);
 	pmen &= ~DMA_PMEN_EPM;
 	writel(pmen, iommu->reg + DMAR_PMEN_REG);
 	/* wait for the protected region status bit to clear */
 	IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
 		readl, !(pmen & DMA_PMEN_PRS), pmen);
 	spin_unlock_irqrestore(&iommu->register_lock, flags);
 }
 static int iommu_enable_translation(struct intel_iommu *iommu)
 {
 	u32 sts;
 	unsigned long flags;
 	spin_lock_irqsave(&iommu->register_lock, flags);
 	iommu->gcmd |= DMA_GCMD_TE;
 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 		      readl, (sts & DMA_GSTS_TES), sts);
 	spin_unlock_irqrestore(&iommu->register_lock, flags);
 	return 0;
 }
 static int iommu_disable_translation(struct intel_iommu *iommu)
 {
 	u32 sts;
 	unsigned long flag;
 	spin_lock_irqsave(&iommu->register_lock, flag);
 	iommu->gcmd &= ~DMA_GCMD_TE;
 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 	/* Make sure hardware complete it */
 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 		      readl, (!(sts & DMA_GSTS_TES)), sts);
 	spin_unlock_irqrestore(&iommu->register_lock, flag);
 	return 0;
 }
 static int iommu_init_domains(struct intel_iommu *iommu)
 {
 	unsigned long ndomains;
 	unsigned long nlongs;
 	ndomains = cap_ndoms(iommu->cap);
 	pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
 			ndomains);
 	nlongs = BITS_TO_LONGS(ndomains);
 	spin_lock_init(&iommu->lock);
 	/* TBD: there might be 64K domains,
 	 * consider other allocation for future chip
 	 */
 	iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
 	if (!iommu->domain_ids) {
 		printk(KERN_ERR "Allocating domain id array failed\n");
 		return -ENOMEM;
 	}
 	iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
 			GFP_KERNEL);
 	if (!iommu->domains) {
 		printk(KERN_ERR "Allocating domain array failed\n");
 		return -ENOMEM;
 	}
 	/*
 	 * if Caching mode is set, then invalid translations are tagged
 	 * with domainid 0. Hence we need to pre-allocate it.
 	 */
 	if (cap_caching_mode(iommu->cap))
 		set_bit(0, iommu->domain_ids);
 	return 0;
 }
 static void domain_exit(struct dmar_domain *domain);
 static void vm_domain_exit(struct dmar_domain *domain);
 void free_dmar_iommu(struct intel_iommu *iommu)
 {
 	struct dmar_domain *domain;
 	int i;
 	unsigned long flags;
 	if ((iommu->domains) && (iommu->domain_ids)) {
 		for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
 			domain = iommu->domains[i];
 			clear_bit(i, iommu->domain_ids);
 			spin_lock_irqsave(&domain->iommu_lock, flags);
 			if (--domain->iommu_count == 0) {
 				if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
 					vm_domain_exit(domain);
 				else
 					domain_exit(domain);
 			}
 			spin_unlock_irqrestore(&domain->iommu_lock, flags);
 		}
 	}
 	if (iommu->gcmd & DMA_GCMD_TE)
 		iommu_disable_translation(iommu);
 	if (iommu->irq) {
 		irq_set_handler_data(iommu->irq, NULL);
 		/* This will mask the irq */
 		free_irq(iommu->irq, iommu);
 		destroy_irq(iommu->irq);
 	}
 	kfree(iommu->domains);
 	kfree(iommu->domain_ids);
 	g_iommus[iommu->seq_id] = NULL;
 	/* if all iommus are freed, free g_iommus */
 	for (i = 0; i < g_num_of_iommus; i++) {
 		if (g_iommus[i])
 			break;
 	}
 	if (i == g_num_of_iommus)
 		kfree(g_iommus);
 	/* free context mapping */
 	free_context_table(iommu);
 }
 static struct dmar_domain *alloc_domain(void)
 {
 	struct dmar_domain *domain;
 	domain = alloc_domain_mem();
 	if (!domain)
 		return NULL;
 	domain->nid = -1;
 	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
 	domain->flags = 0;
 	return domain;
 }
 static int iommu_attach_domain(struct dmar_domain *domain,
 			       struct intel_iommu *iommu)
 {
 	int num;
 	unsigned long ndomains;
 	unsigned long flags;
 	ndomains = cap_ndoms(iommu->cap);
 	spin_lock_irqsave(&iommu->lock, flags);
 	num = find_first_zero_bit(iommu->domain_ids, ndomains);
 	if (num >= ndomains) {
 		spin_unlock_irqrestore(&iommu->lock, flags);
 		printk(KERN_ERR "IOMMU: no free domain ids\n");
 		return -ENOMEM;
 	}
 	domain->id = num;
 	set_bit(num, iommu->domain_ids);
 	set_bit(iommu->seq_id, &domain->iommu_bmp);
 	iommu->domains[num] = domain;
 	spin_unlock_irqrestore(&iommu->lock, flags);
 	return 0;
 }
 static void iommu_detach_domain(struct dmar_domain *domain,
 				struct intel_iommu *iommu)
 {
 	unsigned long flags;
 	int num, ndomains;
 	int found = 0;
 	spin_lock_irqsave(&iommu->lock, flags);
 	ndomains = cap_ndoms(iommu->cap);
 	for_each_set_bit(num, iommu->domain_ids, ndomains) {
 		if (iommu->domains[num] == domain) {
 			found = 1;
 			break;
 		}
 	}
 	if (found) {
 		clear_bit(num, iommu->domain_ids);
 		clear_bit(iommu->seq_id, &domain->iommu_bmp);
 		iommu->domains[num] = NULL;
 	}
 	spin_unlock_irqrestore(&iommu->lock, flags);
 }
 static struct iova_domain reserved_iova_list;
 static struct lock_class_key reserved_rbtree_key;
 static int dmar_init_reserved_ranges(void)
 {
 	struct pci_dev *pdev = NULL;
 	struct iova *iova;
 	int i;
 	init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
 	lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
 		&reserved_rbtree_key);
 	/* IOAPIC ranges shouldn't be accessed by DMA */
 	iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
 		IOVA_PFN(IOAPIC_RANGE_END));
 	if (!iova) {
 		printk(KERN_ERR "Reserve IOAPIC range failed\n");
 		return -ENODEV;
 	}
 	/* Reserve all PCI MMIO to avoid peer-to-peer access */
 	for_each_pci_dev(pdev) {
 		struct resource *r;
 		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
 			r = &pdev->resource[i];
 			if (!r->flags || !(r->flags & IORESOURCE_MEM))
 				continue;
 			iova = reserve_iova(&reserved_iova_list,
 					    IOVA_PFN(r->start),
 					    IOVA_PFN(r->end));
 			if (!iova) {
 				printk(KERN_ERR "Reserve iova failed\n");
 				return -ENODEV;
 			}
 		}
 	}
 	return 0;
 }
 static void domain_reserve_special_ranges(struct dmar_domain *domain)
 {
 	copy_reserved_iova(&reserved_iova_list, &domain->iovad);
 }
 static inline int guestwidth_to_adjustwidth(int gaw)
 {
 	int agaw;
 	int r = (gaw - 12) % 9;
 	if (r == 0)
 		agaw = gaw;
 	else
 		agaw = gaw + 9 - r;
 	if (agaw > 64)
 		agaw = 64;
 	return agaw;
 }
 static int domain_init(struct dmar_domain *domain, int guest_width)
 {
 	struct intel_iommu *iommu;
 	int adjust_width, agaw;
 	unsigned long sagaw;
 	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
 	spin_lock_init(&domain->iommu_lock);
 	domain_reserve_special_ranges(domain);
 	/* calculate AGAW */
 	iommu = domain_get_iommu(domain);
 	if (guest_width > cap_mgaw(iommu->cap))
 		guest_width = cap_mgaw(iommu->cap);
 	domain->gaw = guest_width;
 	adjust_width = guestwidth_to_adjustwidth(guest_width);
 	agaw = width_to_agaw(adjust_width);
 	sagaw = cap_sagaw(iommu->cap);
 	if (!test_bit(agaw, &sagaw)) {
 		/* hardware doesn't support it, choose a bigger one */
 		pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
 		agaw = find_next_bit(&sagaw, 5, agaw);
 		if (agaw >= 5)
 			return -ENODEV;
 	}
 	domain->agaw = agaw;
 	INIT_LIST_HEAD(&domain->devices);
 	if (ecap_coherent(iommu->ecap))
 		domain->iommu_coherency = 1;
 	else
 		domain->iommu_coherency = 0;
 	if (ecap_sc_support(iommu->ecap))
 		domain->iommu_snooping = 1;
 	else
 		domain->iommu_snooping = 0;
 	domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
 	domain->iommu_count = 1;
 	domain->nid = iommu->node;
 	/* always allocate the top pgd */
 	domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
 	if (!domain->pgd)
 		return -ENOMEM;
 	__iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
 	return 0;
 }
 static void domain_exit(struct dmar_domain *domain)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu;
 	/* Domain 0 is reserved, so dont process it */
 	if (!domain)
 		return;
 	/* Flush any lazy unmaps that may reference this domain */
 	if (!intel_iommu_strict)
 		flush_unmaps_timeout(0);
 	domain_remove_dev_info(domain);
 	/* destroy iovas */
 	put_iova_domain(&domain->iovad);
 	/* clear ptes */
 	dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
 	/* free page tables */
 	dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
 	for_each_active_iommu(iommu, drhd)
 		if (test_bit(iommu->seq_id, &domain->iommu_bmp))
 			iommu_detach_domain(domain, iommu);
 	free_domain_mem(domain);
 }
 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
 				 u8 bus, u8 devfn, int translation)
 {
 	struct context_entry *context;
 	unsigned long flags;
 	struct intel_iommu *iommu;
 	struct dma_pte *pgd;
 	unsigned long num;
 	unsigned long ndomains;
 	int id;
 	int agaw;
 	struct device_domain_info *info = NULL;
 	pr_debug("Set context mapping for %02x:%02x.%d\n",
 		bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
 	BUG_ON(!domain->pgd);
 	BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
 	       translation != CONTEXT_TT_MULTI_LEVEL);
 	iommu = device_to_iommu(segment, bus, devfn);
 	if (!iommu)
 		return -ENODEV;
 	context = device_to_context_entry(iommu, bus, devfn);
 	if (!context)
 		return -ENOMEM;
 	spin_lock_irqsave(&iommu->lock, flags);
 	if (context_present(context)) {
 		spin_unlock_irqrestore(&iommu->lock, flags);
 		return 0;
 	}
 	id = domain->id;
 	pgd = domain->pgd;
 	if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
 	    domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
 		int found = 0;
 		/* find an available domain id for this device in iommu */
 		ndomains = cap_ndoms(iommu->cap);
 		for_each_set_bit(num, iommu->domain_ids, ndomains) {
 			if (iommu->domains[num] == domain) {
 				id = num;
 				found = 1;
 				break;
 			}
 		}
 		if (found == 0) {
 			num = find_first_zero_bit(iommu->domain_ids, ndomains);
 			if (num >= ndomains) {
 				spin_unlock_irqrestore(&iommu->lock, flags);
 				printk(KERN_ERR "IOMMU: no free domain ids\n");
 				return -EFAULT;
 			}
 			set_bit(num, iommu->domain_ids);
 			iommu->domains[num] = domain;
 			id = num;
 		}
 		/* Skip top levels of page tables for
 		 * iommu which has less agaw than default.
 		 * Unnecessary for PT mode.
 		 */
 		if (translation != CONTEXT_TT_PASS_THROUGH) {
 			for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
 				pgd = phys_to_virt(dma_pte_addr(pgd));
 				if (!dma_pte_present(pgd)) {
 					spin_unlock_irqrestore(&iommu->lock, flags);
 					return -ENOMEM;
 				}
 			}
 		}
 	}
 	context_set_domain_id(context, id);
 	if (translation != CONTEXT_TT_PASS_THROUGH) {
 		info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
 		translation = info ? CONTEXT_TT_DEV_IOTLB :
 				     CONTEXT_TT_MULTI_LEVEL;
 	}
 	/*
 	 * In pass through mode, AW must be programmed to indicate the largest
 	 * AGAW value supported by hardware. And ASR is ignored by hardware.
 	 */
 	if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
 		context_set_address_width(context, iommu->msagaw);
 	else {
 		context_set_address_root(context, virt_to_phys(pgd));
 		context_set_address_width(context, iommu->agaw);
 	}
 	context_set_translation_type(context, translation);
 	context_set_fault_enable(context);
 	context_set_present(context);
 	domain_flush_cache(domain, context, sizeof(*context));
 	/*
 	 * It's a non-present to present mapping. If hardware doesn't cache
 	 * non-present entry we only need to flush the write-buffer. If the
 	 * _does_ cache non-present entries, then it does so in the special
 	 * domain #0, which we have to flush:
 	 */
 	if (cap_caching_mode(iommu->cap)) {
 		iommu->flush.flush_context(iommu, 0,
 					   (((u16)bus) << 8) | devfn,
 					   DMA_CCMD_MASK_NOBIT,
 					   DMA_CCMD_DEVICE_INVL);
 		iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
 	} else {
 		iommu_flush_write_buffer(iommu);
 	}
 	iommu_enable_dev_iotlb(info);
 	spin_unlock_irqrestore(&iommu->lock, flags);
 	spin_lock_irqsave(&domain->iommu_lock, flags);
 	if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
 		domain->iommu_count++;
 		if (domain->iommu_count == 1)
 			domain->nid = iommu->node;
 		domain_update_iommu_cap(domain);
 	}
 	spin_unlock_irqrestore(&domain->iommu_lock, flags);
 	return 0;
 }
 static int
 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
 			int translation)
 {
 	int ret;
 	struct pci_dev *tmp, *parent;
 	ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
 					 pdev->bus->number, pdev->devfn,
 					 translation);
 	if (ret)
 		return ret;
 	/* dependent device mapping */
 	tmp = pci_find_upstream_pcie_bridge(pdev);
 	if (!tmp)
 		return 0;
 	/* Secondary interface's bus number and devfn 0 */
 	parent = pdev->bus->self;
 	while (parent != tmp) {
 		ret = domain_context_mapping_one(domain,
 						 pci_domain_nr(parent->bus),
 						 parent->bus->number,
 						 parent->devfn, translation);
 		if (ret)
 			return ret;
 		parent = parent->bus->self;
 	}
 	if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
 		return domain_context_mapping_one(domain,
 					pci_domain_nr(tmp->subordinate),
 					tmp->subordinate->number, 0,
 					translation);
 	else /* this is a legacy PCI bridge */
 		return domain_context_mapping_one(domain,
 						  pci_domain_nr(tmp->bus),
 						  tmp->bus->number,
 						  tmp->devfn,
 						  translation);
 }
 static int domain_context_mapped(struct pci_dev *pdev)
 {
 	int ret;
 	struct pci_dev *tmp, *parent;
 	struct intel_iommu *iommu;
 	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
 				pdev->devfn);
 	if (!iommu)
 		return -ENODEV;
 	ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
 	if (!ret)
 		return ret;
 	/* dependent device mapping */
 	tmp = pci_find_upstream_pcie_bridge(pdev);
 	if (!tmp)
 		return ret;
 	/* Secondary interface's bus number and devfn 0 */
 	parent = pdev->bus->self;
 	while (parent != tmp) {
 		ret = device_context_mapped(iommu, parent->bus->number,
 					    parent->devfn);
 		if (!ret)
 			return ret;
 		parent = parent->bus->self;
 	}
 	if (pci_is_pcie(tmp))
 		return device_context_mapped(iommu, tmp->subordinate->number,
 					     0);
 	else
 		return device_context_mapped(iommu, tmp->bus->number,
 					     tmp->devfn);
 }
 /* Returns a number of VTD pages, but aligned to MM page size */
 static inline unsigned long aligned_nrpages(unsigned long host_addr,
 					    size_t size)
 {
 	host_addr &= ~PAGE_MASK;
 	return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
 }
 /* Return largest possible superpage level for a given mapping */
 static inline int hardware_largepage_caps(struct dmar_domain *domain,
 					  unsigned long iov_pfn,
 					  unsigned long phy_pfn,
 					  unsigned long pages)
 {
 	int support, level = 1;
 	unsigned long pfnmerge;
 	support = domain->iommu_superpage;
 	/* To use a large page, the virtual *and* physical addresses
 	   must be aligned to 2MiB/1GiB/etc. Lower bits set in either
 	   of them will mean we have to use smaller pages. So just
 	   merge them and check both at once. */
 	pfnmerge = iov_pfn | phy_pfn;
 	while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
 		pages >>= VTD_STRIDE_SHIFT;
 		if (!pages)
 			break;
 		pfnmerge >>= VTD_STRIDE_SHIFT;
 		level++;
 		support--;
 	}
 	return level;
 }
 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 			    struct scatterlist *sg, unsigned long phys_pfn,
 			    unsigned long nr_pages, int prot)
 {
 	struct dma_pte *first_pte = NULL, *pte = NULL;
 	phys_addr_t uninitialized_var(pteval);
 	int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 	unsigned long sg_res;
 	unsigned int largepage_lvl = 0;
 	unsigned long lvl_pages = 0;
 	BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
 	if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
 		return -EINVAL;
 	prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
 	if (sg)
 		sg_res = 0;
 	else {
 		sg_res = nr_pages + 1;
 		pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
 	}
 	while (nr_pages > 0) {
 		uint64_t tmp;
 		if (!sg_res) {
 			sg_res = aligned_nrpages(sg->offset, sg->length);
 			sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
 			sg->dma_length = sg->length;
 			pteval = page_to_phys(sg_page(sg)) | prot;
 			phys_pfn = pteval >> VTD_PAGE_SHIFT;
 		}
 		if (!pte) {
 			largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
 			first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
 			if (!pte)
 				return -ENOMEM;
 			/* It is large page*/
 			if (largepage_lvl > 1)
 				pteval |= DMA_PTE_LARGE_PAGE;
 			else
 				pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
 		}
 		/* We don't need lock here, nobody else
 		 * touches the iova range
 		 */
 		tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
 		if (tmp) {
 			static int dumps = 5;
 			printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
 			       iov_pfn, tmp, (unsigned long long)pteval);
 			if (dumps) {
 				dumps--;
 				debug_dma_dump_mappings(NULL);
 			}
 			WARN_ON(1);
 		}
 		lvl_pages = lvl_to_nr_pages(largepage_lvl);
 		BUG_ON(nr_pages < lvl_pages);
 		BUG_ON(sg_res < lvl_pages);
 		nr_pages -= lvl_pages;
 		iov_pfn += lvl_pages;
 		phys_pfn += lvl_pages;
 		pteval += lvl_pages * VTD_PAGE_SIZE;
 		sg_res -= lvl_pages;
 		/* If the next PTE would be the first in a new page, then we
 		   need to flush the cache on the entries we've just written.
 		   And then we'll need to recalculate 'pte', so clear it and
 		   let it get set again in the if (!pte) block above.
 		   If we're done (!nr_pages) we need to flush the cache too.
 		   Also if we've been setting superpages, we may need to
 		   recalculate 'pte' and switch back to smaller pages for the
 		   end of the mapping, if the trailing size is not enough to
 		   use another superpage (i.e. sg_res < lvl_pages). */
 		pte++;
 		if (!nr_pages || first_pte_in_page(pte) ||
 		    (largepage_lvl > 1 && sg_res < lvl_pages)) {
 			domain_flush_cache(domain, first_pte,
 					   (void *)pte - (void *)first_pte);
 			pte = NULL;
 		}
 		if (!sg_res && nr_pages)
 			sg = sg_next(sg);
 	}
 	return 0;
 }
 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 				    struct scatterlist *sg, unsigned long nr_pages,
 				    int prot)
 {
 	return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
 }
 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
 				     unsigned long phys_pfn, unsigned long nr_pages,
 				     int prot)
 {
 	return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
 }
 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
 {
 	if (!iommu)
 		return;
 	clear_context_table(iommu, bus, devfn);
 	iommu->flush.flush_context(iommu, 0, 0, 0,
 					   DMA_CCMD_GLOBAL_INVL);
 	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
 }
 static void domain_remove_dev_info(struct dmar_domain *domain)
 {
 	struct device_domain_info *info;
 	unsigned long flags;
 	struct intel_iommu *iommu;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	while (!list_empty(&domain->devices)) {
 		info = list_entry(domain->devices.next,
 			struct device_domain_info, link);
 		list_del(&info->link);
 		list_del(&info->global);
 		if (info->dev)
 			info->dev->dev.archdata.iommu = NULL;
 		spin_unlock_irqrestore(&device_domain_lock, flags);
 		iommu_disable_dev_iotlb(info);
 		iommu = device_to_iommu(info->segment, info->bus, info->devfn);
 		iommu_detach_dev(iommu, info->bus, info->devfn);
 		free_devinfo_mem(info);
 		spin_lock_irqsave(&device_domain_lock, flags);
 	}
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 }
 /*
  * find_domain
  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
  */
 static struct dmar_domain *
 find_domain(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
 	/* No lock here, assumes no domain exit in normal case */
 	info = pdev->dev.archdata.iommu;
 	if (info)
 		return info->domain;
 	return NULL;
 }
 /* domain is initialized */
 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
 {
 	struct dmar_domain *domain, *found = NULL;
 	struct intel_iommu *iommu;
 	struct dmar_drhd_unit *drhd;
 	struct device_domain_info *info, *tmp;
 	struct pci_dev *dev_tmp;
 	unsigned long flags;
 	int bus = 0, devfn = 0;
 	int segment;
 	int ret;
 	domain = find_domain(pdev);
 	if (domain)
 		return domain;
 	segment = pci_domain_nr(pdev->bus);
 	dev_tmp = pci_find_upstream_pcie_bridge(pdev);
 	if (dev_tmp) {
 		if (pci_is_pcie(dev_tmp)) {
 			bus = dev_tmp->subordinate->number;
 			devfn = 0;
 		} else {
 			bus = dev_tmp->bus->number;
 			devfn = dev_tmp->devfn;
 		}
 		spin_lock_irqsave(&device_domain_lock, flags);
 		list_for_each_entry(info, &device_domain_list, global) {
 			if (info->segment == segment &&
 			    info->bus == bus && info->devfn == devfn) {
 				found = info->domain;
 				break;
 			}
 		}
 		spin_unlock_irqrestore(&device_domain_lock, flags);
 		/* pcie-pci bridge already has a domain, uses it */
 		if (found) {
 			domain = found;
 			goto found_domain;
 		}
 	}
 	domain = alloc_domain();
 	if (!domain)
 		goto error;
 	/* Allocate new domain for the device */
 	drhd = dmar_find_matched_drhd_unit(pdev);
 	if (!drhd) {
 		printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
 			pci_name(pdev));
 		return NULL;
 	}
 	iommu = drhd->iommu;
 	ret = iommu_attach_domain(domain, iommu);
 	if (ret) {
 		free_domain_mem(domain);
 		goto error;
 	}
 	if (domain_init(domain, gaw)) {
 		domain_exit(domain);
 		goto error;
 	}
 	/* register pcie-to-pci device */
 	if (dev_tmp) {
 		info = alloc_devinfo_mem();
 		if (!info) {
 			domain_exit(domain);
 			goto error;
 		}
 		info->segment = segment;
 		info->bus = bus;
 		info->devfn = devfn;
 		info->dev = NULL;
 		info->domain = domain;
 		/* This domain is shared by devices under p2p bridge */
 		domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
 		/* pcie-to-pci bridge already has a domain, uses it */
 		found = NULL;
 		spin_lock_irqsave(&device_domain_lock, flags);
 		list_for_each_entry(tmp, &device_domain_list, global) {
 			if (tmp->segment == segment &&
 			    tmp->bus == bus && tmp->devfn == devfn) {
 				found = tmp->domain;
 				break;
 			}
 		}
 		if (found) {
 			spin_unlock_irqrestore(&device_domain_lock, flags);
 			free_devinfo_mem(info);
 			domain_exit(domain);
 			domain = found;
 		} else {
 			list_add(&info->link, &domain->devices);
 			list_add(&info->global, &device_domain_list);
 			spin_unlock_irqrestore(&device_domain_lock, flags);
 		}
 	}
 found_domain:
 	info = alloc_devinfo_mem();
 	if (!info)
 		goto error;
 	info->segment = segment;
 	info->bus = pdev->bus->number;
 	info->devfn = pdev->devfn;
 	info->dev = pdev;
 	info->domain = domain;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	/* somebody is fast */
 	found = find_domain(pdev);
 	if (found != NULL) {
 		spin_unlock_irqrestore(&device_domain_lock, flags);
 		if (found != domain) {
 			domain_exit(domain);
 			domain = found;
 		}
 		free_devinfo_mem(info);
 		return domain;
 	}
 	list_add(&info->link, &domain->devices);
 	list_add(&info->global, &device_domain_list);
 	pdev->dev.archdata.iommu = info;
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 	return domain;
 error:
 	/* recheck it here, maybe others set it */
 	return find_domain(pdev);
 }
 static int iommu_identity_mapping;
 #define IDENTMAP_ALL		1
 #define IDENTMAP_GFX		2
 #define IDENTMAP_AZALIA		4
 static int iommu_domain_identity_map(struct dmar_domain *domain,
 				     unsigned long long start,
 				     unsigned long long end)
 {
 	unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
 	unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
 	if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
 			  dma_to_mm_pfn(last_vpfn))) {
 		printk(KERN_ERR "IOMMU: reserve iova failed\n");
 		return -ENOMEM;
 	}
 	pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
 		 start, end, domain->id);
 	/*
 	 * RMRR range might have overlap with physical memory range,
 	 * clear it first
 	 */
 	dma_pte_clear_range(domain, first_vpfn, last_vpfn);
 	return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
 				  last_vpfn - first_vpfn + 1,
 				  DMA_PTE_READ|DMA_PTE_WRITE);
 }
 static int iommu_prepare_identity_map(struct pci_dev *pdev,
 				      unsigned long long start,
 				      unsigned long long end)
 {
 	struct dmar_domain *domain;
 	int ret;
 	domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 	if (!domain)
 		return -ENOMEM;
 	/* For _hardware_ passthrough, don't bother. But for software
 	   passthrough, we do it anyway -- it may indicate a memory
 	   range which is reserved in E820, so which didn't get set
 	   up to start with in si_domain */
 	if (domain == si_domain && hw_pass_through) {
 		printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
 		       pci_name(pdev), start, end);
 		return 0;
 	}
 	printk(KERN_INFO
 	       "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
 	       pci_name(pdev), start, end);
 	if (end < start) {
 		WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
 			"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 			dmi_get_system_info(DMI_BIOS_VENDOR),
 			dmi_get_system_info(DMI_BIOS_VERSION),
 		     dmi_get_system_info(DMI_PRODUCT_VERSION));
 		ret = -EIO;
 		goto error;
 	}
 	if (end >> agaw_to_width(domain->agaw)) {
 		WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
 		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 		     agaw_to_width(domain->agaw),
 		     dmi_get_system_info(DMI_BIOS_VENDOR),
 		     dmi_get_system_info(DMI_BIOS_VERSION),
 		     dmi_get_system_info(DMI_PRODUCT_VERSION));
 		ret = -EIO;
 		goto error;
 	}
 	ret = iommu_domain_identity_map(domain, start, end);
 	if (ret)
 		goto error;
 	/* context entry init */
 	ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
 	if (ret)
 		goto error;
 	return 0;
  error:
 	domain_exit(domain);
 	return ret;
 }
 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
 	struct pci_dev *pdev)
 {
 	if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
 		return 0;
 	return iommu_prepare_identity_map(pdev, rmrr->base_address,
 		rmrr->end_address);
 }
 #ifdef CONFIG_DMAR_FLOPPY_WA
 static inline void iommu_prepare_isa(void)
 {
 	struct pci_dev *pdev;
 	int ret;
 	pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
 	if (!pdev)
 		return;
 	printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
 	ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
 	if (ret)
 		printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
 		       "floppy might not work\n");
 }
 #else
 static inline void iommu_prepare_isa(void)
 {
 	return;
 }
 #endif /* !CONFIG_DMAR_FLPY_WA */
 static int md_domain_init(struct dmar_domain *domain, int guest_width);
 static int __init si_domain_work_fn(unsigned long start_pfn,
 				    unsigned long end_pfn, void *datax)
 {
 	int *ret = datax;
 	*ret = iommu_domain_identity_map(si_domain,
 					 (uint64_t)start_pfn << PAGE_SHIFT,
 					 (uint64_t)end_pfn << PAGE_SHIFT);
 	return *ret;
 }
 static int __init si_domain_init(int hw)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu;
 	int nid, ret = 0;
 	si_domain = alloc_domain();
 	if (!si_domain)
 		return -EFAULT;
 	pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
 	for_each_active_iommu(iommu, drhd) {
 		ret = iommu_attach_domain(si_domain, iommu);
 		if (ret) {
 			domain_exit(si_domain);
 			return -EFAULT;
 		}
 	}
 	if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
 		domain_exit(si_domain);
 		return -EFAULT;
 	}
 	si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
 	if (hw)
 		return 0;
 	for_each_online_node(nid) {
 		work_with_active_regions(nid, si_domain_work_fn, &ret);
 		if (ret)
 			return ret;
 	}
 	return 0;
 }
 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 					  struct pci_dev *pdev);
 static int identity_mapping(struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
 	if (likely(!iommu_identity_mapping))
 		return 0;
 	info = pdev->dev.archdata.iommu;
 	if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
 		return (info->domain == si_domain);
 	return 0;
 }
 static int domain_add_dev_info(struct dmar_domain *domain,
 			       struct pci_dev *pdev,
 			       int translation)
 {
 	struct device_domain_info *info;
 	unsigned long flags;
 	int ret;
 	info = alloc_devinfo_mem();
 	if (!info)
 		return -ENOMEM;
 	ret = domain_context_mapping(domain, pdev, translation);
 	if (ret) {
 		free_devinfo_mem(info);
 		return ret;
 	}
 	info->segment = pci_domain_nr(pdev->bus);
 	info->bus = pdev->bus->number;
 	info->devfn = pdev->devfn;
 	info->dev = pdev;
 	info->domain = domain;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	list_add(&info->link, &domain->devices);
 	list_add(&info->global, &device_domain_list);
 	pdev->dev.archdata.iommu = info;
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 	return 0;
 }
 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
 {
 	if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
 		return 1;
 	if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
 		return 1;
 	if (!(iommu_identity_mapping & IDENTMAP_ALL))
 		return 0;
 	/*
 	 * We want to start off with all devices in the 1:1 domain, and
 	 * take them out later if we find they can't access all of memory.
 	 *
 	 * However, we can't do this for PCI devices behind bridges,
 	 * because all PCI devices behind the same bridge will end up
 	 * with the same source-id on their transactions.
 	 *
 	 * Practically speaking, we can't change things around for these
 	 * devices at run-time, because we can't be sure there'll be no
 	 * DMA transactions in flight for any of their siblings.
 	 *
 	 * So PCI devices (unless they're on the root bus) as well as
 	 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
 	 * the 1:1 domain, just in _case_ one of their siblings turns out
 	 * not to be able to map all of memory.
 	 */
 	if (!pci_is_pcie(pdev)) {
 		if (!pci_is_root_bus(pdev->bus))
 			return 0;
 		if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
 			return 0;
 	} else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 		return 0;
 	/*
 	 * At boot time, we don't yet know if devices will be 64-bit capable.
 	 * Assume that they will -- if they turn out not to be, then we can
 	 * take them out of the 1:1 domain later.
 	 */
 	if (!startup) {
 		/*
 		 * If the device's dma_mask is less than the system's memory
 		 * size then this is not a candidate for identity mapping.
 		 */
 		u64 dma_mask = pdev->dma_mask;
 		if (pdev->dev.coherent_dma_mask &&
 		    pdev->dev.coherent_dma_mask < dma_mask)
 			dma_mask = pdev->dev.coherent_dma_mask;
 		return dma_mask >= dma_get_required_mask(&pdev->dev);
 	}
 	return 1;
 }
 static int __init iommu_prepare_static_identity_mapping(int hw)
 {
 	struct pci_dev *pdev = NULL;
 	int ret;
 	ret = si_domain_init(hw);
 	if (ret)
 		return -EFAULT;
 	for_each_pci_dev(pdev) {
 		/* Skip Host/PCI Bridge devices */
 		if (IS_BRIDGE_HOST_DEVICE(pdev))
 			continue;
 		if (iommu_should_identity_map(pdev, 1)) {
 			printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
 			       hw ? "hardware" : "software", pci_name(pdev));
 			ret = domain_add_dev_info(si_domain, pdev,
 						     hw ? CONTEXT_TT_PASS_THROUGH :
 						     CONTEXT_TT_MULTI_LEVEL);
 			if (ret)
 				return ret;
 		}
 	}
 	return 0;
 }
 static int __init init_dmars(void)
 {
 	struct dmar_drhd_unit *drhd;
 	struct dmar_rmrr_unit *rmrr;
 	struct pci_dev *pdev;
 	struct intel_iommu *iommu;
 	int i, ret;
 	/*
 	 * for each drhd
 	 *    allocate root
 	 *    initialize and program root entry to not present
 	 * endfor
 	 */
 	for_each_drhd_unit(drhd) {
 		g_num_of_iommus++;
 		/*
 		 * lock not needed as this is only incremented in the single
 		 * threaded kernel __init code path all other access are read
 		 * only
 		 */
 	}
 	g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
 			GFP_KERNEL);
 	if (!g_iommus) {
 		printk(KERN_ERR "Allocating global iommu array failed\n");
 		ret = -ENOMEM;
 		goto error;
 	}
 	deferred_flush = kzalloc(g_num_of_iommus *
 		sizeof(struct deferred_flush_tables), GFP_KERNEL);
 	if (!deferred_flush) {
 		ret = -ENOMEM;
 		goto error;
 	}
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		iommu = drhd->iommu;
 		g_iommus[iommu->seq_id] = iommu;
 		ret = iommu_init_domains(iommu);
 		if (ret)
 			goto error;
 		/*
 		 * TBD:
 		 * we could share the same root & context tables
 		 * among all IOMMU's. Need to Split it later.
 		 */
 		ret = iommu_alloc_root_entry(iommu);
 		if (ret) {
 			printk(KERN_ERR "IOMMU: allocate root entry failed\n");
 			goto error;
 		}
 		if (!ecap_pass_through(iommu->ecap))
 			hw_pass_through = 0;
 	}
 	/*
 	 * Start from the sane iommu hardware state.
 	 */
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		iommu = drhd->iommu;
 		/*
 		 * If the queued invalidation is already initialized by us
 		 * (for example, while enabling interrupt-remapping) then
 		 * we got the things already rolling from a sane state.
 		 */
 		if (iommu->qi)
 			continue;
 		/*
 		 * Clear any previous faults.
 		 */
 		dmar_fault(-1, iommu);
 		/*
 		 * Disable queued invalidation if supported and already enabled
 		 * before OS handover.
 		 */
 		dmar_disable_qi(iommu);
 	}
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		iommu = drhd->iommu;
 		if (dmar_enable_qi(iommu)) {
 			/*
 			 * Queued Invalidate not enabled, use Register Based
 			 * Invalidate
 			 */
 			iommu->flush.flush_context = __iommu_flush_context;
 			iommu->flush.flush_iotlb = __iommu_flush_iotlb;
 			printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
 			       "invalidation\n",
 				iommu->seq_id,
 			       (unsigned long long)drhd->reg_base_addr);
 		} else {
 			iommu->flush.flush_context = qi_flush_context;
 			iommu->flush.flush_iotlb = qi_flush_iotlb;
 			printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
 			       "invalidation\n",
 				iommu->seq_id,
 			       (unsigned long long)drhd->reg_base_addr);
 		}
 	}
 	if (iommu_pass_through)
 		iommu_identity_mapping |= IDENTMAP_ALL;
 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
 	iommu_identity_mapping |= IDENTMAP_GFX;
 #endif
 	check_tylersburg_isoch();
 	/*
 	 * If pass through is not set or not enabled, setup context entries for
 	 * identity mappings for rmrr, gfx, and isa and may fall back to static
 	 * identity mapping if iommu_identity_mapping is set.
 	 */
 	if (iommu_identity_mapping) {
 		ret = iommu_prepare_static_identity_mapping(hw_pass_through);
 		if (ret) {
 			printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
 			goto error;
 		}
 	}
 	/*
 	 * For each rmrr
 	 *   for each dev attached to rmrr
 	 *   do
 	 *     locate drhd for dev, alloc domain for dev
 	 *     allocate free domain
 	 *     allocate page table entries for rmrr
 	 *     if context not allocated for bus
 	 *           allocate and init context
 	 *           set present in root table for this bus
 	 *     init context with domain, translation etc
 	 *    endfor
 	 * endfor
 	 */
 	printk(KERN_INFO "IOMMU: Setting RMRR:\n");
 	for_each_rmrr_units(rmrr) {
 		for (i = 0; i < rmrr->devices_cnt; i++) {
 			pdev = rmrr->devices[i];
 			/*
 			 * some BIOS lists non-exist devices in DMAR
 			 * table.
 			 */
 			if (!pdev)
 				continue;
 			ret = iommu_prepare_rmrr_dev(rmrr, pdev);
 			if (ret)
 				printk(KERN_ERR
 				       "IOMMU: mapping reserved region failed\n");
 		}
 	}
 	iommu_prepare_isa();
 	/*
 	 * for each drhd
 	 *   enable fault log
 	 *   global invalidate context cache
 	 *   global invalidate iotlb
 	 *   enable translation
 	 */
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored) {
 			/*
 			 * we always have to disable PMRs or DMA may fail on
 			 * this device
 			 */
 			if (force_on)
 				iommu_disable_protect_mem_regions(drhd->iommu);
 			continue;
 		}
 		iommu = drhd->iommu;
 		iommu_flush_write_buffer(iommu);
 		ret = dmar_set_interrupt(iommu);
 		if (ret)
 			goto error;
 		iommu_set_root_entry(iommu);
 		iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
 		iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
 		ret = iommu_enable_translation(iommu);
 		if (ret)
 			goto error;
 		iommu_disable_protect_mem_regions(iommu);
 	}
 	return 0;
 error:
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		iommu = drhd->iommu;
 		free_iommu(iommu);
 	}
 	kfree(g_iommus);
 	return ret;
 }
 /* This takes a number of _MM_ pages, not VTD pages */
 static struct iova *intel_alloc_iova(struct device *dev,
 				     struct dmar_domain *domain,
 				     unsigned long nrpages, uint64_t dma_mask)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct iova *iova = NULL;
 	/* Restrict dma_mask to the width that the iommu can handle */
 	dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
 	if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
 		/*
 		 * First try to allocate an io virtual address in
 		 * DMA_BIT_MASK(32) and if that fails then try allocating
 		 * from higher range
 		 */
 		iova = alloc_iova(&domain->iovad, nrpages,
 				  IOVA_PFN(DMA_BIT_MASK(32)), 1);
 		if (iova)
 			return iova;
 	}
 	iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
 	if (unlikely(!iova)) {
 		printk(KERN_ERR "Allocating %ld-page iova for %s failed",
 		       nrpages, pci_name(pdev));
 		return NULL;
 	}
 	return iova;
 }
 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
 {
 	struct dmar_domain *domain;
 	int ret;
 	domain = get_domain_for_dev(pdev,
 			DEFAULT_DOMAIN_ADDRESS_WIDTH);
 	if (!domain) {
 		printk(KERN_ERR
 			"Allocating domain for %s failed", pci_name(pdev));
 		return NULL;
 	}
 	/* make sure context mapping is ok */
 	if (unlikely(!domain_context_mapped(pdev))) {
 		ret = domain_context_mapping(domain, pdev,
 					     CONTEXT_TT_MULTI_LEVEL);
 		if (ret) {
 			printk(KERN_ERR
 				"Domain context map for %s failed",
 				pci_name(pdev));
 			return NULL;
 		}
 	}
 	return domain;
 }
 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
 {
 	struct device_domain_info *info;
 	/* No lock here, assumes no domain exit in normal case */
 	info = dev->dev.archdata.iommu;
 	if (likely(info))
 		return info->domain;
 	return __get_valid_domain_for_dev(dev);
 }
 static int iommu_dummy(struct pci_dev *pdev)
 {
 	return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 }
 /* Check if the pdev needs to go through non-identity map and unmap process.*/
 static int iommu_no_mapping(struct device *dev)
 {
 	struct pci_dev *pdev;
 	int found;
 	if (unlikely(dev->bus != &pci_bus_type))
 		return 1;
 	pdev = to_pci_dev(dev);
 	if (iommu_dummy(pdev))
 		return 1;
 	if (!iommu_identity_mapping)
 		return 0;
 	found = identity_mapping(pdev);
 	if (found) {
 		if (iommu_should_identity_map(pdev, 0))
 			return 1;
 		else {
 			/*
 			 * 32 bit DMA is removed from si_domain and fall back
 			 * to non-identity mapping.
 			 */
 			domain_remove_one_dev_info(si_domain, pdev);
 			printk(KERN_INFO "32bit %s uses non-identity mapping\n",
 			       pci_name(pdev));
 			return 0;
 		}
 	} else {
 		/*
 		 * In case of a detached 64 bit DMA device from vm, the device
 		 * is put into si_domain for identity mapping.
 		 */
 		if (iommu_should_identity_map(pdev, 0)) {
 			int ret;
 			ret = domain_add_dev_info(si_domain, pdev,
 						  hw_pass_through ?
 						  CONTEXT_TT_PASS_THROUGH :
 						  CONTEXT_TT_MULTI_LEVEL);
 			if (!ret) {
 				printk(KERN_INFO "64bit %s uses identity mapping\n",
 				       pci_name(pdev));
 				return 1;
 			}
 		}
 	}
 	return 0;
 }
 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
 				     size_t size, int dir, u64 dma_mask)
 {
 	struct pci_dev *pdev = to_pci_dev(hwdev);
 	struct dmar_domain *domain;
 	phys_addr_t start_paddr;
 	struct iova *iova;
 	int prot = 0;
 	int ret;
 	struct intel_iommu *iommu;
 	unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
 	BUG_ON(dir == DMA_NONE);
 	if (iommu_no_mapping(hwdev))
 		return paddr;
 	domain = get_valid_domain_for_dev(pdev);
 	if (!domain)
 		return 0;
 	iommu = domain_get_iommu(domain);
 	size = aligned_nrpages(paddr, size);
 	iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
 	if (!iova)
 		goto error;
 	/*
 	 * Check if DMAR supports zero-length reads on write only
 	 * mappings..
 	 */
 	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
 			!cap_zlr(iommu->cap))
 		prot |= DMA_PTE_READ;
 	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
 		prot |= DMA_PTE_WRITE;
 	/*
 	 * paddr - (paddr + size) might be partial page, we should map the whole
 	 * page.  Note: if two part of one page are separately mapped, we
 	 * might have two guest_addr mapping to the same host paddr, but this
 	 * is not a big problem
 	 */
 	ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
 				 mm_to_dma_pfn(paddr_pfn), size, prot);
 	if (ret)
 		goto error;
 	/* it's a non-present to present mapping. Only flush if caching mode */
 	if (cap_caching_mode(iommu->cap))
 		iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
 	else
 		iommu_flush_write_buffer(iommu);
 	start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
 	start_paddr += paddr & ~PAGE_MASK;
 	return start_paddr;
 error:
 	if (iova)
 		__free_iova(&domain->iovad, iova);
 	printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
 		pci_name(pdev), size, (unsigned long long)paddr, dir);
 	return 0;
 }
 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
 				 unsigned long offset, size_t size,
 				 enum dma_data_direction dir,
 				 struct dma_attrs *attrs)
 {
 	return __intel_map_single(dev, page_to_phys(page) + offset, size,
 				  dir, to_pci_dev(dev)->dma_mask);
 }
 static void flush_unmaps(void)
 {
 	int i, j;
 	timer_on = 0;
 	/* just flush them all */
 	for (i = 0; i < g_num_of_iommus; i++) {
 		struct intel_iommu *iommu = g_iommus[i];
 		if (!iommu)
 			continue;
 		if (!deferred_flush[i].next)
 			continue;
 		/* In caching mode, global flushes turn emulation expensive */
 		if (!cap_caching_mode(iommu->cap))
 			iommu->flush.flush_iotlb(iommu, 0, 0, 0,
 					 DMA_TLB_GLOBAL_FLUSH);
 		for (j = 0; j < deferred_flush[i].next; j++) {
 			unsigned long mask;
 			struct iova *iova = deferred_flush[i].iova[j];
 			struct dmar_domain *domain = deferred_flush[i].domain[j];
 			/* On real hardware multiple invalidations are expensive */
 			if (cap_caching_mode(iommu->cap))
 				iommu_flush_iotlb_psi(iommu, domain->id,
 				iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
 			else {
 				mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
 				iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
 						(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
 			}
 			__free_iova(&deferred_flush[i].domain[j]->iovad, iova);
 		}
 		deferred_flush[i].next = 0;
 	}
 	list_size = 0;
 }
 static void flush_unmaps_timeout(unsigned long data)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&async_umap_flush_lock, flags);
 	flush_unmaps();
 	spin_unlock_irqrestore(&async_umap_flush_lock, flags);
 }
 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
 {
 	unsigned long flags;
 	int next, iommu_id;
 	struct intel_iommu *iommu;
 	spin_lock_irqsave(&async_umap_flush_lock, flags);
 	if (list_size == HIGH_WATER_MARK)
 		flush_unmaps();
 	iommu = domain_get_iommu(dom);
 	iommu_id = iommu->seq_id;
 	next = deferred_flush[iommu_id].next;
 	deferred_flush[iommu_id].domain[next] = dom;
 	deferred_flush[iommu_id].iova[next] = iova;
 	deferred_flush[iommu_id].next++;
 	if (!timer_on) {
 		mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
 		timer_on = 1;
 	}
 	list_size++;
 	spin_unlock_irqrestore(&async_umap_flush_lock, flags);
 }
 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
 			     size_t size, enum dma_data_direction dir,
 			     struct dma_attrs *attrs)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct dmar_domain *domain;
 	unsigned long start_pfn, last_pfn;
 	struct iova *iova;
 	struct intel_iommu *iommu;
 	if (iommu_no_mapping(dev))
 		return;
 	domain = find_domain(pdev);
 	BUG_ON(!domain);
 	iommu = domain_get_iommu(domain);
 	iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
 	if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
 		      (unsigned long long)dev_addr))
 		return;
 	start_pfn = mm_to_dma_pfn(iova->pfn_lo);
 	last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
 	pr_debug("Device %s unmapping: pfn %lx-%lx\n",
 		 pci_name(pdev), start_pfn, last_pfn);
 	/*  clear the whole page */
 	dma_pte_clear_range(domain, start_pfn, last_pfn);
 	/* free page tables */
 	dma_pte_free_pagetable(domain, start_pfn, last_pfn);
 	if (intel_iommu_strict) {
 		iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
 				      last_pfn - start_pfn + 1, 0);
 		/* free iova */
 		__free_iova(&domain->iovad, iova);
 	} else {
 		add_unmap(domain, iova);
 		/*
 		 * queue up the release of the unmap to save the 1/6th of the
 		 * cpu used up by the iotlb flush operation...
 		 */
 	}
 }
 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
 				  dma_addr_t *dma_handle, gfp_t flags)
 {
 	void *vaddr;
 	int order;
 	size = PAGE_ALIGN(size);
 	order = get_order(size);
 	if (!iommu_no_mapping(hwdev))
 		flags &= ~(GFP_DMA | GFP_DMA32);
 	else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
 		if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
 			flags |= GFP_DMA;
 		else
 			flags |= GFP_DMA32;
 	}
 	vaddr = (void *)__get_free_pages(flags, order);
 	if (!vaddr)
 		return NULL;
 	memset(vaddr, 0, size);
 	*dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
 					 DMA_BIDIRECTIONAL,
 					 hwdev->coherent_dma_mask);
 	if (*dma_handle)
 		return vaddr;
 	free_pages((unsigned long)vaddr, order);
 	return NULL;
 }
 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
 				dma_addr_t dma_handle)
 {
 	int order;
 	size = PAGE_ALIGN(size);
 	order = get_order(size);
 	intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
 	free_pages((unsigned long)vaddr, order);
 }
 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
 			   int nelems, enum dma_data_direction dir,
 			   struct dma_attrs *attrs)
 {
 	struct pci_dev *pdev = to_pci_dev(hwdev);
 	struct dmar_domain *domain;
 	unsigned long start_pfn, last_pfn;
 	struct iova *iova;
 	struct intel_iommu *iommu;
 	if (iommu_no_mapping(hwdev))
 		return;
 	domain = find_domain(pdev);
 	BUG_ON(!domain);
 	iommu = domain_get_iommu(domain);
 	iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
 	if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
 		      (unsigned long long)sglist[0].dma_address))
 		return;
 	start_pfn = mm_to_dma_pfn(iova->pfn_lo);
 	last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
 	/*  clear the whole page */
 	dma_pte_clear_range(domain, start_pfn, last_pfn);
 	/* free page tables */
 	dma_pte_free_pagetable(domain, start_pfn, last_pfn);
 	if (intel_iommu_strict) {
 		iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
 				      last_pfn - start_pfn + 1, 0);
 		/* free iova */
 		__free_iova(&domain->iovad, iova);
 	} else {
 		add_unmap(domain, iova);
 		/*
 		 * queue up the release of the unmap to save the 1/6th of the
 		 * cpu used up by the iotlb flush operation...
 		 */
 	}
 }
 static int intel_nontranslate_map_sg(struct device *hddev,
 	struct scatterlist *sglist, int nelems, int dir)
 {
 	int i;
 	struct scatterlist *sg;
 	for_each_sg(sglist, sg, nelems, i) {
 		BUG_ON(!sg_page(sg));
 		sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
 		sg->dma_length = sg->length;
 	}
 	return nelems;
 }
 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
 			enum dma_data_direction dir, struct dma_attrs *attrs)
 {
 	int i;
 	struct pci_dev *pdev = to_pci_dev(hwdev);
 	struct dmar_domain *domain;
 	size_t size = 0;
 	int prot = 0;
 	struct iova *iova = NULL;
 	int ret;
 	struct scatterlist *sg;
 	unsigned long start_vpfn;
 	struct intel_iommu *iommu;
 	BUG_ON(dir == DMA_NONE);
 	if (iommu_no_mapping(hwdev))
 		return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
 	domain = get_valid_domain_for_dev(pdev);
 	if (!domain)
 		return 0;
 	iommu = domain_get_iommu(domain);
 	for_each_sg(sglist, sg, nelems, i)
 		size += aligned_nrpages(sg->offset, sg->length);
 	iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
 				pdev->dma_mask);
 	if (!iova) {
 		sglist->dma_length = 0;
 		return 0;
 	}
 	/*
 	 * Check if DMAR supports zero-length reads on write only
 	 * mappings..
 	 */
 	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
 			!cap_zlr(iommu->cap))
 		prot |= DMA_PTE_READ;
 	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
 		prot |= DMA_PTE_WRITE;
 	start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
 	ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
 	if (unlikely(ret)) {
 		/*  clear the page */
 		dma_pte_clear_range(domain, start_vpfn,
 				    start_vpfn + size - 1);
 		/* free page tables */
 		dma_pte_free_pagetable(domain, start_vpfn,
 				       start_vpfn + size - 1);
 		/* free iova */
 		__free_iova(&domain->iovad, iova);
 		return 0;
 	}
 	/* it's a non-present to present mapping. Only flush if caching mode */
 	if (cap_caching_mode(iommu->cap))
 		iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
 	else
 		iommu_flush_write_buffer(iommu);
 	return nelems;
 }
 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
 	return !dma_addr;
 }
 struct dma_map_ops intel_dma_ops = {
 	.alloc_coherent = intel_alloc_coherent,
 	.free_coherent = intel_free_coherent,
 	.map_sg = intel_map_sg,
 	.unmap_sg = intel_unmap_sg,
 	.map_page = intel_map_page,
 	.unmap_page = intel_unmap_page,
 	.mapping_error = intel_mapping_error,
 };
 static inline int iommu_domain_cache_init(void)
 {
 	int ret = 0;
 	iommu_domain_cache = kmem_cache_create("iommu_domain",
 					 sizeof(struct dmar_domain),
 					 0,
 					 SLAB_HWCACHE_ALIGN,
 					 NULL);
 	if (!iommu_domain_cache) {
 		printk(KERN_ERR "Couldn't create iommu_domain cache\n");
 		ret = -ENOMEM;
 	}
 	return ret;
 }
 static inline int iommu_devinfo_cache_init(void)
 {
 	int ret = 0;
 	iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
 					 sizeof(struct device_domain_info),
 					 0,
 					 SLAB_HWCACHE_ALIGN,
 					 NULL);
 	if (!iommu_devinfo_cache) {
 		printk(KERN_ERR "Couldn't create devinfo cache\n");
 		ret = -ENOMEM;
 	}
 	return ret;
 }
 static inline int iommu_iova_cache_init(void)
 {
 	int ret = 0;
 	iommu_iova_cache = kmem_cache_create("iommu_iova",
 					 sizeof(struct iova),
 					 0,
 					 SLAB_HWCACHE_ALIGN,
 					 NULL);
 	if (!iommu_iova_cache) {
 		printk(KERN_ERR "Couldn't create iova cache\n");
 		ret = -ENOMEM;
 	}
 	return ret;
 }
 static int __init iommu_init_mempool(void)
 {
 	int ret;
 	ret = iommu_iova_cache_init();
 	if (ret)
 		return ret;
 	ret = iommu_domain_cache_init();
 	if (ret)
 		goto domain_error;
 	ret = iommu_devinfo_cache_init();
 	if (!ret)
 		return ret;
 	kmem_cache_destroy(iommu_domain_cache);
 domain_error:
 	kmem_cache_destroy(iommu_iova_cache);
 	return -ENOMEM;
 }
 static void __init iommu_exit_mempool(void)
 {
 	kmem_cache_destroy(iommu_devinfo_cache);
 	kmem_cache_destroy(iommu_domain_cache);
 	kmem_cache_destroy(iommu_iova_cache);
 }
 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
 {
 	struct dmar_drhd_unit *drhd;
 	u32 vtbar;
 	int rc;
 	/* We know that this device on this chipset has its own IOMMU.
 	 * If we find it under a different IOMMU, then the BIOS is lying
 	 * to us. Hope that the IOMMU for this device is actually
 	 * disabled, and it needs no translation...
 	 */
 	rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
 	if (rc) {
 		/* "can't" happen */
 		dev_info(&pdev->dev, "failed to run vt-d quirk\n");
 		return;
 	}
 	vtbar &= 0xffff0000;
 	/* we know that the this iommu should be at offset 0xa000 from vtbar */
 	drhd = dmar_find_matched_drhd_unit(pdev);
 	if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
 			    TAINT_FIRMWARE_WORKAROUND,
 			    "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
 		pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
 }
 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
 static void __init init_no_remapping_devices(void)
 {
 	struct dmar_drhd_unit *drhd;
 	for_each_drhd_unit(drhd) {
 		if (!drhd->include_all) {
 			int i;
 			for (i = 0; i < drhd->devices_cnt; i++)
 				if (drhd->devices[i] != NULL)
 					break;
 			/* ignore DMAR unit if no pci devices exist */
 			if (i == drhd->devices_cnt)
 				drhd->ignored = 1;
 		}
 	}
 	for_each_drhd_unit(drhd) {
 		int i;
 		if (drhd->ignored || drhd->include_all)
 			continue;
 		for (i = 0; i < drhd->devices_cnt; i++)
 			if (drhd->devices[i] &&
 			    !IS_GFX_DEVICE(drhd->devices[i]))
 				break;
 		if (i < drhd->devices_cnt)
 			continue;
 		/* This IOMMU has *only* gfx devices. Either bypass it or
 		   set the gfx_mapped flag, as appropriate */
 		if (dmar_map_gfx) {
 			intel_iommu_gfx_mapped = 1;
 		} else {
 			drhd->ignored = 1;
 			for (i = 0; i < drhd->devices_cnt; i++) {
 				if (!drhd->devices[i])
 					continue;
 				drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
 			}
 		}
 	}
 }
 #ifdef CONFIG_SUSPEND
 static int init_iommu_hw(void)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu = NULL;
 	for_each_active_iommu(iommu, drhd)
 		if (iommu->qi)
 			dmar_reenable_qi(iommu);
 	for_each_iommu(iommu, drhd) {
 		if (drhd->ignored) {
 			/*
 			 * we always have to disable PMRs or DMA may fail on
 			 * this device
 			 */
 			if (force_on)
 				iommu_disable_protect_mem_regions(iommu);
 			continue;
 		}
 		iommu_flush_write_buffer(iommu);
 		iommu_set_root_entry(iommu);
 		iommu->flush.flush_context(iommu, 0, 0, 0,
 					   DMA_CCMD_GLOBAL_INVL);
 		iommu->flush.flush_iotlb(iommu, 0, 0, 0,
 					 DMA_TLB_GLOBAL_FLUSH);
 		if (iommu_enable_translation(iommu))
 			return 1;
 		iommu_disable_protect_mem_regions(iommu);
 	}
 	return 0;
 }
 static void iommu_flush_all(void)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu;
 	for_each_active_iommu(iommu, drhd) {
 		iommu->flush.flush_context(iommu, 0, 0, 0,
 					   DMA_CCMD_GLOBAL_INVL);
 		iommu->flush.flush_iotlb(iommu, 0, 0, 0,
 					 DMA_TLB_GLOBAL_FLUSH);
 	}
 }
 static int iommu_suspend(void)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu = NULL;
 	unsigned long flag;
 	for_each_active_iommu(iommu, drhd) {
 		iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
 						 GFP_ATOMIC);
 		if (!iommu->iommu_state)
 			goto nomem;
 	}
 	iommu_flush_all();
 	for_each_active_iommu(iommu, drhd) {
 		iommu_disable_translation(iommu);
 		spin_lock_irqsave(&iommu->register_lock, flag);
 		iommu->iommu_state[SR_DMAR_FECTL_REG] =
 			readl(iommu->reg + DMAR_FECTL_REG);
 		iommu->iommu_state[SR_DMAR_FEDATA_REG] =
 			readl(iommu->reg + DMAR_FEDATA_REG);
 		iommu->iommu_state[SR_DMAR_FEADDR_REG] =
 			readl(iommu->reg + DMAR_FEADDR_REG);
 		iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
 			readl(iommu->reg + DMAR_FEUADDR_REG);
 		spin_unlock_irqrestore(&iommu->register_lock, flag);
 	}
 	return 0;
 nomem:
 	for_each_active_iommu(iommu, drhd)
 		kfree(iommu->iommu_state);
 	return -ENOMEM;
 }
 static void iommu_resume(void)
 {
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu = NULL;
 	unsigned long flag;
 	if (init_iommu_hw()) {
 		if (force_on)
 			panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
 		else
 			WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
 		return;
 	}
 	for_each_active_iommu(iommu, drhd) {
 		spin_lock_irqsave(&iommu->register_lock, flag);
 		writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
 			iommu->reg + DMAR_FECTL_REG);
 		writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
 			iommu->reg + DMAR_FEDATA_REG);
 		writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
 			iommu->reg + DMAR_FEADDR_REG);
 		writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
 			iommu->reg + DMAR_FEUADDR_REG);
 		spin_unlock_irqrestore(&iommu->register_lock, flag);
 	}
 	for_each_active_iommu(iommu, drhd)
 		kfree(iommu->iommu_state);
 }
 static struct syscore_ops iommu_syscore_ops = {
 	.resume		= iommu_resume,
 	.suspend	= iommu_suspend,
 };
 static void __init init_iommu_pm_ops(void)
 {
 	register_syscore_ops(&iommu_syscore_ops);
 }
 #else
 static inline void init_iommu_pm_ops(void) {}
 #endif	/* CONFIG_PM */
 /*
  * Here we only respond to action of unbound device from driver.
  *
  * Added device is not attached to its DMAR domain here yet. That will happen
  * when mapping the device to iova.
  */
 static int device_notifier(struct notifier_block *nb,
 				  unsigned long action, void *data)
 {
 	struct device *dev = data;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct dmar_domain *domain;
 	if (iommu_no_mapping(dev))
 		return 0;
 	domain = find_domain(pdev);
 	if (!domain)
 		return 0;
 	if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
 		domain_remove_one_dev_info(domain, pdev);
 		if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
 		    !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
 		    list_empty(&domain->devices))
 			domain_exit(domain);
 	}
 	return 0;
 }
 static struct notifier_block device_nb = {
 	.notifier_call = device_notifier,
 };
 int __init intel_iommu_init(void)
 {
 	int ret = 0;
 	/* VT-d is required for a TXT/tboot launch, so enforce that */
 	force_on = tboot_force_iommu();
 	if (dmar_table_init()) {
 		if (force_on)
 			panic("tboot: Failed to initialize DMAR table\n");
 		return 	-ENODEV;
 	}
 	if (dmar_dev_scope_init()) {
 		if (force_on)
 			panic("tboot: Failed to initialize DMAR device scope\n");
 		return 	-ENODEV;
 	}
 	/*
 	 * Check the need for DMA-remapping initialization now.
 	 * Above initialization will also be used by Interrupt-remapping.
 	 */
 	if (no_iommu || dmar_disabled)
 		return -ENODEV;
 	if (iommu_init_mempool()) {
 		if (force_on)
 			panic("tboot: Failed to initialize iommu memory\n");
 		return 	-ENODEV;
 	}
 	if (dmar_init_reserved_ranges()) {
 		if (force_on)
 			panic("tboot: Failed to reserve iommu ranges\n");
 		return 	-ENODEV;
 	}
 	init_no_remapping_devices();
 	ret = init_dmars();
 	if (ret) {
 		if (force_on)
 			panic("tboot: Failed to initialize DMARs\n");
 		printk(KERN_ERR "IOMMU: dmar init failed\n");
 		put_iova_domain(&reserved_iova_list);
 		iommu_exit_mempool();
 		return ret;
 	}
 	printk(KERN_INFO
 	"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
 	init_timer(&unmap_timer);
 #ifdef CONFIG_SWIOTLB
 	swiotlb = 0;
 #endif
 	dma_ops = &intel_dma_ops;
 	init_iommu_pm_ops();
 	register_iommu(&intel_iommu_ops);
 	bus_register_notifier(&pci_bus_type, &device_nb);
 	return 0;
 }
 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
 					   struct pci_dev *pdev)
 {
 	struct pci_dev *tmp, *parent;
 	if (!iommu || !pdev)
 		return;
 	/* dependent device detach */
 	tmp = pci_find_upstream_pcie_bridge(pdev);
 	/* Secondary interface's bus number and devfn 0 */
 	if (tmp) {
 		parent = pdev->bus->self;
 		while (parent != tmp) {
 			iommu_detach_dev(iommu, parent->bus->number,
 					 parent->devfn);
 			parent = parent->bus->self;
 		}
 		if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
 			iommu_detach_dev(iommu,
 				tmp->subordinate->number, 0);
 		else /* this is a legacy PCI bridge */
 			iommu_detach_dev(iommu, tmp->bus->number,
 					 tmp->devfn);
 	}
 }
 static void domain_remove_one_dev_info(struct dmar_domain *domain,
 					  struct pci_dev *pdev)
 {
 	struct device_domain_info *info;
 	struct intel_iommu *iommu;
 	unsigned long flags;
 	int found = 0;
 	struct list_head *entry, *tmp;
 	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
 				pdev->devfn);
 	if (!iommu)
 		return;
 	spin_lock_irqsave(&device_domain_lock, flags);
 	list_for_each_safe(entry, tmp, &domain->devices) {
 		info = list_entry(entry, struct device_domain_info, link);
 		if (info->segment == pci_domain_nr(pdev->bus) &&
 		    info->bus == pdev->bus->number &&
 		    info->devfn == pdev->devfn) {
 			list_del(&info->link);
 			list_del(&info->global);
 			if (info->dev)
 				info->dev->dev.archdata.iommu = NULL;
 			spin_unlock_irqrestore(&device_domain_lock, flags);
 			iommu_disable_dev_iotlb(info);
 			iommu_detach_dev(iommu, info->bus, info->devfn);
 			iommu_detach_dependent_devices(iommu, pdev);
 			free_devinfo_mem(info);
 			spin_lock_irqsave(&device_domain_lock, flags);
 			if (found)
 				break;
 			else
 				continue;
 		}
 		/* if there is no other devices under the same iommu
 		 * owned by this domain, clear this iommu in iommu_bmp
 		 * update iommu count and coherency
 		 */
 		if (iommu == device_to_iommu(info->segment, info->bus,
 					    info->devfn))
 			found = 1;
 	}
 	spin_unlock_irqrestore(&device_domain_lock, flags);
 	if (found == 0) {
 		unsigned long tmp_flags;
 		spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
 		clear_bit(iommu->seq_id, &domain->iommu_bmp);
 		domain->iommu_count--;
 		domain_update_iommu_cap(domain);
 		spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
 		if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
 		    !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
 			spin_lock_irqsave(&iommu->lock, tmp_flags);
 			clear_bit(domain->id, iommu->domain_ids);
 			iommu->domains[domain->id] = NULL;
 			spin_unlock_irqrestore(&iommu->lock, tmp_flags);
 		}
 	}
 }
 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
 {
 	struct device_domain_info *info;
 	struct intel_iommu *iommu;
 	unsigned long flags1, flags2;
 	spin_lock_irqsave(&device_domain_lock, flags1);
 	while (!list_empty(&domain->devices)) {
 		info = list_entry(domain->devices.next,
 			struct device_domain_info, link);
 		list_del(&info->link);
 		list_del(&info->global);
 		if (info->dev)
 			info->dev->dev.archdata.iommu = NULL;
 		spin_unlock_irqrestore(&device_domain_lock, flags1);
 		iommu_disable_dev_iotlb(info);
 		iommu = device_to_iommu(info->segment, info->bus, info->devfn);
 		iommu_detach_dev(iommu, info->bus, info->devfn);
 		iommu_detach_dependent_devices(iommu, info->dev);
 		/* clear this iommu in iommu_bmp, update iommu count
 		 * and capabilities
 		 */
 		spin_lock_irqsave(&domain->iommu_lock, flags2);
 		if (test_and_clear_bit(iommu->seq_id,
 				       &domain->iommu_bmp)) {
 			domain->iommu_count--;
 			domain_update_iommu_cap(domain);
 		}
 		spin_unlock_irqrestore(&domain->iommu_lock, flags2);
 		free_devinfo_mem(info);
 		spin_lock_irqsave(&device_domain_lock, flags1);
 	}
 	spin_unlock_irqrestore(&device_domain_lock, flags1);
 }
 /* domain id for virtual machine, it won't be set in context */
 static unsigned long vm_domid;
 static struct dmar_domain *iommu_alloc_vm_domain(void)
 {
 	struct dmar_domain *domain;
 	domain = alloc_domain_mem();
 	if (!domain)
 		return NULL;
 	domain->id = vm_domid++;
 	domain->nid = -1;
 	memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
 	domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
 	return domain;
 }
 static int md_domain_init(struct dmar_domain *domain, int guest_width)
 {
 	int adjust_width;
 	init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
 	spin_lock_init(&domain->iommu_lock);
 	domain_reserve_special_ranges(domain);
 	/* calculate AGAW */
 	domain->gaw = guest_width;
 	adjust_width = guestwidth_to_adjustwidth(guest_width);
 	domain->agaw = width_to_agaw(adjust_width);
 	INIT_LIST_HEAD(&domain->devices);
 	domain->iommu_count = 0;
 	domain->iommu_coherency = 0;
 	domain->iommu_snooping = 0;
 	domain->iommu_superpage = 0;
 	domain->max_addr = 0;
 	domain->nid = -1;
 	/* always allocate the top pgd */
 	domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
 	if (!domain->pgd)
 		return -ENOMEM;
 	domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
 	return 0;
 }
 static void iommu_free_vm_domain(struct dmar_domain *domain)
 {
 	unsigned long flags;
 	struct dmar_drhd_unit *drhd;
 	struct intel_iommu *iommu;
 	unsigned long i;
 	unsigned long ndomains;
 	for_each_drhd_unit(drhd) {
 		if (drhd->ignored)
 			continue;
 		iommu = drhd->iommu;
 		ndomains = cap_ndoms(iommu->cap);
 		for_each_set_bit(i, iommu->domain_ids, ndomains) {
 			if (iommu->domains[i] == domain) {
 				spin_lock_irqsave(&iommu->lock, flags);
 				clear_bit(i, iommu->domain_ids);
 				iommu->domains[i] = NULL;
 				spin_unlock_irqrestore(&iommu->lock, flags);
 				break;
 			}
 		}
 	}
 }
 static void vm_domain_exit(struct dmar_domain *domain)
 {
 	/* Domain 0 is reserved, so dont process it */
 	if (!domain)
 		return;
 	vm_domain_remove_all_dev_info(domain);
 	/* destroy iovas */
 	put_iova_domain(&domain->iovad);
 	/* clear ptes */
 	dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
 	/* free page tables */
 	dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
 	iommu_free_vm_domain(domain);
 	free_domain_mem(domain);
 }
 static int intel_iommu_domain_init(struct iommu_domain *domain)
 {
 	struct dmar_domain *dmar_domain;
 	dmar_domain = iommu_alloc_vm_domain();
 	if (!dmar_domain) {
 		printk(KERN_ERR
 			"intel_iommu_domain_init: dmar_domain == NULL\n");
 		return -ENOMEM;
 	}
 	if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
 		printk(KERN_ERR
 			"intel_iommu_domain_init() failed\n");
 		vm_domain_exit(dmar_domain);
 		return -ENOMEM;
 	}
 	domain_update_iommu_cap(dmar_domain);
 	domain->priv = dmar_domain;
 	return 0;
 }
 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	domain->priv = NULL;
 	vm_domain_exit(dmar_domain);
 }
 static int intel_iommu_attach_device(struct iommu_domain *domain,
 				     struct device *dev)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct intel_iommu *iommu;
 	int addr_width;
 	/* normally pdev is not mapped */
 	if (unlikely(domain_context_mapped(pdev))) {
 		struct dmar_domain *old_domain;
 		old_domain = find_domain(pdev);
 		if (old_domain) {
 			if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
 			    dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
 				domain_remove_one_dev_info(old_domain, pdev);
 			else
 				domain_remove_dev_info(old_domain);
 		}
 	}
 	iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
 				pdev->devfn);
 	if (!iommu)
 		return -ENODEV;
 	/* check if this iommu agaw is sufficient for max mapped address */
 	addr_width = agaw_to_width(iommu->agaw);
 	if (addr_width > cap_mgaw(iommu->cap))
 		addr_width = cap_mgaw(iommu->cap);
 	if (dmar_domain->max_addr > (1LL << addr_width)) {
 		printk(KERN_ERR "%s: iommu width (%d) is not "
 		       "sufficient for the mapped address (%llx)\n",
 		       __func__, addr_width, dmar_domain->max_addr);
 		return -EFAULT;
 	}
 	dmar_domain->gaw = addr_width;
 	/*
 	 * Knock out extra levels of page tables if necessary
 	 */
 	while (iommu->agaw < dmar_domain->agaw) {
 		struct dma_pte *pte;
 		pte = dmar_domain->pgd;
 		if (dma_pte_present(pte)) {
 			dmar_domain->pgd = (struct dma_pte *)
 				phys_to_virt(dma_pte_addr(pte));
 			free_pgtable_page(pte);
 		}
 		dmar_domain->agaw--;
 	}
 	return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
 }
 static void intel_iommu_detach_device(struct iommu_domain *domain,
 				      struct device *dev)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	domain_remove_one_dev_info(dmar_domain, pdev);
 }
 static int intel_iommu_map(struct iommu_domain *domain,
 			   unsigned long iova, phys_addr_t hpa,
 			   int gfp_order, int iommu_prot)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	u64 max_addr;
 	int prot = 0;
 	size_t size;
 	int ret;
 	if (iommu_prot & IOMMU_READ)
 		prot |= DMA_PTE_READ;
 	if (iommu_prot & IOMMU_WRITE)
 		prot |= DMA_PTE_WRITE;
 	if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
 		prot |= DMA_PTE_SNP;
 	size     = PAGE_SIZE << gfp_order;
 	max_addr = iova + size;
 	if (dmar_domain->max_addr < max_addr) {
 		u64 end;
 		/* check if minimum agaw is sufficient for mapped address */
 		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
 		if (end < max_addr) {
 			printk(KERN_ERR "%s: iommu width (%d) is not "
 			       "sufficient for the mapped address (%llx)\n",
 			       __func__, dmar_domain->gaw, max_addr);
 			return -EFAULT;
 		}
 		dmar_domain->max_addr = max_addr;
 	}
 	/* Round up size to next multiple of PAGE_SIZE, if it and
 	   the low bits of hpa would take us onto the next page */
 	size = aligned_nrpages(hpa, size);
 	ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
 				 hpa >> VTD_PAGE_SHIFT, size, prot);
 	return ret;
 }
 static int intel_iommu_unmap(struct iommu_domain *domain,
 			     unsigned long iova, int gfp_order)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	size_t size = PAGE_SIZE << gfp_order;
 	int order;
 	order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
 			    (iova + size - 1) >> VTD_PAGE_SHIFT);
 	if (dmar_domain->max_addr == iova + size)
 		dmar_domain->max_addr = iova;
 	return order;
 }
 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
 					    unsigned long iova)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	struct dma_pte *pte;
 	u64 phys = 0;
 	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
 	if (pte)
 		phys = dma_pte_addr(pte);
 	return phys;
 }
 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
 				      unsigned long cap)
 {
 	struct dmar_domain *dmar_domain = domain->priv;
 	if (cap == IOMMU_CAP_CACHE_COHERENCY)
 		return dmar_domain->iommu_snooping;
 	if (cap == IOMMU_CAP_INTR_REMAP)
 		return intr_remapping_enabled;
 	return 0;
 }
 static struct iommu_ops intel_iommu_ops = {
 	.domain_init	= intel_iommu_domain_init,
 	.domain_destroy = intel_iommu_domain_destroy,
 	.attach_dev	= intel_iommu_attach_device,
 	.detach_dev	= intel_iommu_detach_device,
 	.map		= intel_iommu_map,
 	.unmap		= intel_iommu_unmap,
 	.iova_to_phys	= intel_iommu_iova_to_phys,
 	.domain_has_cap = intel_iommu_domain_has_cap,
 };
 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
 {
 	/*
 	 * Mobile 4 Series Chipset neglects to set RWBF capability,
 	 * but needs it:
 	 */
 	printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
 	rwbf_quirk = 1;
 	/* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
 	if (dev->revision == 0x07) {
 		printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
 		dmar_map_gfx = 0;
 	}
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
 #define GGC 0x52
 #define GGC_MEMORY_SIZE_MASK	(0xf << 8)
 #define GGC_MEMORY_SIZE_NONE	(0x0 << 8)
 #define GGC_MEMORY_SIZE_1M	(0x1 << 8)
 #define GGC_MEMORY_SIZE_2M	(0x3 << 8)
 #define GGC_MEMORY_VT_ENABLED	(0x8 << 8)
 #define GGC_MEMORY_SIZE_2M_VT	(0x9 << 8)
 #define GGC_MEMORY_SIZE_3M_VT	(0xa << 8)
 #define GGC_MEMORY_SIZE_4M_VT	(0xb << 8)
 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
 {
 	unsigned short ggc;
 	if (pci_read_config_word(dev, GGC, &ggc))
 		return;
 	if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
 		printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
 		dmar_map_gfx = 0;
 	} else if (dmar_map_gfx) {
 		/* we have to ensure the gfx device is idle before we flush */
 		printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
 		intel_iommu_strict = 1;
        }
 }
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
 /* On Tylersburg chipsets, some BIOSes have been known to enable the
    ISOCH DMAR unit for the Azalia sound device, but not give it any
    TLB entries, which causes it to deadlock. Check for that.  We do
    this in a function called from init_dmars(), instead of in a PCI
    quirk, because we don't want to print the obnoxious "BIOS broken"
    message if VT-d is actually disabled.
 */
 static void __init check_tylersburg_isoch(void)
 {
 	struct pci_dev *pdev;
 	uint32_t vtisochctrl;
 	/* If there's no Azalia in the system anyway, forget it. */
 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
 	if (!pdev)
 		return;
 	pci_dev_put(pdev);
 	/* System Management Registers. Might be hidden, in which case
 	   we can't do the sanity check. But that's OK, because the
 	   known-broken BIOSes _don't_ actually hide it, so far. */
 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
 	if (!pdev)
 		return;
 	if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
 		pci_dev_put(pdev);
 		return;
 	}
 	pci_dev_put(pdev);
 	/* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
 	if (vtisochctrl & 1)
 		return;
 	/* Drop all bits other than the number of TLB entries */
 	vtisochctrl &= 0x1c;
 	/* If we have the recommended number of TLB entries (16), fine. */
 	if (vtisochctrl == 0x10)
 		return;
 	/* Zero TLB entries? You get to ride the short bus to school. */
 	if (!vtisochctrl) {
 		WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
 		     "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 		     dmi_get_system_info(DMI_BIOS_VENDOR),
 		     dmi_get_system_info(DMI_BIOS_VERSION),
 		     dmi_get_system_info(DMI_PRODUCT_VERSION));
 		iommu_identity_mapping |= IDENTMAP_AZALIA;
 		return;
 	}