Eric Lee / smarc-ti-linux-kernel

1

/*

1

/*

2

* linux/arch/arm/mm/dma-mapping.c

2

* linux/arch/arm/mm/dma-mapping.c

3

*

3

*

4

5

*

5

*

6

* This program is free software; you can redistribute it and/or modify

6

* This program is free software; you can redistribute it and/or modify

7

* it under the terms of the GNU General Public License version 2 as

7

* it under the terms of the GNU General Public License version 2 as

8

* published by the Free Software Foundation.

8

* published by the Free Software Foundation.

9

*

9

*

10

* DMA uncached mapping support.

10

* DMA uncached mapping support.

11

*/

11

*/

12

#include <linux/bootmem.h>

12

#include <linux/bootmem.h>

13

#include <linux/module.h>

13

#include <linux/module.h>

14

#include <linux/mm.h>

14

#include <linux/mm.h>

15

#include <linux/genalloc.h>

15

#include <linux/genalloc.h>

16

#include <linux/gfp.h>

16

#include <linux/gfp.h>

17

#include <linux/errno.h>

17

#include <linux/errno.h>

18

#include <linux/list.h>

18

#include <linux/list.h>

19

#include <linux/init.h>

19

#include <linux/init.h>

20

#include <linux/device.h>

20

#include <linux/device.h>

21

#include <linux/dma-mapping.h>

21

#include <linux/dma-mapping.h>

22

#include <linux/dma-contiguous.h>

22

#include <linux/dma-contiguous.h>

23

#include <linux/highmem.h>

23

#include <linux/highmem.h>

24

#include <linux/memblock.h>

24

#include <linux/memblock.h>

25

#include <linux/slab.h>

25

#include <linux/slab.h>

26

#include <linux/iommu.h>

26

#include <linux/iommu.h>

27

#include <linux/io.h>

27

#include <linux/io.h>

28

#include <linux/vmalloc.h>

28

#include <linux/vmalloc.h>

29

#include <linux/sizes.h>

29

#include <linux/sizes.h>

30

#include <linux/cma.h>

30

#include <linux/cma.h>

31

32

#include <asm/memory.h>

32

#include <asm/memory.h>

33

#include <asm/highmem.h>

33

#include <asm/highmem.h>

34

#include <asm/cacheflush.h>

34

#include <asm/cacheflush.h>

35

#include <asm/tlbflush.h>

35

#include <asm/tlbflush.h>

36

#include <asm/mach/arch.h>

36

#include <asm/mach/arch.h>

37

#include <asm/dma-iommu.h>

37

#include <asm/dma-iommu.h>

38

#include <asm/mach/map.h>

38

#include <asm/mach/map.h>

39

#include <asm/system_info.h>

39

#include <asm/system_info.h>

40

#include <asm/dma-contiguous.h>

40

#include <asm/dma-contiguous.h>

41

42

#include "mm.h"

42

#include "mm.h"

43

44

/*

44

/*

45

* The DMA API is built upon the notion of "buffer ownership". A buffer

45

* The DMA API is built upon the notion of "buffer ownership". A buffer

46

* is either exclusively owned by the CPU (and therefore may be accessed

46

* is either exclusively owned by the CPU (and therefore may be accessed

47

* by it) or exclusively owned by the DMA device. These helper functions

47

* by it) or exclusively owned by the DMA device. These helper functions

48

* represent the transitions between these two ownership states.

48

* represent the transitions between these two ownership states.

49

*

49

*

50

* Note, however, that on later ARMs, this notion does not work due to

50

* Note, however, that on later ARMs, this notion does not work due to

51

* speculative prefetches. We model our approach on the assumption that

51

* speculative prefetches. We model our approach on the assumption that

52

* the CPU does do speculative prefetches, which means we clean caches

52

* the CPU does do speculative prefetches, which means we clean caches

53

* before transfers and delay cache invalidation until transfer completion.

53

* before transfers and delay cache invalidation until transfer completion.

54

*

54

*

55

*/

55

*/

56

static void __dma_page_cpu_to_dev(struct page *, unsigned long,

56

static void __dma_page_cpu_to_dev(struct page *, unsigned long,

57

size_t, enum dma_data_direction);

57

size_t, enum dma_data_direction);

58

static void __dma_page_dev_to_cpu(struct page *, unsigned long,

58

static void __dma_page_dev_to_cpu(struct page *, unsigned long,

59

size_t, enum dma_data_direction);

59

size_t, enum dma_data_direction);

60

61

/**

61

/**

62

* arm_dma_map_page - map a portion of a page for streaming DMA

62

* arm_dma_map_page - map a portion of a page for streaming DMA

63

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

63

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

64

* @page: page that buffer resides in

64

* @page: page that buffer resides in

65

* @offset: offset into page for start of buffer

65

* @offset: offset into page for start of buffer

66

* @size: size of buffer to map

66

* @size: size of buffer to map

67

* @dir: DMA transfer direction

67

* @dir: DMA transfer direction

68

*

68

*

69

* Ensure that any data held in the cache is appropriately discarded

69

* Ensure that any data held in the cache is appropriately discarded

70

* or written back.

70

* or written back.

71

*

71

*

72

* The device owns this memory once this call has completed. The CPU

72

* The device owns this memory once this call has completed. The CPU

73

* can regain ownership by calling dma_unmap_page().

73

* can regain ownership by calling dma_unmap_page().

74

*/

74

*/

75

static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,

75

static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,

76

unsigned long offset, size_t size, enum dma_data_direction dir,

76

unsigned long offset, size_t size, enum dma_data_direction dir,

77

struct dma_attrs *attrs)

77

struct dma_attrs *attrs)

78

{

78

{

79

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

79

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

80

__dma_page_cpu_to_dev(page, offset, size, dir);

80

__dma_page_cpu_to_dev(page, offset, size, dir);

81

return pfn_to_dma(dev, page_to_pfn(page)) + offset;

81

return pfn_to_dma(dev, page_to_pfn(page)) + offset;

82

}

82

}

83

84

static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,

84

static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,

85

unsigned long offset, size_t size, enum dma_data_direction dir,

85

unsigned long offset, size_t size, enum dma_data_direction dir,

86

struct dma_attrs *attrs)

86

struct dma_attrs *attrs)

87

{

87

{

88

return pfn_to_dma(dev, page_to_pfn(page)) + offset;

88

return pfn_to_dma(dev, page_to_pfn(page)) + offset;

89

}

89

}

90

91

/**

91

/**

92

* arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()

92

* arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()

93

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

93

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

94

* @handle: DMA address of buffer

94

* @handle: DMA address of buffer

95

* @size: size of buffer (same as passed to dma_map_page)

95

* @size: size of buffer (same as passed to dma_map_page)

96

* @dir: DMA transfer direction (same as passed to dma_map_page)

96

* @dir: DMA transfer direction (same as passed to dma_map_page)

97

*

97

*

98

* Unmap a page streaming mode DMA translation. The handle and size

98

* Unmap a page streaming mode DMA translation. The handle and size

99

* must match what was provided in the previous dma_map_page() call.

99

* must match what was provided in the previous dma_map_page() call.

100

* All other usages are undefined.

100

* All other usages are undefined.

101

*

101

*

102

* After this call, reads by the CPU to the buffer are guaranteed to see

102

* After this call, reads by the CPU to the buffer are guaranteed to see

103

* whatever the device wrote there.

103

* whatever the device wrote there.

104

*/

104

*/

105

static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,

105

static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,

106

size_t size, enum dma_data_direction dir,

106

size_t size, enum dma_data_direction dir,

107

struct dma_attrs *attrs)

107

struct dma_attrs *attrs)

108

{

108

{

109

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

109

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

110

__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),

110

__dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),

111

handle & ~PAGE_MASK, size, dir);

111

handle & ~PAGE_MASK, size, dir);

112

}

112

}

113

114

static void arm_dma_sync_single_for_cpu(struct device *dev,

114

static void arm_dma_sync_single_for_cpu(struct device *dev,

115

dma_addr_t handle, size_t size, enum dma_data_direction dir)

115

dma_addr_t handle, size_t size, enum dma_data_direction dir)

116

{

116

{

117

unsigned int offset = handle & (PAGE_SIZE - 1);

117

unsigned int offset = handle & (PAGE_SIZE - 1);

118

struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));

118

struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));

119

__dma_page_dev_to_cpu(page, offset, size, dir);

119

__dma_page_dev_to_cpu(page, offset, size, dir);

120

}

120

}

121

122

static void arm_dma_sync_single_for_device(struct device *dev,

122

static void arm_dma_sync_single_for_device(struct device *dev,

123

dma_addr_t handle, size_t size, enum dma_data_direction dir)

123

dma_addr_t handle, size_t size, enum dma_data_direction dir)

124

{

124

{

125

unsigned int offset = handle & (PAGE_SIZE - 1);

125

unsigned int offset = handle & (PAGE_SIZE - 1);

126

struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));

126

struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));

127

__dma_page_cpu_to_dev(page, offset, size, dir);

127

__dma_page_cpu_to_dev(page, offset, size, dir);

128

}

128

}

129

130

struct dma_map_ops arm_dma_ops = {

130

struct dma_map_ops arm_dma_ops = {

131

.alloc = arm_dma_alloc,

131

.alloc = arm_dma_alloc,

132

.free = arm_dma_free,

132

.free = arm_dma_free,

133

.mmap = arm_dma_mmap,

133

.mmap = arm_dma_mmap,

134

.get_sgtable = arm_dma_get_sgtable,

134

.get_sgtable = arm_dma_get_sgtable,

135

.map_page = arm_dma_map_page,

135

.map_page = arm_dma_map_page,

136

.unmap_page = arm_dma_unmap_page,

136

.unmap_page = arm_dma_unmap_page,

137

.map_sg = arm_dma_map_sg,

137

.map_sg = arm_dma_map_sg,

138

.unmap_sg = arm_dma_unmap_sg,

138

.unmap_sg = arm_dma_unmap_sg,

139

.sync_single_for_cpu = arm_dma_sync_single_for_cpu,

139

.sync_single_for_cpu = arm_dma_sync_single_for_cpu,

140

.sync_single_for_device = arm_dma_sync_single_for_device,

140

.sync_single_for_device = arm_dma_sync_single_for_device,

141

.sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,

141

.sync_sg_for_cpu = arm_dma_sync_sg_for_cpu,

142

.sync_sg_for_device = arm_dma_sync_sg_for_device,

142

.sync_sg_for_device = arm_dma_sync_sg_for_device,

143

.set_dma_mask = arm_dma_set_mask,

143

.set_dma_mask = arm_dma_set_mask,

144

};

144

};

145

EXPORT_SYMBOL(arm_dma_ops);

145

EXPORT_SYMBOL(arm_dma_ops);

146

147

static void *arm_coherent_dma_alloc(struct device *dev, size_t size,

147

static void *arm_coherent_dma_alloc(struct device *dev, size_t size,

148

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);

148

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);

149

static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,

149

static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,

150

dma_addr_t handle, struct dma_attrs *attrs);

150

dma_addr_t handle, struct dma_attrs *attrs);

151

152

struct dma_map_ops arm_coherent_dma_ops = {

152

struct dma_map_ops arm_coherent_dma_ops = {

153

.alloc = arm_coherent_dma_alloc,

153

.alloc = arm_coherent_dma_alloc,

154

.free = arm_coherent_dma_free,

154

.free = arm_coherent_dma_free,

155

.mmap = arm_dma_mmap,

155

.mmap = arm_dma_mmap,

156

.get_sgtable = arm_dma_get_sgtable,

156

.get_sgtable = arm_dma_get_sgtable,

157

.map_page = arm_coherent_dma_map_page,

157

.map_page = arm_coherent_dma_map_page,

158

.map_sg = arm_dma_map_sg,

158

.map_sg = arm_dma_map_sg,

159

.set_dma_mask = arm_dma_set_mask,

159

.set_dma_mask = arm_dma_set_mask,

160

};

160

};

161

EXPORT_SYMBOL(arm_coherent_dma_ops);

161

EXPORT_SYMBOL(arm_coherent_dma_ops);

162

163

static int __dma_supported(struct device *dev, u64 mask, bool warn)

163

static int __dma_supported(struct device *dev, u64 mask, bool warn)

164

{

164

{

165

unsigned long max_dma_pfn;

165

unsigned long max_dma_pfn;

166

167

/*

167

/*

168

* If the mask allows for more memory than we can address,

168

* If the mask allows for more memory than we can address,

169

* and we actually have that much memory, then we must

169

* and we actually have that much memory, then we must

170

* indicate that DMA to this device is not supported.

170

* indicate that DMA to this device is not supported.

171

*/

171

*/

172

if (sizeof(mask) != sizeof(dma_addr_t) &&

172

if (sizeof(mask) != sizeof(dma_addr_t) &&

173

mask > (dma_addr_t)~0 &&

173

mask > (dma_addr_t)~0 &&

174

dma_to_pfn(dev, ~0) < max_pfn) {

174

dma_to_pfn(dev, ~0) < max_pfn) {

175

if (warn) {

175

if (warn) {

176

dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",

176

dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",

177

mask);

177

mask);

178

dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");

178

dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");

179

}

179

}

180

return 0;

180

return 0;

181

}

181

}

182

183

max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);

183

max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);

184

185

/*

185

/*

186

* Translate the device's DMA mask to a PFN limit. This

186

* Translate the device's DMA mask to a PFN limit. This

187

* PFN number includes the page which we can DMA to.

187

* PFN number includes the page which we can DMA to.

188

*/

188

*/

189

if (dma_to_pfn(dev, mask) < max_dma_pfn) {

189

if (dma_to_pfn(dev, mask) < max_dma_pfn) {

190

if (warn)

190

if (warn)

191

dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",

191

dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",

192

mask,

192

mask,

193

dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,

193

dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,

194

max_dma_pfn + 1);

194

max_dma_pfn + 1);

195

return 0;

195

return 0;

196

}

196

}

197

198

return 1;

198

return 1;

199

}

199

}

200

201

static u64 get_coherent_dma_mask(struct device *dev)

201

static u64 get_coherent_dma_mask(struct device *dev)

202

{

202

{

203

u64 mask = (u64)DMA_BIT_MASK(32);

203

u64 mask = (u64)DMA_BIT_MASK(32);

204

205

if (dev) {

205

if (dev) {

206

mask = dev->coherent_dma_mask;

206

mask = dev->coherent_dma_mask;

207

208

/*

208

/*

209

* Sanity check the DMA mask - it must be non-zero, and

209

* Sanity check the DMA mask - it must be non-zero, and

210

* must be able to be satisfied by a DMA allocation.

210

* must be able to be satisfied by a DMA allocation.

211

*/

211

*/

212

if (mask == 0) {

212

if (mask == 0) {

213

dev_warn(dev, "coherent DMA mask is unset\n");

213

dev_warn(dev, "coherent DMA mask is unset\n");

214

return 0;

214

return 0;

215

}

215

}

216

217

if (!__dma_supported(dev, mask, true))

217

if (!__dma_supported(dev, mask, true))

218

return 0;

218

return 0;

219

}

219

}

220

221

return mask;

221

return mask;

222

}

222

}

223

224

static void __dma_clear_buffer(struct page *page, size_t size)

224

static void __dma_clear_buffer(struct page *page, size_t size)

225

{

225

{

226

/*

226

/*

227

* Ensure that the allocated pages are zeroed, and that any data

227

* Ensure that the allocated pages are zeroed, and that any data

228

* lurking in the kernel direct-mapped region is invalidated.

228

* lurking in the kernel direct-mapped region is invalidated.

229

*/

229

*/

230

if (PageHighMem(page)) {

230

if (PageHighMem(page)) {

231

phys_addr_t base = __pfn_to_phys(page_to_pfn(page));

231

phys_addr_t base = __pfn_to_phys(page_to_pfn(page));

232

phys_addr_t end = base + size;

232

phys_addr_t end = base + size;

233

while (size > 0) {

233

while (size > 0) {

234

void *ptr = kmap_atomic(page);

234

void *ptr = kmap_atomic(page);

235

memset(ptr, 0, PAGE_SIZE);

235

memset(ptr, 0, PAGE_SIZE);

236

dmac_flush_range(ptr, ptr + PAGE_SIZE);

236

dmac_flush_range(ptr, ptr + PAGE_SIZE);

237

kunmap_atomic(ptr);

237

kunmap_atomic(ptr);

238

page++;

238

page++;

239

size -= PAGE_SIZE;

239

size -= PAGE_SIZE;

240

}

240

}

241

outer_flush_range(base, end);

241

outer_flush_range(base, end);

242

} else {

242

} else {

243

void *ptr = page_address(page);

243

void *ptr = page_address(page);

244

memset(ptr, 0, size);

244

memset(ptr, 0, size);

245

dmac_flush_range(ptr, ptr + size);

245

dmac_flush_range(ptr, ptr + size);

246

outer_flush_range(__pa(ptr), __pa(ptr) + size);

246

outer_flush_range(__pa(ptr), __pa(ptr) + size);

247

}

247

}

248

}

248

}

249

250

/*

250

/*

251

* Allocate a DMA buffer for 'dev' of size 'size' using the

251

* Allocate a DMA buffer for 'dev' of size 'size' using the

252

* specified gfp mask. Note that 'size' must be page aligned.

252

* specified gfp mask. Note that 'size' must be page aligned.

253

*/

253

*/

254

static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)

254

static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)

255

{

255

{

256

unsigned long order = get_order(size);

256

unsigned long order = get_order(size);

257

struct page *page, *p, *e;

257

struct page *page, *p, *e;

258

259

page = alloc_pages(gfp, order);

259

page = alloc_pages(gfp, order);

260

if (!page)

260

if (!page)

261

return NULL;

261

return NULL;

262

263

/*

263

/*

264

* Now split the huge page and free the excess pages

264

* Now split the huge page and free the excess pages

265

*/

265

*/

266

split_page(page, order);

266

split_page(page, order);

267

for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)

267

for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)

268

__free_page(p);

268

__free_page(p);

269

270

__dma_clear_buffer(page, size);

270

__dma_clear_buffer(page, size);

271

272

return page;

272

return page;

273

}

273

}

274

275

/*

275

/*

276

* Free a DMA buffer. 'size' must be page aligned.

276

* Free a DMA buffer. 'size' must be page aligned.

277

*/

277

*/

278

static void __dma_free_buffer(struct page *page, size_t size)

278

static void __dma_free_buffer(struct page *page, size_t size)

279

{

279

{

280

struct page *e = page + (size >> PAGE_SHIFT);

280

struct page *e = page + (size >> PAGE_SHIFT);

281

282

while (page < e) {

282

while (page < e) {

283

__free_page(page);

283

__free_page(page);

284

page++;

284

page++;

285

}

285

}

286

}

286

}

287

288

#ifdef CONFIG_MMU

288

#ifdef CONFIG_MMU

289

290

static void *__alloc_from_contiguous(struct device *dev, size_t size,

290

static void *__alloc_from_contiguous(struct device *dev, size_t size,

291

pgprot_t prot, struct page **ret_page,

291

pgprot_t prot, struct page **ret_page,

292

const void *caller);

292

const void *caller);

293

294

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,

294

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,

295

pgprot_t prot, struct page **ret_page,

295

pgprot_t prot, struct page **ret_page,

296

const void *caller);

296

const void *caller);

297

298

static void *

298

static void *

299

__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,

299

__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,

300

const void *caller)

300

const void *caller)

301

{

301

{

302

/*

302

/*

303

* DMA allocation can be mapped to user space, so lets

303

* DMA allocation can be mapped to user space, so lets

304

* set VM_USERMAP flags too.

304

* set VM_USERMAP flags too.

305

*/

305

*/

306

return dma_common_contiguous_remap(page, size,

306

return dma_common_contiguous_remap(page, size,

307

VM_ARM_DMA_CONSISTENT | VM_USERMAP,

307

VM_ARM_DMA_CONSISTENT | VM_USERMAP,

308

prot, caller);

308

prot, caller);

309

}

309

}

310

311

static void __dma_free_remap(void *cpu_addr, size_t size)

311

static void __dma_free_remap(void *cpu_addr, size_t size)

312

{

312

{

313

dma_common_free_remap(cpu_addr, size,

313

dma_common_free_remap(cpu_addr, size,

314

VM_ARM_DMA_CONSISTENT | VM_USERMAP);

314

VM_ARM_DMA_CONSISTENT | VM_USERMAP);

315

}

315

}

316

317

#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K

317

#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K

318

static struct gen_pool *atomic_pool;

318

static struct gen_pool *atomic_pool;

319

320

static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;

320

static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;

321

322

static int __init early_coherent_pool(char *p)

322

static int __init early_coherent_pool(char *p)

323

{

323

{

324

atomic_pool_size = memparse(p, &p);

324

atomic_pool_size = memparse(p, &p);

325

return 0;

325

return 0;

326

}

326

}

327

early_param("coherent_pool", early_coherent_pool);

327

early_param("coherent_pool", early_coherent_pool);

328

329

void __init init_dma_coherent_pool_size(unsigned long size)

329

void __init init_dma_coherent_pool_size(unsigned long size)

330

{

330

{

331

/*

331

/*

332

* Catch any attempt to set the pool size too late.

332

* Catch any attempt to set the pool size too late.

333

*/

333

*/

334

BUG_ON(atomic_pool);

334

BUG_ON(atomic_pool);

335

336

/*

336

/*

337

* Set architecture specific coherent pool size only if

337

* Set architecture specific coherent pool size only if

338

* it has not been changed by kernel command line parameter.

338

* it has not been changed by kernel command line parameter.

339

*/

339

*/

340

if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)

340

if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)

341

atomic_pool_size = size;

341

atomic_pool_size = size;

342

}

342

}

343

344

/*

344

/*

345

* Initialise the coherent pool for atomic allocations.

345

* Initialise the coherent pool for atomic allocations.

346

*/

346

*/

347

static int __init atomic_pool_init(void)

347

static int __init atomic_pool_init(void)

348

{

348

{

349

pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);

349

pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);

350

gfp_t gfp = GFP_KERNEL | GFP_DMA;

350

gfp_t gfp = GFP_KERNEL | GFP_DMA;

351

struct page *page;

351

struct page *page;

352

void *ptr;

352

void *ptr;

353

354

atomic_pool = gen_pool_create(PAGE_SHIFT, -1);

354

atomic_pool = gen_pool_create(PAGE_SHIFT, -1);

355

if (!atomic_pool)

355

if (!atomic_pool)

356

goto out;

356

goto out;

357

358

if (dev_get_cma_area(NULL))

358

if (dev_get_cma_area(NULL))

359

ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,

359

ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,

360

&page, atomic_pool_init);

360

&page, atomic_pool_init);

361

else

361

else

362

ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,

362

ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,

363

&page, atomic_pool_init);

363

&page, atomic_pool_init);

364

if (ptr) {

364

if (ptr) {

365

int ret;

365

int ret;

366

367

ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,

367

ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,

368

page_to_phys(page),

368

page_to_phys(page),

369

atomic_pool_size, -1);

369

atomic_pool_size, -1);

370

if (ret)

370

if (ret)

371

goto destroy_genpool;

371

goto destroy_genpool;

372

373

gen_pool_set_algo(atomic_pool,

373

gen_pool_set_algo(atomic_pool,

374

gen_pool_first_fit_order_align,

374

gen_pool_first_fit_order_align,

375

(void *)PAGE_SHIFT);

375

(void *)PAGE_SHIFT);

376

pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",

376

pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",

377

atomic_pool_size / 1024);

377

atomic_pool_size / 1024);

378

return 0;

378

return 0;

379

}

379

}

380

381

destroy_genpool:

381

destroy_genpool:

382

gen_pool_destroy(atomic_pool);

382

gen_pool_destroy(atomic_pool);

383

atomic_pool = NULL;

383

atomic_pool = NULL;

384

out:

384

out:

385

pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",

385

pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",

386

atomic_pool_size / 1024);

386

atomic_pool_size / 1024);

387

return -ENOMEM;

387

return -ENOMEM;

388

}

388

}

389

/*

389

/*

390

* CMA is activated by core_initcall, so we must be called after it.

390

* CMA is activated by core_initcall, so we must be called after it.

391

*/

391

*/

392

postcore_initcall(atomic_pool_init);

392

postcore_initcall(atomic_pool_init);

393

394

struct dma_contig_early_reserve {

394

struct dma_contig_early_reserve {

395

phys_addr_t base;

395

phys_addr_t base;

396

unsigned long size;

396

unsigned long size;

397

};

397

};

398

399

static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;

399

static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;

400

401

static int dma_mmu_remap_num __initdata;

401

static int dma_mmu_remap_num __initdata;

402

403

void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)

403

void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)

404

{

404

{

405

dma_mmu_remap[dma_mmu_remap_num].base = base;

405

dma_mmu_remap[dma_mmu_remap_num].base = base;

406

dma_mmu_remap[dma_mmu_remap_num].size = size;

406

dma_mmu_remap[dma_mmu_remap_num].size = size;

407

dma_mmu_remap_num++;

407

dma_mmu_remap_num++;

408

}

408

}

409

410

void __init dma_contiguous_remap(void)

410

void __init dma_contiguous_remap(void)

411

{

411

{

412

int i;

412

int i;

413

for (i = 0; i < dma_mmu_remap_num; i++) {

413

for (i = 0; i < dma_mmu_remap_num; i++) {

414

phys_addr_t start = dma_mmu_remap[i].base;

414

phys_addr_t start = dma_mmu_remap[i].base;

415

phys_addr_t end = start + dma_mmu_remap[i].size;

415

phys_addr_t end = start + dma_mmu_remap[i].size;

416

struct map_desc map;

416

struct map_desc map;

417

unsigned long addr;

417

unsigned long addr;

418

419

if (end > arm_lowmem_limit)

419

if (end > arm_lowmem_limit)

420

end = arm_lowmem_limit;

420

end = arm_lowmem_limit;

421

if (start >= end)

421

if (start >= end)

422

continue;

422

continue;

423

424

map.pfn = __phys_to_pfn(start);

424

map.pfn = __phys_to_pfn(start);

425

map.virtual = __phys_to_virt(start);

425

map.virtual = __phys_to_virt(start);

426

map.length = end - start;

426

map.length = end - start;

427

map.type = MT_MEMORY_DMA_READY;

427

map.type = MT_MEMORY_DMA_READY;

428

429

/*

429

/*

430

* Clear previous low-memory mapping to ensure that the

430

* Clear previous low-memory mapping to ensure that the

431

* TLB does not see any conflicting entries, then flush

431

* TLB does not see any conflicting entries, then flush

432

* the TLB of the old entries before creating new mappings.

432

* the TLB of the old entries before creating new mappings.

433

*

433

*

434

* This ensures that any speculatively loaded TLB entries

434

* This ensures that any speculatively loaded TLB entries

435

* (even though they may be rare) can not cause any problems,

435

* (even though they may be rare) can not cause any problems,

436

* and ensures that this code is architecturally compliant.

436

* and ensures that this code is architecturally compliant.

437

*/

437

*/

438

for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);

438

for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);

439

addr += PMD_SIZE)

439

addr += PMD_SIZE)

440

pmd_clear(pmd_off_k(addr));

440

pmd_clear(pmd_off_k(addr));

441

442

flush_tlb_kernel_range(__phys_to_virt(start),

442

flush_tlb_kernel_range(__phys_to_virt(start),

443

__phys_to_virt(end));

443

__phys_to_virt(end));

444

445

iotable_init(&map, 1);

445

iotable_init(&map, 1);

446

}

446

}

447

}

447

}

448

449

static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,

449

static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,

450

void *data)

450

void *data)

451

{

451

{

452

struct page *page = virt_to_page(addr);

452

struct page *page = virt_to_page(addr);

453

pgprot_t prot = *(pgprot_t *)data;

453

pgprot_t prot = *(pgprot_t *)data;

454

455

set_pte_ext(pte, mk_pte(page, prot), 0);

455

set_pte_ext(pte, mk_pte(page, prot), 0);

456

return 0;

456

return 0;

457

}

457

}

458

459

static void __dma_remap(struct page *page, size_t size, pgprot_t prot)

459

static void __dma_remap(struct page *page, size_t size, pgprot_t prot)

460

{

460

{

461

unsigned long start = (unsigned long) page_address(page);

461

unsigned long start = (unsigned long) page_address(page);

462

unsigned end = start + size;

462

unsigned end = start + size;

463

464

apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);

464

apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);

465

flush_tlb_kernel_range(start, end);

465

flush_tlb_kernel_range(start, end);

466

}

466

}

467

468

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,

468

static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,

469

pgprot_t prot, struct page **ret_page,

469

pgprot_t prot, struct page **ret_page,

470

const void *caller)

470

const void *caller)

471

{

471

{

472

struct page *page;

472

struct page *page;

473

void *ptr;

473

void *ptr;

474

page = __dma_alloc_buffer(dev, size, gfp);

474

page = __dma_alloc_buffer(dev, size, gfp);

475

if (!page)

475

if (!page)

476

return NULL;

476

return NULL;

477

478

ptr = __dma_alloc_remap(page, size, gfp, prot, caller);

478

ptr = __dma_alloc_remap(page, size, gfp, prot, caller);

479

if (!ptr) {

479

if (!ptr) {

480

__dma_free_buffer(page, size);

480

__dma_free_buffer(page, size);

481

return NULL;

481

return NULL;

482

}

482

}

483

484

*ret_page = page;

484

*ret_page = page;

485

return ptr;

485

return ptr;

486

}

486

}

487

488

static void *__alloc_from_pool(size_t size, struct page **ret_page)

488

static void *__alloc_from_pool(size_t size, struct page **ret_page)

489

{

489

{

490

unsigned long val;

490

unsigned long val;

491

void *ptr = NULL;

491

void *ptr = NULL;

492

493

if (!atomic_pool) {

493

if (!atomic_pool) {

494

WARN(1, "coherent pool not initialised!\n");

494

WARN(1, "coherent pool not initialised!\n");

495

return NULL;

495

return NULL;

496

}

496

}

497

498

val = gen_pool_alloc(atomic_pool, size);

498

val = gen_pool_alloc(atomic_pool, size);

499

if (val) {

499

if (val) {

500

phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);

500

phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);

501

502

*ret_page = phys_to_page(phys);

502

*ret_page = phys_to_page(phys);

503

ptr = (void *)val;

503

ptr = (void *)val;

504

}

504

}

505

506

return ptr;

506

return ptr;

507

}

507

}

508

509

static bool __in_atomic_pool(void *start, size_t size)

509

static bool __in_atomic_pool(void *start, size_t size)

510

{

510

{

511

return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);

511

return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);

512

}

512

}

513

514

static int __free_from_pool(void *start, size_t size)

514

static int __free_from_pool(void *start, size_t size)

515

{

515

{

516

if (!__in_atomic_pool(start, size))

516

if (!__in_atomic_pool(start, size))

517

return 0;

517

return 0;

518

519

gen_pool_free(atomic_pool, (unsigned long)start, size);

519

gen_pool_free(atomic_pool, (unsigned long)start, size);

520

521

return 1;

521

return 1;

522

}

522

}

523

524

static void *__alloc_from_contiguous(struct device *dev, size_t size,

524

static void *__alloc_from_contiguous(struct device *dev, size_t size,

525

pgprot_t prot, struct page **ret_page,

525

pgprot_t prot, struct page **ret_page,

526

const void *caller)

526

const void *caller)

527

{

527

{

528

unsigned long order = get_order(size);

528

unsigned long order = get_order(size);

529

size_t count = size >> PAGE_SHIFT;

529

size_t count = size >> PAGE_SHIFT;

530

struct page *page;

530

struct page *page;

531

void *ptr;

531

void *ptr;

532

533

page = dma_alloc_from_contiguous(dev, count, order);

533

page = dma_alloc_from_contiguous(dev, count, order);

534

if (!page)

534

if (!page)

535

return NULL;

535

return NULL;

536

537

__dma_clear_buffer(page, size);

537

__dma_clear_buffer(page, size);

538

539

if (PageHighMem(page)) {

539

if (PageHighMem(page)) {

540

ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);

540

ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);

541

if (!ptr) {

541

if (!ptr) {

542

dma_release_from_contiguous(dev, page, count);

542

dma_release_from_contiguous(dev, page, count);

543

return NULL;

543

return NULL;

544

}

544

}

545

} else {

545

} else {

546

__dma_remap(page, size, prot);

546

__dma_remap(page, size, prot);

547

ptr = page_address(page);

547

ptr = page_address(page);

548

}

548

}

549

*ret_page = page;

549

*ret_page = page;

550

return ptr;

550

return ptr;

551

}

551

}

552

553

static void __free_from_contiguous(struct device *dev, struct page *page,

553

static void __free_from_contiguous(struct device *dev, struct page *page,

554

void *cpu_addr, size_t size)

554

void *cpu_addr, size_t size)

555

{

555

{

556

if (PageHighMem(page))

556

if (PageHighMem(page))

557

__dma_free_remap(cpu_addr, size);

557

__dma_free_remap(cpu_addr, size);

558

else

558

else

559

__dma_remap(page, size, PAGE_KERNEL);

559

__dma_remap(page, size, PAGE_KERNEL);

560

dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);

560

dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);

561

}

561

}

562

563

static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)

563

static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)

564

{

564

{

565

prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?

565

prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?

566

pgprot_writecombine(prot) :

566

pgprot_writecombine(prot) :

567

pgprot_dmacoherent(prot);

567

pgprot_dmacoherent(prot);

568

return prot;

568

return prot;

569

}

569

}

570

571

#define nommu() 0

571

#define nommu() 0

572

573

#else /* !CONFIG_MMU */

573

#else /* !CONFIG_MMU */

574

575

#define nommu() 1

575

#define nommu() 1

576

577

#define __get_dma_pgprot(attrs, prot) __pgprot(0)

577

#define __get_dma_pgprot(attrs, prot) __pgprot(0)

578

#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL

578

#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL

579

#define __alloc_from_pool(size, ret_page) NULL

579

#define __alloc_from_pool(size, ret_page) NULL

580

#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL

580

#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL

581

#define __free_from_pool(cpu_addr, size) 0

581

#define __free_from_pool(cpu_addr, size) 0

582

#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0)

582

#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0)

583

#define __dma_free_remap(cpu_addr, size) do { } while (0)

583

#define __dma_free_remap(cpu_addr, size) do { } while (0)

584

585

#endif /* CONFIG_MMU */

585

#endif /* CONFIG_MMU */

586

587

static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,

587

static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,

588

struct page **ret_page)

588

struct page **ret_page)

589

{

589

{

590

struct page *page;

590

struct page *page;

591

page = __dma_alloc_buffer(dev, size, gfp);

591

page = __dma_alloc_buffer(dev, size, gfp);

592

if (!page)

592

if (!page)

593

return NULL;

593

return NULL;

594

595

*ret_page = page;

595

*ret_page = page;

596

return page_address(page);

596

return page_address(page);

597

}

597

}

598

599

600

601

static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,

601

static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,

602

gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)

602

gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)

603

{

603

{

604

u64 mask = get_coherent_dma_mask(dev);

604

u64 mask = get_coherent_dma_mask(dev);

605

struct page *page = NULL;

605

struct page *page = NULL;

606

void *addr;

606

void *addr;

607

608

#ifdef CONFIG_DMA_API_DEBUG

608

#ifdef CONFIG_DMA_API_DEBUG

609

u64 limit = (mask + 1) & ~mask;

609

u64 limit = (mask + 1) & ~mask;

610

if (limit && size >= limit) {

610

if (limit && size >= limit) {

611

dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",

611

dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",

612

size, mask);

612

size, mask);

613

return NULL;

613

return NULL;

614

}

614

}

615

#endif

615

#endif

616

617

if (!mask)

617

if (!mask)

618

return NULL;

618

return NULL;

619

620

if (mask < 0xffffffffULL)

620

if (mask < 0xffffffffULL)

621

gfp |= GFP_DMA;

621

gfp |= GFP_DMA;

622

623

/*

623

/*

624

* Following is a work-around (a.k.a. hack) to prevent pages

624

* Following is a work-around (a.k.a. hack) to prevent pages

625

* with __GFP_COMP being passed to split_page() which cannot

625

* with __GFP_COMP being passed to split_page() which cannot

626

* handle them. The real problem is that this flag probably

626

* handle them. The real problem is that this flag probably

627

* should be 0 on ARM as it is not supported on this

627

* should be 0 on ARM as it is not supported on this

628

* platform; see CONFIG_HUGETLBFS.

628

* platform; see CONFIG_HUGETLBFS.

629

*/

629

*/

630

gfp &= ~(__GFP_COMP);

630

gfp &= ~(__GFP_COMP);

631

632

*handle = DMA_ERROR_CODE;

632

*handle = DMA_ERROR_CODE;

633

size = PAGE_ALIGN(size);

633

size = PAGE_ALIGN(size);

634

635

if (is_coherent || nommu())

635

if (is_coherent || nommu())

636

addr = __alloc_simple_buffer(dev, size, gfp, &page);

636

addr = __alloc_simple_buffer(dev, size, gfp, &page);

637

else if (!(gfp & __GFP_WAIT))

637

else if (!(gfp & __GFP_WAIT))

638

addr = __alloc_from_pool(size, &page);

638

addr = __alloc_from_pool(size, &page);

639

else if (!dev_get_cma_area(dev))

639

else if (!dev_get_cma_area(dev))

640

addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);

640

addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);

641

else

641

else

642

addr = __alloc_from_contiguous(dev, size, prot, &page, caller);

642

addr = __alloc_from_contiguous(dev, size, prot, &page, caller);

643

644

if (addr)

644

if (addr)

645

*handle = pfn_to_dma(dev, page_to_pfn(page));

645

*handle = pfn_to_dma(dev, page_to_pfn(page));

646

647

return addr;

647

return addr;

648

}

648

}

649

650

/*

650

/*

651

* Allocate DMA-coherent memory space and return both the kernel remapped

651

* Allocate DMA-coherent memory space and return both the kernel remapped

652

* virtual and bus address for that space.

652

* virtual and bus address for that space.

653

*/

653

*/

654

void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,

654

void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,

655

gfp_t gfp, struct dma_attrs *attrs)

655

gfp_t gfp, struct dma_attrs *attrs)

656

{

656

{

657

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

657

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

658

void *memory;

658

void *memory;

659

660

if (dma_alloc_from_coherent(dev, size, handle, &memory))

660

if (dma_alloc_from_coherent(dev, size, handle, &memory))

661

return memory;

661

return memory;

662

663

return __dma_alloc(dev, size, handle, gfp, prot, false,

663

return __dma_alloc(dev, size, handle, gfp, prot, false,

664

__builtin_return_address(0));

664

__builtin_return_address(0));

665

}

665

}

666

667

static void *arm_coherent_dma_alloc(struct device *dev, size_t size,

667

static void *arm_coherent_dma_alloc(struct device *dev, size_t size,

668

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)

668

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)

669

{

669

{

670

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

670

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

671

void *memory;

671

void *memory;

672

673

if (dma_alloc_from_coherent(dev, size, handle, &memory))

673

if (dma_alloc_from_coherent(dev, size, handle, &memory))

674

return memory;

674

return memory;

675

676

return __dma_alloc(dev, size, handle, gfp, prot, true,

676

return __dma_alloc(dev, size, handle, gfp, prot, true,

677

__builtin_return_address(0));

677

__builtin_return_address(0));

678

}

678

}

679

680

/*

680

/*

681

* Create userspace mapping for the DMA-coherent memory.

681

* Create userspace mapping for the DMA-coherent memory.

682

*/

682

*/

683

int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,

683

int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,

684

void *cpu_addr, dma_addr_t dma_addr, size_t size,

684

void *cpu_addr, dma_addr_t dma_addr, size_t size,

685

struct dma_attrs *attrs)

685

struct dma_attrs *attrs)

686

{

686

{

687

int ret = -ENXIO;

687

int ret = -ENXIO;

688

#ifdef CONFIG_MMU

688

#ifdef CONFIG_MMU

689

unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

689

unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

690

unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;

690

unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;

691

unsigned long pfn = dma_to_pfn(dev, dma_addr);

691

unsigned long pfn = dma_to_pfn(dev, dma_addr);

692

unsigned long off = vma->vm_pgoff;

692

unsigned long off = vma->vm_pgoff;

693

694

vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

694

vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

695

696

if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))

696

if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))

697

return ret;

697

return ret;

698

699

if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {

699

if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {

700

ret = remap_pfn_range(vma, vma->vm_start,

700

ret = remap_pfn_range(vma, vma->vm_start,

701

pfn + off,

701

pfn + off,

702

vma->vm_end - vma->vm_start,

702

vma->vm_end - vma->vm_start,

703

vma->vm_page_prot);

703

vma->vm_page_prot);

704

}

704

}

705

#endif /* CONFIG_MMU */

705

#endif /* CONFIG_MMU */

706

707

return ret;

707

return ret;

708

}

708

}

709

710

/*

710

/*

711

* Free a buffer as defined by the above mapping.

711

* Free a buffer as defined by the above mapping.

712

*/

712

*/

713

static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,

713

static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,

714

dma_addr_t handle, struct dma_attrs *attrs,

714

dma_addr_t handle, struct dma_attrs *attrs,

715

bool is_coherent)

715

bool is_coherent)

716

{

716

{

717

struct page *page = pfn_to_page(dma_to_pfn(dev, handle));

717

struct page *page = pfn_to_page(dma_to_pfn(dev, handle));

718

719

if (dma_release_from_coherent(dev, get_order(size), cpu_addr))

719

if (dma_release_from_coherent(dev, get_order(size), cpu_addr))

720

return;

720

return;

721

722

size = PAGE_ALIGN(size);

722

size = PAGE_ALIGN(size);

723

724

if (is_coherent || nommu()) {

724

if (is_coherent || nommu()) {

725

__dma_free_buffer(page, size);

725

__dma_free_buffer(page, size);

726

} else if (__free_from_pool(cpu_addr, size)) {

726

} else if (__free_from_pool(cpu_addr, size)) {

727

return;

727

return;

728

} else if (!dev_get_cma_area(dev)) {

728

} else if (!dev_get_cma_area(dev)) {

729

__dma_free_remap(cpu_addr, size);

729

__dma_free_remap(cpu_addr, size);

730

__dma_free_buffer(page, size);

730

__dma_free_buffer(page, size);

731

} else {

731

} else {

732

/*

732

/*

733

* Non-atomic allocations cannot be freed with IRQs disabled

733

* Non-atomic allocations cannot be freed with IRQs disabled

734

*/

734

*/

735

WARN_ON(irqs_disabled());

735

WARN_ON(irqs_disabled());

736

__free_from_contiguous(dev, page, cpu_addr, size);

736

__free_from_contiguous(dev, page, cpu_addr, size);

737

}

737

}

738

}

738

}

739

740

void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,

740

void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,

741

dma_addr_t handle, struct dma_attrs *attrs)

741

dma_addr_t handle, struct dma_attrs *attrs)

742

{

742

{

743

__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);

743

__arm_dma_free(dev, size, cpu_addr, handle, attrs, false);

744

}

744

}

745

746

static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,

746

static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,

747

dma_addr_t handle, struct dma_attrs *attrs)

747

dma_addr_t handle, struct dma_attrs *attrs)

748

{

748

{

749

__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);

749

__arm_dma_free(dev, size, cpu_addr, handle, attrs, true);

750

}

750

}

751

752

int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,

752

int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,

753

void *cpu_addr, dma_addr_t handle, size_t size,

753

void *cpu_addr, dma_addr_t handle, size_t size,

754

struct dma_attrs *attrs)

754

struct dma_attrs *attrs)

755

{

755

{

756

struct page *page = pfn_to_page(dma_to_pfn(dev, handle));

756

struct page *page = pfn_to_page(dma_to_pfn(dev, handle));

757

int ret;

757

int ret;

758

759

ret = sg_alloc_table(sgt, 1, GFP_KERNEL);

759

ret = sg_alloc_table(sgt, 1, GFP_KERNEL);

760

if (unlikely(ret))

760

if (unlikely(ret))

761

return ret;

761

return ret;

762

763

sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);

763

sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);

764

return 0;

764

return 0;

765

}

765

}

766

767

static void dma_cache_maint_page(struct page *page, unsigned long offset,

767

static void dma_cache_maint_page(struct page *page, unsigned long offset,

768

size_t size, enum dma_data_direction dir,

768

size_t size, enum dma_data_direction dir,

769

void (*op)(const void *, size_t, int))

769

void (*op)(const void *, size_t, int))

770

{

770

{

771

unsigned long pfn;

771

unsigned long pfn;

772

size_t left = size;

772

size_t left = size;

773

774

pfn = page_to_pfn(page) + offset / PAGE_SIZE;

774

pfn = page_to_pfn(page) + offset / PAGE_SIZE;

775

offset %= PAGE_SIZE;

775

offset %= PAGE_SIZE;

776

777

/*

777

/*

778

* A single sg entry may refer to multiple physically contiguous

778

* A single sg entry may refer to multiple physically contiguous

779

* pages. But we still need to process highmem pages individually.

779

* pages. But we still need to process highmem pages individually.

780

* If highmem is not configured then the bulk of this loop gets

780

* If highmem is not configured then the bulk of this loop gets

781

* optimized out.

781

* optimized out.

782

*/

782

*/

783

do {

783

do {

784

size_t len = left;

784

size_t len = left;

785

void *vaddr;

785

void *vaddr;

786

787

page = pfn_to_page(pfn);

787

page = pfn_to_page(pfn);

788

789

if (PageHighMem(page)) {

789

if (PageHighMem(page)) {

790

if (len + offset > PAGE_SIZE)

790

if (len + offset > PAGE_SIZE)

791

len = PAGE_SIZE - offset;

791

len = PAGE_SIZE - offset;

792

793

if (cache_is_vipt_nonaliasing()) {

793

if (cache_is_vipt_nonaliasing()) {

794

vaddr = kmap_atomic(page);

794

vaddr = kmap_atomic(page);

795

op(vaddr + offset, len, dir);

795

op(vaddr + offset, len, dir);

796

kunmap_atomic(vaddr);

796

kunmap_atomic(vaddr);

797

} else {

797

} else {

798

vaddr = kmap_high_get(page);

798

vaddr = kmap_high_get(page);

799

if (vaddr) {

799

if (vaddr) {

800

op(vaddr + offset, len, dir);

800

op(vaddr + offset, len, dir);

801

kunmap_high(page);

801

kunmap_high(page);

802

}

802

}

803

}

803

}

804

} else {

804

} else {

805

vaddr = page_address(page) + offset;

805

vaddr = page_address(page) + offset;

806

op(vaddr, len, dir);

806

op(vaddr, len, dir);

807

}

807

}

808

offset = 0;

808

offset = 0;

809

pfn++;

809

pfn++;

810

left -= len;

810

left -= len;

811

} while (left);

811

} while (left);

812

}

812

}

813

814

/*

814

/*

815

* Make an area consistent for devices.

815

* Make an area consistent for devices.

816

* Note: Drivers should NOT use this function directly, as it will break

816

* Note: Drivers should NOT use this function directly, as it will break

817

* platforms with CONFIG_DMABOUNCE.

817

* platforms with CONFIG_DMABOUNCE.

818

* Use the driver DMA support - see dma-mapping.h (dma_sync_*)

818

* Use the driver DMA support - see dma-mapping.h (dma_sync_*)

819

*/

819

*/

820

static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,

820

static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,

821

size_t size, enum dma_data_direction dir)

821

size_t size, enum dma_data_direction dir)

822

{

822

{

823

phys_addr_t paddr;

823

phys_addr_t paddr;

824

825

dma_cache_maint_page(page, off, size, dir, dmac_map_area);

825

dma_cache_maint_page(page, off, size, dir, dmac_map_area);

826

827

paddr = page_to_phys(page) + off;

827

paddr = page_to_phys(page) + off;

828

if (dir == DMA_FROM_DEVICE) {

828

if (dir == DMA_FROM_DEVICE) {

829

outer_inv_range(paddr, paddr + size);

829

outer_inv_range(paddr, paddr + size);

830

} else {

830

} else {

831

outer_clean_range(paddr, paddr + size);

831

outer_clean_range(paddr, paddr + size);

832

}

832

}

833

/* FIXME: non-speculating: flush on bidirectional mappings? */

833

/* FIXME: non-speculating: flush on bidirectional mappings? */

834

}

834

}

835

836

static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,

836

static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,

837

size_t size, enum dma_data_direction dir)

837

size_t size, enum dma_data_direction dir)

838

{

838

{

839

phys_addr_t paddr = page_to_phys(page) + off;

839

phys_addr_t paddr = page_to_phys(page) + off;

840

841

/* FIXME: non-speculating: not required */

841

/* FIXME: non-speculating: not required */

842

/* in any case, don't bother invalidating if DMA to device */

842

/* in any case, don't bother invalidating if DMA to device */

843

if (dir != DMA_TO_DEVICE) {

843

if (dir != DMA_TO_DEVICE) {

844

outer_inv_range(paddr, paddr + size);

844

outer_inv_range(paddr, paddr + size);

845

846

dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);

846

dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);

847

}

847

}

848

849

/*

849

/*

850

* Mark the D-cache clean for these pages to avoid extra flushing.

850

* Mark the D-cache clean for these pages to avoid extra flushing.

851

*/

851

*/

852

if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {

852

if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {

853

unsigned long pfn;

853

unsigned long pfn;

854

size_t left = size;

854

size_t left = size;

855

856

pfn = page_to_pfn(page) + off / PAGE_SIZE;

856

pfn = page_to_pfn(page) + off / PAGE_SIZE;

857

off %= PAGE_SIZE;

857

off %= PAGE_SIZE;

858

if (off) {

858

if (off) {

859

pfn++;

859

pfn++;

860

left -= PAGE_SIZE - off;

860

left -= PAGE_SIZE - off;

861

}

861

}

862

while (left >= PAGE_SIZE) {

862

while (left >= PAGE_SIZE) {

863

page = pfn_to_page(pfn++);

863

page = pfn_to_page(pfn++);

864

set_bit(PG_dcache_clean, &page->flags);

864

set_bit(PG_dcache_clean, &page->flags);

865

left -= PAGE_SIZE;

865

left -= PAGE_SIZE;

866

}

866

}

867

}

867

}

868

}

868

}

869

870

/**

870

/**

871

* arm_dma_map_sg - map a set of SG buffers for streaming mode DMA

871

* arm_dma_map_sg - map a set of SG buffers for streaming mode DMA

872

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

872

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

873

* @sg: list of buffers

873

* @sg: list of buffers

874

* @nents: number of buffers to map

874

* @nents: number of buffers to map

875

* @dir: DMA transfer direction

875

* @dir: DMA transfer direction

876

*

876

*

877

* Map a set of buffers described by scatterlist in streaming mode for DMA.

877

* Map a set of buffers described by scatterlist in streaming mode for DMA.

878

* This is the scatter-gather version of the dma_map_single interface.

878

* This is the scatter-gather version of the dma_map_single interface.

879

* Here the scatter gather list elements are each tagged with the

879

* Here the scatter gather list elements are each tagged with the

880

* appropriate dma address and length. They are obtained via

880

* appropriate dma address and length. They are obtained via

881

* sg_dma_{address,length}.

881

* sg_dma_{address,length}.

882

*

882

*

883

* Device ownership issues as mentioned for dma_map_single are the same

883

* Device ownership issues as mentioned for dma_map_single are the same

884

* here.

884

* here.

885

*/

885

*/

886

int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,

886

int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,

887

enum dma_data_direction dir, struct dma_attrs *attrs)

887

enum dma_data_direction dir, struct dma_attrs *attrs)

888

{

888

{

889

struct dma_map_ops *ops = get_dma_ops(dev);

889

struct dma_map_ops *ops = get_dma_ops(dev);

890

struct scatterlist *s;

890

struct scatterlist *s;

891

int i, j;

891

int i, j;

892

893

for_each_sg(sg, s, nents, i) {

893

for_each_sg(sg, s, nents, i) {

894

#ifdef CONFIG_NEED_SG_DMA_LENGTH

894

#ifdef CONFIG_NEED_SG_DMA_LENGTH

895

s->dma_length = s->length;

895

s->dma_length = s->length;

896

#endif

896

#endif

897

s->dma_address = ops->map_page(dev, sg_page(s), s->offset,

897

s->dma_address = ops->map_page(dev, sg_page(s), s->offset,

898

s->length, dir, attrs);

898

s->length, dir, attrs);

899

if (dma_mapping_error(dev, s->dma_address))

899

if (dma_mapping_error(dev, s->dma_address))

900

goto bad_mapping;

900

goto bad_mapping;

901

}

901

}

902

return nents;

902

return nents;

903

904

bad_mapping:

904

bad_mapping:

905

for_each_sg(sg, s, i, j)

905

for_each_sg(sg, s, i, j)

906

ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);

906

ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);

907

return 0;

907

return 0;

908

}

908

}

909

910

/**

910

/**

911

* arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

911

* arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

912

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

912

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

913

* @sg: list of buffers

913

* @sg: list of buffers

914

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

914

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

915

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

915

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

916

*

916

*

917

* Unmap a set of streaming mode DMA translations. Again, CPU access

917

* Unmap a set of streaming mode DMA translations. Again, CPU access

918

* rules concerning calls here are the same as for dma_unmap_single().

918

* rules concerning calls here are the same as for dma_unmap_single().

919

*/

919

*/

920

void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,

920

void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,

921

enum dma_data_direction dir, struct dma_attrs *attrs)

921

enum dma_data_direction dir, struct dma_attrs *attrs)

922

{

922

{

923

struct dma_map_ops *ops = get_dma_ops(dev);

923

struct dma_map_ops *ops = get_dma_ops(dev);

924

struct scatterlist *s;

924

struct scatterlist *s;

925

926

int i;

926

int i;

927

928

for_each_sg(sg, s, nents, i)

928

for_each_sg(sg, s, nents, i)

929

ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);

929

ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);

930

}

930

}

931

932

/**

932

/**

933

* arm_dma_sync_sg_for_cpu

933

* arm_dma_sync_sg_for_cpu

934

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

934

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

935

* @sg: list of buffers

935

* @sg: list of buffers

936

* @nents: number of buffers to map (returned from dma_map_sg)

936

* @nents: number of buffers to map (returned from dma_map_sg)

937

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

937

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

938

*/

938

*/

939

void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

939

void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

940

int nents, enum dma_data_direction dir)

940

int nents, enum dma_data_direction dir)

941

{

941

{

942

struct dma_map_ops *ops = get_dma_ops(dev);

942

struct dma_map_ops *ops = get_dma_ops(dev);

943

struct scatterlist *s;

943

struct scatterlist *s;

944

int i;

944

int i;

945

946

for_each_sg(sg, s, nents, i)

946

for_each_sg(sg, s, nents, i)

947

ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,

947

ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,

948

dir);

948

dir);

949

}

949

}

950

951

/**

951

/**

952

* arm_dma_sync_sg_for_device

952

* arm_dma_sync_sg_for_device

953

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

953

* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices

954

* @sg: list of buffers

954

* @sg: list of buffers

955

* @nents: number of buffers to map (returned from dma_map_sg)

955

* @nents: number of buffers to map (returned from dma_map_sg)

956

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

956

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

957

*/

957

*/

958

void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

958

void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

959

int nents, enum dma_data_direction dir)

959

int nents, enum dma_data_direction dir)

960

{

960

{

961

struct dma_map_ops *ops = get_dma_ops(dev);

961

struct dma_map_ops *ops = get_dma_ops(dev);

962

struct scatterlist *s;

962

struct scatterlist *s;

963

int i;

963

int i;

964

965

for_each_sg(sg, s, nents, i)

965

for_each_sg(sg, s, nents, i)

966

ops->sync_single_for_device(dev, sg_dma_address(s), s->length,

966

ops->sync_single_for_device(dev, sg_dma_address(s), s->length,

967

dir);

967

dir);

968

}

968

}

969

970

/*

970

/*

971

* Return whether the given device DMA address mask can be supported

971

* Return whether the given device DMA address mask can be supported

972

* properly. For example, if your device can only drive the low 24-bits

972

* properly. For example, if your device can only drive the low 24-bits

973

* during bus mastering, then you would pass 0x00ffffff as the mask

973

* during bus mastering, then you would pass 0x00ffffff as the mask

974

* to this function.

974

* to this function.

975

*/

975

*/

976

int dma_supported(struct device *dev, u64 mask)

976

int dma_supported(struct device *dev, u64 mask)

977

{

977

{

978

return __dma_supported(dev, mask, false);

978

return __dma_supported(dev, mask, false);

979

}

979

}

980

EXPORT_SYMBOL(dma_supported);

980

EXPORT_SYMBOL(dma_supported);

981

982

int arm_dma_set_mask(struct device *dev, u64 dma_mask)

982

int arm_dma_set_mask(struct device *dev, u64 dma_mask)

983

{

983

{

984

if (!dev->dma_mask || !dma_supported(dev, dma_mask))

984

if (!dev->dma_mask || !dma_supported(dev, dma_mask))

985

return -EIO;

985

return -EIO;

986

987

*dev->dma_mask = dma_mask;

987

*dev->dma_mask = dma_mask;

988

989

return 0;

989

return 0;

990

}

990

}

991

992

#define PREALLOC_DMA_DEBUG_ENTRIES 4096

992

#define PREALLOC_DMA_DEBUG_ENTRIES 4096

993

994

static int __init dma_debug_do_init(void)

994

static int __init dma_debug_do_init(void)

995

{

995

{

996

dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

996

dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

997

return 0;

997

return 0;

998

}

998

}

999

fs_initcall(dma_debug_do_init);

999

fs_initcall(dma_debug_do_init);

1000

1001

#ifdef CONFIG_ARM_DMA_USE_IOMMU

1001

#ifdef CONFIG_ARM_DMA_USE_IOMMU

1002

1003

/* IOMMU */

1003

/* IOMMU */

1004

1005

static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);

1005

static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);

1006

1007

static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,

1007

static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,

1008

size_t size)

1008

size_t size)

1009

{

1009

{

1010

unsigned int order = get_order(size);

1010

unsigned int order = get_order(size);

1011

unsigned int align = 0;

1011

unsigned int align = 0;

1012

unsigned int count, start;

1012

unsigned int count, start;

1013

size_t mapping_size = mapping->bits << PAGE_SHIFT;

1013

size_t mapping_size = mapping->bits << PAGE_SHIFT;

1014

unsigned long flags;

1014

unsigned long flags;

1015

dma_addr_t iova;

1015

dma_addr_t iova;

1016

int i;

1016

int i;

1017

1018

if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)

1018

if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)

1019

order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;

1019

order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;

1020

1021

count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1021

count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1022

align = (1 << order) - 1;

1022

align = (1 << order) - 1;

1023

1024

spin_lock_irqsave(&mapping->lock, flags);

1024

spin_lock_irqsave(&mapping->lock, flags);

1025

for (i = 0; i < mapping->nr_bitmaps; i++) {

1025

for (i = 0; i < mapping->nr_bitmaps; i++) {

1026

start = bitmap_find_next_zero_area(mapping->bitmaps[i],

1026

start = bitmap_find_next_zero_area(mapping->bitmaps[i],

1027

mapping->bits, 0, count, align);

1027

mapping->bits, 0, count, align);

1028

1029

if (start > mapping->bits)

1029

if (start > mapping->bits)

1030

continue;

1030

continue;

1031

1032

bitmap_set(mapping->bitmaps[i], start, count);

1032

bitmap_set(mapping->bitmaps[i], start, count);

1033

break;

1033

break;

1034

}

1034

}

1035

1036

/*

1036

/*

1037

* No unused range found. Try to extend the existing mapping

1037

* No unused range found. Try to extend the existing mapping

1038

* and perform a second attempt to reserve an IO virtual

1038

* and perform a second attempt to reserve an IO virtual

1039

* address range of size bytes.

1039

* address range of size bytes.

1040

*/

1040

*/

1041

if (i == mapping->nr_bitmaps) {

1041

if (i == mapping->nr_bitmaps) {

1042

if (extend_iommu_mapping(mapping)) {

1042

if (extend_iommu_mapping(mapping)) {

1043

spin_unlock_irqrestore(&mapping->lock, flags);

1043

spin_unlock_irqrestore(&mapping->lock, flags);

1044

return DMA_ERROR_CODE;

1044

return DMA_ERROR_CODE;

1045

}

1045

}

1046

1047

start = bitmap_find_next_zero_area(mapping->bitmaps[i],

1047

start = bitmap_find_next_zero_area(mapping->bitmaps[i],

1048

mapping->bits, 0, count, align);

1048

mapping->bits, 0, count, align);

1049

1050

if (start > mapping->bits) {

1050

if (start > mapping->bits) {

1051

spin_unlock_irqrestore(&mapping->lock, flags);

1051

spin_unlock_irqrestore(&mapping->lock, flags);

1052

return DMA_ERROR_CODE;

1052

return DMA_ERROR_CODE;

1053

}

1053

}

1054

1055

bitmap_set(mapping->bitmaps[i], start, count);

1055

bitmap_set(mapping->bitmaps[i], start, count);

1056

}

1056

}

1057

spin_unlock_irqrestore(&mapping->lock, flags);

1057

spin_unlock_irqrestore(&mapping->lock, flags);

1058

1059

iova = mapping->base + (mapping_size * i);

1059

iova = mapping->base + (mapping_size * i);

1060

iova += start << PAGE_SHIFT;

1060

iova += start << PAGE_SHIFT;

1061

1062

return iova;

1062

return iova;

1063

}

1063

}

1064

1065

static inline void __free_iova(struct dma_iommu_mapping *mapping,

1065

static inline void __free_iova(struct dma_iommu_mapping *mapping,

1066

dma_addr_t addr, size_t size)

1066

dma_addr_t addr, size_t size)

1067

{

1067

{

1068

unsigned int start, count;

1068

unsigned int start, count;

1069

size_t mapping_size = mapping->bits << PAGE_SHIFT;

1069

size_t mapping_size = mapping->bits << PAGE_SHIFT;

1070

unsigned long flags;

1070

unsigned long flags;

1071

dma_addr_t bitmap_base;

1071

dma_addr_t bitmap_base;

1072

u32 bitmap_index;

1072

u32 bitmap_index;

1073

1074

if (!size)

1074

if (!size)

1075

return;

1075

return;

1076

1077

bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;

1077

bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;

1078

BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);

1078

BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);

1079

1080

bitmap_base = mapping->base + mapping_size * bitmap_index;

1080

bitmap_base = mapping->base + mapping_size * bitmap_index;

1081

1082

start = (addr - bitmap_base) >> PAGE_SHIFT;

1082

start = (addr - bitmap_base) >> PAGE_SHIFT;

1083

1084

if (addr + size > bitmap_base + mapping_size) {

1084

if (addr + size > bitmap_base + mapping_size) {

1085

/*

1085

/*

1086

* The address range to be freed reaches into the iova

1086

* The address range to be freed reaches into the iova

1087

* range of the next bitmap. This should not happen as

1087

* range of the next bitmap. This should not happen as

1088

* we don't allow this in __alloc_iova (at the

1088

* we don't allow this in __alloc_iova (at the

1089

* moment).

1089

* moment).

1090

*/

1090

*/

1091

BUG();

1091

BUG();

1092

} else

1092

} else

1093

count = size >> PAGE_SHIFT;

1093

count = size >> PAGE_SHIFT;

1094

1095

spin_lock_irqsave(&mapping->lock, flags);

1095

spin_lock_irqsave(&mapping->lock, flags);

1096

bitmap_clear(mapping->bitmaps[bitmap_index], start, count);

1096

bitmap_clear(mapping->bitmaps[bitmap_index], start, count);

1097

spin_unlock_irqrestore(&mapping->lock, flags);

1097

spin_unlock_irqrestore(&mapping->lock, flags);

1098

}

1098

}

1099

1100

static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,

1100

static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,

1101

gfp_t gfp, struct dma_attrs *attrs)

1101

gfp_t gfp, struct dma_attrs *attrs)

1102

{

1102

{

1103

struct page **pages;

1103

struct page **pages;

1104

int count = size >> PAGE_SHIFT;

1104

int count = size >> PAGE_SHIFT;

1105

int array_size = count * sizeof(struct page *);

1105

int array_size = count * sizeof(struct page *);

1106

int i = 0;

1106

int i = 0;

1107

1108

if (array_size <= PAGE_SIZE)

1108

if (array_size <= PAGE_SIZE)

1109

pages = kzalloc(array_size, gfp);

1109

pages = kzalloc(array_size, GFP_KERNEL);

1110

else

1110

else

1111

pages = vzalloc(array_size);

1111

pages = vzalloc(array_size);

1112

if (!pages)

1112

if (!pages)

1113

return NULL;

1113

return NULL;

1114

1115

if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))

1115

if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))

1116

{

1116

{

1117

unsigned long order = get_order(size);

1117

unsigned long order = get_order(size);

1118

struct page *page;

1118

struct page *page;

1119

1120

page = dma_alloc_from_contiguous(dev, count, order);

1120

page = dma_alloc_from_contiguous(dev, count, order);

1121

if (!page)

1121

if (!page)

1122

goto error;

1122

goto error;

1123

1124

__dma_clear_buffer(page, size);

1124

__dma_clear_buffer(page, size);

1125

1126

for (i = 0; i < count; i++)

1126

for (i = 0; i < count; i++)

1127

pages[i] = page + i;

1127

pages[i] = page + i;

1128

1129

return pages;

1129

return pages;

1130

}

1130

}

1131

1132

/*

1132

/*

1133

* IOMMU can map any pages, so himem can also be used here

1133

* IOMMU can map any pages, so himem can also be used here

1134

*/

1134

*/

1135

gfp |= __GFP_NOWARN | __GFP_HIGHMEM;

1135

gfp |= __GFP_NOWARN | __GFP_HIGHMEM;

1136

1137

while (count) {

1137

while (count) {

1138

int j, order = __fls(count);

1138

int j, order = __fls(count);

1139

1140

pages[i] = alloc_pages(gfp, order);

1140

pages[i] = alloc_pages(gfp, order);

1141

while (!pages[i] && order)

1141

while (!pages[i] && order)

1142

pages[i] = alloc_pages(gfp, --order);

1142

pages[i] = alloc_pages(gfp, --order);

1143

if (!pages[i])

1143

if (!pages[i])

1144

goto error;

1144

goto error;

1145

1146

if (order) {

1146

if (order) {

1147

split_page(pages[i], order);

1147

split_page(pages[i], order);

1148

j = 1 << order;

1148

j = 1 << order;

1149

while (--j)

1149

while (--j)

1150

pages[i + j] = pages[i] + j;

1150

pages[i + j] = pages[i] + j;

1151

}

1151

}

1152

1153

__dma_clear_buffer(pages[i], PAGE_SIZE << order);

1153

__dma_clear_buffer(pages[i], PAGE_SIZE << order);

1154

i += 1 << order;

1154

i += 1 << order;

1155

count -= 1 << order;

1155

count -= 1 << order;

1156

}

1156

}

1157

1158

return pages;

1158

return pages;

1159

error:

1159

error:

1160

while (i--)

1160

while (i--)

1161

if (pages[i])

1161

if (pages[i])

1162

__free_pages(pages[i], 0);

1162

__free_pages(pages[i], 0);

1163

if (array_size <= PAGE_SIZE)

1163

if (array_size <= PAGE_SIZE)

1164

kfree(pages);

1164

kfree(pages);

1165

else

1165

else

1166

vfree(pages);

1166

vfree(pages);

1167

return NULL;

1167

return NULL;

1168

}

1168

}

1169

1170

static int __iommu_free_buffer(struct device *dev, struct page **pages,

1170

static int __iommu_free_buffer(struct device *dev, struct page **pages,

1171

size_t size, struct dma_attrs *attrs)

1171

size_t size, struct dma_attrs *attrs)

1172

{

1172

{

1173

int count = size >> PAGE_SHIFT;

1173

int count = size >> PAGE_SHIFT;

1174

int array_size = count * sizeof(struct page *);

1174

int array_size = count * sizeof(struct page *);

1175

int i;

1175

int i;

1176

1177

if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {

1177

if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {

1178

dma_release_from_contiguous(dev, pages[0], count);

1178

dma_release_from_contiguous(dev, pages[0], count);

1179

} else {

1179

} else {

1180

for (i = 0; i < count; i++)

1180

for (i = 0; i < count; i++)

1181

if (pages[i])

1181

if (pages[i])

1182

__free_pages(pages[i], 0);

1182

__free_pages(pages[i], 0);

1183

}

1183

}

1184

1185

if (array_size <= PAGE_SIZE)

1185

if (array_size <= PAGE_SIZE)

1186

kfree(pages);

1186

kfree(pages);

1187

else

1187

else

1188

vfree(pages);

1188

vfree(pages);

1189

return 0;

1189

return 0;

1190

}

1190

}

1191

1192

/*

1192

/*

1193

* Create a CPU mapping for a specified pages

1193

* Create a CPU mapping for a specified pages

1194

*/

1194

*/

1195

static void *

1195

static void *

1196

__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,

1196

__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,

1197

const void *caller)

1197

const void *caller)

1198

{

1198

{

1199

return dma_common_pages_remap(pages, size,

1199

return dma_common_pages_remap(pages, size,

1200

VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller);

1200

VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller);

1201

}

1201

}

1202

1203

/*

1203

/*

1204

* Create a mapping in device IO address space for specified pages

1204

* Create a mapping in device IO address space for specified pages

1205

*/

1205

*/

1206

static dma_addr_t

1206

static dma_addr_t

1207

__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)

1207

__iommu_create_mapping(struct device *dev, struct page **pages, size_t size)

1208

{

1208

{

1209

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1209

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1210

unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1210

unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1211

dma_addr_t dma_addr, iova;

1211

dma_addr_t dma_addr, iova;

1212

int i, ret = DMA_ERROR_CODE;

1212

int i, ret = DMA_ERROR_CODE;

1213

1214

dma_addr = __alloc_iova(mapping, size);

1214

dma_addr = __alloc_iova(mapping, size);

1215

if (dma_addr == DMA_ERROR_CODE)

1215

if (dma_addr == DMA_ERROR_CODE)

1216

return dma_addr;

1216

return dma_addr;

1217

1218

iova = dma_addr;

1218

iova = dma_addr;

1219

for (i = 0; i < count; ) {

1219

for (i = 0; i < count; ) {

1220

unsigned int next_pfn = page_to_pfn(pages[i]) + 1;

1220

unsigned int next_pfn = page_to_pfn(pages[i]) + 1;

1221

phys_addr_t phys = page_to_phys(pages[i]);

1221

phys_addr_t phys = page_to_phys(pages[i]);

1222

unsigned int len, j;

1222

unsigned int len, j;

1223

1224

for (j = i + 1; j < count; j++, next_pfn++)

1224

for (j = i + 1; j < count; j++, next_pfn++)

1225

if (page_to_pfn(pages[j]) != next_pfn)

1225

if (page_to_pfn(pages[j]) != next_pfn)

1226

break;

1226

break;

1227

1228

len = (j - i) << PAGE_SHIFT;

1228

len = (j - i) << PAGE_SHIFT;

1229

ret = iommu_map(mapping->domain, iova, phys, len,

1229

ret = iommu_map(mapping->domain, iova, phys, len,

1230

IOMMU_READ|IOMMU_WRITE);

1230

IOMMU_READ|IOMMU_WRITE);

1231

if (ret < 0)

1231

if (ret < 0)

1232

goto fail;

1232

goto fail;

1233

iova += len;

1233

iova += len;

1234

i = j;

1234

i = j;

1235

}

1235

}

1236

return dma_addr;

1236

return dma_addr;

1237

fail:

1237

fail:

1238

iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);

1238

iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);

1239

__free_iova(mapping, dma_addr, size);

1239

__free_iova(mapping, dma_addr, size);

1240

return DMA_ERROR_CODE;

1240

return DMA_ERROR_CODE;

1241

}

1241

}

1242

1243

static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)

1243

static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)

1244

{

1244

{

1245

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1245

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1246

1247

/*

1247

/*

1248

* add optional in-page offset from iova to size and align

1248

* add optional in-page offset from iova to size and align

1249

* result to page size

1249

* result to page size

1250

*/

1250

*/

1251

size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);

1251

size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);

1252

iova &= PAGE_MASK;

1252

iova &= PAGE_MASK;

1253

1254

iommu_unmap(mapping->domain, iova, size);

1254

iommu_unmap(mapping->domain, iova, size);

1255

__free_iova(mapping, iova, size);

1255

__free_iova(mapping, iova, size);

1256

return 0;

1256

return 0;

1257

}

1257

}

1258

1259

static struct page **__atomic_get_pages(void *addr)

1259

static struct page **__atomic_get_pages(void *addr)

1260

{

1260

{

1261

struct page *page;

1261

struct page *page;

1262

phys_addr_t phys;

1262

phys_addr_t phys;

1263

1264

phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);

1264

phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);

1265

page = phys_to_page(phys);

1265

page = phys_to_page(phys);

1266

1267

return (struct page **)page;

1267

return (struct page **)page;

1268

}

1268

}

1269

1270

static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)

1270

static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)

1271

{

1271

{

1272

struct vm_struct *area;

1272

struct vm_struct *area;

1273

1274

if (__in_atomic_pool(cpu_addr, PAGE_SIZE))

1274

if (__in_atomic_pool(cpu_addr, PAGE_SIZE))

1275

return __atomic_get_pages(cpu_addr);

1275

return __atomic_get_pages(cpu_addr);

1276

1277

if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))

1277

if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))

1278

return cpu_addr;

1278

return cpu_addr;

1279

1280

area = find_vm_area(cpu_addr);

1280

area = find_vm_area(cpu_addr);

1281

if (area && (area->flags & VM_ARM_DMA_CONSISTENT))

1281

if (area && (area->flags & VM_ARM_DMA_CONSISTENT))

1282

return area->pages;

1282

return area->pages;

1283

return NULL;

1283

return NULL;

1284

}

1284

}

1285

1286

static void *__iommu_alloc_atomic(struct device *dev, size_t size,

1286

static void *__iommu_alloc_atomic(struct device *dev, size_t size,

1287

dma_addr_t *handle)

1287

dma_addr_t *handle)

1288

{

1288

{

1289

struct page *page;

1289

struct page *page;

1290

void *addr;

1290

void *addr;

1291

1292

addr = __alloc_from_pool(size, &page);

1292

addr = __alloc_from_pool(size, &page);

1293

if (!addr)

1293

if (!addr)

1294

return NULL;

1294

return NULL;

1295

1296

*handle = __iommu_create_mapping(dev, &page, size);

1296

*handle = __iommu_create_mapping(dev, &page, size);

1297

if (*handle == DMA_ERROR_CODE)

1297

if (*handle == DMA_ERROR_CODE)

1298

goto err_mapping;

1298

goto err_mapping;

1299

1300

return addr;

1300

return addr;

1301

1302

err_mapping:

1302

err_mapping:

1303

__free_from_pool(addr, size);

1303

__free_from_pool(addr, size);

1304

return NULL;

1304

return NULL;

1305

}

1305

}

1306

1307

static void __iommu_free_atomic(struct device *dev, void *cpu_addr,

1307

static void __iommu_free_atomic(struct device *dev, void *cpu_addr,

1308

dma_addr_t handle, size_t size)

1308

dma_addr_t handle, size_t size)

1309

{

1309

{

1310

__iommu_remove_mapping(dev, handle, size);

1310

__iommu_remove_mapping(dev, handle, size);

1311

__free_from_pool(cpu_addr, size);

1311

__free_from_pool(cpu_addr, size);

1312

}

1312

}

1313

1314

static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,

1314

static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,

1315

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)

1315

dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)

1316

{

1316

{

1317

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

1317

pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

1318

struct page **pages;

1318

struct page **pages;

1319

void *addr = NULL;

1319

void *addr = NULL;

1320

1321

*handle = DMA_ERROR_CODE;

1321

*handle = DMA_ERROR_CODE;

1322

size = PAGE_ALIGN(size);

1322

size = PAGE_ALIGN(size);

1323

1324

if (!(gfp & __GFP_WAIT))

1324

if (!(gfp & __GFP_WAIT))

1325

return __iommu_alloc_atomic(dev, size, handle);

1325

return __iommu_alloc_atomic(dev, size, handle);

1326

1327

/*

1327

/*

1328

* Following is a work-around (a.k.a. hack) to prevent pages

1328

* Following is a work-around (a.k.a. hack) to prevent pages

1329

* with __GFP_COMP being passed to split_page() which cannot

1329

* with __GFP_COMP being passed to split_page() which cannot

1330

* handle them. The real problem is that this flag probably

1330

* handle them. The real problem is that this flag probably

1331

* should be 0 on ARM as it is not supported on this

1331

* should be 0 on ARM as it is not supported on this

1332

* platform; see CONFIG_HUGETLBFS.

1332

* platform; see CONFIG_HUGETLBFS.

1333

*/

1333

*/

1334

gfp &= ~(__GFP_COMP);

1334

gfp &= ~(__GFP_COMP);

1335

1336

pages = __iommu_alloc_buffer(dev, size, gfp, attrs);

1336

pages = __iommu_alloc_buffer(dev, size, gfp, attrs);

1337

if (!pages)

1337

if (!pages)

1338

return NULL;

1338

return NULL;

1339

1340

*handle = __iommu_create_mapping(dev, pages, size);

1340

*handle = __iommu_create_mapping(dev, pages, size);

1341

if (*handle == DMA_ERROR_CODE)

1341

if (*handle == DMA_ERROR_CODE)

1342

goto err_buffer;

1342

goto err_buffer;

1343

1344

if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))

1344

if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))

1345

return pages;

1345

return pages;

1346

1347

addr = __iommu_alloc_remap(pages, size, gfp, prot,

1347

addr = __iommu_alloc_remap(pages, size, gfp, prot,

1348

__builtin_return_address(0));

1348

__builtin_return_address(0));

1349

if (!addr)

1349

if (!addr)

1350

goto err_mapping;

1350

goto err_mapping;

1351

1352

return addr;

1352

return addr;

1353

1354

err_mapping:

1354

err_mapping:

1355

__iommu_remove_mapping(dev, *handle, size);

1355

__iommu_remove_mapping(dev, *handle, size);

1356

err_buffer:

1356

err_buffer:

1357

__iommu_free_buffer(dev, pages, size, attrs);

1357

__iommu_free_buffer(dev, pages, size, attrs);

1358

return NULL;

1358

return NULL;

1359

}

1359

}

1360

1361

static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,

1361

static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,

1362

void *cpu_addr, dma_addr_t dma_addr, size_t size,

1362

void *cpu_addr, dma_addr_t dma_addr, size_t size,

1363

struct dma_attrs *attrs)

1363

struct dma_attrs *attrs)

1364

{

1364

{

1365

unsigned long uaddr = vma->vm_start;

1365

unsigned long uaddr = vma->vm_start;

1366

unsigned long usize = vma->vm_end - vma->vm_start;

1366

unsigned long usize = vma->vm_end - vma->vm_start;

1367

struct page **pages = __iommu_get_pages(cpu_addr, attrs);

1367

struct page **pages = __iommu_get_pages(cpu_addr, attrs);

1368

1369

vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

1369

vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);

1370

1371

if (!pages)

1371

if (!pages)

1372

return -ENXIO;

1372

return -ENXIO;

1373

1374

do {

1374

do {

1375

int ret = vm_insert_page(vma, uaddr, *pages++);

1375

int ret = vm_insert_page(vma, uaddr, *pages++);

1376

if (ret) {

1376

if (ret) {

1377

pr_err("Remapping memory failed: %d\n", ret);

1377

pr_err("Remapping memory failed: %d\n", ret);

1378

return ret;

1378

return ret;

1379

}

1379

}

1380

uaddr += PAGE_SIZE;

1380

uaddr += PAGE_SIZE;

1381

usize -= PAGE_SIZE;

1381

usize -= PAGE_SIZE;

1382

} while (usize > 0);

1382

} while (usize > 0);

1383

1384

return 0;

1384

return 0;

1385

}

1385

}

1386

1387

/*

1387

/*

1388

* free a page as defined by the above mapping.

1388

* free a page as defined by the above mapping.

1389

* Must not be called with IRQs disabled.

1389

* Must not be called with IRQs disabled.

1390

*/

1390

*/

1391

void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,

1391

void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,

1392

dma_addr_t handle, struct dma_attrs *attrs)

1392

dma_addr_t handle, struct dma_attrs *attrs)

1393

{

1393

{

1394

struct page **pages;

1394

struct page **pages;

1395

size = PAGE_ALIGN(size);

1395

size = PAGE_ALIGN(size);

1396

1397

if (__in_atomic_pool(cpu_addr, size)) {

1397

if (__in_atomic_pool(cpu_addr, size)) {

1398

__iommu_free_atomic(dev, cpu_addr, handle, size);

1398

__iommu_free_atomic(dev, cpu_addr, handle, size);

1399

return;

1399

return;

1400

}

1400

}

1401

1402

pages = __iommu_get_pages(cpu_addr, attrs);

1402

pages = __iommu_get_pages(cpu_addr, attrs);

1403

if (!pages) {

1403

if (!pages) {

1404

WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);

1404

WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);

1405

return;

1405

return;

1406

}

1406

}

1407

1408

if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {

1408

if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {

1409

dma_common_free_remap(cpu_addr, size,

1409

dma_common_free_remap(cpu_addr, size,

1410

VM_ARM_DMA_CONSISTENT | VM_USERMAP);

1410

VM_ARM_DMA_CONSISTENT | VM_USERMAP);

1411

}

1411

}

1412

1413

__iommu_remove_mapping(dev, handle, size);

1413

__iommu_remove_mapping(dev, handle, size);

1414

__iommu_free_buffer(dev, pages, size, attrs);

1414

__iommu_free_buffer(dev, pages, size, attrs);

1415

}

1415

}

1416

1417

static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,

1417

static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,

1418

void *cpu_addr, dma_addr_t dma_addr,

1418

void *cpu_addr, dma_addr_t dma_addr,

1419

size_t size, struct dma_attrs *attrs)

1419

size_t size, struct dma_attrs *attrs)

1420

{

1420

{

1421

unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1421

unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

1422

struct page **pages = __iommu_get_pages(cpu_addr, attrs);

1422

struct page **pages = __iommu_get_pages(cpu_addr, attrs);

1423

1424

if (!pages)

1424

if (!pages)

1425

return -ENXIO;

1425

return -ENXIO;

1426

1427

return sg_alloc_table_from_pages(sgt, pages, count, 0, size,

1427

return sg_alloc_table_from_pages(sgt, pages, count, 0, size,

1428

GFP_KERNEL);

1428

GFP_KERNEL);

1429

}

1429

}

1430

1431

static int __dma_direction_to_prot(enum dma_data_direction dir)

1431

static int __dma_direction_to_prot(enum dma_data_direction dir)

1432

{

1432

{

1433

int prot;

1433

int prot;

1434

1435

switch (dir) {

1435

switch (dir) {

1436

case DMA_BIDIRECTIONAL:

1436

case DMA_BIDIRECTIONAL:

1437

prot = IOMMU_READ | IOMMU_WRITE;

1437

prot = IOMMU_READ | IOMMU_WRITE;

1438

break;

1438

break;

1439

case DMA_TO_DEVICE:

1439

case DMA_TO_DEVICE:

1440

prot = IOMMU_READ;

1440

prot = IOMMU_READ;

1441

break;

1441

break;

1442

case DMA_FROM_DEVICE:

1442

case DMA_FROM_DEVICE:

1443

prot = IOMMU_WRITE;

1443

prot = IOMMU_WRITE;

1444

break;

1444

break;

1445

default:

1445

default:

1446

prot = 0;

1446

prot = 0;

1447

}

1447

}

1448

1449

return prot;

1449

return prot;

1450

}

1450

}

1451

1452

/*

1452

/*

1453

* Map a part of the scatter-gather list into contiguous io address space

1453

* Map a part of the scatter-gather list into contiguous io address space

1454

*/

1454

*/

1455

static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,

1455

static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,

1456

size_t size, dma_addr_t *handle,

1456

size_t size, dma_addr_t *handle,

1457

enum dma_data_direction dir, struct dma_attrs *attrs,

1457

enum dma_data_direction dir, struct dma_attrs *attrs,

1458

bool is_coherent)

1458

bool is_coherent)

1459

{

1459

{

1460

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1460

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1461

dma_addr_t iova, iova_base;

1461

dma_addr_t iova, iova_base;

1462

int ret = 0;

1462

int ret = 0;

1463

unsigned int count;

1463

unsigned int count;

1464

struct scatterlist *s;

1464

struct scatterlist *s;

1465

int prot;

1465

int prot;

1466

1467

size = PAGE_ALIGN(size);

1467

size = PAGE_ALIGN(size);

1468

*handle = DMA_ERROR_CODE;

1468

*handle = DMA_ERROR_CODE;

1469

1470

iova_base = iova = __alloc_iova(mapping, size);

1470

iova_base = iova = __alloc_iova(mapping, size);

1471

if (iova == DMA_ERROR_CODE)

1471

if (iova == DMA_ERROR_CODE)

1472

return -ENOMEM;

1472

return -ENOMEM;

1473

1474

for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {

1474

for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {

1475

phys_addr_t phys = page_to_phys(sg_page(s));

1475

phys_addr_t phys = page_to_phys(sg_page(s));

1476

unsigned int len = PAGE_ALIGN(s->offset + s->length);

1476

unsigned int len = PAGE_ALIGN(s->offset + s->length);

1477

1478

if (!is_coherent &&

1478

if (!is_coherent &&

1479

!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1479

!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1480

__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

1480

__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

1481

1482

prot = __dma_direction_to_prot(dir);

1482

prot = __dma_direction_to_prot(dir);

1483

1484

ret = iommu_map(mapping->domain, iova, phys, len, prot);

1484

ret = iommu_map(mapping->domain, iova, phys, len, prot);

1485

if (ret < 0)

1485

if (ret < 0)

1486

goto fail;

1486

goto fail;

1487

count += len >> PAGE_SHIFT;

1487

count += len >> PAGE_SHIFT;

1488

iova += len;

1488

iova += len;

1489

}

1489

}

1490

*handle = iova_base;

1490

*handle = iova_base;

1491

1492

return 0;

1492

return 0;

1493

fail:

1493

fail:

1494

iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);

1494

iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);

1495

__free_iova(mapping, iova_base, size);

1495

__free_iova(mapping, iova_base, size);

1496

return ret;

1496

return ret;

1497

}

1497

}

1498

1499

static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,

1499

static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,

1500

enum dma_data_direction dir, struct dma_attrs *attrs,

1500

enum dma_data_direction dir, struct dma_attrs *attrs,

1501

bool is_coherent)

1501

bool is_coherent)

1502

{

1502

{

1503

struct scatterlist *s = sg, *dma = sg, *start = sg;

1503

struct scatterlist *s = sg, *dma = sg, *start = sg;

1504

int i, count = 0;

1504

int i, count = 0;

1505

unsigned int offset = s->offset;

1505

unsigned int offset = s->offset;

1506

unsigned int size = s->offset + s->length;

1506

unsigned int size = s->offset + s->length;

1507

unsigned int max = dma_get_max_seg_size(dev);

1507

unsigned int max = dma_get_max_seg_size(dev);

1508

1509

for (i = 1; i < nents; i++) {

1509

for (i = 1; i < nents; i++) {

1510

s = sg_next(s);

1510

s = sg_next(s);

1511

1512

s->dma_address = DMA_ERROR_CODE;

1512

s->dma_address = DMA_ERROR_CODE;

1513

s->dma_length = 0;

1513

s->dma_length = 0;

1514

1515

if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {

1515

if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {

1516

if (__map_sg_chunk(dev, start, size, &dma->dma_address,

1516

if (__map_sg_chunk(dev, start, size, &dma->dma_address,

1517

dir, attrs, is_coherent) < 0)

1517

dir, attrs, is_coherent) < 0)

1518

goto bad_mapping;

1518

goto bad_mapping;

1519

1520

dma->dma_address += offset;

1520

dma->dma_address += offset;

1521

dma->dma_length = size - offset;

1521

dma->dma_length = size - offset;

1522

1523

size = offset = s->offset;

1523

size = offset = s->offset;

1524

start = s;

1524

start = s;

1525

dma = sg_next(dma);

1525

dma = sg_next(dma);

1526

count += 1;

1526

count += 1;

1527

}

1527

}

1528

size += s->length;

1528

size += s->length;

1529

}

1529

}

1530

if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,

1530

if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,

1531

is_coherent) < 0)

1531

is_coherent) < 0)

1532

goto bad_mapping;

1532

goto bad_mapping;

1533

1534

dma->dma_address += offset;

1534

dma->dma_address += offset;

1535

dma->dma_length = size - offset;

1535

dma->dma_length = size - offset;

1536

1537

return count+1;

1537

return count+1;

1538

1539

bad_mapping:

1539

bad_mapping:

1540

for_each_sg(sg, s, count, i)

1540

for_each_sg(sg, s, count, i)

1541

__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));

1541

__iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));

1542

return 0;

1542

return 0;

1543

}

1543

}

1544

1545

/**

1545

/**

1546

* arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA

1546

* arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA

1547

* @dev: valid struct device pointer

1547

* @dev: valid struct device pointer

1548

* @sg: list of buffers

1548

* @sg: list of buffers

1549

* @nents: number of buffers to map

1549

* @nents: number of buffers to map

1550

* @dir: DMA transfer direction

1550

* @dir: DMA transfer direction

1551

*

1551

*

1552

* Map a set of i/o coherent buffers described by scatterlist in streaming

1552

* Map a set of i/o coherent buffers described by scatterlist in streaming

1553

* mode for DMA. The scatter gather list elements are merged together (if

1553

* mode for DMA. The scatter gather list elements are merged together (if

1554

* possible) and tagged with the appropriate dma address and length. They are

1554

* possible) and tagged with the appropriate dma address and length. They are

1555

* obtained via sg_dma_{address,length}.

1555

* obtained via sg_dma_{address,length}.

1556

*/

1556

*/

1557

int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,

1557

int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,

1558

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1558

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1559

{

1559

{

1560

return __iommu_map_sg(dev, sg, nents, dir, attrs, true);

1560

return __iommu_map_sg(dev, sg, nents, dir, attrs, true);

1561

}

1561

}

1562

1563

/**

1563

/**

1564

* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA

1564

* arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA

1565

* @dev: valid struct device pointer

1565

* @dev: valid struct device pointer

1566

* @sg: list of buffers

1566

* @sg: list of buffers

1567

* @nents: number of buffers to map

1567

* @nents: number of buffers to map

1568

* @dir: DMA transfer direction

1568

* @dir: DMA transfer direction

1569

*

1569

*

1570

* Map a set of buffers described by scatterlist in streaming mode for DMA.

1570

* Map a set of buffers described by scatterlist in streaming mode for DMA.

1571

* The scatter gather list elements are merged together (if possible) and

1571

* The scatter gather list elements are merged together (if possible) and

1572

* tagged with the appropriate dma address and length. They are obtained via

1572

* tagged with the appropriate dma address and length. They are obtained via

1573

* sg_dma_{address,length}.

1573

* sg_dma_{address,length}.

1574

*/

1574

*/

1575

int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,

1575

int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,

1576

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1576

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1577

{

1577

{

1578

return __iommu_map_sg(dev, sg, nents, dir, attrs, false);

1578

return __iommu_map_sg(dev, sg, nents, dir, attrs, false);

1579

}

1579

}

1580

1581

static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,

1581

static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,

1582

int nents, enum dma_data_direction dir, struct dma_attrs *attrs,

1582

int nents, enum dma_data_direction dir, struct dma_attrs *attrs,

1583

bool is_coherent)

1583

bool is_coherent)

1584

{

1584

{

1585

struct scatterlist *s;

1585

struct scatterlist *s;

1586

int i;

1586

int i;

1587

1588

for_each_sg(sg, s, nents, i) {

1588

for_each_sg(sg, s, nents, i) {

1589

if (sg_dma_len(s))

1589

if (sg_dma_len(s))

1590

__iommu_remove_mapping(dev, sg_dma_address(s),

1590

__iommu_remove_mapping(dev, sg_dma_address(s),

1591

sg_dma_len(s));

1591

sg_dma_len(s));

1592

if (!is_coherent &&

1592

if (!is_coherent &&

1593

!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1593

!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1594

__dma_page_dev_to_cpu(sg_page(s), s->offset,

1594

__dma_page_dev_to_cpu(sg_page(s), s->offset,

1595

s->length, dir);

1595

s->length, dir);

1596

}

1596

}

1597

}

1597

}

1598

1599

/**

1599

/**

1600

* arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

1600

* arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

1601

* @dev: valid struct device pointer

1601

* @dev: valid struct device pointer

1602

* @sg: list of buffers

1602

* @sg: list of buffers

1603

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

1603

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

1604

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1604

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1605

*

1605

*

1606

* Unmap a set of streaming mode DMA translations. Again, CPU access

1606

* Unmap a set of streaming mode DMA translations. Again, CPU access

1607

* rules concerning calls here are the same as for dma_unmap_single().

1607

* rules concerning calls here are the same as for dma_unmap_single().

1608

*/

1608

*/

1609

void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,

1609

void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,

1610

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1610

int nents, enum dma_data_direction dir, struct dma_attrs *attrs)

1611

{

1611

{

1612

__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);

1612

__iommu_unmap_sg(dev, sg, nents, dir, attrs, true);

1613

}

1613

}

1614

1615

/**

1615

/**

1616

* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

1616

* arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg

1617

* @dev: valid struct device pointer

1617

* @dev: valid struct device pointer

1618

* @sg: list of buffers

1618

* @sg: list of buffers

1619

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

1619

* @nents: number of buffers to unmap (same as was passed to dma_map_sg)

1620

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1620

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1621

*

1621

*

1622

* Unmap a set of streaming mode DMA translations. Again, CPU access

1622

* Unmap a set of streaming mode DMA translations. Again, CPU access

1623

* rules concerning calls here are the same as for dma_unmap_single().

1623

* rules concerning calls here are the same as for dma_unmap_single().

1624

*/

1624

*/

1625

void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,

1625

void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,

1626

enum dma_data_direction dir, struct dma_attrs *attrs)

1626

enum dma_data_direction dir, struct dma_attrs *attrs)

1627

{

1627

{

1628

__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);

1628

__iommu_unmap_sg(dev, sg, nents, dir, attrs, false);

1629

}

1629

}

1630

1631

/**

1631

/**

1632

* arm_iommu_sync_sg_for_cpu

1632

* arm_iommu_sync_sg_for_cpu

1633

* @dev: valid struct device pointer

1633

* @dev: valid struct device pointer

1634

* @sg: list of buffers

1634

* @sg: list of buffers

1635

* @nents: number of buffers to map (returned from dma_map_sg)

1635

* @nents: number of buffers to map (returned from dma_map_sg)

1636

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1636

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1637

*/

1637

*/

1638

void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

1638

void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,

1639

int nents, enum dma_data_direction dir)

1639

int nents, enum dma_data_direction dir)

1640

{

1640

{

1641

struct scatterlist *s;

1641

struct scatterlist *s;

1642

int i;

1642

int i;

1643

1644

for_each_sg(sg, s, nents, i)

1644

for_each_sg(sg, s, nents, i)

1645

__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);

1645

__dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);

1646

1647

}

1647

}

1648

1649

/**

1649

/**

1650

* arm_iommu_sync_sg_for_device

1650

* arm_iommu_sync_sg_for_device

1651

* @dev: valid struct device pointer

1651

* @dev: valid struct device pointer

1652

* @sg: list of buffers

1652

* @sg: list of buffers

1653

* @nents: number of buffers to map (returned from dma_map_sg)

1653

* @nents: number of buffers to map (returned from dma_map_sg)

1654

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1654

* @dir: DMA transfer direction (same as was passed to dma_map_sg)

1655

*/

1655

*/

1656

void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

1656

void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,

1657

int nents, enum dma_data_direction dir)

1657

int nents, enum dma_data_direction dir)

1658

{

1658

{

1659

struct scatterlist *s;

1659

struct scatterlist *s;

1660

int i;

1660

int i;

1661

1662

for_each_sg(sg, s, nents, i)

1662

for_each_sg(sg, s, nents, i)

1663

__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

1663

__dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);

1664

}

1664

}

1665

1666

1667

/**

1667

/**

1668

* arm_coherent_iommu_map_page

1668

* arm_coherent_iommu_map_page

1669

* @dev: valid struct device pointer

1669

* @dev: valid struct device pointer

1670

* @page: page that buffer resides in

1670

* @page: page that buffer resides in

1671

* @offset: offset into page for start of buffer

1671

* @offset: offset into page for start of buffer

1672

* @size: size of buffer to map

1672

* @size: size of buffer to map

1673

* @dir: DMA transfer direction

1673

* @dir: DMA transfer direction

1674

*

1674

*

1675

* Coherent IOMMU aware version of arm_dma_map_page()

1675

* Coherent IOMMU aware version of arm_dma_map_page()

1676

*/

1676

*/

1677

static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,

1677

static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,

1678

unsigned long offset, size_t size, enum dma_data_direction dir,

1678

unsigned long offset, size_t size, enum dma_data_direction dir,

1679

struct dma_attrs *attrs)

1679

struct dma_attrs *attrs)

1680

{

1680

{

1681

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1681

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1682

dma_addr_t dma_addr;

1682

dma_addr_t dma_addr;

1683

int ret, prot, len = PAGE_ALIGN(size + offset);

1683

int ret, prot, len = PAGE_ALIGN(size + offset);

1684

1685

dma_addr = __alloc_iova(mapping, len);

1685

dma_addr = __alloc_iova(mapping, len);

1686

if (dma_addr == DMA_ERROR_CODE)

1686

if (dma_addr == DMA_ERROR_CODE)

1687

return dma_addr;

1687

return dma_addr;

1688

1689

prot = __dma_direction_to_prot(dir);

1689

prot = __dma_direction_to_prot(dir);

1690

1691

ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);

1691

ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);

1692

if (ret < 0)

1692

if (ret < 0)

1693

goto fail;

1693

goto fail;

1694

1695

return dma_addr + offset;

1695

return dma_addr + offset;

1696

fail:

1696

fail:

1697

__free_iova(mapping, dma_addr, len);

1697

__free_iova(mapping, dma_addr, len);

1698

return DMA_ERROR_CODE;

1698

return DMA_ERROR_CODE;

1699

}

1699

}

1700

1701

/**

1701

/**

1702

* arm_iommu_map_page

1702

* arm_iommu_map_page

1703

* @dev: valid struct device pointer

1703

* @dev: valid struct device pointer

1704

* @page: page that buffer resides in

1704

* @page: page that buffer resides in

1705

* @offset: offset into page for start of buffer

1705

* @offset: offset into page for start of buffer

1706

* @size: size of buffer to map

1706

* @size: size of buffer to map

1707

* @dir: DMA transfer direction

1707

* @dir: DMA transfer direction

1708

*

1708

*

1709

* IOMMU aware version of arm_dma_map_page()

1709

* IOMMU aware version of arm_dma_map_page()

1710

*/

1710

*/

1711

static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,

1711

static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,

1712

unsigned long offset, size_t size, enum dma_data_direction dir,

1712

unsigned long offset, size_t size, enum dma_data_direction dir,

1713

struct dma_attrs *attrs)

1713

struct dma_attrs *attrs)

1714

{

1714

{

1715

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1715

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1716

__dma_page_cpu_to_dev(page, offset, size, dir);

1716

__dma_page_cpu_to_dev(page, offset, size, dir);

1717

1718

return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);

1718

return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);

1719

}

1719

}

1720

1721

/**

1721

/**

1722

* arm_coherent_iommu_unmap_page

1722

* arm_coherent_iommu_unmap_page

1723

* @dev: valid struct device pointer

1723

* @dev: valid struct device pointer

1724

* @handle: DMA address of buffer

1724

* @handle: DMA address of buffer

1725

* @size: size of buffer (same as passed to dma_map_page)

1725

* @size: size of buffer (same as passed to dma_map_page)

1726

* @dir: DMA transfer direction (same as passed to dma_map_page)

1726

* @dir: DMA transfer direction (same as passed to dma_map_page)

1727

*

1727

*

1728

* Coherent IOMMU aware version of arm_dma_unmap_page()

1728

* Coherent IOMMU aware version of arm_dma_unmap_page()

1729

*/

1729

*/

1730

static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,

1730

static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,

1731

size_t size, enum dma_data_direction dir,

1731

size_t size, enum dma_data_direction dir,

1732

struct dma_attrs *attrs)

1732

struct dma_attrs *attrs)

1733

{

1733

{

1734

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1734

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1735

dma_addr_t iova = handle & PAGE_MASK;

1735

dma_addr_t iova = handle & PAGE_MASK;

1736

int offset = handle & ~PAGE_MASK;

1736

int offset = handle & ~PAGE_MASK;

1737

int len = PAGE_ALIGN(size + offset);

1737

int len = PAGE_ALIGN(size + offset);

1738

1739

if (!iova)

1739

if (!iova)

1740

return;

1740

return;

1741

1742

iommu_unmap(mapping->domain, iova, len);

1742

iommu_unmap(mapping->domain, iova, len);

1743

__free_iova(mapping, iova, len);

1743

__free_iova(mapping, iova, len);

1744

}

1744

}

1745

1746

/**

1746

/**

1747

* arm_iommu_unmap_page

1747

* arm_iommu_unmap_page

1748

* @dev: valid struct device pointer

1748

* @dev: valid struct device pointer

1749

* @handle: DMA address of buffer

1749

* @handle: DMA address of buffer

1750

* @size: size of buffer (same as passed to dma_map_page)

1750

* @size: size of buffer (same as passed to dma_map_page)

1751

* @dir: DMA transfer direction (same as passed to dma_map_page)

1751

* @dir: DMA transfer direction (same as passed to dma_map_page)

1752

*

1752

*

1753

* IOMMU aware version of arm_dma_unmap_page()

1753

* IOMMU aware version of arm_dma_unmap_page()

1754

*/

1754

*/

1755

static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,

1755

static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,

1756

size_t size, enum dma_data_direction dir,

1756

size_t size, enum dma_data_direction dir,

1757

struct dma_attrs *attrs)

1757

struct dma_attrs *attrs)

1758

{

1758

{

1759

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1759

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1760

dma_addr_t iova = handle & PAGE_MASK;

1760

dma_addr_t iova = handle & PAGE_MASK;

1761

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1761

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1762

int offset = handle & ~PAGE_MASK;

1762

int offset = handle & ~PAGE_MASK;

1763

int len = PAGE_ALIGN(size + offset);

1763

int len = PAGE_ALIGN(size + offset);

1764

1765

if (!iova)

1765

if (!iova)

1766

return;

1766

return;

1767

1768

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1768

if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))

1769

__dma_page_dev_to_cpu(page, offset, size, dir);

1769

__dma_page_dev_to_cpu(page, offset, size, dir);

1770

1771

iommu_unmap(mapping->domain, iova, len);

1771

iommu_unmap(mapping->domain, iova, len);

1772

__free_iova(mapping, iova, len);

1772

__free_iova(mapping, iova, len);

1773

}

1773

}

1774

1775

static void arm_iommu_sync_single_for_cpu(struct device *dev,

1775

static void arm_iommu_sync_single_for_cpu(struct device *dev,

1776

dma_addr_t handle, size_t size, enum dma_data_direction dir)

1776

dma_addr_t handle, size_t size, enum dma_data_direction dir)

1777

{

1777

{

1778

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1778

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1779

dma_addr_t iova = handle & PAGE_MASK;

1779

dma_addr_t iova = handle & PAGE_MASK;

1780

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1780

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1781

unsigned int offset = handle & ~PAGE_MASK;

1781

unsigned int offset = handle & ~PAGE_MASK;

1782

1783

if (!iova)

1783

if (!iova)

1784

return;

1784

return;

1785

1786

__dma_page_dev_to_cpu(page, offset, size, dir);

1786

__dma_page_dev_to_cpu(page, offset, size, dir);

1787

}

1787

}

1788

1789

static void arm_iommu_sync_single_for_device(struct device *dev,

1789

static void arm_iommu_sync_single_for_device(struct device *dev,

1790

dma_addr_t handle, size_t size, enum dma_data_direction dir)

1790

dma_addr_t handle, size_t size, enum dma_data_direction dir)

1791

{

1791

{

1792

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1792

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

1793

dma_addr_t iova = handle & PAGE_MASK;

1793

dma_addr_t iova = handle & PAGE_MASK;

1794

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1794

struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));

1795

unsigned int offset = handle & ~PAGE_MASK;

1795

unsigned int offset = handle & ~PAGE_MASK;

1796

1797

if (!iova)

1797

if (!iova)

1798

return;

1798

return;

1799

1800

__dma_page_cpu_to_dev(page, offset, size, dir);

1800

__dma_page_cpu_to_dev(page, offset, size, dir);

1801

}

1801

}

1802

1803

struct dma_map_ops iommu_ops = {

1803

struct dma_map_ops iommu_ops = {

1804

.alloc = arm_iommu_alloc_attrs,

1804

.alloc = arm_iommu_alloc_attrs,

1805

.free = arm_iommu_free_attrs,

1805

.free = arm_iommu_free_attrs,

1806

.mmap = arm_iommu_mmap_attrs,

1806

.mmap = arm_iommu_mmap_attrs,

1807

.get_sgtable = arm_iommu_get_sgtable,

1807

.get_sgtable = arm_iommu_get_sgtable,

1808

1809

.map_page = arm_iommu_map_page,

1809

.map_page = arm_iommu_map_page,

1810

.unmap_page = arm_iommu_unmap_page,

1810

.unmap_page = arm_iommu_unmap_page,

1811

.sync_single_for_cpu = arm_iommu_sync_single_for_cpu,

1811

.sync_single_for_cpu = arm_iommu_sync_single_for_cpu,

1812

.sync_single_for_device = arm_iommu_sync_single_for_device,

1812

.sync_single_for_device = arm_iommu_sync_single_for_device,

1813

1814

.map_sg = arm_iommu_map_sg,

1814

.map_sg = arm_iommu_map_sg,

1815

.unmap_sg = arm_iommu_unmap_sg,

1815

.unmap_sg = arm_iommu_unmap_sg,

1816

.sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,

1816

.sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu,

1817

.sync_sg_for_device = arm_iommu_sync_sg_for_device,

1817

.sync_sg_for_device = arm_iommu_sync_sg_for_device,

1818

1819

.set_dma_mask = arm_dma_set_mask,

1819

.set_dma_mask = arm_dma_set_mask,

1820

};

1820

};

1821

1822

struct dma_map_ops iommu_coherent_ops = {

1822

struct dma_map_ops iommu_coherent_ops = {

1823

.alloc = arm_iommu_alloc_attrs,

1823

.alloc = arm_iommu_alloc_attrs,

1824

.free = arm_iommu_free_attrs,

1824

.free = arm_iommu_free_attrs,

1825

.mmap = arm_iommu_mmap_attrs,

1825

.mmap = arm_iommu_mmap_attrs,

1826

.get_sgtable = arm_iommu_get_sgtable,

1826

.get_sgtable = arm_iommu_get_sgtable,

1827

1828

.map_page = arm_coherent_iommu_map_page,

1828

.map_page = arm_coherent_iommu_map_page,

1829

.unmap_page = arm_coherent_iommu_unmap_page,

1829

.unmap_page = arm_coherent_iommu_unmap_page,

1830

1831

.map_sg = arm_coherent_iommu_map_sg,

1831

.map_sg = arm_coherent_iommu_map_sg,

1832

.unmap_sg = arm_coherent_iommu_unmap_sg,

1832

.unmap_sg = arm_coherent_iommu_unmap_sg,

1833

1834

.set_dma_mask = arm_dma_set_mask,

1834

.set_dma_mask = arm_dma_set_mask,

1835

};

1835

};

1836

1837

/**

1837

/**

1838

* arm_iommu_create_mapping

1838

* arm_iommu_create_mapping

1839

* @bus: pointer to the bus holding the client device (for IOMMU calls)

1839

* @bus: pointer to the bus holding the client device (for IOMMU calls)

1840

* @base: start address of the valid IO address space

1840

* @base: start address of the valid IO address space

1841

* @size: maximum size of the valid IO address space

1841

* @size: maximum size of the valid IO address space

1842

*

1842

*

1843

* Creates a mapping structure which holds information about used/unused

1843

* Creates a mapping structure which holds information about used/unused

1844

* IO address ranges, which is required to perform memory allocation and

1844

* IO address ranges, which is required to perform memory allocation and

1845

* mapping with IOMMU aware functions.

1845

* mapping with IOMMU aware functions.

1846

*

1846

*

1847

* The client device need to be attached to the mapping with

1847

* The client device need to be attached to the mapping with

1848

* arm_iommu_attach_device function.

1848

* arm_iommu_attach_device function.

1849

*/

1849

*/

1850

struct dma_iommu_mapping *

1850

struct dma_iommu_mapping *

1851

arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)

1851

arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size)

1852

{

1852

{

1853

unsigned int bits = size >> PAGE_SHIFT;

1853

unsigned int bits = size >> PAGE_SHIFT;

1854

unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);

1854

unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);

1855

struct dma_iommu_mapping *mapping;

1855

struct dma_iommu_mapping *mapping;

1856

int extensions = 1;

1856

int extensions = 1;

1857

int err = -ENOMEM;

1857

int err = -ENOMEM;

1858

1859

if (!bitmap_size)

1859

if (!bitmap_size)

1860

return ERR_PTR(-EINVAL);

1860

return ERR_PTR(-EINVAL);

1861

1862

if (bitmap_size > PAGE_SIZE) {

1862

if (bitmap_size > PAGE_SIZE) {

1863

extensions = bitmap_size / PAGE_SIZE;

1863

extensions = bitmap_size / PAGE_SIZE;

1864

bitmap_size = PAGE_SIZE;

1864

bitmap_size = PAGE_SIZE;

1865

}

1865

}

1866

1867

mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);

1867

mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);

1868

if (!mapping)

1868

if (!mapping)

1869

goto err;

1869

goto err;

1870

1871

mapping->bitmap_size = bitmap_size;

1871

mapping->bitmap_size = bitmap_size;

1872

mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),

1872

mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *),

1873

GFP_KERNEL);

1873

GFP_KERNEL);

1874

if (!mapping->bitmaps)

1874

if (!mapping->bitmaps)

1875

goto err2;

1875

goto err2;

1876

1877

mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);

1877

mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);

1878

if (!mapping->bitmaps[0])

1878

if (!mapping->bitmaps[0])

1879

goto err3;

1879

goto err3;

1880

1881

mapping->nr_bitmaps = 1;

1881

mapping->nr_bitmaps = 1;

1882

mapping->extensions = extensions;

1882

mapping->extensions = extensions;

1883

mapping->base = base;

1883

mapping->base = base;

1884

mapping->bits = BITS_PER_BYTE * bitmap_size;

1884

mapping->bits = BITS_PER_BYTE * bitmap_size;

1885

1886

spin_lock_init(&mapping->lock);

1886

spin_lock_init(&mapping->lock);

1887

1888

mapping->domain = iommu_domain_alloc(bus);

1888

mapping->domain = iommu_domain_alloc(bus);

1889

if (!mapping->domain)

1889

if (!mapping->domain)

1890

goto err4;

1890

goto err4;

1891

1892

kref_init(&mapping->kref);

1892

kref_init(&mapping->kref);

1893

return mapping;

1893

return mapping;

1894

err4:

1894

err4:

1895

kfree(mapping->bitmaps[0]);

1895

kfree(mapping->bitmaps[0]);

1896

err3:

1896

err3:

1897

kfree(mapping->bitmaps);

1897

kfree(mapping->bitmaps);

1898

err2:

1898

err2:

1899

kfree(mapping);

1899

kfree(mapping);

1900

err:

1900

err:

1901

return ERR_PTR(err);

1901

return ERR_PTR(err);

1902

}

1902

}

1903

EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);

1903

EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);

1904

1905

static void release_iommu_mapping(struct kref *kref)

1905

static void release_iommu_mapping(struct kref *kref)

1906

{

1906

{

1907

int i;

1907

int i;

1908

struct dma_iommu_mapping *mapping =

1908

struct dma_iommu_mapping *mapping =

1909

container_of(kref, struct dma_iommu_mapping, kref);

1909

container_of(kref, struct dma_iommu_mapping, kref);

1910

1911

iommu_domain_free(mapping->domain);

1911

iommu_domain_free(mapping->domain);

1912

for (i = 0; i < mapping->nr_bitmaps; i++)

1912

for (i = 0; i < mapping->nr_bitmaps; i++)

1913

kfree(mapping->bitmaps[i]);

1913

kfree(mapping->bitmaps[i]);

1914

kfree(mapping->bitmaps);

1914

kfree(mapping->bitmaps);

1915

kfree(mapping);

1915

kfree(mapping);

1916

}

1916

}

1917

1918

static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)

1918

static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)

1919

{

1919

{

1920

int next_bitmap;

1920

int next_bitmap;

1921

1922

if (mapping->nr_bitmaps > mapping->extensions)

1922

if (mapping->nr_bitmaps > mapping->extensions)

1923

return -EINVAL;

1923

return -EINVAL;

1924

1925

next_bitmap = mapping->nr_bitmaps;

1925

next_bitmap = mapping->nr_bitmaps;

1926

mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,

1926

mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,

1927

GFP_ATOMIC);

1927

GFP_ATOMIC);

1928

if (!mapping->bitmaps[next_bitmap])

1928

if (!mapping->bitmaps[next_bitmap])

1929

return -ENOMEM;

1929

return -ENOMEM;

1930

1931

mapping->nr_bitmaps++;

1931

mapping->nr_bitmaps++;

1932

1933

return 0;

1933

return 0;

1934

}

1934

}

1935

1936

void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)

1936

void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)

1937

{

1937

{

1938

if (mapping)

1938

if (mapping)

1939

kref_put(&mapping->kref, release_iommu_mapping);

1939

kref_put(&mapping->kref, release_iommu_mapping);

1940

}

1940

}

1941

EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);

1941

EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);

1942

1943

static int __arm_iommu_attach_device(struct device *dev,

1943

static int __arm_iommu_attach_device(struct device *dev,

1944

struct dma_iommu_mapping *mapping)

1944

struct dma_iommu_mapping *mapping)

1945

{

1945

{

1946

int err;

1946

int err;

1947

1948

err = iommu_attach_device(mapping->domain, dev);

1948

err = iommu_attach_device(mapping->domain, dev);

1949

if (err)

1949

if (err)

1950

return err;

1950

return err;

1951

1952

kref_get(&mapping->kref);

1952

kref_get(&mapping->kref);

1953

dev->archdata.mapping = mapping;

1953

dev->archdata.mapping = mapping;

1954

1955

pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));

1955

pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));

1956

return 0;

1956

return 0;

1957

}

1957

}

1958

1959

/**

1959

/**

1960

* arm_iommu_attach_device

1960

* arm_iommu_attach_device

1961

* @dev: valid struct device pointer

1961

* @dev: valid struct device pointer

1962

* @mapping: io address space mapping structure (returned from

1962

* @mapping: io address space mapping structure (returned from

1963

* arm_iommu_create_mapping)

1963

* arm_iommu_create_mapping)

1964

*

1964

*

1965

* Attaches specified io address space mapping to the provided device.

1965

* Attaches specified io address space mapping to the provided device.

1966

* This replaces the dma operations (dma_map_ops pointer) with the

1966

* This replaces the dma operations (dma_map_ops pointer) with the

1967

* IOMMU aware version.

1967

* IOMMU aware version.

1968

*

1968

*

1969

* More than one client might be attached to the same io address space

1969

* More than one client might be attached to the same io address space

1970

* mapping.

1970

* mapping.

1971

*/

1971

*/

1972

int arm_iommu_attach_device(struct device *dev,

1972

int arm_iommu_attach_device(struct device *dev,

1973

struct dma_iommu_mapping *mapping)

1973

struct dma_iommu_mapping *mapping)

1974

{

1974

{

1975

int err;

1975

int err;

1976

1977

err = __arm_iommu_attach_device(dev, mapping);

1977

err = __arm_iommu_attach_device(dev, mapping);

1978

if (err)

1978

if (err)

1979

return err;

1979

return err;

1980

1981

set_dma_ops(dev, &iommu_ops);

1981

set_dma_ops(dev, &iommu_ops);

1982

return 0;

1982

return 0;

1983

}

1983

}

1984

EXPORT_SYMBOL_GPL(arm_iommu_attach_device);

1984

EXPORT_SYMBOL_GPL(arm_iommu_attach_device);

1985

1986

static void __arm_iommu_detach_device(struct device *dev)

1986

static void __arm_iommu_detach_device(struct device *dev)

1987

{

1987

{

1988

struct dma_iommu_mapping *mapping;

1988

struct dma_iommu_mapping *mapping;

1989

1990

mapping = to_dma_iommu_mapping(dev);

1990

mapping = to_dma_iommu_mapping(dev);

1991

if (!mapping) {

1991

if (!mapping) {

1992

dev_warn(dev, "Not attached\n");

1992

dev_warn(dev, "Not attached\n");

1993

return;

1993

return;

1994

}

1994

}

1995

1996

iommu_detach_device(mapping->domain, dev);

1996

iommu_detach_device(mapping->domain, dev);

1997

kref_put(&mapping->kref, release_iommu_mapping);

1997

kref_put(&mapping->kref, release_iommu_mapping);

1998

dev->archdata.mapping = NULL;

1998

dev->archdata.mapping = NULL;

1999

2000

pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));

2000

pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));

2001

}

2001

}

2002

2003

/**

2003

/**

2004

* arm_iommu_detach_device

2004

* arm_iommu_detach_device

2005

* @dev: valid struct device pointer

2005

* @dev: valid struct device pointer

2006

*

2006

*

2007

* Detaches the provided device from a previously attached map.

2007

* Detaches the provided device from a previously attached map.

2008

* This voids the dma operations (dma_map_ops pointer)

2008

* This voids the dma operations (dma_map_ops pointer)

2009

*/

2009

*/

2010

void arm_iommu_detach_device(struct device *dev)

2010

void arm_iommu_detach_device(struct device *dev)

2011

{

2011

{

2012

__arm_iommu_detach_device(dev);

2012

__arm_iommu_detach_device(dev);

2013

set_dma_ops(dev, NULL);

2013

set_dma_ops(dev, NULL);

2014

}

2014

}

2015

EXPORT_SYMBOL_GPL(arm_iommu_detach_device);

2015

EXPORT_SYMBOL_GPL(arm_iommu_detach_device);

2016

2017

static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)

2017

static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)

2018

{

2018

{

2019

return coherent ? &iommu_coherent_ops : &iommu_ops;

2019

return coherent ? &iommu_coherent_ops : &iommu_ops;

2020

}

2020

}

2021

2022

static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,

2022

static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,

2023

struct iommu_ops *iommu)

2023

struct iommu_ops *iommu)

2024

{

2024

{

2025

struct dma_iommu_mapping *mapping;

2025

struct dma_iommu_mapping *mapping;

2026

2027

if (!iommu)

2027

if (!iommu)

2028

return false;

2028

return false;

2029

2030

mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);

2030

mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);

2031

if (IS_ERR(mapping)) {

2031

if (IS_ERR(mapping)) {

2032

pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",

2032

pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",

2033

size, dev_name(dev));

2033

size, dev_name(dev));

2034

return false;

2034

return false;

2035

}

2035

}

2036

2037

if (__arm_iommu_attach_device(dev, mapping)) {

2037

if (__arm_iommu_attach_device(dev, mapping)) {

2038

pr_warn("Failed to attached device %s to IOMMU_mapping\n",

2038

pr_warn("Failed to attached device %s to IOMMU_mapping\n",

2039

dev_name(dev));

2039

dev_name(dev));

2040

arm_iommu_release_mapping(mapping);

2040

arm_iommu_release_mapping(mapping);

2041

return false;

2041

return false;

2042

}

2042

}

2043

2044

return true;

2044

return true;

2045

}

2045

}

2046

2047

static void arm_teardown_iommu_dma_ops(struct device *dev)

2047

static void arm_teardown_iommu_dma_ops(struct device *dev)

2048

{

2048

{

2049

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

2049

struct dma_iommu_mapping *mapping = dev->archdata.mapping;

2050

2051

if (!mapping)

2051

if (!mapping)

2052

return;

2052

return;

2053

2054

__arm_iommu_detach_device(dev);

2054

__arm_iommu_detach_device(dev);

2055

arm_iommu_release_mapping(mapping);

2055

arm_iommu_release_mapping(mapping);

2056

}

2056

}

2057

2058

#else

2058

#else

2059

2060

static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,

2060

static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,

2061

struct iommu_ops *iommu)

2061

struct iommu_ops *iommu)

2062

{

2062

{

2063

return false;

2063

return false;

2064

}

2064

}

2065

2066

static void arm_teardown_iommu_dma_ops(struct device *dev) { }

2066

static void arm_teardown_iommu_dma_ops(struct device *dev) { }

2067

2068

#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops

2068

#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops

2069

2070

#endif /* CONFIG_ARM_DMA_USE_IOMMU */

2070

#endif /* CONFIG_ARM_DMA_USE_IOMMU */

2071

2072

static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)

2072

static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)

2073

{

2073

{

2074

return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;

2074

return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;

2075

}

2075

}

2076

2077

void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,

2077

void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,

2078

struct iommu_ops *iommu, bool coherent)

2078

struct iommu_ops *iommu, bool coherent)

2079

{

2079

{

2080

struct dma_map_ops *dma_ops;

2080

struct dma_map_ops *dma_ops;

2081

2082

dev->archdata.dma_coherent = coherent;

2082

dev->archdata.dma_coherent = coherent;

2083

if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))

2083

if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))

2084

dma_ops = arm_get_iommu_dma_map_ops(coherent);

2084

dma_ops = arm_get_iommu_dma_map_ops(coherent);

2085

else

2085

else

2086

dma_ops = arm_get_dma_map_ops(coherent);

2086

dma_ops = arm_get_dma_map_ops(coherent);

2087

2088

set_dma_ops(dev, dma_ops);

2088

set_dma_ops(dev, dma_ops);

2089

}

2089

}

2090

2091

void arch_teardown_dma_ops(struct device *dev)

2091

void arch_teardown_dma_ops(struct device *dev)

2092

{

2092

{

2093

arm_teardown_iommu_dma_ops(dev);

2093

arm_teardown_iommu_dma_ops(dev);

2094

}

2094

}

2095

GITLAB

Merge branch 'fixes' of git://ftp.arm.linux.org.uk/~rmk/linux-arm