Blame view

kernel/dma/mapping.c 18.2 KB
989d42e85   Greg Kroah-Hartman   driver core: add ...
1
  // SPDX-License-Identifier: GPL-2.0
9ac7849e3   Tejun Heo   devres: device re...
2
  /*
cf65a0f6f   Christoph Hellwig   dma-mapping: move...
3
   * arch-independent dma-mapping routines
9ac7849e3   Tejun Heo   devres: device re...
4
5
6
   *
   * Copyright (c) 2006  SUSE Linux Products GmbH
   * Copyright (c) 2006  Tejun Heo <teheo@suse.de>
9ac7849e3   Tejun Heo   devres: device re...
7
   */
05887cb61   Christoph Hellwig   dma-mapping: move...
8
  #include <linux/memblock.h> /* for max_pfn */
09515ef5d   Sricharan R   of/acpi: Configur...
9
  #include <linux/acpi.h>
0a0f0d8be   Christoph Hellwig   dma-mapping: spli...
10
  #include <linux/dma-map-ops.h>
1b6bc32f0   Paul Gortmaker   drivers/base: Add...
11
  #include <linux/export.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
12
  #include <linux/gfp.h>
09515ef5d   Sricharan R   of/acpi: Configur...
13
  #include <linux/of_device.h>
513510ddb   Laura Abbott   common: dma-mappi...
14
15
  #include <linux/slab.h>
  #include <linux/vmalloc.h>
a1fd09e8e   Christoph Hellwig   dma-mapping: move...
16
  #include "debug.h"
19c65c3d3   Christoph Hellwig   dma-mapping: move...
17
  #include "direct.h"
9ac7849e3   Tejun Heo   devres: device re...
18
19
20
21
22
23
24
25
  
  /*
   * Managed DMA API
   */
  struct dma_devres {
  	size_t		size;
  	void		*vaddr;
  	dma_addr_t	dma_handle;
63d36c955   Christoph Hellwig   dma-mapping: repl...
26
  	unsigned long	attrs;
9ac7849e3   Tejun Heo   devres: device re...
27
  };
63d36c955   Christoph Hellwig   dma-mapping: repl...
28
  static void dmam_release(struct device *dev, void *res)
9ac7849e3   Tejun Heo   devres: device re...
29
30
  {
  	struct dma_devres *this = res;
63d36c955   Christoph Hellwig   dma-mapping: repl...
31
32
  	dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,
  			this->attrs);
9ac7849e3   Tejun Heo   devres: device re...
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
  }
  
  static int dmam_match(struct device *dev, void *res, void *match_data)
  {
  	struct dma_devres *this = res, *match = match_data;
  
  	if (this->vaddr == match->vaddr) {
  		WARN_ON(this->size != match->size ||
  			this->dma_handle != match->dma_handle);
  		return 1;
  	}
  	return 0;
  }
  
  /**
9ac7849e3   Tejun Heo   devres: device re...
48
49
50
51
52
53
54
55
56
57
58
59
60
61
   * dmam_free_coherent - Managed dma_free_coherent()
   * @dev: Device to free coherent memory for
   * @size: Size of allocation
   * @vaddr: Virtual address of the memory to free
   * @dma_handle: DMA handle of the memory to free
   *
   * Managed dma_free_coherent().
   */
  void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
  			dma_addr_t dma_handle)
  {
  	struct dma_devres match_data = { size, vaddr, dma_handle };
  
  	dma_free_coherent(dev, size, vaddr, dma_handle);
63d36c955   Christoph Hellwig   dma-mapping: repl...
62
  	WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data));
9ac7849e3   Tejun Heo   devres: device re...
63
64
65
66
  }
  EXPORT_SYMBOL(dmam_free_coherent);
  
  /**
63d36c955   Christoph Hellwig   dma-mapping: repl...
67
   * dmam_alloc_attrs - Managed dma_alloc_attrs()
9ac7849e3   Tejun Heo   devres: device re...
68
69
70
71
   * @dev: Device to allocate non_coherent memory for
   * @size: Size of allocation
   * @dma_handle: Out argument for allocated DMA handle
   * @gfp: Allocation flags
63d36c955   Christoph Hellwig   dma-mapping: repl...
72
   * @attrs: Flags in the DMA_ATTR_* namespace.
9ac7849e3   Tejun Heo   devres: device re...
73
   *
63d36c955   Christoph Hellwig   dma-mapping: repl...
74
75
   * Managed dma_alloc_attrs().  Memory allocated using this function will be
   * automatically released on driver detach.
9ac7849e3   Tejun Heo   devres: device re...
76
77
78
79
   *
   * RETURNS:
   * Pointer to allocated memory on success, NULL on failure.
   */
63d36c955   Christoph Hellwig   dma-mapping: repl...
80
81
  void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
  		gfp_t gfp, unsigned long attrs)
9ac7849e3   Tejun Heo   devres: device re...
82
83
84
  {
  	struct dma_devres *dr;
  	void *vaddr;
63d36c955   Christoph Hellwig   dma-mapping: repl...
85
  	dr = devres_alloc(dmam_release, sizeof(*dr), gfp);
9ac7849e3   Tejun Heo   devres: device re...
86
87
  	if (!dr)
  		return NULL;
63d36c955   Christoph Hellwig   dma-mapping: repl...
88
  	vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);
9ac7849e3   Tejun Heo   devres: device re...
89
90
91
92
93
94
95
96
  	if (!vaddr) {
  		devres_free(dr);
  		return NULL;
  	}
  
  	dr->vaddr = vaddr;
  	dr->dma_handle = *dma_handle;
  	dr->size = size;
63d36c955   Christoph Hellwig   dma-mapping: repl...
97
  	dr->attrs = attrs;
9ac7849e3   Tejun Heo   devres: device re...
98
99
100
101
102
  
  	devres_add(dev, dr);
  
  	return vaddr;
  }
63d36c955   Christoph Hellwig   dma-mapping: repl...
103
  EXPORT_SYMBOL(dmam_alloc_attrs);
9ac7849e3   Tejun Heo   devres: device re...
104

d35834c64   Christoph Hellwig   dma-mapping: add ...
105
106
  static bool dma_go_direct(struct device *dev, dma_addr_t mask,
  		const struct dma_map_ops *ops)
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
107
  {
d35834c64   Christoph Hellwig   dma-mapping: add ...
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
  	if (likely(!ops))
  		return true;
  #ifdef CONFIG_DMA_OPS_BYPASS
  	if (dev->dma_ops_bypass)
  		return min_not_zero(mask, dev->bus_dma_limit) >=
  			    dma_direct_get_required_mask(dev);
  #endif
  	return false;
  }
  
  
  /*
   * Check if the devices uses a direct mapping for streaming DMA operations.
   * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
   * enough.
   */
  static inline bool dma_alloc_direct(struct device *dev,
  		const struct dma_map_ops *ops)
  {
  	return dma_go_direct(dev, dev->coherent_dma_mask, ops);
  }
  
  static inline bool dma_map_direct(struct device *dev,
  		const struct dma_map_ops *ops)
  {
  	return dma_go_direct(dev, *dev->dma_mask, ops);
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
134
135
136
137
138
139
140
141
142
143
  }
  
  dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
  		size_t offset, size_t size, enum dma_data_direction dir,
  		unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	dma_addr_t addr;
  
  	BUG_ON(!valid_dma_direction(dir));
f959dcd6d   Thomas Tai   dma-direct: Fix p...
144
145
146
  
  	if (WARN_ON_ONCE(!dev->dma_mask))
  		return DMA_MAPPING_ERROR;
d35834c64   Christoph Hellwig   dma-mapping: add ...
147
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
  		addr = dma_direct_map_page(dev, page, offset, size, dir, attrs);
  	else
  		addr = ops->map_page(dev, page, offset, size, dir, attrs);
  	debug_dma_map_page(dev, page, offset, size, dir, addr);
  
  	return addr;
  }
  EXPORT_SYMBOL(dma_map_page_attrs);
  
  void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
  		enum dma_data_direction dir, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
163
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
  		dma_direct_unmap_page(dev, addr, size, dir, attrs);
  	else if (ops->unmap_page)
  		ops->unmap_page(dev, addr, size, dir, attrs);
  	debug_dma_unmap_page(dev, addr, size, dir);
  }
  EXPORT_SYMBOL(dma_unmap_page_attrs);
  
  /*
   * dma_maps_sg_attrs returns 0 on error and > 0 on success.
   * It should never return a value < 0.
   */
  int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents,
  		enum dma_data_direction dir, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	int ents;
  
  	BUG_ON(!valid_dma_direction(dir));
f959dcd6d   Thomas Tai   dma-direct: Fix p...
182
183
184
  
  	if (WARN_ON_ONCE(!dev->dma_mask))
  		return 0;
d35834c64   Christoph Hellwig   dma-mapping: add ...
185
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
  		ents = dma_direct_map_sg(dev, sg, nents, dir, attrs);
  	else
  		ents = ops->map_sg(dev, sg, nents, dir, attrs);
  	BUG_ON(ents < 0);
  	debug_dma_map_sg(dev, sg, nents, ents, dir);
  
  	return ents;
  }
  EXPORT_SYMBOL(dma_map_sg_attrs);
  
  void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
  				      int nents, enum dma_data_direction dir,
  				      unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
  	debug_dma_unmap_sg(dev, sg, nents, dir);
d35834c64   Christoph Hellwig   dma-mapping: add ...
204
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
205
206
207
208
209
210
211
212
213
214
215
216
217
  		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
  	else if (ops->unmap_sg)
  		ops->unmap_sg(dev, sg, nents, dir, attrs);
  }
  EXPORT_SYMBOL(dma_unmap_sg_attrs);
  
  dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
  		size_t size, enum dma_data_direction dir, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	dma_addr_t addr = DMA_MAPPING_ERROR;
  
  	BUG_ON(!valid_dma_direction(dir));
f959dcd6d   Thomas Tai   dma-direct: Fix p...
218
219
  	if (WARN_ON_ONCE(!dev->dma_mask))
  		return DMA_MAPPING_ERROR;
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
220
221
222
  	/* Don't allow RAM to be mapped */
  	if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
  		return DMA_MAPPING_ERROR;
d35834c64   Christoph Hellwig   dma-mapping: add ...
223
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
  		addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
  	else if (ops->map_resource)
  		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);
  
  	debug_dma_map_resource(dev, phys_addr, size, dir, addr);
  	return addr;
  }
  EXPORT_SYMBOL(dma_map_resource);
  
  void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
  		enum dma_data_direction dir, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
239
  	if (!dma_map_direct(dev, ops) && ops->unmap_resource)
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
240
241
242
243
244
245
246
247
248
249
250
  		ops->unmap_resource(dev, addr, size, dir, attrs);
  	debug_dma_unmap_resource(dev, addr, size, dir);
  }
  EXPORT_SYMBOL(dma_unmap_resource);
  
  void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
  		enum dma_data_direction dir)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
251
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
252
253
254
255
256
257
258
259
260
261
262
263
264
  		dma_direct_sync_single_for_cpu(dev, addr, size, dir);
  	else if (ops->sync_single_for_cpu)
  		ops->sync_single_for_cpu(dev, addr, size, dir);
  	debug_dma_sync_single_for_cpu(dev, addr, size, dir);
  }
  EXPORT_SYMBOL(dma_sync_single_for_cpu);
  
  void dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
  		size_t size, enum dma_data_direction dir)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
265
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
266
267
268
269
270
271
272
273
274
275
276
277
278
  		dma_direct_sync_single_for_device(dev, addr, size, dir);
  	else if (ops->sync_single_for_device)
  		ops->sync_single_for_device(dev, addr, size, dir);
  	debug_dma_sync_single_for_device(dev, addr, size, dir);
  }
  EXPORT_SYMBOL(dma_sync_single_for_device);
  
  void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
  		    int nelems, enum dma_data_direction dir)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
279
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
280
281
282
283
284
285
286
287
288
289
290
291
292
  		dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir);
  	else if (ops->sync_sg_for_cpu)
  		ops->sync_sg_for_cpu(dev, sg, nelems, dir);
  	debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
  }
  EXPORT_SYMBOL(dma_sync_sg_for_cpu);
  
  void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
  		       int nelems, enum dma_data_direction dir)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	BUG_ON(!valid_dma_direction(dir));
d35834c64   Christoph Hellwig   dma-mapping: add ...
293
  	if (dma_map_direct(dev, ops))
d3fa60d7b   Christoph Hellwig   dma-mapping: move...
294
295
296
297
298
299
  		dma_direct_sync_sg_for_device(dev, sg, nelems, dir);
  	else if (ops->sync_sg_for_device)
  		ops->sync_sg_for_device(dev, sg, nelems, dir);
  	debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
  }
  EXPORT_SYMBOL(dma_sync_sg_for_device);
d2b7428eb   Marek Szyprowski   common: dma-mappi...
300
  /*
144514670   Christoph Hellwig   dma-mapping: move...
301
302
303
304
305
306
307
308
309
310
   * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems
   * that the intention is to allow exporting memory allocated via the
   * coherent DMA APIs through the dma_buf API, which only accepts a
   * scattertable.  This presents a couple of problems:
   * 1. Not all memory allocated via the coherent DMA APIs is backed by
   *    a struct page
   * 2. Passing coherent DMA memory into the streaming APIs is not allowed
   *    as we will try to flush the memory through a different alias to that
   *    actually being used (and the flushes are redundant.)
   */
7249c1a52   Christoph Hellwig   dma-mapping: move...
311
312
313
314
315
  int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
  		void *cpu_addr, dma_addr_t dma_addr, size_t size,
  		unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
316

d35834c64   Christoph Hellwig   dma-mapping: add ...
317
  	if (dma_alloc_direct(dev, ops))
34dc0ea6b   Christoph Hellwig   dma-direct: provi...
318
  		return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr,
f9f3232a7   Christoph Hellwig   dma-mapping: expl...
319
320
321
322
  				size, attrs);
  	if (!ops->get_sgtable)
  		return -ENXIO;
  	return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);
7249c1a52   Christoph Hellwig   dma-mapping: move...
323
324
  }
  EXPORT_SYMBOL(dma_get_sgtable_attrs);
d2b7428eb   Marek Szyprowski   common: dma-mappi...
325

33dcb37ce   Christoph Hellwig   dma-mapping: fix ...
326
327
328
329
330
331
332
  #ifdef CONFIG_MMU
  /*
   * Return the page attributes used for mapping dma_alloc_* memory, either in
   * kernel space if remapping is needed, or to userspace through dma_mmap_*.
   */
  pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
  {
17c4a2ae1   Thomas Hellstrom   dma-mapping: Fix ...
333
334
  	if (force_dma_unencrypted(dev))
  		prot = pgprot_decrypted(prot);
efa70f2fd   Christoph Hellwig   dma-mapping: add ...
335
  	if (dev_is_dma_coherent(dev))
33dcb37ce   Christoph Hellwig   dma-mapping: fix ...
336
  		return prot;
419e2f183   Christoph Hellwig   dma-mapping: remo...
337
338
339
340
  #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
  	if (attrs & DMA_ATTR_WRITE_COMBINE)
  		return pgprot_writecombine(prot);
  #endif
7cfc6861b   Isaac J. Manjarres   ANDROID: iommu/dm...
341
342
  	if (attrs & DMA_ATTR_SYS_CACHE_ONLY ||
  	    attrs & DMA_ATTR_SYS_CACHE_ONLY_NWA)
0b653f27b   Isaac J. Manjarres   ANDROID: iommu/dm...
343
  		return pgprot_syscached(prot);
419e2f183   Christoph Hellwig   dma-mapping: remo...
344
  	return pgprot_dmacoherent(prot);
33dcb37ce   Christoph Hellwig   dma-mapping: fix ...
345
346
  }
  #endif /* CONFIG_MMU */
7249c1a52   Christoph Hellwig   dma-mapping: move...
347
  /**
e29ccc188   Christoph Hellwig   dma-mapping: add ...
348
349
350
351
352
353
354
355
356
   * dma_can_mmap - check if a given device supports dma_mmap_*
   * @dev: device to check
   *
   * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to
   * map DMA allocations to userspace.
   */
  bool dma_can_mmap(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
d35834c64   Christoph Hellwig   dma-mapping: add ...
357
  	if (dma_alloc_direct(dev, ops))
34dc0ea6b   Christoph Hellwig   dma-direct: provi...
358
  		return dma_direct_can_mmap(dev);
e29ccc188   Christoph Hellwig   dma-mapping: add ...
359
360
361
362
363
  	return ops->mmap != NULL;
  }
  EXPORT_SYMBOL_GPL(dma_can_mmap);
  
  /**
7249c1a52   Christoph Hellwig   dma-mapping: move...
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
   * dma_mmap_attrs - map a coherent DMA allocation into user space
   * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
   * @vma: vm_area_struct describing requested user mapping
   * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
   * @dma_addr: device-view address returned from dma_alloc_attrs
   * @size: size of memory originally requested in dma_alloc_attrs
   * @attrs: attributes of mapping properties requested in dma_alloc_attrs
   *
   * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user
   * space.  The coherent DMA buffer must not be freed by the driver until the
   * user space mapping has been released.
   */
  int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
  		void *cpu_addr, dma_addr_t dma_addr, size_t size,
  		unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
381

d35834c64   Christoph Hellwig   dma-mapping: add ...
382
  	if (dma_alloc_direct(dev, ops))
34dc0ea6b   Christoph Hellwig   dma-direct: provi...
383
  		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,
f9f3232a7   Christoph Hellwig   dma-mapping: expl...
384
385
386
387
  				attrs);
  	if (!ops->mmap)
  		return -ENXIO;
  	return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
7249c1a52   Christoph Hellwig   dma-mapping: move...
388
389
  }
  EXPORT_SYMBOL(dma_mmap_attrs);
05887cb61   Christoph Hellwig   dma-mapping: move...
390

05887cb61   Christoph Hellwig   dma-mapping: move...
391
392
393
  u64 dma_get_required_mask(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
d35834c64   Christoph Hellwig   dma-mapping: add ...
394
  	if (dma_alloc_direct(dev, ops))
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
395
  		return dma_direct_get_required_mask(dev);
05887cb61   Christoph Hellwig   dma-mapping: move...
396
397
  	if (ops->get_required_mask)
  		return ops->get_required_mask(dev);
249baa547   Christoph Hellwig   dma-mapping: prov...
398
399
400
401
402
403
404
405
406
407
  
  	/*
  	 * We require every DMA ops implementation to at least support a 32-bit
  	 * DMA mask (and use bounce buffering if that isn't supported in
  	 * hardware).  As the direct mapping code has its own routine to
  	 * actually report an optimal mask we default to 32-bit here as that
  	 * is the right thing for most IOMMUs, and at least not actively
  	 * harmful in general.
  	 */
  	return DMA_BIT_MASK(32);
05887cb61   Christoph Hellwig   dma-mapping: move...
408
409
  }
  EXPORT_SYMBOL_GPL(dma_get_required_mask);
05887cb61   Christoph Hellwig   dma-mapping: move...
410

7249c1a52   Christoph Hellwig   dma-mapping: move...
411
412
413
414
415
  void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
  		gfp_t flag, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	void *cpu_addr;
148a97d5a   Dan Carpenter   dma-mapping: remo...
416
  	WARN_ON_ONCE(!dev->coherent_dma_mask);
7249c1a52   Christoph Hellwig   dma-mapping: move...
417
418
419
420
421
422
  
  	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
  		return cpu_addr;
  
  	/* let the implementation decide on the zone to allocate from: */
  	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
d35834c64   Christoph Hellwig   dma-mapping: add ...
423
  	if (dma_alloc_direct(dev, ops))
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
424
425
426
427
  		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
  	else if (ops->alloc)
  		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
  	else
7249c1a52   Christoph Hellwig   dma-mapping: move...
428
  		return NULL;
7249c1a52   Christoph Hellwig   dma-mapping: move...
429
430
431
432
433
434
435
436
437
  	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
  	return cpu_addr;
  }
  EXPORT_SYMBOL(dma_alloc_attrs);
  
  void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
  		dma_addr_t dma_handle, unsigned long attrs)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
7249c1a52   Christoph Hellwig   dma-mapping: move...
438
439
440
441
442
443
444
445
446
447
  	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))
  		return;
  	/*
  	 * On non-coherent platforms which implement DMA-coherent buffers via
  	 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting
  	 * this far in IRQ context is a) at risk of a BUG_ON() or trying to
  	 * sleep on some machines, and b) an indication that the driver is
  	 * probably misusing the coherent API anyway.
  	 */
  	WARN_ON(irqs_disabled());
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
448
  	if (!cpu_addr)
7249c1a52   Christoph Hellwig   dma-mapping: move...
449
450
451
  		return;
  
  	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
d35834c64   Christoph Hellwig   dma-mapping: add ...
452
  	if (dma_alloc_direct(dev, ops))
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
453
454
455
  		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
  	else if (ops->free)
  		ops->free(dev, size, cpu_addr, dma_handle, attrs);
7249c1a52   Christoph Hellwig   dma-mapping: move...
456
457
  }
  EXPORT_SYMBOL(dma_free_attrs);
efa70f2fd   Christoph Hellwig   dma-mapping: add ...
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
  struct page *dma_alloc_pages(struct device *dev, size_t size,
  		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	struct page *page;
  
  	if (WARN_ON_ONCE(!dev->coherent_dma_mask))
  		return NULL;
  	if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)))
  		return NULL;
  
  	size = PAGE_ALIGN(size);
  	if (dma_alloc_direct(dev, ops))
  		page = dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);
  	else if (ops->alloc_pages)
  		page = ops->alloc_pages(dev, size, dma_handle, dir, gfp);
  	else
  		return NULL;
  
  	debug_dma_map_page(dev, page, 0, size, dir, *dma_handle);
  
  	return page;
  }
  EXPORT_SYMBOL_GPL(dma_alloc_pages);
  
  void dma_free_pages(struct device *dev, size_t size, struct page *page,
  		dma_addr_t dma_handle, enum dma_data_direction dir)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	size = PAGE_ALIGN(size);
  	debug_dma_unmap_page(dev, dma_handle, size, dir);
  
  	if (dma_alloc_direct(dev, ops))
  		dma_direct_free_pages(dev, size, page, dma_handle, dir);
  	else if (ops->free_pages)
  		ops->free_pages(dev, size, page, dma_handle, dir);
  }
  EXPORT_SYMBOL_GPL(dma_free_pages);
  
  void *dma_alloc_noncoherent(struct device *dev, size_t size,
  		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
  {
de7cf9177   Christoph Hellwig   dma-mapping: add ...
501
502
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	void *vaddr;
efa70f2fd   Christoph Hellwig   dma-mapping: add ...
503

de7cf9177   Christoph Hellwig   dma-mapping: add ...
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
  	if (!ops || !ops->alloc_noncoherent) {
  		struct page *page;
  
  		page = dma_alloc_pages(dev, size, dma_handle, dir, gfp);
  		if (!page)
  			return NULL;
  		return page_address(page);
  	}
  
  	size = PAGE_ALIGN(size);
  	vaddr = ops->alloc_noncoherent(dev, size, dma_handle, dir, gfp);
  	if (vaddr)
  		debug_dma_map_page(dev, virt_to_page(vaddr), 0, size, dir,
  				   *dma_handle);
  	return vaddr;
efa70f2fd   Christoph Hellwig   dma-mapping: add ...
519
520
521
522
523
524
  }
  EXPORT_SYMBOL_GPL(dma_alloc_noncoherent);
  
  void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
  		dma_addr_t dma_handle, enum dma_data_direction dir)
  {
de7cf9177   Christoph Hellwig   dma-mapping: add ...
525
526
527
528
529
530
531
532
533
534
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	if (!ops || !ops->free_noncoherent) {
  		dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir);
  		return;
  	}
  
  	size = PAGE_ALIGN(size);
  	debug_dma_unmap_page(dev, dma_handle, size, dir);
  	ops->free_noncoherent(dev, size, vaddr, dma_handle, dir);
efa70f2fd   Christoph Hellwig   dma-mapping: add ...
535
536
  }
  EXPORT_SYMBOL_GPL(dma_free_noncoherent);
7249c1a52   Christoph Hellwig   dma-mapping: move...
537
538
539
  int dma_supported(struct device *dev, u64 mask)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
d35834c64   Christoph Hellwig   dma-mapping: add ...
540
541
542
543
544
  	/*
  	 * ->dma_supported sets the bypass flag, so we must always call
  	 * into the method here unless the device is truly direct mapped.
  	 */
  	if (!ops)
356da6d0c   Christoph Hellwig   dma-mapping: bypa...
545
  		return dma_direct_supported(dev, mask);
8b1cce9f5   Thierry Reding   dma-mapping: fix ...
546
  	if (!ops->dma_supported)
7249c1a52   Christoph Hellwig   dma-mapping: move...
547
548
549
550
  		return 1;
  	return ops->dma_supported(dev, mask);
  }
  EXPORT_SYMBOL(dma_supported);
11ddce154   Christoph Hellwig   dma-mapping, powe...
551
552
553
554
555
  #ifdef CONFIG_ARCH_HAS_DMA_SET_MASK
  void arch_dma_set_mask(struct device *dev, u64 mask);
  #else
  #define arch_dma_set_mask(dev, mask)	do { } while (0)
  #endif
7249c1a52   Christoph Hellwig   dma-mapping: move...
556
557
  int dma_set_mask(struct device *dev, u64 mask)
  {
4a54d16f6   Christoph Hellwig   dma-mapping: trun...
558
559
560
561
562
  	/*
  	 * Truncate the mask to the actually supported dma_addr_t width to
  	 * avoid generating unsupportable addresses.
  	 */
  	mask = (dma_addr_t)mask;
7249c1a52   Christoph Hellwig   dma-mapping: move...
563
564
  	if (!dev->dma_mask || !dma_supported(dev, mask))
  		return -EIO;
11ddce154   Christoph Hellwig   dma-mapping, powe...
565
  	arch_dma_set_mask(dev, mask);
7249c1a52   Christoph Hellwig   dma-mapping: move...
566
567
568
569
  	*dev->dma_mask = mask;
  	return 0;
  }
  EXPORT_SYMBOL(dma_set_mask);
7249c1a52   Christoph Hellwig   dma-mapping: move...
570
571
572
573
  
  #ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK
  int dma_set_coherent_mask(struct device *dev, u64 mask)
  {
4a54d16f6   Christoph Hellwig   dma-mapping: trun...
574
575
576
577
578
  	/*
  	 * Truncate the mask to the actually supported dma_addr_t width to
  	 * avoid generating unsupportable addresses.
  	 */
  	mask = (dma_addr_t)mask;
7249c1a52   Christoph Hellwig   dma-mapping: move...
579
580
  	if (!dma_supported(dev, mask))
  		return -EIO;
7249c1a52   Christoph Hellwig   dma-mapping: move...
581
582
583
584
585
  	dev->coherent_dma_mask = mask;
  	return 0;
  }
  EXPORT_SYMBOL(dma_set_coherent_mask);
  #endif
8ddbe5943   Christoph Hellwig   dma-mapping: move...
586

133d624b1   Joerg Roedel   dma: Introduce dm...
587
588
589
590
  size_t dma_max_mapping_size(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  	size_t size = SIZE_MAX;
d35834c64   Christoph Hellwig   dma-mapping: add ...
591
  	if (dma_map_direct(dev, ops))
133d624b1   Joerg Roedel   dma: Introduce dm...
592
593
594
595
596
597
598
  		size = dma_direct_max_mapping_size(dev);
  	else if (ops && ops->max_mapping_size)
  		size = ops->max_mapping_size(dev);
  
  	return size;
  }
  EXPORT_SYMBOL_GPL(dma_max_mapping_size);
6ba99411b   Yoshihiro Shimoda   dma-mapping: intr...
599

3aa916250   Christoph Hellwig   dma-mapping: Add ...
600
601
602
  bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
d35834c64   Christoph Hellwig   dma-mapping: add ...
603
  	if (dma_map_direct(dev, ops))
3aa916250   Christoph Hellwig   dma-mapping: Add ...
604
605
606
607
  		return dma_direct_need_sync(dev, dma_addr);
  	return ops->sync_single_for_cpu || ops->sync_single_for_device;
  }
  EXPORT_SYMBOL_GPL(dma_need_sync);
6ba99411b   Yoshihiro Shimoda   dma-mapping: intr...
608
609
610
611
612
613
614
615
616
617
  unsigned long dma_get_merge_boundary(struct device *dev)
  {
  	const struct dma_map_ops *ops = get_dma_ops(dev);
  
  	if (!ops || !ops->get_merge_boundary)
  		return 0;	/* can't merge */
  
  	return ops->get_merge_boundary(dev);
  }
  EXPORT_SYMBOL_GPL(dma_get_merge_boundary);