Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 22d48269984fc93a71f65a54aa422aacf5fdb926

Authored by Konrad Rzeszutek Wilk
1 parent bfc5501f6d
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

swiotlb: search and replace "int dir" with "enum dma_data_direction dir" 

.. to catch anybody doing something funky.

See "swiotlb: swiotlb: add swiotlb_tbl_map_single library function" for
full description of patchset.

[v2: swiotlb_sync_single_range_* no more - removed usage]
[v3: enum dma_data_direction direction -> enum dma_data_direction dir]

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Acked-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp>
Tested-by: Albert Herranz <albert_herranz@yahoo.es>

Showing 2 changed files with 15 additions and 12 deletions Inline Diff

include/linux/swiotlb.h
Diff comments   View file @ 22d4826
1 #ifndef __LINUX_SWIOTLB_H 1 #ifndef __LINUX_SWIOTLB_H
2 #define __LINUX_SWIOTLB_H 2 #define __LINUX_SWIOTLB_H
3 3
4 #include <linux/types.h> 4 #include <linux/types.h>
5 5
6 struct device; 6 struct device;
7 struct dma_attrs; 7 struct dma_attrs;
8 struct scatterlist; 8 struct scatterlist;
9 9
10 extern int swiotlb_force; 10 extern int swiotlb_force;
11 11
12 /* 12 /*
13 * Maximum allowable number of contiguous slabs to map, 13 * Maximum allowable number of contiguous slabs to map,
14 * must be a power of 2. What is the appropriate value ? 14 * must be a power of 2. What is the appropriate value ?
15 * The complexity of {map,unmap}_single is linearly dependent on this value. 15 * The complexity of {map,unmap}_single is linearly dependent on this value.
16 */ 16 */
17 #define IO_TLB_SEGSIZE 128 17 #define IO_TLB_SEGSIZE 128
18 18
19 /* 19 /*
20 * log of the size of each IO TLB slab. The number of slabs is command line 20 * log of the size of each IO TLB slab. The number of slabs is command line
21 * controllable. 21 * controllable.
22 */ 22 */
23 #define IO_TLB_SHIFT 11 23 #define IO_TLB_SHIFT 11
24 24
25 extern void swiotlb_init(int verbose); 25 extern void swiotlb_init(int verbose);
26 extern void swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose); 26 extern void swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose);
27 27
28 extern void 28 extern void
29 *swiotlb_alloc_coherent(struct device *hwdev, size_t size, 29 *swiotlb_alloc_coherent(struct device *hwdev, size_t size,
30 dma_addr_t *dma_handle, gfp_t flags); 30 dma_addr_t *dma_handle, gfp_t flags);
31 31
32 extern void 32 extern void
33 swiotlb_free_coherent(struct device *hwdev, size_t size, 33 swiotlb_free_coherent(struct device *hwdev, size_t size,
34 void *vaddr, dma_addr_t dma_handle); 34 void *vaddr, dma_addr_t dma_handle);
35 35
36 extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, 36 extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
37 unsigned long offset, size_t size, 37 unsigned long offset, size_t size,
38 enum dma_data_direction dir, 38 enum dma_data_direction dir,
39 struct dma_attrs *attrs); 39 struct dma_attrs *attrs);
40 extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, 40 extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
41 size_t size, enum dma_data_direction dir, 41 size_t size, enum dma_data_direction dir,
42 struct dma_attrs *attrs); 42 struct dma_attrs *attrs);
43 43
44 extern int 44 extern int
45 swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nents, 45 swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
46 int direction); 46 enum dma_data_direction dir);
47 47
48 extern void 48 extern void
49 swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, 49 swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
50 int direction); 50 enum dma_data_direction dir);
51 51
52 extern int 52 extern int
53 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, 53 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
54 enum dma_data_direction dir, struct dma_attrs *attrs); 54 enum dma_data_direction dir, struct dma_attrs *attrs);
55 55
56 extern void 56 extern void
57 swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 57 swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
58 int nelems, enum dma_data_direction dir, 58 int nelems, enum dma_data_direction dir,
59 struct dma_attrs *attrs); 59 struct dma_attrs *attrs);
60 60
61 extern void 61 extern void
62 swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, 62 swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
63 size_t size, enum dma_data_direction dir); 63 size_t size, enum dma_data_direction dir);
64 64
65 extern void 65 extern void
66 swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, 66 swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
67 int nelems, enum dma_data_direction dir); 67 int nelems, enum dma_data_direction dir);
68 68
69 extern void 69 extern void
70 swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, 70 swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
71 size_t size, enum dma_data_direction dir); 71 size_t size, enum dma_data_direction dir);
72 72
73 extern void 73 extern void
74 swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, 74 swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
75 int nelems, enum dma_data_direction dir); 75 int nelems, enum dma_data_direction dir);
76 76
77 extern int 77 extern int
78 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr); 78 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr);
79 79
80 extern int 80 extern int
81 swiotlb_dma_supported(struct device *hwdev, u64 mask); 81 swiotlb_dma_supported(struct device *hwdev, u64 mask);
82 82
83 #ifdef CONFIG_SWIOTLB 83 #ifdef CONFIG_SWIOTLB
84 extern void __init swiotlb_free(void); 84 extern void __init swiotlb_free(void);
85 #else 85 #else
86 static inline void swiotlb_free(void) { } 86 static inline void swiotlb_free(void) { }
87 #endif 87 #endif
88 88
89 extern void swiotlb_print_info(void); 89 extern void swiotlb_print_info(void);
90 #endif /* __LINUX_SWIOTLB_H */ 90 #endif /* __LINUX_SWIOTLB_H */
91 91
1 /* 1 /*
2 * Dynamic DMA mapping support. 2 * Dynamic DMA mapping support.
3 * 3 *
4 * This implementation is a fallback for platforms that do not support 4 * This implementation is a fallback for platforms that do not support
5 * I/O TLBs (aka DMA address translation hardware). 5 * I/O TLBs (aka DMA address translation hardware).
6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com> 6 * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com> 7 * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
8 * Copyright (C) 2000, 2003 Hewlett-Packard Co 8 * Copyright (C) 2000, 2003 Hewlett-Packard Co
9 * David Mosberger-Tang <davidm@hpl.hp.com> 9 * David Mosberger-Tang <davidm@hpl.hp.com>
10 * 10 *
11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API. 11 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid 12 * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
13 * unnecessary i-cache flushing. 13 * unnecessary i-cache flushing.
14 * 04/07/.. ak Better overflow handling. Assorted fixes. 14 * 04/07/.. ak Better overflow handling. Assorted fixes.
15 * 05/09/10 linville Add support for syncing ranges, support syncing for 15 * 05/09/10 linville Add support for syncing ranges, support syncing for
16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup. 16 * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
17 * 08/12/11 beckyb Add highmem support 17 * 08/12/11 beckyb Add highmem support
18 */ 18 */
19 19
20 #include <linux/cache.h> 20 #include <linux/cache.h>
21 #include <linux/dma-mapping.h> 21 #include <linux/dma-mapping.h>
22 #include <linux/mm.h> 22 #include <linux/mm.h>
23 #include <linux/module.h> 23 #include <linux/module.h>
24 #include <linux/spinlock.h> 24 #include <linux/spinlock.h>
25 #include <linux/string.h> 25 #include <linux/string.h>
26 #include <linux/swiotlb.h> 26 #include <linux/swiotlb.h>
27 #include <linux/pfn.h> 27 #include <linux/pfn.h>
28 #include <linux/types.h> 28 #include <linux/types.h>
29 #include <linux/ctype.h> 29 #include <linux/ctype.h>
30 #include <linux/highmem.h> 30 #include <linux/highmem.h>
31 #include <linux/gfp.h> 31 #include <linux/gfp.h>
32 32
33 #include <asm/io.h> 33 #include <asm/io.h>
34 #include <asm/dma.h> 34 #include <asm/dma.h>
35 #include <asm/scatterlist.h> 35 #include <asm/scatterlist.h>
36 36
37 #include <linux/init.h> 37 #include <linux/init.h>
38 #include <linux/bootmem.h> 38 #include <linux/bootmem.h>
39 #include <linux/iommu-helper.h> 39 #include <linux/iommu-helper.h>
40 40
41 #define OFFSET(val,align) ((unsigned long) \ 41 #define OFFSET(val,align) ((unsigned long) \
42 ( (val) & ( (align) - 1))) 42 ( (val) & ( (align) - 1)))
43 43
44 #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) 44 #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
45 45
46 /* 46 /*
47 * Minimum IO TLB size to bother booting with. Systems with mainly 47 * Minimum IO TLB size to bother booting with. Systems with mainly
48 * 64bit capable cards will only lightly use the swiotlb. If we can't 48 * 64bit capable cards will only lightly use the swiotlb. If we can't
49 * allocate a contiguous 1MB, we're probably in trouble anyway. 49 * allocate a contiguous 1MB, we're probably in trouble anyway.
50 */ 50 */
51 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) 51 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
52 52
53 /* 53 /*
54 * Enumeration for sync targets 54 * Enumeration for sync targets
55 */ 55 */
56 enum dma_sync_target { 56 enum dma_sync_target {
57 SYNC_FOR_CPU = 0, 57 SYNC_FOR_CPU = 0,
58 SYNC_FOR_DEVICE = 1, 58 SYNC_FOR_DEVICE = 1,
59 }; 59 };
60 60
61 int swiotlb_force; 61 int swiotlb_force;
62 62
63 /* 63 /*
64 * Used to do a quick range check in swiotlb_tbl_unmap_single and 64 * Used to do a quick range check in swiotlb_tbl_unmap_single and
65 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this 65 * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
66 * API. 66 * API.
67 */ 67 */
68 static char *io_tlb_start, *io_tlb_end; 68 static char *io_tlb_start, *io_tlb_end;
69 69
70 /* 70 /*
71 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and 71 * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
72 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages. 72 * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
73 */ 73 */
74 static unsigned long io_tlb_nslabs; 74 static unsigned long io_tlb_nslabs;
75 75
76 /* 76 /*
77 * When the IOMMU overflows we return a fallback buffer. This sets the size. 77 * When the IOMMU overflows we return a fallback buffer. This sets the size.
78 */ 78 */
79 static unsigned long io_tlb_overflow = 32*1024; 79 static unsigned long io_tlb_overflow = 32*1024;
80 80
81 void *io_tlb_overflow_buffer; 81 void *io_tlb_overflow_buffer;
82 82
83 /* 83 /*
84 * This is a free list describing the number of free entries available from 84 * This is a free list describing the number of free entries available from
85 * each index 85 * each index
86 */ 86 */
87 static unsigned int *io_tlb_list; 87 static unsigned int *io_tlb_list;
88 static unsigned int io_tlb_index; 88 static unsigned int io_tlb_index;
89 89
90 /* 90 /*
91 * We need to save away the original address corresponding to a mapped entry 91 * We need to save away the original address corresponding to a mapped entry
92 * for the sync operations. 92 * for the sync operations.
93 */ 93 */
94 static phys_addr_t *io_tlb_orig_addr; 94 static phys_addr_t *io_tlb_orig_addr;
95 95
96 /* 96 /*
97 * Protect the above data structures in the map and unmap calls 97 * Protect the above data structures in the map and unmap calls
98 */ 98 */
99 static DEFINE_SPINLOCK(io_tlb_lock); 99 static DEFINE_SPINLOCK(io_tlb_lock);
100 100
101 static int late_alloc; 101 static int late_alloc;
102 102
103 static int __init 103 static int __init
104 setup_io_tlb_npages(char *str) 104 setup_io_tlb_npages(char *str)
105 { 105 {
106 if (isdigit(*str)) { 106 if (isdigit(*str)) {
107 io_tlb_nslabs = simple_strtoul(str, &str, 0); 107 io_tlb_nslabs = simple_strtoul(str, &str, 0);
108 /* avoid tail segment of size < IO_TLB_SEGSIZE */ 108 /* avoid tail segment of size < IO_TLB_SEGSIZE */
109 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE); 109 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
110 } 110 }
111 if (*str == ',') 111 if (*str == ',')
112 ++str; 112 ++str;
113 if (!strcmp(str, "force")) 113 if (!strcmp(str, "force"))
114 swiotlb_force = 1; 114 swiotlb_force = 1;
115 115
116 return 1; 116 return 1;
117 } 117 }
118 __setup("swiotlb=", setup_io_tlb_npages); 118 __setup("swiotlb=", setup_io_tlb_npages);
119 /* make io_tlb_overflow tunable too? */ 119 /* make io_tlb_overflow tunable too? */
120 120
121 /* Note that this doesn't work with highmem page */ 121 /* Note that this doesn't work with highmem page */
122 static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev, 122 static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
123 volatile void *address) 123 volatile void *address)
124 { 124 {
125 return phys_to_dma(hwdev, virt_to_phys(address)); 125 return phys_to_dma(hwdev, virt_to_phys(address));
126 } 126 }
127 127
128 void swiotlb_print_info(void) 128 void swiotlb_print_info(void)
129 { 129 {
130 unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT; 130 unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;
131 phys_addr_t pstart, pend; 131 phys_addr_t pstart, pend;
132 132
133 pstart = virt_to_phys(io_tlb_start); 133 pstart = virt_to_phys(io_tlb_start);
134 pend = virt_to_phys(io_tlb_end); 134 pend = virt_to_phys(io_tlb_end);
135 135
136 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n", 136 printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
137 bytes >> 20, io_tlb_start, io_tlb_end); 137 bytes >> 20, io_tlb_start, io_tlb_end);
138 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n", 138 printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
139 (unsigned long long)pstart, 139 (unsigned long long)pstart,
140 (unsigned long long)pend); 140 (unsigned long long)pend);
141 } 141 }
142 142
143 void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose) 143 void __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
144 { 144 {
145 unsigned long i, bytes; 145 unsigned long i, bytes;
146 146
147 bytes = nslabs << IO_TLB_SHIFT; 147 bytes = nslabs << IO_TLB_SHIFT;
148 148
149 io_tlb_nslabs = nslabs; 149 io_tlb_nslabs = nslabs;
150 io_tlb_start = tlb; 150 io_tlb_start = tlb;
151 io_tlb_end = io_tlb_start + bytes; 151 io_tlb_end = io_tlb_start + bytes;
152 152
153 /* 153 /*
154 * Allocate and initialize the free list array. This array is used 154 * Allocate and initialize the free list array. This array is used
155 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE 155 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
156 * between io_tlb_start and io_tlb_end. 156 * between io_tlb_start and io_tlb_end.
157 */ 157 */
158 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int)); 158 io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
159 for (i = 0; i < io_tlb_nslabs; i++) 159 for (i = 0; i < io_tlb_nslabs; i++)
160 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 160 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
161 io_tlb_index = 0; 161 io_tlb_index = 0;
162 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t)); 162 io_tlb_orig_addr = alloc_bootmem(io_tlb_nslabs * sizeof(phys_addr_t));
163 163
164 /* 164 /*
165 * Get the overflow emergency buffer 165 * Get the overflow emergency buffer
166 */ 166 */
167 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow); 167 io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
168 if (!io_tlb_overflow_buffer) 168 if (!io_tlb_overflow_buffer)
169 panic("Cannot allocate SWIOTLB overflow buffer!\n"); 169 panic("Cannot allocate SWIOTLB overflow buffer!\n");
170 if (verbose) 170 if (verbose)
171 swiotlb_print_info(); 171 swiotlb_print_info();
172 } 172 }
173 173
174 /* 174 /*
175 * Statically reserve bounce buffer space and initialize bounce buffer data 175 * Statically reserve bounce buffer space and initialize bounce buffer data
176 * structures for the software IO TLB used to implement the DMA API. 176 * structures for the software IO TLB used to implement the DMA API.
177 */ 177 */
178 void __init 178 void __init
179 swiotlb_init_with_default_size(size_t default_size, int verbose) 179 swiotlb_init_with_default_size(size_t default_size, int verbose)
180 { 180 {
181 unsigned long bytes; 181 unsigned long bytes;
182 182
183 if (!io_tlb_nslabs) { 183 if (!io_tlb_nslabs) {
184 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT); 184 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
185 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE); 185 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
186 } 186 }
187 187
188 bytes = io_tlb_nslabs << IO_TLB_SHIFT; 188 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
189 189
190 /* 190 /*
191 * Get IO TLB memory from the low pages 191 * Get IO TLB memory from the low pages
192 */ 192 */
193 io_tlb_start = alloc_bootmem_low_pages(bytes); 193 io_tlb_start = alloc_bootmem_low_pages(bytes);
194 if (!io_tlb_start) 194 if (!io_tlb_start)
195 panic("Cannot allocate SWIOTLB buffer"); 195 panic("Cannot allocate SWIOTLB buffer");
196 196
197 swiotlb_init_with_tbl(io_tlb_start, io_tlb_nslabs, verbose); 197 swiotlb_init_with_tbl(io_tlb_start, io_tlb_nslabs, verbose);
198 } 198 }
199 199
200 void __init 200 void __init
201 swiotlb_init(int verbose) 201 swiotlb_init(int verbose)
202 { 202 {
203 swiotlb_init_with_default_size(64 * (1<<20), verbose); /* default to 64MB */ 203 swiotlb_init_with_default_size(64 * (1<<20), verbose); /* default to 64MB */
204 } 204 }
205 205
206 /* 206 /*
207 * Systems with larger DMA zones (those that don't support ISA) can 207 * Systems with larger DMA zones (those that don't support ISA) can
208 * initialize the swiotlb later using the slab allocator if needed. 208 * initialize the swiotlb later using the slab allocator if needed.
209 * This should be just like above, but with some error catching. 209 * This should be just like above, but with some error catching.
210 */ 210 */
211 int 211 int
212 swiotlb_late_init_with_default_size(size_t default_size) 212 swiotlb_late_init_with_default_size(size_t default_size)
213 { 213 {
214 unsigned long i, bytes, req_nslabs = io_tlb_nslabs; 214 unsigned long i, bytes, req_nslabs = io_tlb_nslabs;
215 unsigned int order; 215 unsigned int order;
216 216
217 if (!io_tlb_nslabs) { 217 if (!io_tlb_nslabs) {
218 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT); 218 io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
219 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE); 219 io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
220 } 220 }
221 221
222 /* 222 /*
223 * Get IO TLB memory from the low pages 223 * Get IO TLB memory from the low pages
224 */ 224 */
225 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT); 225 order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
226 io_tlb_nslabs = SLABS_PER_PAGE << order; 226 io_tlb_nslabs = SLABS_PER_PAGE << order;
227 bytes = io_tlb_nslabs << IO_TLB_SHIFT; 227 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
228 228
229 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { 229 while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
230 io_tlb_start = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, 230 io_tlb_start = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
231 order); 231 order);
232 if (io_tlb_start) 232 if (io_tlb_start)
233 break; 233 break;
234 order--; 234 order--;
235 } 235 }
236 236
237 if (!io_tlb_start) 237 if (!io_tlb_start)
238 goto cleanup1; 238 goto cleanup1;
239 239
240 if (order != get_order(bytes)) { 240 if (order != get_order(bytes)) {
241 printk(KERN_WARNING "Warning: only able to allocate %ld MB " 241 printk(KERN_WARNING "Warning: only able to allocate %ld MB "
242 "for software IO TLB\n", (PAGE_SIZE << order) >> 20); 242 "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
243 io_tlb_nslabs = SLABS_PER_PAGE << order; 243 io_tlb_nslabs = SLABS_PER_PAGE << order;
244 bytes = io_tlb_nslabs << IO_TLB_SHIFT; 244 bytes = io_tlb_nslabs << IO_TLB_SHIFT;
245 } 245 }
246 io_tlb_end = io_tlb_start + bytes; 246 io_tlb_end = io_tlb_start + bytes;
247 memset(io_tlb_start, 0, bytes); 247 memset(io_tlb_start, 0, bytes);
248 248
249 /* 249 /*
250 * Allocate and initialize the free list array. This array is used 250 * Allocate and initialize the free list array. This array is used
251 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE 251 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
252 * between io_tlb_start and io_tlb_end. 252 * between io_tlb_start and io_tlb_end.
253 */ 253 */
254 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL, 254 io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
255 get_order(io_tlb_nslabs * sizeof(int))); 255 get_order(io_tlb_nslabs * sizeof(int)));
256 if (!io_tlb_list) 256 if (!io_tlb_list)
257 goto cleanup2; 257 goto cleanup2;
258 258
259 for (i = 0; i < io_tlb_nslabs; i++) 259 for (i = 0; i < io_tlb_nslabs; i++)
260 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE); 260 io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
261 io_tlb_index = 0; 261 io_tlb_index = 0;
262 262
263 io_tlb_orig_addr = (phys_addr_t *) 263 io_tlb_orig_addr = (phys_addr_t *)
264 __get_free_pages(GFP_KERNEL, 264 __get_free_pages(GFP_KERNEL,
265 get_order(io_tlb_nslabs * 265 get_order(io_tlb_nslabs *
266 sizeof(phys_addr_t))); 266 sizeof(phys_addr_t)));
267 if (!io_tlb_orig_addr) 267 if (!io_tlb_orig_addr)
268 goto cleanup3; 268 goto cleanup3;
269 269
270 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t)); 270 memset(io_tlb_orig_addr, 0, io_tlb_nslabs * sizeof(phys_addr_t));
271 271
272 /* 272 /*
273 * Get the overflow emergency buffer 273 * Get the overflow emergency buffer
274 */ 274 */
275 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA, 275 io_tlb_overflow_buffer = (void *)__get_free_pages(GFP_DMA,
276 get_order(io_tlb_overflow)); 276 get_order(io_tlb_overflow));
277 if (!io_tlb_overflow_buffer) 277 if (!io_tlb_overflow_buffer)
278 goto cleanup4; 278 goto cleanup4;
279 279
280 swiotlb_print_info(); 280 swiotlb_print_info();
281 281
282 late_alloc = 1; 282 late_alloc = 1;
283 283
284 return 0; 284 return 0;
285 285
286 cleanup4: 286 cleanup4:
287 free_pages((unsigned long)io_tlb_orig_addr, 287 free_pages((unsigned long)io_tlb_orig_addr,
288 get_order(io_tlb_nslabs * sizeof(phys_addr_t))); 288 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
289 io_tlb_orig_addr = NULL; 289 io_tlb_orig_addr = NULL;
290 cleanup3: 290 cleanup3:
291 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * 291 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
292 sizeof(int))); 292 sizeof(int)));
293 io_tlb_list = NULL; 293 io_tlb_list = NULL;
294 cleanup2: 294 cleanup2:
295 io_tlb_end = NULL; 295 io_tlb_end = NULL;
296 free_pages((unsigned long)io_tlb_start, order); 296 free_pages((unsigned long)io_tlb_start, order);
297 io_tlb_start = NULL; 297 io_tlb_start = NULL;
298 cleanup1: 298 cleanup1:
299 io_tlb_nslabs = req_nslabs; 299 io_tlb_nslabs = req_nslabs;
300 return -ENOMEM; 300 return -ENOMEM;
301 } 301 }
302 302
303 void __init swiotlb_free(void) 303 void __init swiotlb_free(void)
304 { 304 {
305 if (!io_tlb_overflow_buffer) 305 if (!io_tlb_overflow_buffer)
306 return; 306 return;
307 307
308 if (late_alloc) { 308 if (late_alloc) {
309 free_pages((unsigned long)io_tlb_overflow_buffer, 309 free_pages((unsigned long)io_tlb_overflow_buffer,
310 get_order(io_tlb_overflow)); 310 get_order(io_tlb_overflow));
311 free_pages((unsigned long)io_tlb_orig_addr, 311 free_pages((unsigned long)io_tlb_orig_addr,
312 get_order(io_tlb_nslabs * sizeof(phys_addr_t))); 312 get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
313 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs * 313 free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
314 sizeof(int))); 314 sizeof(int)));
315 free_pages((unsigned long)io_tlb_start, 315 free_pages((unsigned long)io_tlb_start,
316 get_order(io_tlb_nslabs << IO_TLB_SHIFT)); 316 get_order(io_tlb_nslabs << IO_TLB_SHIFT));
317 } else { 317 } else {
318 free_bootmem_late(__pa(io_tlb_overflow_buffer), 318 free_bootmem_late(__pa(io_tlb_overflow_buffer),
319 io_tlb_overflow); 319 io_tlb_overflow);
320 free_bootmem_late(__pa(io_tlb_orig_addr), 320 free_bootmem_late(__pa(io_tlb_orig_addr),
321 io_tlb_nslabs * sizeof(phys_addr_t)); 321 io_tlb_nslabs * sizeof(phys_addr_t));
322 free_bootmem_late(__pa(io_tlb_list), 322 free_bootmem_late(__pa(io_tlb_list),
323 io_tlb_nslabs * sizeof(int)); 323 io_tlb_nslabs * sizeof(int));
324 free_bootmem_late(__pa(io_tlb_start), 324 free_bootmem_late(__pa(io_tlb_start),
325 io_tlb_nslabs << IO_TLB_SHIFT); 325 io_tlb_nslabs << IO_TLB_SHIFT);
326 } 326 }
327 } 327 }
328 328
329 static int is_swiotlb_buffer(phys_addr_t paddr) 329 static int is_swiotlb_buffer(phys_addr_t paddr)
330 { 330 {
331 return paddr >= virt_to_phys(io_tlb_start) && 331 return paddr >= virt_to_phys(io_tlb_start) &&
332 paddr < virt_to_phys(io_tlb_end); 332 paddr < virt_to_phys(io_tlb_end);
333 } 333 }
334 334
335 /* 335 /*
336 * Bounce: copy the swiotlb buffer back to the original dma location 336 * Bounce: copy the swiotlb buffer back to the original dma location
337 */ 337 */
338 static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size, 338 static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
339 enum dma_data_direction dir) 339 enum dma_data_direction dir)
340 { 340 {
341 unsigned long pfn = PFN_DOWN(phys); 341 unsigned long pfn = PFN_DOWN(phys);
342 342
343 if (PageHighMem(pfn_to_page(pfn))) { 343 if (PageHighMem(pfn_to_page(pfn))) {
344 /* The buffer does not have a mapping. Map it in and copy */ 344 /* The buffer does not have a mapping. Map it in and copy */
345 unsigned int offset = phys & ~PAGE_MASK; 345 unsigned int offset = phys & ~PAGE_MASK;
346 char *buffer; 346 char *buffer;
347 unsigned int sz = 0; 347 unsigned int sz = 0;
348 unsigned long flags; 348 unsigned long flags;
349 349
350 while (size) { 350 while (size) {
351 sz = min_t(size_t, PAGE_SIZE - offset, size); 351 sz = min_t(size_t, PAGE_SIZE - offset, size);
352 352
353 local_irq_save(flags); 353 local_irq_save(flags);
354 buffer = kmap_atomic(pfn_to_page(pfn), 354 buffer = kmap_atomic(pfn_to_page(pfn),
355 KM_BOUNCE_READ); 355 KM_BOUNCE_READ);
356 if (dir == DMA_TO_DEVICE) 356 if (dir == DMA_TO_DEVICE)
357 memcpy(dma_addr, buffer + offset, sz); 357 memcpy(dma_addr, buffer + offset, sz);
358 else 358 else
359 memcpy(buffer + offset, dma_addr, sz); 359 memcpy(buffer + offset, dma_addr, sz);
360 kunmap_atomic(buffer, KM_BOUNCE_READ); 360 kunmap_atomic(buffer, KM_BOUNCE_READ);
361 local_irq_restore(flags); 361 local_irq_restore(flags);
362 362
363 size -= sz; 363 size -= sz;
364 pfn++; 364 pfn++;
365 dma_addr += sz; 365 dma_addr += sz;
366 offset = 0; 366 offset = 0;
367 } 367 }
368 } else { 368 } else {
369 if (dir == DMA_TO_DEVICE) 369 if (dir == DMA_TO_DEVICE)
370 memcpy(dma_addr, phys_to_virt(phys), size); 370 memcpy(dma_addr, phys_to_virt(phys), size);
371 else 371 else
372 memcpy(phys_to_virt(phys), dma_addr, size); 372 memcpy(phys_to_virt(phys), dma_addr, size);
373 } 373 }
374 } 374 }
375 375
376 void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr, 376 void *swiotlb_tbl_map_single(struct device *hwdev, dma_addr_t tbl_dma_addr,
377 phys_addr_t phys, size_t size, int dir) 377 phys_addr_t phys, size_t size,
378 enum dma_data_direction dir)
378 { 379 {
379 unsigned long flags; 380 unsigned long flags;
380 char *dma_addr; 381 char *dma_addr;
381 unsigned int nslots, stride, index, wrap; 382 unsigned int nslots, stride, index, wrap;
382 int i; 383 int i;
383 unsigned long mask; 384 unsigned long mask;
384 unsigned long offset_slots; 385 unsigned long offset_slots;
385 unsigned long max_slots; 386 unsigned long max_slots;
386 387
387 mask = dma_get_seg_boundary(hwdev); 388 mask = dma_get_seg_boundary(hwdev);
388 389
389 tbl_dma_addr &= mask; 390 tbl_dma_addr &= mask;
390 391
391 offset_slots = ALIGN(tbl_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 392 offset_slots = ALIGN(tbl_dma_addr, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
392 393
393 /* 394 /*
394 * Carefully handle integer overflow which can occur when mask == ~0UL. 395 * Carefully handle integer overflow which can occur when mask == ~0UL.
395 */ 396 */
396 max_slots = mask + 1 397 max_slots = mask + 1
397 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT 398 ? ALIGN(mask + 1, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT
398 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT); 399 : 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
399 400
400 /* 401 /*
401 * For mappings greater than a page, we limit the stride (and 402 * For mappings greater than a page, we limit the stride (and
402 * hence alignment) to a page size. 403 * hence alignment) to a page size.
403 */ 404 */
404 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 405 nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
405 if (size > PAGE_SIZE) 406 if (size > PAGE_SIZE)
406 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT)); 407 stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
407 else 408 else
408 stride = 1; 409 stride = 1;
409 410
410 BUG_ON(!nslots); 411 BUG_ON(!nslots);
411 412
412 /* 413 /*
413 * Find suitable number of IO TLB entries size that will fit this 414 * Find suitable number of IO TLB entries size that will fit this
414 * request and allocate a buffer from that IO TLB pool. 415 * request and allocate a buffer from that IO TLB pool.
415 */ 416 */
416 spin_lock_irqsave(&io_tlb_lock, flags); 417 spin_lock_irqsave(&io_tlb_lock, flags);
417 index = ALIGN(io_tlb_index, stride); 418 index = ALIGN(io_tlb_index, stride);
418 if (index >= io_tlb_nslabs) 419 if (index >= io_tlb_nslabs)
419 index = 0; 420 index = 0;
420 wrap = index; 421 wrap = index;
421 422
422 do { 423 do {
423 while (iommu_is_span_boundary(index, nslots, offset_slots, 424 while (iommu_is_span_boundary(index, nslots, offset_slots,
424 max_slots)) { 425 max_slots)) {
425 index += stride; 426 index += stride;
426 if (index >= io_tlb_nslabs) 427 if (index >= io_tlb_nslabs)
427 index = 0; 428 index = 0;
428 if (index == wrap) 429 if (index == wrap)
429 goto not_found; 430 goto not_found;
430 } 431 }
431 432
432 /* 433 /*
433 * If we find a slot that indicates we have 'nslots' number of 434 * If we find a slot that indicates we have 'nslots' number of
434 * contiguous buffers, we allocate the buffers from that slot 435 * contiguous buffers, we allocate the buffers from that slot
435 * and mark the entries as '0' indicating unavailable. 436 * and mark the entries as '0' indicating unavailable.
436 */ 437 */
437 if (io_tlb_list[index] >= nslots) { 438 if (io_tlb_list[index] >= nslots) {
438 int count = 0; 439 int count = 0;
439 440
440 for (i = index; i < (int) (index + nslots); i++) 441 for (i = index; i < (int) (index + nslots); i++)
441 io_tlb_list[i] = 0; 442 io_tlb_list[i] = 0;
442 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--) 443 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE - 1) && io_tlb_list[i]; i--)
443 io_tlb_list[i] = ++count; 444 io_tlb_list[i] = ++count;
444 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT); 445 dma_addr = io_tlb_start + (index << IO_TLB_SHIFT);
445 446
446 /* 447 /*
447 * Update the indices to avoid searching in the next 448 * Update the indices to avoid searching in the next
448 * round. 449 * round.
449 */ 450 */
450 io_tlb_index = ((index + nslots) < io_tlb_nslabs 451 io_tlb_index = ((index + nslots) < io_tlb_nslabs
451 ? (index + nslots) : 0); 452 ? (index + nslots) : 0);
452 453
453 goto found; 454 goto found;
454 } 455 }
455 index += stride; 456 index += stride;
456 if (index >= io_tlb_nslabs) 457 if (index >= io_tlb_nslabs)
457 index = 0; 458 index = 0;
458 } while (index != wrap); 459 } while (index != wrap);
459 460
460 not_found: 461 not_found:
461 spin_unlock_irqrestore(&io_tlb_lock, flags); 462 spin_unlock_irqrestore(&io_tlb_lock, flags);
462 return NULL; 463 return NULL;
463 found: 464 found:
464 spin_unlock_irqrestore(&io_tlb_lock, flags); 465 spin_unlock_irqrestore(&io_tlb_lock, flags);
465 466
466 /* 467 /*
467 * Save away the mapping from the original address to the DMA address. 468 * Save away the mapping from the original address to the DMA address.
468 * This is needed when we sync the memory. Then we sync the buffer if 469 * This is needed when we sync the memory. Then we sync the buffer if
469 * needed. 470 * needed.
470 */ 471 */
471 for (i = 0; i < nslots; i++) 472 for (i = 0; i < nslots; i++)
472 io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT); 473 io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
473 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) 474 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
474 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); 475 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
475 476
476 return dma_addr; 477 return dma_addr;
477 } 478 }
478 479
479 /* 480 /*
480 * Allocates bounce buffer and returns its kernel virtual address. 481 * Allocates bounce buffer and returns its kernel virtual address.
481 */ 482 */
482 483
483 static void * 484 static void *
484 map_single(struct device *hwdev, phys_addr_t phys, size_t size, int dir) 485 map_single(struct device *hwdev, phys_addr_t phys, size_t size,
486 enum dma_data_direction dir)
485 { 487 {
486 dma_addr_t start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start); 488 dma_addr_t start_dma_addr = swiotlb_virt_to_bus(hwdev, io_tlb_start);
487 489
488 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir); 490 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir);
489 } 491 }
490 492
491 /* 493 /*
492 * dma_addr is the kernel virtual address of the bounce buffer to unmap. 494 * dma_addr is the kernel virtual address of the bounce buffer to unmap.
493 */ 495 */
494 static void 496 static void
495 swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size, 497 swiotlb_tbl_unmap_single(struct device *hwdev, char *dma_addr, size_t size,
496 int dir) 498 enum dma_data_direction dir)
497 { 499 {
498 unsigned long flags; 500 unsigned long flags;
499 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; 501 int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
500 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 502 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
501 phys_addr_t phys = io_tlb_orig_addr[index]; 503 phys_addr_t phys = io_tlb_orig_addr[index];
502 504
503 /* 505 /*
504 * First, sync the memory before unmapping the entry 506 * First, sync the memory before unmapping the entry
505 */ 507 */
506 if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) 508 if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
507 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); 509 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
508 510
509 /* 511 /*
510 * Return the buffer to the free list by setting the corresponding 512 * Return the buffer to the free list by setting the corresponding
511 * entries to indicate the number of contiguous entries available. 513 * entries to indicate the number of contiguous entries available.
512 * While returning the entries to the free list, we merge the entries 514 * While returning the entries to the free list, we merge the entries
513 * with slots below and above the pool being returned. 515 * with slots below and above the pool being returned.
514 */ 516 */
515 spin_lock_irqsave(&io_tlb_lock, flags); 517 spin_lock_irqsave(&io_tlb_lock, flags);
516 { 518 {
517 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ? 519 count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
518 io_tlb_list[index + nslots] : 0); 520 io_tlb_list[index + nslots] : 0);
519 /* 521 /*
520 * Step 1: return the slots to the free list, merging the 522 * Step 1: return the slots to the free list, merging the
521 * slots with superceeding slots 523 * slots with superceeding slots
522 */ 524 */
523 for (i = index + nslots - 1; i >= index; i--) 525 for (i = index + nslots - 1; i >= index; i--)
524 io_tlb_list[i] = ++count; 526 io_tlb_list[i] = ++count;
525 /* 527 /*
526 * Step 2: merge the returned slots with the preceding slots, 528 * Step 2: merge the returned slots with the preceding slots,
527 * if available (non zero) 529 * if available (non zero)
528 */ 530 */
529 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--) 531 for (i = index - 1; (OFFSET(i, IO_TLB_SEGSIZE) != IO_TLB_SEGSIZE -1) && io_tlb_list[i]; i--)
530 io_tlb_list[i] = ++count; 532 io_tlb_list[i] = ++count;
531 } 533 }
532 spin_unlock_irqrestore(&io_tlb_lock, flags); 534 spin_unlock_irqrestore(&io_tlb_lock, flags);
533 } 535 }
534 536
535 static void 537 static void
536 swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr, size_t size, 538 swiotlb_tbl_sync_single(struct device *hwdev, char *dma_addr, size_t size,
537 int dir, int target) 539 enum dma_data_direction dir, int target)
538 { 540 {
539 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT; 541 int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
540 phys_addr_t phys = io_tlb_orig_addr[index]; 542 phys_addr_t phys = io_tlb_orig_addr[index];
541 543
542 phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1)); 544 phys += ((unsigned long)dma_addr & ((1 << IO_TLB_SHIFT) - 1));
543 545
544 switch (target) { 546 switch (target) {
545 case SYNC_FOR_CPU: 547 case SYNC_FOR_CPU:
546 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) 548 if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
547 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE); 549 swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
548 else 550 else
549 BUG_ON(dir != DMA_TO_DEVICE); 551 BUG_ON(dir != DMA_TO_DEVICE);
550 break; 552 break;
551 case SYNC_FOR_DEVICE: 553 case SYNC_FOR_DEVICE:
552 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) 554 if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
553 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE); 555 swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
554 else 556 else
555 BUG_ON(dir != DMA_FROM_DEVICE); 557 BUG_ON(dir != DMA_FROM_DEVICE);
556 break; 558 break;
557 default: 559 default:
558 BUG(); 560 BUG();
559 } 561 }
560 } 562 }
561 563
562 void * 564 void *
563 swiotlb_alloc_coherent(struct device *hwdev, size_t size, 565 swiotlb_alloc_coherent(struct device *hwdev, size_t size,
564 dma_addr_t *dma_handle, gfp_t flags) 566 dma_addr_t *dma_handle, gfp_t flags)
565 { 567 {
566 dma_addr_t dev_addr; 568 dma_addr_t dev_addr;
567 void *ret; 569 void *ret;
568 int order = get_order(size); 570 int order = get_order(size);
569 u64 dma_mask = DMA_BIT_MASK(32); 571 u64 dma_mask = DMA_BIT_MASK(32);
570 572
571 if (hwdev && hwdev->coherent_dma_mask) 573 if (hwdev && hwdev->coherent_dma_mask)
572 dma_mask = hwdev->coherent_dma_mask; 574 dma_mask = hwdev->coherent_dma_mask;
573 575
574 ret = (void *)__get_free_pages(flags, order); 576 ret = (void *)__get_free_pages(flags, order);
575 if (ret && swiotlb_virt_to_bus(hwdev, ret) + size - 1 > dma_mask) { 577 if (ret && swiotlb_virt_to_bus(hwdev, ret) + size - 1 > dma_mask) {
576 /* 578 /*
577 * The allocated memory isn't reachable by the device. 579 * The allocated memory isn't reachable by the device.
578 */ 580 */
579 free_pages((unsigned long) ret, order); 581 free_pages((unsigned long) ret, order);
580 ret = NULL; 582 ret = NULL;
581 } 583 }
582 if (!ret) { 584 if (!ret) {
583 /* 585 /*
584 * We are either out of memory or the device can't DMA to 586 * We are either out of memory or the device can't DMA to
585 * GFP_DMA memory; fall back on map_single(), which 587 * GFP_DMA memory; fall back on map_single(), which
586 * will grab memory from the lowest available address range. 588 * will grab memory from the lowest available address range.
587 */ 589 */
588 ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE); 590 ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
589 if (!ret) 591 if (!ret)
590 return NULL; 592 return NULL;
591 } 593 }
592 594
593 memset(ret, 0, size); 595 memset(ret, 0, size);
594 dev_addr = swiotlb_virt_to_bus(hwdev, ret); 596 dev_addr = swiotlb_virt_to_bus(hwdev, ret);
595 597
596 /* Confirm address can be DMA'd by device */ 598 /* Confirm address can be DMA'd by device */
597 if (dev_addr + size - 1 > dma_mask) { 599 if (dev_addr + size - 1 > dma_mask) {
598 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", 600 printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
599 (unsigned long long)dma_mask, 601 (unsigned long long)dma_mask,
600 (unsigned long long)dev_addr); 602 (unsigned long long)dev_addr);
601 603
602 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */ 604 /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
603 swiotlb_tbl_unmap_single(hwdev, ret, size, DMA_TO_DEVICE); 605 swiotlb_tbl_unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
604 return NULL; 606 return NULL;
605 } 607 }
606 *dma_handle = dev_addr; 608 *dma_handle = dev_addr;
607 return ret; 609 return ret;
608 } 610 }
609 EXPORT_SYMBOL(swiotlb_alloc_coherent); 611 EXPORT_SYMBOL(swiotlb_alloc_coherent);
610 612
611 void 613 void
612 swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, 614 swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
613 dma_addr_t dev_addr) 615 dma_addr_t dev_addr)
614 { 616 {
615 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); 617 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
616 618
617 WARN_ON(irqs_disabled()); 619 WARN_ON(irqs_disabled());
618 if (!is_swiotlb_buffer(paddr)) 620 if (!is_swiotlb_buffer(paddr))
619 free_pages((unsigned long)vaddr, get_order(size)); 621 free_pages((unsigned long)vaddr, get_order(size));
620 else 622 else
621 /* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */ 623 /* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */
622 swiotlb_tbl_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE); 624 swiotlb_tbl_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
623 } 625 }
624 EXPORT_SYMBOL(swiotlb_free_coherent); 626 EXPORT_SYMBOL(swiotlb_free_coherent);
625 627
626 static void 628 static void
627 swiotlb_full(struct device *dev, size_t size, int dir, int do_panic) 629 swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
630 int do_panic)
628 { 631 {
629 /* 632 /*
630 * Ran out of IOMMU space for this operation. This is very bad. 633 * Ran out of IOMMU space for this operation. This is very bad.
631 * Unfortunately the drivers cannot handle this operation properly. 634 * Unfortunately the drivers cannot handle this operation properly.
632 * unless they check for dma_mapping_error (most don't) 635 * unless they check for dma_mapping_error (most don't)
633 * When the mapping is small enough return a static buffer to limit 636 * When the mapping is small enough return a static buffer to limit
634 * the damage, or panic when the transfer is too big. 637 * the damage, or panic when the transfer is too big.
635 */ 638 */
636 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at " 639 printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
637 "device %s\n", size, dev ? dev_name(dev) : "?"); 640 "device %s\n", size, dev ? dev_name(dev) : "?");
638 641
639 if (size <= io_tlb_overflow || !do_panic) 642 if (size <= io_tlb_overflow || !do_panic)
640 return; 643 return;
641 644
642 if (dir == DMA_BIDIRECTIONAL) 645 if (dir == DMA_BIDIRECTIONAL)
643 panic("DMA: Random memory could be DMA accessed\n"); 646 panic("DMA: Random memory could be DMA accessed\n");
644 if (dir == DMA_FROM_DEVICE) 647 if (dir == DMA_FROM_DEVICE)
645 panic("DMA: Random memory could be DMA written\n"); 648 panic("DMA: Random memory could be DMA written\n");
646 if (dir == DMA_TO_DEVICE) 649 if (dir == DMA_TO_DEVICE)
647 panic("DMA: Random memory could be DMA read\n"); 650 panic("DMA: Random memory could be DMA read\n");
648 } 651 }
649 652
650 /* 653 /*
651 * Map a single buffer of the indicated size for DMA in streaming mode. The 654 * Map a single buffer of the indicated size for DMA in streaming mode. The
652 * physical address to use is returned. 655 * physical address to use is returned.
653 * 656 *
654 * Once the device is given the dma address, the device owns this memory until 657 * Once the device is given the dma address, the device owns this memory until
655 * either swiotlb_unmap_page or swiotlb_dma_sync_single is performed. 658 * either swiotlb_unmap_page or swiotlb_dma_sync_single is performed.
656 */ 659 */
657 dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, 660 dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
658 unsigned long offset, size_t size, 661 unsigned long offset, size_t size,
659 enum dma_data_direction dir, 662 enum dma_data_direction dir,
660 struct dma_attrs *attrs) 663 struct dma_attrs *attrs)
661 { 664 {
662 phys_addr_t phys = page_to_phys(page) + offset; 665 phys_addr_t phys = page_to_phys(page) + offset;
663 dma_addr_t dev_addr = phys_to_dma(dev, phys); 666 dma_addr_t dev_addr = phys_to_dma(dev, phys);
664 void *map; 667 void *map;
665 668
666 BUG_ON(dir == DMA_NONE); 669 BUG_ON(dir == DMA_NONE);
667 /* 670 /*
668 * If the address happens to be in the device's DMA window, 671 * If the address happens to be in the device's DMA window,
669 * we can safely return the device addr and not worry about bounce 672 * we can safely return the device addr and not worry about bounce
670 * buffering it. 673 * buffering it.
671 */ 674 */
672 if (dma_capable(dev, dev_addr, size) && !swiotlb_force) 675 if (dma_capable(dev, dev_addr, size) && !swiotlb_force)
673 return dev_addr; 676 return dev_addr;
674 677
675 /* 678 /*
676 * Oh well, have to allocate and map a bounce buffer. 679 * Oh well, have to allocate and map a bounce buffer.
677 */ 680 */
678 map = map_single(dev, phys, size, dir); 681 map = map_single(dev, phys, size, dir);
679 if (!map) { 682 if (!map) {
680 swiotlb_full(dev, size, dir, 1); 683 swiotlb_full(dev, size, dir, 1);
681 map = io_tlb_overflow_buffer; 684 map = io_tlb_overflow_buffer;
682 } 685 }
683 686
684 dev_addr = swiotlb_virt_to_bus(dev, map); 687 dev_addr = swiotlb_virt_to_bus(dev, map);
685 688
686 /* 689 /*
687 * Ensure that the address returned is DMA'ble 690 * Ensure that the address returned is DMA'ble
688 */ 691 */
689 if (!dma_capable(dev, dev_addr, size)) 692 if (!dma_capable(dev, dev_addr, size))
690 panic("map_single: bounce buffer is not DMA'ble"); 693 panic("map_single: bounce buffer is not DMA'ble");
691 694
692 return dev_addr; 695 return dev_addr;
693 } 696 }
694 EXPORT_SYMBOL_GPL(swiotlb_map_page); 697 EXPORT_SYMBOL_GPL(swiotlb_map_page);
695 698
696 /* 699 /*
697 * Unmap a single streaming mode DMA translation. The dma_addr and size must 700 * Unmap a single streaming mode DMA translation. The dma_addr and size must
698 * match what was provided for in a previous swiotlb_map_page call. All 701 * match what was provided for in a previous swiotlb_map_page call. All
699 * other usages are undefined. 702 * other usages are undefined.
700 * 703 *
701 * After this call, reads by the cpu to the buffer are guaranteed to see 704 * After this call, reads by the cpu to the buffer are guaranteed to see
702 * whatever the device wrote there. 705 * whatever the device wrote there.
703 */ 706 */
704 static void unmap_single(struct device *hwdev, dma_addr_t dev_addr, 707 static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
705 size_t size, int dir) 708 size_t size, enum dma_data_direction dir)
706 { 709 {
707 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); 710 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
708 711
709 BUG_ON(dir == DMA_NONE); 712 BUG_ON(dir == DMA_NONE);
710 713
711 if (is_swiotlb_buffer(paddr)) { 714 if (is_swiotlb_buffer(paddr)) {
712 swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir); 715 swiotlb_tbl_unmap_single(hwdev, phys_to_virt(paddr), size, dir);
713 return; 716 return;
714 } 717 }
715 718
716 if (dir != DMA_FROM_DEVICE) 719 if (dir != DMA_FROM_DEVICE)
717 return; 720 return;
718 721
719 /* 722 /*
720 * phys_to_virt doesn't work with hihgmem page but we could 723 * phys_to_virt doesn't work with hihgmem page but we could
721 * call dma_mark_clean() with hihgmem page here. However, we 724 * call dma_mark_clean() with hihgmem page here. However, we
722 * are fine since dma_mark_clean() is null on POWERPC. We can 725 * are fine since dma_mark_clean() is null on POWERPC. We can
723 * make dma_mark_clean() take a physical address if necessary. 726 * make dma_mark_clean() take a physical address if necessary.
724 */ 727 */
725 dma_mark_clean(phys_to_virt(paddr), size); 728 dma_mark_clean(phys_to_virt(paddr), size);
726 } 729 }
727 730
728 void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, 731 void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
729 size_t size, enum dma_data_direction dir, 732 size_t size, enum dma_data_direction dir,
730 struct dma_attrs *attrs) 733 struct dma_attrs *attrs)
731 { 734 {
732 unmap_single(hwdev, dev_addr, size, dir); 735 unmap_single(hwdev, dev_addr, size, dir);
733 } 736 }
734 EXPORT_SYMBOL_GPL(swiotlb_unmap_page); 737 EXPORT_SYMBOL_GPL(swiotlb_unmap_page);
735 738
736 /* 739 /*
737 * Make physical memory consistent for a single streaming mode DMA translation 740 * Make physical memory consistent for a single streaming mode DMA translation
738 * after a transfer. 741 * after a transfer.
739 * 742 *
740 * If you perform a swiotlb_map_page() but wish to interrogate the buffer 743 * If you perform a swiotlb_map_page() but wish to interrogate the buffer
741 * using the cpu, yet do not wish to teardown the dma mapping, you must 744 * using the cpu, yet do not wish to teardown the dma mapping, you must
742 * call this function before doing so. At the next point you give the dma 745 * call this function before doing so. At the next point you give the dma
743 * address back to the card, you must first perform a 746 * address back to the card, you must first perform a
744 * swiotlb_dma_sync_for_device, and then the device again owns the buffer 747 * swiotlb_dma_sync_for_device, and then the device again owns the buffer
745 */ 748 */
746 static void 749 static void
747 swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr, 750 swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
748 size_t size, int dir, int target) 751 size_t size, enum dma_data_direction dir, int target)
749 { 752 {
750 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); 753 phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
751 754
752 BUG_ON(dir == DMA_NONE); 755 BUG_ON(dir == DMA_NONE);
753 756
754 if (is_swiotlb_buffer(paddr)) { 757 if (is_swiotlb_buffer(paddr)) {
755 swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir, 758 swiotlb_tbl_sync_single(hwdev, phys_to_virt(paddr), size, dir,
756 target); 759 target);
757 return; 760 return;
758 } 761 }
759 762
760 if (dir != DMA_FROM_DEVICE) 763 if (dir != DMA_FROM_DEVICE)
761 return; 764 return;
762 765
763 dma_mark_clean(phys_to_virt(paddr), size); 766 dma_mark_clean(phys_to_virt(paddr), size);
764 } 767 }
765 768
766 void 769 void
767 swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, 770 swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
768 size_t size, enum dma_data_direction dir) 771 size_t size, enum dma_data_direction dir)
769 { 772 {
770 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); 773 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
771 } 774 }
772 EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); 775 EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
773 776
774 void 777 void
775 swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, 778 swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
776 size_t size, enum dma_data_direction dir) 779 size_t size, enum dma_data_direction dir)
777 { 780 {
778 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); 781 swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
779 } 782 }
780 EXPORT_SYMBOL(swiotlb_sync_single_for_device); 783 EXPORT_SYMBOL(swiotlb_sync_single_for_device);
781 784
782 /* 785 /*
783 * Map a set of buffers described by scatterlist in streaming mode for DMA. 786 * Map a set of buffers described by scatterlist in streaming mode for DMA.
784 * This is the scatter-gather version of the above swiotlb_map_page 787 * This is the scatter-gather version of the above swiotlb_map_page
785 * interface. Here the scatter gather list elements are each tagged with the 788 * interface. Here the scatter gather list elements are each tagged with the
786 * appropriate dma address and length. They are obtained via 789 * appropriate dma address and length. They are obtained via
787 * sg_dma_{address,length}(SG). 790 * sg_dma_{address,length}(SG).
788 * 791 *
789 * NOTE: An implementation may be able to use a smaller number of 792 * NOTE: An implementation may be able to use a smaller number of
790 * DMA address/length pairs than there are SG table elements. 793 * DMA address/length pairs than there are SG table elements.
791 * (for example via virtual mapping capabilities) 794 * (for example via virtual mapping capabilities)
792 * The routine returns the number of addr/length pairs actually 795 * The routine returns the number of addr/length pairs actually
793 * used, at most nents. 796 * used, at most nents.
794 * 797 *
795 * Device ownership issues as mentioned above for swiotlb_map_page are the 798 * Device ownership issues as mentioned above for swiotlb_map_page are the
796 * same here. 799 * same here.
797 */ 800 */
798 int 801 int
799 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, 802 swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
800 enum dma_data_direction dir, struct dma_attrs *attrs) 803 enum dma_data_direction dir, struct dma_attrs *attrs)
801 { 804 {
802 struct scatterlist *sg; 805 struct scatterlist *sg;
803 int i; 806 int i;
804 807
805 BUG_ON(dir == DMA_NONE); 808 BUG_ON(dir == DMA_NONE);
806 809
807 for_each_sg(sgl, sg, nelems, i) { 810 for_each_sg(sgl, sg, nelems, i) {
808 phys_addr_t paddr = sg_phys(sg); 811 phys_addr_t paddr = sg_phys(sg);
809 dma_addr_t dev_addr = phys_to_dma(hwdev, paddr); 812 dma_addr_t dev_addr = phys_to_dma(hwdev, paddr);
810 813
811 if (swiotlb_force || 814 if (swiotlb_force ||
812 !dma_capable(hwdev, dev_addr, sg->length)) { 815 !dma_capable(hwdev, dev_addr, sg->length)) {
813 void *map = map_single(hwdev, sg_phys(sg), 816 void *map = map_single(hwdev, sg_phys(sg),
814 sg->length, dir); 817 sg->length, dir);
815 if (!map) { 818 if (!map) {
816 /* Don't panic here, we expect map_sg users 819 /* Don't panic here, we expect map_sg users
817 to do proper error handling. */ 820 to do proper error handling. */
818 swiotlb_full(hwdev, sg->length, dir, 0); 821 swiotlb_full(hwdev, sg->length, dir, 0);
819 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, 822 swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
820 attrs); 823 attrs);
821 sgl[0].dma_length = 0; 824 sgl[0].dma_length = 0;
822 return 0; 825 return 0;
823 } 826 }
824 sg->dma_address = swiotlb_virt_to_bus(hwdev, map); 827 sg->dma_address = swiotlb_virt_to_bus(hwdev, map);
825 } else 828 } else
826 sg->dma_address = dev_addr; 829 sg->dma_address = dev_addr;
827 sg->dma_length = sg->length; 830 sg->dma_length = sg->length;
828 } 831 }
829 return nelems; 832 return nelems;
830 } 833 }
831 EXPORT_SYMBOL(swiotlb_map_sg_attrs); 834 EXPORT_SYMBOL(swiotlb_map_sg_attrs);
832 835
833 int 836 int
834 swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, 837 swiotlb_map_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
835 int dir) 838 enum dma_data_direction dir)
836 { 839 {
837 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL); 840 return swiotlb_map_sg_attrs(hwdev, sgl, nelems, dir, NULL);
838 } 841 }
839 EXPORT_SYMBOL(swiotlb_map_sg); 842 EXPORT_SYMBOL(swiotlb_map_sg);
840 843
841 /* 844 /*
842 * Unmap a set of streaming mode DMA translations. Again, cpu read rules 845 * Unmap a set of streaming mode DMA translations. Again, cpu read rules
843 * concerning calls here are the same as for swiotlb_unmap_page() above. 846 * concerning calls here are the same as for swiotlb_unmap_page() above.
844 */ 847 */
845 void 848 void
846 swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, 849 swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
847 int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) 850 int nelems, enum dma_data_direction dir, struct dma_attrs *attrs)
848 { 851 {
849 struct scatterlist *sg; 852 struct scatterlist *sg;
850 int i; 853 int i;
851 854
852 BUG_ON(dir == DMA_NONE); 855 BUG_ON(dir == DMA_NONE);
853 856
854 for_each_sg(sgl, sg, nelems, i) 857 for_each_sg(sgl, sg, nelems, i)
855 unmap_single(hwdev, sg->dma_address, sg->dma_length, dir); 858 unmap_single(hwdev, sg->dma_address, sg->dma_length, dir);
856 859
857 } 860 }
858 EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); 861 EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
859 862
860 void 863 void
861 swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, 864 swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems,
862 int dir) 865 enum dma_data_direction dir)
863 { 866 {
864 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL); 867 return swiotlb_unmap_sg_attrs(hwdev, sgl, nelems, dir, NULL);
865 } 868 }
866 EXPORT_SYMBOL(swiotlb_unmap_sg); 869 EXPORT_SYMBOL(swiotlb_unmap_sg);
867 870
868 /* 871 /*
869 * Make physical memory consistent for a set of streaming mode DMA translations 872 * Make physical memory consistent for a set of streaming mode DMA translations
870 * after a transfer. 873 * after a transfer.
871 * 874 *
872 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules 875 * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
873 * and usage. 876 * and usage.
874 */ 877 */
875 static void 878 static void
876 swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl, 879 swiotlb_sync_sg(struct device *hwdev, struct scatterlist *sgl,
877 int nelems, int dir, int target) 880 int nelems, enum dma_data_direction dir, int target)
878 { 881 {
879 struct scatterlist *sg; 882 struct scatterlist *sg;
880 int i; 883 int i;
881 884
882 for_each_sg(sgl, sg, nelems, i) 885 for_each_sg(sgl, sg, nelems, i)
883 swiotlb_sync_single(hwdev, sg->dma_address, 886 swiotlb_sync_single(hwdev, sg->dma_address,
884 sg->dma_length, dir, target); 887 sg->dma_length, dir, target);
885 } 888 }
886 889
887 void 890 void
888 swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, 891 swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
889 int nelems, enum dma_data_direction dir) 892 int nelems, enum dma_data_direction dir)
890 { 893 {
891 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); 894 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
892 } 895 }
893 EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); 896 EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
894 897
895 void 898 void
896 swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, 899 swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
897 int nelems, enum dma_data_direction dir) 900 int nelems, enum dma_data_direction dir)
898 { 901 {
899 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); 902 swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
900 } 903 }
901 EXPORT_SYMBOL(swiotlb_sync_sg_for_device); 904 EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
902 905
903 int 906 int
904 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) 907 swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
905 { 908 {
906 return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer)); 909 return (dma_addr == swiotlb_virt_to_bus(hwdev, io_tlb_overflow_buffer));
907 } 910 }
908 EXPORT_SYMBOL(swiotlb_dma_mapping_error); 911 EXPORT_SYMBOL(swiotlb_dma_mapping_error);
909 912
910 /* 913 /*
911 * Return whether the given device DMA address mask can be supported 914 * Return whether the given device DMA address mask can be supported
912 * properly. For example, if your device can only drive the low 24-bits 915 * properly. For example, if your device can only drive the low 24-bits
913 * during bus mastering, then you would pass 0x00ffffff as the mask to 916 * during bus mastering, then you would pass 0x00ffffff as the mask to
914 * this function. 917 * this function.
915 */ 918 */
916 int 919 int
917 swiotlb_dma_supported(struct device *hwdev, u64 mask) 920 swiotlb_dma_supported(struct device *hwdev, u64 mask)
918 { 921 {
919 return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask; 922 return swiotlb_virt_to_bus(hwdev, io_tlb_end - 1) <= mask;
920 } 923 }
921 EXPORT_SYMBOL(swiotlb_dma_supported); 924 EXPORT_SYMBOL(swiotlb_dma_supported);
922 925