swiotlb: add support for systems with highmem

Impact: extend code for highmem - existing users unaffected On highmem systems, the original dma buffer might not have a virtual mapping - we need to kmap it in to perform the bounce. Extract the code that does the actual copy into a function that does the kmap if highmem is enabled, and default to the normal swiotlb memcpy if not. [ ported by Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> ] Signed-off-by: Becky Bruce <beckyb@kernel.crashing.org> Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>

swiotlb: add support for systems with highmem
Impact: extend code for highmem - existing users unaffected On highmem systems, the original dma buffer might not have a virtual mapping - we need to kmap it in to perform the bounce. Extract the code that does the actual copy into a function that does the kmap if highmem is enabled, and default to the normal swiotlb memcpy if not. [ ported by Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> ] Signed-off-by: Becky Bruce <beckyb@kernel.crashing.org> Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Becky Bruce · Ingo Molnar
1 parent bc40ac6698
Showing 1 changed file with 51 additions and 17 deletions Side-by-side Diff
lib/swiotlb.c
@@ -14,6 +14,7 @@
  * 04/07/.. ak		Better overflow handling. Assorted fixes.
  * 05/09/10 linville	Add support for syncing ranges, support syncing for
  *			DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
+ * 08/12/11 beckyb	Add highmem support
  */
  
 #include <linux/cache.h>
@@ -24,6 +25,7 @@
 #include <linux/swiotlb.h>
 #include <linux/string.h>
 #include <linux/swiotlb.h>
+#include <linux/pfn.h>
 #include <linux/types.h>
 #include <linux/ctype.h>
 #include <linux/highmem.h>
@@ -149,11 +151,6 @@
 	return 0;
 }
  
-static dma_addr_t swiotlb_sg_to_bus(struct device *hwdev, struct scatterlist *sg)
-{
-	return swiotlb_phys_to_bus(hwdev, page_to_phys(sg_page(sg)) + sg->offset);
-}
-
 static void swiotlb_print_info(unsigned long bytes)
 {
 	phys_addr_t pstart, pend;
@@ -330,6 +327,47 @@
 }
  
 /*
+ * Bounce: copy the swiotlb buffer back to the original dma location
+ */
+static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
+			   enum dma_data_direction dir)
+{
+	unsigned long pfn = PFN_DOWN(phys);
+
+	if (PageHighMem(pfn_to_page(pfn))) {
+		/* The buffer does not have a mapping.  Map it in and copy */
+		unsigned int offset = phys & ~PAGE_MASK;
+		char *buffer;
+		unsigned int sz = 0;
+		unsigned long flags;
+
+		while (size) {
+			sz = min(PAGE_SIZE - offset, size);
+
+			local_irq_save(flags);
+			buffer = kmap_atomic(pfn_to_page(pfn),
+					     KM_BOUNCE_READ);
+			if (dir == DMA_TO_DEVICE)
+				memcpy(dma_addr, buffer + offset, sz);
+			else
+				memcpy(buffer + offset, dma_addr, sz);
+			kunmap_atomic(buffer, KM_BOUNCE_READ);
+			local_irq_restore(flags);
+
+			size -= sz;
+			pfn++;
+			dma_addr += sz;
+			offset = 0;
+		}
+	} else {
+		if (dir == DMA_TO_DEVICE)
+			memcpy(dma_addr, phys_to_virt(phys), size);
+		else
+			memcpy(phys_to_virt(phys), dma_addr, size);
+	}
+}
+
+/*
  * Allocates bounce buffer and returns its kernel virtual address.
  */
 static void *
@@ -430,7 +468,7 @@
 	for (i = 0; i < nslots; i++)
 		io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
 	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
-		memcpy(dma_addr, phys_to_virt(phys), size);
+		swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
  
 	return dma_addr;
 }
@@ -450,11 +488,7 @@
 	 * First, sync the memory before unmapping the entry
 	 */
 	if (phys && ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
-		/*
-		 * bounce... copy the data back into the original buffer * and
-		 * delete the bounce buffer.
-		 */
-		memcpy(phys_to_virt(phys), dma_addr, size);
+		swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
  
 	/*
 	 * Return the buffer to the free list by setting the corresponding
  
@@ -494,13 +528,13 @@
 	switch (target) {
 	case SYNC_FOR_CPU:
 		if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
-			memcpy(phys_to_virt(phys), dma_addr, size);
+			swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
 		else
 			BUG_ON(dir != DMA_TO_DEVICE);
 		break;
 	case SYNC_FOR_DEVICE:
 		if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
-			memcpy(dma_addr, phys_to_virt(phys), size);
+			swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
 		else
 			BUG_ON(dir != DMA_FROM_DEVICE);
 		break;
  
@@ -817,11 +851,11 @@
 	BUG_ON(dir == DMA_NONE);
  
 	for_each_sg(sgl, sg, nelems, i) {
-		if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg))
+		if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
 			unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
 				     sg->dma_length, dir);
 		else if (dir == DMA_FROM_DEVICE)
-			dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
+			dma_mark_clean(sg_virt(sg), sg->dma_length);
 	}
 }
 EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
  
@@ -850,11 +884,11 @@
 	BUG_ON(dir == DMA_NONE);
  
 	for_each_sg(sgl, sg, nelems, i) {
-		if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg))
+		if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
 			sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
 				    sg->dma_length, dir, target);
 		else if (dir == DMA_FROM_DEVICE)
-			dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
+			dma_mark_clean(sg_virt(sg), sg->dma_length);
 	}
 }
...	...	@@ -14,6 +14,7 @@
14	14	* 04/07/.. ak Better overflow handling. Assorted fixes.
15	15	* 05/09/10 linville Add support for syncing ranges, support syncing for
16	16	* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
	17	+ * 08/12/11 beckyb Add highmem support
17	18	*/
18	19
19	20	#include <linux/cache.h>
...	...	@@ -24,6 +25,7 @@
24	25	#include <linux/swiotlb.h>
25	26	#include <linux/string.h>
26	27	#include <linux/swiotlb.h>
	28	+#include <linux/pfn.h>
27	29	#include <linux/types.h>
28	30	#include <linux/ctype.h>
29	31	#include <linux/highmem.h>
...	...	@@ -149,11 +151,6 @@
149	151	return 0;
150	152	}
151	153
152		-static dma_addr_t swiotlb_sg_to_bus(struct device hwdev, struct scatterlist sg)
153		-{
154		- return swiotlb_phys_to_bus(hwdev, page_to_phys(sg_page(sg)) + sg->offset);
155		-}
156		-
157	154	static void swiotlb_print_info(unsigned long bytes)
158	155	{
159	156	phys_addr_t pstart, pend;
...	...	@@ -330,6 +327,47 @@
330	327	}
331	328
332	329	/*
	330	+ * Bounce: copy the swiotlb buffer back to the original dma location
	331	+ */
	332	+static void swiotlb_bounce(phys_addr_t phys, char *dma_addr, size_t size,
	333	+ enum dma_data_direction dir)
	334	+{
	335	+ unsigned long pfn = PFN_DOWN(phys);
	336	+
	337	+ if (PageHighMem(pfn_to_page(pfn))) {
	338	+ /* The buffer does not have a mapping. Map it in and copy */
	339	+ unsigned int offset = phys & ~PAGE_MASK;
	340	+ char *buffer;
	341	+ unsigned int sz = 0;
	342	+ unsigned long flags;
	343	+
	344	+ while (size) {
	345	+ sz = min(PAGE_SIZE - offset, size);
	346	+
	347	+ local_irq_save(flags);
	348	+ buffer = kmap_atomic(pfn_to_page(pfn),
	349	+ KM_BOUNCE_READ);
	350	+ if (dir == DMA_TO_DEVICE)
	351	+ memcpy(dma_addr, buffer + offset, sz);
	352	+ else
	353	+ memcpy(buffer + offset, dma_addr, sz);
	354	+ kunmap_atomic(buffer, KM_BOUNCE_READ);
	355	+ local_irq_restore(flags);
	356	+
	357	+ size -= sz;
	358	+ pfn++;
	359	+ dma_addr += sz;
	360	+ offset = 0;
	361	+ }
	362	+ } else {
	363	+ if (dir == DMA_TO_DEVICE)
	364	+ memcpy(dma_addr, phys_to_virt(phys), size);
	365	+ else
	366	+ memcpy(phys_to_virt(phys), dma_addr, size);
	367	+ }
	368	+}
	369	+
	370	+/*
333	371	* Allocates bounce buffer and returns its kernel virtual address.
334	372	*/
335	373	static void *
...	...	@@ -430,7 +468,7 @@
430	468	for (i = 0; i < nslots; i++)
431	469	io_tlb_orig_addr[index+i] = phys + (i << IO_TLB_SHIFT);
432	470	if (dir == DMA_TO_DEVICE \|\| dir == DMA_BIDIRECTIONAL)
433		- memcpy(dma_addr, phys_to_virt(phys), size);
	471	+ swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
434	472
435	473	return dma_addr;
436	474	}
...	...	@@ -450,11 +488,7 @@
450	488	* First, sync the memory before unmapping the entry
451	489	*/
452	490	if (phys && ((dir == DMA_FROM_DEVICE) \|\| (dir == DMA_BIDIRECTIONAL)))
453		- /*
454		- * bounce... copy the data back into the original buffer * and
455		- * delete the bounce buffer.
456		- */
457		- memcpy(phys_to_virt(phys), dma_addr, size);
	491	+ swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
458	492
459	493	/*
460	494	* Return the buffer to the free list by setting the corresponding
461	495
...	...	@@ -494,13 +528,13 @@
494	528	switch (target) {
495	529	case SYNC_FOR_CPU:
496	530	if (likely(dir == DMA_FROM_DEVICE \|\| dir == DMA_BIDIRECTIONAL))
497		- memcpy(phys_to_virt(phys), dma_addr, size);
	531	+ swiotlb_bounce(phys, dma_addr, size, DMA_FROM_DEVICE);
498	532	else
499	533	BUG_ON(dir != DMA_TO_DEVICE);
500	534	break;
501	535	case SYNC_FOR_DEVICE:
502	536	if (likely(dir == DMA_TO_DEVICE \|\| dir == DMA_BIDIRECTIONAL))
503		- memcpy(dma_addr, phys_to_virt(phys), size);
	537	+ swiotlb_bounce(phys, dma_addr, size, DMA_TO_DEVICE);
504	538	else
505	539	BUG_ON(dir != DMA_FROM_DEVICE);
506	540	break;
507	541
...	...	@@ -817,11 +851,11 @@
817	851	BUG_ON(dir == DMA_NONE);
818	852
819	853	for_each_sg(sgl, sg, nelems, i) {
820		- if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg))
	854	+ if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
821	855	unmap_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
822	856	sg->dma_length, dir);
823	857	else if (dir == DMA_FROM_DEVICE)
824		- dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
	858	+ dma_mark_clean(sg_virt(sg), sg->dma_length);
825	859	}
826	860	}
827	861	EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
828	862
...	...	@@ -850,11 +884,11 @@
850	884	BUG_ON(dir == DMA_NONE);
851	885
852	886	for_each_sg(sgl, sg, nelems, i) {
853		- if (sg->dma_address != swiotlb_sg_to_bus(hwdev, sg))
	887	+ if (sg->dma_address != swiotlb_virt_to_bus(hwdev, sg_virt(sg)))
854	888	sync_single(hwdev, swiotlb_bus_to_virt(sg->dma_address),
855	889	sg->dma_length, dir, target);
856	890	else if (dir == DMA_FROM_DEVICE)
857		- dma_mark_clean(swiotlb_bus_to_virt(sg->dma_address), sg->dma_length);
	891	+ dma_mark_clean(sg_virt(sg), sg->dma_length);
858	892	}
859	893	}
860	894