Merge branch 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping

Pull DMA-mapping fixes from Marek Szyprowski: "Another set of fixes for ARM dma-mapping subsystem. Commit e9da6e9905e6 replaced custom consistent buffer remapping code with generic vmalloc areas. It however introduced some regressions caused by limited support for allocations in atomic context. This series contains fixes for those regressions. For some subplatforms the default, pre-allocated pool for atomic allocations turned out to be too small, so a function for setting its size has been added. Another set of patches adds support for atomic allocations to IOMMU-aware DMA-mapping implementation. The last part of this pull request contains two fixes for Contiguous Memory Allocator, which relax too strict requirements." * 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping: ARM: dma-mapping: IOMMU allocates pages from atomic_pool with GFP_ATOMIC ARM: dma-mapping: Introduce __atomic_get_pages() for __iommu_get_pages() ARM: dma-mapping: Refactor out to introduce __in_atomic_pool ARM: dma-mapping: atomic_pool with struct page **pages ARM: Kirkwood: increase atomic coherent pool size ARM: DMA-Mapping: print warning when atomic coherent allocation fails ARM: DMA-Mapping: add function for setting coherent pool size from platform code ARM: relax conditions required for enabling Contiguous Memory Allocator mm: cma: fix alignment requirements for contiguous regions

Merge branch 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping
Pull DMA-mapping fixes from Marek Szyprowski: "Another set of fixes for ARM dma-mapping subsystem. Commit e9da6e9905e6 replaced custom consistent buffer remapping code with generic vmalloc areas. It however introduced some regressions caused by limited support for allocations in atomic context. This series contains fixes for those regressions. For some subplatforms the default, pre-allocated pool for atomic allocations turned out to be too small, so a function for setting its size has been added. Another set of patches adds support for atomic allocations to IOMMU-aware DMA-mapping implementation. The last part of this pull request contains two fixes for Contiguous Memory Allocator, which relax too strict requirements." * 'fixes-for-3.6' of git://git.linaro.org/people/mszyprowski/linux-dma-mapping: ARM: dma-mapping: IOMMU allocates pages from atomic_pool with GFP_ATOMIC ARM: dma-mapping: Introduce __atomic_get_pages() for __iommu_get_pages() ARM: dma-mapping: Refactor out to introduce __in_atomic_pool ARM: dma-mapping: atomic_pool with struct page **pages ARM: Kirkwood: increase atomic coherent pool size ARM: DMA-Mapping: print warning when atomic coherent allocation fails ARM: DMA-Mapping: add function for setting coherent pool size from platform code ARM: relax conditions required for enabling Contiguous Memory Allocator mm: cma: fix alignment requirements for contiguous regions
Linus Torvalds
2 parents 11be4bc6a1 479ed93a4b
Showing 5 changed files Side-by-side Diff
arch/arm/Kconfig
arch/arm/include/asm/dma-mapping.h
arch/arm/mach-kirkwood/common.c
arch/arm/mm/dma-mapping.c
drivers/base/dma-contiguous.c
@@ -6,7 +6,7 @@
 	select HAVE_DMA_API_DEBUG
 	select HAVE_IDE if PCI || ISA || PCMCIA
 	select HAVE_DMA_ATTRS
-	select HAVE_DMA_CONTIGUOUS if (CPU_V6 || CPU_V6K || CPU_V7)
+	select HAVE_DMA_CONTIGUOUS if MMU
 	select HAVE_MEMBLOCK
 	select RTC_LIB
 	select SYS_SUPPORTS_APM_EMULATION
@@ -203,6 +203,13 @@
 }
  
 /*
+ * This can be called during early boot to increase the size of the atomic
+ * coherent DMA pool above the default value of 256KiB. It must be called
+ * before postcore_initcall.
+ */
+extern void __init init_dma_coherent_pool_size(unsigned long size);
+
+/*
  * This can be called during boot to increase the size of the consistent
  * DMA region above it's default value of 2MB. It must be called before the
  * memory allocator is initialised, i.e. before any core_initcall.
@@ -517,6 +517,13 @@
 void __init kirkwood_init_early(void)
 {
 	orion_time_set_base(TIMER_VIRT_BASE);
+
+	/*
+	 * Some Kirkwood devices allocate their coherent buffers from atomic
+	 * context. Increase size of atomic coherent pool to make sure such
+	 * the allocations won't fail.
+	 */
+	init_dma_coherent_pool_size(SZ_1M);
 }
  
 int kirkwood_tclk;
@@ -267,17 +267,19 @@
 	vunmap(cpu_addr);
 }
  
+#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K
+
 struct dma_pool {
 	size_t size;
 	spinlock_t lock;
 	unsigned long *bitmap;
 	unsigned long nr_pages;
 	void *vaddr;
-	struct page *page;
+	struct page **pages;
 };
  
 static struct dma_pool atomic_pool = {
-	.size = SZ_256K,
+	.size = DEFAULT_DMA_COHERENT_POOL_SIZE,
 };
  
 static int __init early_coherent_pool(char *p)
@@ -287,6 +289,21 @@
 }
 early_param("coherent_pool", early_coherent_pool);
  
+void __init init_dma_coherent_pool_size(unsigned long size)
+{
+	/*
+	 * Catch any attempt to set the pool size too late.
+	 */
+	BUG_ON(atomic_pool.vaddr);
+
+	/*
+	 * Set architecture specific coherent pool size only if
+	 * it has not been changed by kernel command line parameter.
+	 */
+	if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
+		atomic_pool.size = size;
+}
+
 /*
  * Initialise the coherent pool for atomic allocations.
  */
@@ -297,6 +314,7 @@
 	unsigned long nr_pages = pool->size >> PAGE_SHIFT;
 	unsigned long *bitmap;
 	struct page *page;
+	struct page **pages;
 	void *ptr;
 	int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
  
  
  
  
@@ -304,21 +322,31 @@
 	if (!bitmap)
 		goto no_bitmap;
  
+	pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
+	if (!pages)
+		goto no_pages;
+
 	if (IS_ENABLED(CONFIG_CMA))
 		ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
 	else
 		ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot,
 					   &page, NULL);
 	if (ptr) {
+		int i;
+
+		for (i = 0; i < nr_pages; i++)
+			pages[i] = page + i;
+
 		spin_lock_init(&pool->lock);
 		pool->vaddr = ptr;
-		pool->page = page;
+		pool->pages = pages;
 		pool->bitmap = bitmap;
 		pool->nr_pages = nr_pages;
 		pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
 		       (unsigned)pool->size / 1024);
 		return 0;
 	}
+no_pages:
 	kfree(bitmap);
 no_bitmap:
 	pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
  
  
  
@@ -443,27 +471,45 @@
 	if (pageno < pool->nr_pages) {
 		bitmap_set(pool->bitmap, pageno, count);
 		ptr = pool->vaddr + PAGE_SIZE * pageno;
-		*ret_page = pool->page + pageno;
+		*ret_page = pool->pages[pageno];
+	} else {
+		pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
+			    "Please increase it with coherent_pool= kernel parameter!\n",
+			    (unsigned)pool->size / 1024);
 	}
 	spin_unlock_irqrestore(&pool->lock, flags);
  
 	return ptr;
 }
  
+static bool __in_atomic_pool(void *start, size_t size)
+{
+	struct dma_pool *pool = &atomic_pool;
+	void *end = start + size;
+	void *pool_start = pool->vaddr;
+	void *pool_end = pool->vaddr + pool->size;
+
+	if (start < pool_start || start > pool_end)
+		return false;
+
+	if (end <= pool_end)
+		return true;
+
+	WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
+	     start, end - 1, pool_start, pool_end - 1);
+
+	return false;
+}
+
 static int __free_from_pool(void *start, size_t size)
 {
 	struct dma_pool *pool = &atomic_pool;
 	unsigned long pageno, count;
 	unsigned long flags;
  
-	if (start < pool->vaddr || start > pool->vaddr + pool->size)
+	if (!__in_atomic_pool(start, size))
 		return 0;
  
-	if (start + size > pool->vaddr + pool->size) {
-		WARN(1, "freeing wrong coherent size from pool\n");
-		return 0;
-	}
-
 	pageno = (start - pool->vaddr) >> PAGE_SHIFT;
 	count = size >> PAGE_SHIFT;
  
  
@@ -1090,10 +1136,22 @@
 	return 0;
 }
  
+static struct page **__atomic_get_pages(void *addr)
+{
+	struct dma_pool *pool = &atomic_pool;
+	struct page **pages = pool->pages;
+	int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
+
+	return pages + offs;
+}
+
 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
 {
 	struct vm_struct *area;
  
+	if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
+		return __atomic_get_pages(cpu_addr);
+
 	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
 		return cpu_addr;
  
@@ -1103,6 +1161,34 @@
 	return NULL;
 }
  
+static void *__iommu_alloc_atomic(struct device *dev, size_t size,
+				  dma_addr_t *handle)
+{
+	struct page *page;
+	void *addr;
+
+	addr = __alloc_from_pool(size, &page);
+	if (!addr)
+		return NULL;
+
+	*handle = __iommu_create_mapping(dev, &page, size);
+	if (*handle == DMA_ERROR_CODE)
+		goto err_mapping;
+
+	return addr;
+
+err_mapping:
+	__free_from_pool(addr, size);
+	return NULL;
+}
+
+static void __iommu_free_atomic(struct device *dev, struct page **pages,
+				dma_addr_t handle, size_t size)
+{
+	__iommu_remove_mapping(dev, handle, size);
+	__free_from_pool(page_address(pages[0]), size);
+}
+
 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
 	    dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
 {
@@ -1113,6 +1199,9 @@
 	*handle = DMA_ERROR_CODE;
 	size = PAGE_ALIGN(size);
  
+	if (gfp & GFP_ATOMIC)
+		return __iommu_alloc_atomic(dev, size, handle);
+
 	pages = __iommu_alloc_buffer(dev, size, gfp);
 	if (!pages)
 		return NULL;
@@ -1176,6 +1265,11 @@
  
 	if (!pages) {
 		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
+		return;
+	}
+
+	if (__in_atomic_pool(cpu_addr, size)) {
+		__iommu_free_atomic(dev, pages, handle, size);
 		return;
 	}
  
@@ -250,7 +250,7 @@
 		return -EINVAL;
  
 	/* Sanitise input arguments */
-	alignment = PAGE_SIZE << max(MAX_ORDER, pageblock_order);
+	alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order);
 	base = ALIGN(base, alignment);
 	size = ALIGN(size, alignment);
 	limit &= ~(alignment - 1);
...	...	@@ -6,7 +6,7 @@
6	6	select HAVE_DMA_API_DEBUG
7	7	select HAVE_IDE if PCI \|\| ISA \|\| PCMCIA
8	8	select HAVE_DMA_ATTRS
9		- select HAVE_DMA_CONTIGUOUS if (CPU_V6 \|\| CPU_V6K \|\| CPU_V7)
	9	+ select HAVE_DMA_CONTIGUOUS if MMU
10	10	select HAVE_MEMBLOCK
11	11	select RTC_LIB
12	12	select SYS_SUPPORTS_APM_EMULATION
...	...	@@ -203,6 +203,13 @@
203	203	}
204	204
205	205	/*
	206	+ * This can be called during early boot to increase the size of the atomic
	207	+ * coherent DMA pool above the default value of 256KiB. It must be called
	208	+ * before postcore_initcall.
	209	+ */
	210	+extern void __init init_dma_coherent_pool_size(unsigned long size);
	211	+
	212	+/*
206	213	* This can be called during boot to increase the size of the consistent
207	214	* DMA region above it's default value of 2MB. It must be called before the
208	215	* memory allocator is initialised, i.e. before any core_initcall.
...	...	@@ -517,6 +517,13 @@
517	517	void __init kirkwood_init_early(void)
518	518	{
519	519	orion_time_set_base(TIMER_VIRT_BASE);
	520	+
	521	+ /*
	522	+ * Some Kirkwood devices allocate their coherent buffers from atomic
	523	+ * context. Increase size of atomic coherent pool to make sure such
	524	+ * the allocations won't fail.
	525	+ */
	526	+ init_dma_coherent_pool_size(SZ_1M);
520	527	}
521	528
522	529	int kirkwood_tclk;
...	...	@@ -267,17 +267,19 @@
267	267	vunmap(cpu_addr);
268	268	}
269	269
	270	+#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
	271	+
270	272	struct dma_pool {
271	273	size_t size;
272	274	spinlock_t lock;
273	275	unsigned long *bitmap;
274	276	unsigned long nr_pages;
275	277	void *vaddr;
276		- struct page *page;
	278	+ struct page **pages;
277	279	};
278	280
279	281	static struct dma_pool atomic_pool = {
280		- .size = SZ_256K,
	282	+ .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
281	283	};
282	284
283	285	static int __init early_coherent_pool(char *p)
...	...	@@ -287,6 +289,21 @@
287	289	}
288	290	early_param("coherent_pool", early_coherent_pool);
289	291
	292	+void __init init_dma_coherent_pool_size(unsigned long size)
	293	+{
	294	+ /*
	295	+ * Catch any attempt to set the pool size too late.
	296	+ */
	297	+ BUG_ON(atomic_pool.vaddr);
	298	+
	299	+ /*
	300	+ * Set architecture specific coherent pool size only if
	301	+ * it has not been changed by kernel command line parameter.
	302	+ */
	303	+ if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
	304	+ atomic_pool.size = size;
	305	+}
	306	+
290	307	/*
291	308	* Initialise the coherent pool for atomic allocations.
292	309	*/
...	...	@@ -297,6 +314,7 @@
297	314	unsigned long nr_pages = pool->size >> PAGE_SHIFT;
298	315	unsigned long *bitmap;
299	316	struct page *page;
	317	+ struct page **pages;
300	318	void *ptr;
301	319	int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
302	320
303	321
304	322
305	323
...	...	@@ -304,21 +322,31 @@
304	322	if (!bitmap)
305	323	goto no_bitmap;
306	324
	325	+ pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
	326	+ if (!pages)
	327	+ goto no_pages;
	328	+
307	329	if (IS_ENABLED(CONFIG_CMA))
308	330	ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
309	331	else
310	332	ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot,
311	333	&page, NULL);
312	334	if (ptr) {
	335	+ int i;
	336	+
	337	+ for (i = 0; i < nr_pages; i++)
	338	+ pages[i] = page + i;
	339	+
313	340	spin_lock_init(&pool->lock);
314	341	pool->vaddr = ptr;
315		- pool->page = page;
	342	+ pool->pages = pages;
316	343	pool->bitmap = bitmap;
317	344	pool->nr_pages = nr_pages;
318	345	pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
319	346	(unsigned)pool->size / 1024);
320	347	return 0;
321	348	}
	349	+no_pages:
322	350	kfree(bitmap);
323	351	no_bitmap:
324	352	pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
325	353
326	354
327	355
...	...	@@ -443,27 +471,45 @@
443	471	if (pageno < pool->nr_pages) {
444	472	bitmap_set(pool->bitmap, pageno, count);
445	473	ptr = pool->vaddr + PAGE_SIZE * pageno;
446		- *ret_page = pool->page + pageno;
	474	+ *ret_page = pool->pages[pageno];
	475	+ } else {
	476	+ pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
	477	+ "Please increase it with coherent_pool= kernel parameter!\n",
	478	+ (unsigned)pool->size / 1024);
447	479	}
448	480	spin_unlock_irqrestore(&pool->lock, flags);
449	481
450	482	return ptr;
451	483	}
452	484
	485	+static bool __in_atomic_pool(void *start, size_t size)
	486	+{
	487	+ struct dma_pool *pool = &atomic_pool;
	488	+ void *end = start + size;
	489	+ void *pool_start = pool->vaddr;
	490	+ void *pool_end = pool->vaddr + pool->size;
	491	+
	492	+ if (start < pool_start \|\| start > pool_end)
	493	+ return false;
	494	+
	495	+ if (end <= pool_end)
	496	+ return true;
	497	+
	498	+ WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
	499	+ start, end - 1, pool_start, pool_end - 1);
	500	+
	501	+ return false;
	502	+}
	503	+
453	504	static int __free_from_pool(void *start, size_t size)
454	505	{
455	506	struct dma_pool *pool = &atomic_pool;
456	507	unsigned long pageno, count;
457	508	unsigned long flags;
458	509
459		- if (start < pool->vaddr \|\| start > pool->vaddr + pool->size)
	510	+ if (!__in_atomic_pool(start, size))
460	511	return 0;
461	512
462		- if (start + size > pool->vaddr + pool->size) {
463		- WARN(1, "freeing wrong coherent size from pool\n");
464		- return 0;
465		- }
466		-
467	513	pageno = (start - pool->vaddr) >> PAGE_SHIFT;
468	514	count = size >> PAGE_SHIFT;
469	515
470	516
...	...	@@ -1090,10 +1136,22 @@
1090	1136	return 0;
1091	1137	}
1092	1138
	1139	+static struct page *__atomic_get_pages(void addr)
	1140	+{
	1141	+ struct dma_pool *pool = &atomic_pool;
	1142	+ struct page **pages = pool->pages;
	1143	+ int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
	1144	+
	1145	+ return pages + offs;
	1146	+}
	1147	+
1093	1148	static struct page *__iommu_get_pages(void cpu_addr, struct dma_attrs *attrs)
1094	1149	{
1095	1150	struct vm_struct *area;
1096	1151
	1152	+ if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
	1153	+ return __atomic_get_pages(cpu_addr);
	1154	+
1097	1155	if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1098	1156	return cpu_addr;
1099	1157
...	...	@@ -1103,6 +1161,34 @@
1103	1161	return NULL;
1104	1162	}
1105	1163
	1164	+static void __iommu_alloc_atomic(struct device dev, size_t size,
	1165	+ dma_addr_t *handle)
	1166	+{
	1167	+ struct page *page;
	1168	+ void *addr;
	1169	+
	1170	+ addr = __alloc_from_pool(size, &page);
	1171	+ if (!addr)
	1172	+ return NULL;
	1173	+
	1174	+ *handle = __iommu_create_mapping(dev, &page, size);
	1175	+ if (*handle == DMA_ERROR_CODE)
	1176	+ goto err_mapping;
	1177	+
	1178	+ return addr;
	1179	+
	1180	+err_mapping:
	1181	+ __free_from_pool(addr, size);
	1182	+ return NULL;
	1183	+}
	1184	+
	1185	+static void __iommu_free_atomic(struct device dev, struct page *pages,
	1186	+ dma_addr_t handle, size_t size)
	1187	+{
	1188	+ __iommu_remove_mapping(dev, handle, size);
	1189	+ __free_from_pool(page_address(pages[0]), size);
	1190	+}
	1191	+
1106	1192	static void arm_iommu_alloc_attrs(struct device dev, size_t size,
1107	1193	dma_addr_t handle, gfp_t gfp, struct dma_attrs attrs)
1108	1194	{
...	...	@@ -1113,6 +1199,9 @@
1113	1199	*handle = DMA_ERROR_CODE;
1114	1200	size = PAGE_ALIGN(size);
1115	1201
	1202	+ if (gfp & GFP_ATOMIC)
	1203	+ return __iommu_alloc_atomic(dev, size, handle);
	1204	+
1116	1205	pages = __iommu_alloc_buffer(dev, size, gfp);
1117	1206	if (!pages)
1118	1207	return NULL;
...	...	@@ -1176,6 +1265,11 @@
1176	1265
1177	1266	if (!pages) {
1178	1267	WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
	1268	+ return;
	1269	+ }
	1270	+
	1271	+ if (__in_atomic_pool(cpu_addr, size)) {
	1272	+ __iommu_free_atomic(dev, pages, handle, size);
1179	1273	return;
1180	1274	}
1181	1275
...	...	@@ -250,7 +250,7 @@
250	250	return -EINVAL;
251	251
252	252	/* Sanitise input arguments */
253		- alignment = PAGE_SIZE << max(MAX_ORDER, pageblock_order);
	253	+ alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order);
254	254	base = ALIGN(base, alignment);
255	255	size = ALIGN(size, alignment);
256	256	limit &= ~(alignment - 1);