generic: per-device coherent dma allocator

Currently x86_32, sh and cris-v32 provide per-device coherent dma memory allocator. However their implementation is nearly identical. Refactor out common code to be reused by them. Signed-off-by: Dmitry Baryshkov <dbaryshkov@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>

generic: per-device coherent dma allocator
Currently x86_32, sh and cris-v32 provide per-device coherent dma memory allocator. However their implementation is nearly identical. Refactor out common code to be reused by them. Signed-off-by: Dmitry Baryshkov <dbaryshkov@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Dmitry Baryshkov · Ingo Molnar
1 parent 543cf4cb3f
Showing 4 changed files with 164 additions and 0 deletions Side-by-side Diff
include/asm-generic/dma-coherent.h
init/Kconfig
kernel/Makefile
kernel/dma-coherent.c
+#ifndef DMA_COHERENT_H
+#define DMA_COHERENT_H
+
+#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
+/*
+ * These two functions are only for dma allocator.
+ * Don't use them in device drivers.
+ */
+int dma_alloc_from_coherent(struct device *dev, ssize_t size,
+				       dma_addr_t *dma_handle, void **ret);
+int dma_release_from_coherent(struct device *dev, int order, void *vaddr);
+
+/*
+ * Standard interface
+ */
+#define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY
+extern int
+dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
+			    dma_addr_t device_addr, size_t size, int flags);
+
+extern void
+dma_release_declared_memory(struct device *dev);
+
+extern void *
+dma_mark_declared_memory_occupied(struct device *dev,
+				  dma_addr_t device_addr, size_t size);
+#else
+#define dma_alloc_from_coherent(dev, size, handle, ret) (0)
+#define dma_release_from_coherent(dev, order, vaddr) (0)
+#endif
+
+#endif
@@ -802,6 +802,10 @@
  
 endmenu		# General setup
  
+config HAVE_GENERIC_DMA_COHERENT
+	bool
+	default n
+
 config SLABINFO
 	bool
 	depends on PROC_FS
@@ -69,6 +69,7 @@
 obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
 obj-$(CONFIG_MARKERS) += marker.o
 obj-$(CONFIG_LATENCYTOP) += latencytop.o
+obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
  
 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
 # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
+/*
+ * Coherent per-device memory handling.
+ * Borrowed from i386
+ */
+#include <linux/kernel.h>
+#include <linux/dma-mapping.h>
+
+struct dma_coherent_mem {
+	void		*virt_base;
+	u32		device_base;
+	int		size;
+	int		flags;
+	unsigned long	*bitmap;
+};
+
+int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
+				dma_addr_t device_addr, size_t size, int flags)
+{
+	void __iomem *mem_base = NULL;
+	int pages = size >> PAGE_SHIFT;
+	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+
+	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
+		goto out;
+	if (!size)
+		goto out;
+	if (dev->dma_mem)
+		goto out;
+
+	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
+
+	mem_base = ioremap(bus_addr, size);
+	if (!mem_base)
+		goto out;
+
+	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
+	if (!dev->dma_mem)
+		goto out;
+	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+	if (!dev->dma_mem->bitmap)
+		goto free1_out;
+
+	dev->dma_mem->virt_base = mem_base;
+	dev->dma_mem->device_base = device_addr;
+	dev->dma_mem->size = pages;
+	dev->dma_mem->flags = flags;
+
+	if (flags & DMA_MEMORY_MAP)
+		return DMA_MEMORY_MAP;
+
+	return DMA_MEMORY_IO;
+
+ free1_out:
+	kfree(dev->dma_mem);
+ out:
+	if (mem_base)
+		iounmap(mem_base);
+	return 0;
+}
+EXPORT_SYMBOL(dma_declare_coherent_memory);
+
+void dma_release_declared_memory(struct device *dev)
+{
+	struct dma_coherent_mem *mem = dev->dma_mem;
+
+	if (!mem)
+		return;
+	dev->dma_mem = NULL;
+	iounmap(mem->virt_base);
+	kfree(mem->bitmap);
+	kfree(mem);
+}
+EXPORT_SYMBOL(dma_release_declared_memory);
+
+void *dma_mark_declared_memory_occupied(struct device *dev,
+					dma_addr_t device_addr, size_t size)
+{
+	struct dma_coherent_mem *mem = dev->dma_mem;
+	int pos, err;
+	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
+
+	pages >>= PAGE_SHIFT;
+
+	if (!mem)
+		return ERR_PTR(-EINVAL);
+
+	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
+	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
+	if (err != 0)
+		return ERR_PTR(err);
+	return mem->virt_base + (pos << PAGE_SHIFT);
+}
+EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
+
+int dma_alloc_from_coherent(struct device *dev, ssize_t size,
+				       dma_addr_t *dma_handle, void **ret)
+{
+	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
+	int order = get_order(size);
+
+	if (mem) {
+		int page = bitmap_find_free_region(mem->bitmap, mem->size,
+						     order);
+		if (page >= 0) {
+			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
+			*ret = mem->virt_base + (page << PAGE_SHIFT);
+			memset(*ret, 0, size);
+		}
+		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
+			*ret = NULL;
+	}
+	return (mem != NULL);
+}
+
+int dma_release_from_coherent(struct device *dev, int order, void *vaddr)
+{
+	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
+
+	if (mem && vaddr >= mem->virt_base && vaddr <
+		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
+		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
+
+		bitmap_release_region(mem->bitmap, page, order);
+		return 1;
+	}
+	return 0;
+}
	1	+#ifndef DMA_COHERENT_H
	2	+#define DMA_COHERENT_H
	3	+
	4	+#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
	5	+/*
	6	+ * These two functions are only for dma allocator.
	7	+ * Don't use them in device drivers.
	8	+ */
	9	+int dma_alloc_from_coherent(struct device *dev, ssize_t size,
	10	+ dma_addr_t dma_handle, void *ret);
	11	+int dma_release_from_coherent(struct device dev, int order, void vaddr);
	12	+
	13	+/*
	14	+ * Standard interface
	15	+ */
	16	+#define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY
	17	+extern int
	18	+dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
	19	+ dma_addr_t device_addr, size_t size, int flags);
	20	+
	21	+extern void
	22	+dma_release_declared_memory(struct device *dev);
	23	+
	24	+extern void *
	25	+dma_mark_declared_memory_occupied(struct device *dev,
	26	+ dma_addr_t device_addr, size_t size);
	27	+#else
	28	+#define dma_alloc_from_coherent(dev, size, handle, ret) (0)
	29	+#define dma_release_from_coherent(dev, order, vaddr) (0)
	30	+#endif
	31	+
	32	+#endif
...	...	@@ -802,6 +802,10 @@
802	802
803	803	endmenu # General setup
804	804
	805	+config HAVE_GENERIC_DMA_COHERENT
	806	+ bool
	807	+ default n
	808	+
805	809	config SLABINFO
806	810	bool
807	811	depends on PROC_FS
...	...	@@ -69,6 +69,7 @@
69	69	obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
70	70	obj-$(CONFIG_MARKERS) += marker.o
71	71	obj-$(CONFIG_LATENCYTOP) += latencytop.o
	72	+obj-$(CONFIG_HAVE_GENERIC_DMA_COHERENT) += dma-coherent.o
72	73
73	74	ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
74	75	# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
	1	+/*
	2	+ * Coherent per-device memory handling.
	3	+ * Borrowed from i386
	4	+ */
	5	+#include <linux/kernel.h>
	6	+#include <linux/dma-mapping.h>
	7	+
	8	+struct dma_coherent_mem {
	9	+ void *virt_base;
	10	+ u32 device_base;
	11	+ int size;
	12	+ int flags;
	13	+ unsigned long *bitmap;
	14	+};
	15	+
	16	+int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
	17	+ dma_addr_t device_addr, size_t size, int flags)
	18	+{
	19	+ void __iomem *mem_base = NULL;
	20	+ int pages = size >> PAGE_SHIFT;
	21	+ int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
	22	+
	23	+ if ((flags & (DMA_MEMORY_MAP \| DMA_MEMORY_IO)) == 0)
	24	+ goto out;
	25	+ if (!size)
	26	+ goto out;
	27	+ if (dev->dma_mem)
	28	+ goto out;
	29	+
	30	+ /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
	31	+
	32	+ mem_base = ioremap(bus_addr, size);
	33	+ if (!mem_base)
	34	+ goto out;
	35	+
	36	+ dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	37	+ if (!dev->dma_mem)
	38	+ goto out;
	39	+ dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	40	+ if (!dev->dma_mem->bitmap)
	41	+ goto free1_out;
	42	+
	43	+ dev->dma_mem->virt_base = mem_base;
	44	+ dev->dma_mem->device_base = device_addr;
	45	+ dev->dma_mem->size = pages;
	46	+ dev->dma_mem->flags = flags;
	47	+
	48	+ if (flags & DMA_MEMORY_MAP)
	49	+ return DMA_MEMORY_MAP;
	50	+
	51	+ return DMA_MEMORY_IO;
	52	+
	53	+ free1_out:
	54	+ kfree(dev->dma_mem);
	55	+ out:
	56	+ if (mem_base)
	57	+ iounmap(mem_base);
	58	+ return 0;
	59	+}
	60	+EXPORT_SYMBOL(dma_declare_coherent_memory);
	61	+
	62	+void dma_release_declared_memory(struct device *dev)
	63	+{
	64	+ struct dma_coherent_mem *mem = dev->dma_mem;
	65	+
	66	+ if (!mem)
	67	+ return;
	68	+ dev->dma_mem = NULL;
	69	+ iounmap(mem->virt_base);
	70	+ kfree(mem->bitmap);
	71	+ kfree(mem);
	72	+}
	73	+EXPORT_SYMBOL(dma_release_declared_memory);
	74	+
	75	+void dma_mark_declared_memory_occupied(struct device dev,
	76	+ dma_addr_t device_addr, size_t size)
	77	+{
	78	+ struct dma_coherent_mem *mem = dev->dma_mem;
	79	+ int pos, err;
	80	+ int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
	81	+
	82	+ pages >>= PAGE_SHIFT;
	83	+
	84	+ if (!mem)
	85	+ return ERR_PTR(-EINVAL);
	86	+
	87	+ pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
	88	+ err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
	89	+ if (err != 0)
	90	+ return ERR_PTR(err);
	91	+ return mem->virt_base + (pos << PAGE_SHIFT);
	92	+}
	93	+EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
	94	+
	95	+int dma_alloc_from_coherent(struct device *dev, ssize_t size,
	96	+ dma_addr_t dma_handle, void *ret)
	97	+{
	98	+ struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	99	+ int order = get_order(size);
	100	+
	101	+ if (mem) {
	102	+ int page = bitmap_find_free_region(mem->bitmap, mem->size,
	103	+ order);
	104	+ if (page >= 0) {
	105	+ *dma_handle = mem->device_base + (page << PAGE_SHIFT);
	106	+ *ret = mem->virt_base + (page << PAGE_SHIFT);
	107	+ memset(*ret, 0, size);
	108	+ }
	109	+ if (mem->flags & DMA_MEMORY_EXCLUSIVE)
	110	+ *ret = NULL;
	111	+ }
	112	+ return (mem != NULL);
	113	+}
	114	+
	115	+int dma_release_from_coherent(struct device dev, int order, void vaddr)
	116	+{
	117	+ struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	118	+
	119	+ if (mem && vaddr >= mem->virt_base && vaddr <
	120	+ (mem->virt_base + (mem->size << PAGE_SHIFT))) {
	121	+ int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
	122	+
	123	+ bitmap_release_region(mem->bitmap, page, order);
	124	+ return 1;
	125	+ }
	126	+ return 0;
	127	+}