lib: scatterlist: add sg splitting function

Sometimes a scatter-gather has to be split into several chunks, or sub scatter lists. This happens for example if a scatter list will be handled by multiple DMA channels, each one filling a part of it. A concrete example comes with the media V4L2 API, where the scatter list is allocated from userspace to hold an image, regardless of the knowledge of how many DMAs will fill it : - in a simple RGB565 case, one DMA will pump data from the camera ISP to memory - in the trickier YUV422 case, 3 DMAs will pump data from the camera ISP pipes, one for pipe Y, one for pipe U and one for pipe V For these cases, it is necessary to split the original scatter list into multiple scatter lists, which is the purpose of this patch. The guarantees that are required for this patch are : - the intersection of spans of any couple of resulting scatter lists is empty. - the union of spans of all resulting scatter lists is a subrange of the span of the original scatter list. - streaming DMA API operations (mapping, unmapping) should not happen both on both the resulting and the original scatter list. It's either the first or the later ones. - the caller is reponsible to call kfree() on the resulting scatterlists. Signed-off-by: Robert Jarzmik <robert.jarzmik@free.fr> Signed-off-by: Jens Axboe <axboe@fb.com>

lib: scatterlist: add sg splitting function
Sometimes a scatter-gather has to be split into several chunks, or sub scatter lists. This happens for example if a scatter list will be handled by multiple DMA channels, each one filling a part of it. A concrete example comes with the media V4L2 API, where the scatter list is allocated from userspace to hold an image, regardless of the knowledge of how many DMAs will fill it : - in a simple RGB565 case, one DMA will pump data from the camera ISP to memory - in the trickier YUV422 case, 3 DMAs will pump data from the camera ISP pipes, one for pipe Y, one for pipe U and one for pipe V For these cases, it is necessary to split the original scatter list into multiple scatter lists, which is the purpose of this patch. The guarantees that are required for this patch are : - the intersection of spans of any couple of resulting scatter lists is empty. - the union of spans of all resulting scatter lists is a subrange of the span of the original scatter list. - streaming DMA API operations (mapping, unmapping) should not happen both on both the resulting and the original scatter list. It's either the first or the later ones. - the caller is reponsible to call kfree() on the resulting scatterlists. Signed-off-by: Robert Jarzmik <robert.jarzmik@free.fr> Signed-off-by: Jens Axboe <axboe@fb.com>
Robert Jarzmik · Jens Axboe
1 parent db0fa0cb01
Showing 4 changed files with 215 additions and 0 deletions Side-by-side Diff
include/linux/scatterlist.h
lib/Kconfig
lib/Makefile
lib/sg_split.c
@@ -247,6 +247,11 @@
 struct scatterlist *sg_last(struct scatterlist *s, unsigned int);
 void sg_init_table(struct scatterlist *, unsigned int);
 void sg_init_one(struct scatterlist *, const void *, unsigned int);
+int sg_split(struct scatterlist *in, const int in_mapped_nents,
+	     const off_t skip, const int nb_splits,
+	     const size_t *split_sizes,
+	     struct scatterlist **out, int *out_mapped_nents,
+	     gfp_t gfp_mask);
  
 typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
 typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
@@ -521,6 +521,13 @@
  
 source "lib/fonts/Kconfig"
  
+config SG_SPLIT
+	def_bool n
+	help
+	 Provides a heler to split scatterlists into chunks, each chunk being a
+	 scatterlist. This should be selected by a driver or an API which
+	 whishes to split a scatterlist amongst multiple DMA channel.
+
 #
 # sg chaining option
 #
@@ -160,6 +160,7 @@
  
 obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
  
+obj-$(CONFIG_SG_SPLIT) += sg_split.o
 obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
  
 libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
+/*
+ * Copyright (C) 2015 Robert Jarzmik <robert.jarzmik@free.fr>
+ *
+ * Scatterlist splitting helpers.
+ *
+ * This source code is licensed under the GNU General Public License,
+ * Version 2. See the file COPYING for more details.
+ */
+
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+
+struct sg_splitter {
+	struct scatterlist *in_sg0;
+	int nents;
+	off_t skip_sg0;
+	unsigned int length_last_sg;
+
+	struct scatterlist *out_sg;
+};
+
+static int sg_calculate_split(struct scatterlist *in, int nents, int nb_splits,
+			      off_t skip, const size_t *sizes,
+			      struct sg_splitter *splitters, bool mapped)
+{
+	int i;
+	unsigned int sglen;
+	size_t size = sizes[0], len;
+	struct sg_splitter *curr = splitters;
+	struct scatterlist *sg;
+
+	for (i = 0; i < nb_splits; i++) {
+		splitters[i].in_sg0 = NULL;
+		splitters[i].nents = 0;
+	}
+
+	for_each_sg(in, sg, nents, i) {
+		sglen = mapped ? sg_dma_len(sg) : sg->length;
+		if (skip > sglen) {
+			skip -= sglen;
+			continue;
+		}
+
+		len = min_t(size_t, size, sglen - skip);
+		if (!curr->in_sg0) {
+			curr->in_sg0 = sg;
+			curr->skip_sg0 = skip;
+		}
+		size -= len;
+		curr->nents++;
+		curr->length_last_sg = len;
+
+		while (!size && (skip + len < sglen) && (--nb_splits > 0)) {
+			curr++;
+			size = *(++sizes);
+			skip += len;
+			len = min_t(size_t, size, sglen - skip);
+
+			curr->in_sg0 = sg;
+			curr->skip_sg0 = skip;
+			curr->nents = 1;
+			curr->length_last_sg = len;
+			size -= len;
+		}
+		skip = 0;
+
+		if (!size && --nb_splits > 0) {
+			curr++;
+			size = *(++sizes);
+		}
+
+		if (!nb_splits)
+			break;
+	}
+
+	return (size || !splitters[0].in_sg0) ? -EINVAL : 0;
+}
+
+static void sg_split_phys(struct sg_splitter *splitters, const int nb_splits)
+{
+	int i, j;
+	struct scatterlist *in_sg, *out_sg;
+	struct sg_splitter *split;
+
+	for (i = 0, split = splitters; i < nb_splits; i++, split++) {
+		in_sg = split->in_sg0;
+		out_sg = split->out_sg;
+		for (j = 0; j < split->nents; j++, out_sg++) {
+			*out_sg = *in_sg;
+			if (!j) {
+				out_sg->offset += split->skip_sg0;
+				out_sg->length -= split->skip_sg0;
+			} else {
+				out_sg->offset = 0;
+			}
+			sg_dma_address(out_sg) = 0;
+			sg_dma_len(out_sg) = 0;
+			in_sg = sg_next(in_sg);
+		}
+		out_sg[-1].length = split->length_last_sg;
+		sg_mark_end(out_sg - 1);
+	}
+}
+
+static void sg_split_mapped(struct sg_splitter *splitters, const int nb_splits)
+{
+	int i, j;
+	struct scatterlist *in_sg, *out_sg;
+	struct sg_splitter *split;
+
+	for (i = 0, split = splitters; i < nb_splits; i++, split++) {
+		in_sg = split->in_sg0;
+		out_sg = split->out_sg;
+		for (j = 0; j < split->nents; j++, out_sg++) {
+			sg_dma_address(out_sg) = sg_dma_address(in_sg);
+			sg_dma_len(out_sg) = sg_dma_len(in_sg);
+			if (!j) {
+				sg_dma_address(out_sg) += split->skip_sg0;
+				sg_dma_len(out_sg) -= split->skip_sg0;
+			}
+			in_sg = sg_next(in_sg);
+		}
+		sg_dma_len(--out_sg) = split->length_last_sg;
+	}
+}
+
+/**
+ * sg_split - split a scatterlist into several scatterlists
+ * @in: the input sg list
+ * @in_mapped_nents: the result of a dma_map_sg(in, ...), or 0 if not mapped.
+ * @skip: the number of bytes to skip in the input sg list
+ * @nb_splits: the number of desired sg outputs
+ * @split_sizes: the respective size of each output sg list in bytes
+ * @out: an array where to store the allocated output sg lists
+ * @out_mapped_nents: the resulting sg lists mapped number of sg entries. Might
+ *                    be NULL if sglist not already mapped (in_mapped_nents = 0)
+ * @gfp_mask: the allocation flag
+ *
+ * This function splits the input sg list into nb_splits sg lists, which are
+ * allocated and stored into out.
+ * The @in is split into :
+ *  - @out[0], which covers bytes [@skip .. @skip + @split_sizes[0] - 1] of @in
+ *  - @out[1], which covers bytes [@skip + split_sizes[0] ..
+ *                                 @skip + @split_sizes[0] + @split_sizes[1] -1]
+ * etc ...
+ * It will be the caller's duty to kfree() out array members.
+ *
+ * Returns 0 upon success, or error code
+ */
+int sg_split(struct scatterlist *in, const int in_mapped_nents,
+	     const off_t skip, const int nb_splits,
+	     const size_t *split_sizes,
+	     struct scatterlist **out, int *out_mapped_nents,
+	     gfp_t gfp_mask)
+{
+	int i, ret;
+	struct sg_splitter *splitters;
+
+	splitters = kcalloc(nb_splits, sizeof(*splitters), gfp_mask);
+	if (!splitters)
+		return -ENOMEM;
+
+	ret = sg_calculate_split(in, sg_nents(in), nb_splits, skip, split_sizes,
+			   splitters, false);
+	if (ret < 0)
+		goto err;
+
+	ret = -ENOMEM;
+	for (i = 0; i < nb_splits; i++) {
+		splitters[i].out_sg = kmalloc_array(splitters[i].nents,
+						    sizeof(struct scatterlist),
+						    gfp_mask);
+		if (!splitters[i].out_sg)
+			goto err;
+	}
+
+	/*
+	 * The order of these 3 calls is important and should be kept.
+	 */
+	sg_split_phys(splitters, nb_splits);
+	ret = sg_calculate_split(in, in_mapped_nents, nb_splits, skip,
+				 split_sizes, splitters, true);
+	if (ret < 0)
+		goto err;
+	sg_split_mapped(splitters, nb_splits);
+
+	for (i = 0; i < nb_splits; i++) {
+		out[i] = splitters[i].out_sg;
+		if (out_mapped_nents)
+			out_mapped_nents[i] = splitters[i].nents;
+	}
+
+	kfree(splitters);
+	return 0;
+
+err:
+	for (i = 0; i < nb_splits; i++)
+		kfree(splitters[i].out_sg);
+	kfree(splitters);
+	return ret;
+}
+EXPORT_SYMBOL(sg_split);
...	...	@@ -247,6 +247,11 @@
247	247	struct scatterlist sg_last(struct scatterlist s, unsigned int);
248	248	void sg_init_table(struct scatterlist *, unsigned int);
249	249	void sg_init_one(struct scatterlist , const void , unsigned int);
	250	+int sg_split(struct scatterlist *in, const int in_mapped_nents,
	251	+ const off_t skip, const int nb_splits,
	252	+ const size_t *split_sizes,
	253	+ struct scatterlist *out, int out_mapped_nents,
	254	+ gfp_t gfp_mask);
250	255
251	256	typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);
252	257	typedef void (sg_free_fn)(struct scatterlist *, unsigned int);
...	...	@@ -521,6 +521,13 @@
521	521
522	522	source "lib/fonts/Kconfig"
523	523
	524	+config SG_SPLIT
	525	+ def_bool n
	526	+ help
	527	+ Provides a heler to split scatterlists into chunks, each chunk being a
	528	+ scatterlist. This should be selected by a driver or an API which
	529	+ whishes to split a scatterlist amongst multiple DMA channel.
	530	+
524	531	#
525	532	# sg chaining option
526	533	#
...	...	@@ -160,6 +160,7 @@
160	160
161	161	obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
162	162
	163	+obj-$(CONFIG_SG_SPLIT) += sg_split.o
163	164	obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
164	165
165	166	libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
	1	+/*
	2	+ * Copyright (C) 2015 Robert Jarzmik <robert.jarzmik@free.fr>
	3	+ *
	4	+ * Scatterlist splitting helpers.
	5	+ *
	6	+ * This source code is licensed under the GNU General Public License,
	7	+ * Version 2. See the file COPYING for more details.
	8	+ */
	9	+
	10	+#include <linux/scatterlist.h>
	11	+#include <linux/slab.h>
	12	+
	13	+struct sg_splitter {
	14	+ struct scatterlist *in_sg0;
	15	+ int nents;
	16	+ off_t skip_sg0;
	17	+ unsigned int length_last_sg;
	18	+
	19	+ struct scatterlist *out_sg;
	20	+};
	21	+
	22	+static int sg_calculate_split(struct scatterlist *in, int nents, int nb_splits,
	23	+ off_t skip, const size_t *sizes,
	24	+ struct sg_splitter *splitters, bool mapped)
	25	+{
	26	+ int i;
	27	+ unsigned int sglen;
	28	+ size_t size = sizes[0], len;
	29	+ struct sg_splitter *curr = splitters;
	30	+ struct scatterlist *sg;
	31	+
	32	+ for (i = 0; i < nb_splits; i++) {
	33	+ splitters[i].in_sg0 = NULL;
	34	+ splitters[i].nents = 0;
	35	+ }
	36	+
	37	+ for_each_sg(in, sg, nents, i) {
	38	+ sglen = mapped ? sg_dma_len(sg) : sg->length;
	39	+ if (skip > sglen) {
	40	+ skip -= sglen;
	41	+ continue;
	42	+ }
	43	+
	44	+ len = min_t(size_t, size, sglen - skip);
	45	+ if (!curr->in_sg0) {
	46	+ curr->in_sg0 = sg;
	47	+ curr->skip_sg0 = skip;
	48	+ }
	49	+ size -= len;
	50	+ curr->nents++;
	51	+ curr->length_last_sg = len;
	52	+
	53	+ while (!size && (skip + len < sglen) && (--nb_splits > 0)) {
	54	+ curr++;
	55	+ size = *(++sizes);
	56	+ skip += len;
	57	+ len = min_t(size_t, size, sglen - skip);
	58	+
	59	+ curr->in_sg0 = sg;
	60	+ curr->skip_sg0 = skip;
	61	+ curr->nents = 1;
	62	+ curr->length_last_sg = len;
	63	+ size -= len;
	64	+ }
	65	+ skip = 0;
	66	+
	67	+ if (!size && --nb_splits > 0) {
	68	+ curr++;
	69	+ size = *(++sizes);
	70	+ }
	71	+
	72	+ if (!nb_splits)
	73	+ break;
	74	+ }
	75	+
	76	+ return (size \|\| !splitters[0].in_sg0) ? -EINVAL : 0;
	77	+}
	78	+
	79	+static void sg_split_phys(struct sg_splitter *splitters, const int nb_splits)
	80	+{
	81	+ int i, j;
	82	+ struct scatterlist in_sg, out_sg;
	83	+ struct sg_splitter *split;
	84	+
	85	+ for (i = 0, split = splitters; i < nb_splits; i++, split++) {
	86	+ in_sg = split->in_sg0;
	87	+ out_sg = split->out_sg;
	88	+ for (j = 0; j < split->nents; j++, out_sg++) {
	89	+ out_sg = in_sg;
	90	+ if (!j) {
	91	+ out_sg->offset += split->skip_sg0;
	92	+ out_sg->length -= split->skip_sg0;
	93	+ } else {
	94	+ out_sg->offset = 0;
	95	+ }
	96	+ sg_dma_address(out_sg) = 0;
	97	+ sg_dma_len(out_sg) = 0;
	98	+ in_sg = sg_next(in_sg);
	99	+ }
	100	+ out_sg[-1].length = split->length_last_sg;
	101	+ sg_mark_end(out_sg - 1);
	102	+ }
	103	+}
	104	+
	105	+static void sg_split_mapped(struct sg_splitter *splitters, const int nb_splits)
	106	+{
	107	+ int i, j;
	108	+ struct scatterlist in_sg, out_sg;
	109	+ struct sg_splitter *split;
	110	+
	111	+ for (i = 0, split = splitters; i < nb_splits; i++, split++) {
	112	+ in_sg = split->in_sg0;
	113	+ out_sg = split->out_sg;
	114	+ for (j = 0; j < split->nents; j++, out_sg++) {
	115	+ sg_dma_address(out_sg) = sg_dma_address(in_sg);
	116	+ sg_dma_len(out_sg) = sg_dma_len(in_sg);
	117	+ if (!j) {
	118	+ sg_dma_address(out_sg) += split->skip_sg0;
	119	+ sg_dma_len(out_sg) -= split->skip_sg0;
	120	+ }
	121	+ in_sg = sg_next(in_sg);
	122	+ }
	123	+ sg_dma_len(--out_sg) = split->length_last_sg;
	124	+ }
	125	+}
	126	+
	127	+/**
	128	+ * sg_split - split a scatterlist into several scatterlists
	129	+ * @in: the input sg list
	130	+ * @in_mapped_nents: the result of a dma_map_sg(in, ...), or 0 if not mapped.
	131	+ * @skip: the number of bytes to skip in the input sg list
	132	+ * @nb_splits: the number of desired sg outputs
	133	+ * @split_sizes: the respective size of each output sg list in bytes
	134	+ * @out: an array where to store the allocated output sg lists
	135	+ * @out_mapped_nents: the resulting sg lists mapped number of sg entries. Might
	136	+ * be NULL if sglist not already mapped (in_mapped_nents = 0)
	137	+ * @gfp_mask: the allocation flag
	138	+ *
	139	+ * This function splits the input sg list into nb_splits sg lists, which are
	140	+ * allocated and stored into out.
	141	+ * The @in is split into :
	142	+ * - @out[0], which covers bytes [@skip .. @skip + @split_sizes[0] - 1] of @in
	143	+ * - @out[1], which covers bytes [@skip + split_sizes[0] ..
	144	+ * @skip + @split_sizes[0] + @split_sizes[1] -1]
	145	+ * etc ...
	146	+ * It will be the caller's duty to kfree() out array members.
	147	+ *
	148	+ * Returns 0 upon success, or error code
	149	+ */
	150	+int sg_split(struct scatterlist *in, const int in_mapped_nents,
	151	+ const off_t skip, const int nb_splits,
	152	+ const size_t *split_sizes,
	153	+ struct scatterlist *out, int out_mapped_nents,
	154	+ gfp_t gfp_mask)
	155	+{
	156	+ int i, ret;
	157	+ struct sg_splitter *splitters;
	158	+
	159	+ splitters = kcalloc(nb_splits, sizeof(*splitters), gfp_mask);
	160	+ if (!splitters)
	161	+ return -ENOMEM;
	162	+
	163	+ ret = sg_calculate_split(in, sg_nents(in), nb_splits, skip, split_sizes,
	164	+ splitters, false);
	165	+ if (ret < 0)
	166	+ goto err;
	167	+
	168	+ ret = -ENOMEM;
	169	+ for (i = 0; i < nb_splits; i++) {
	170	+ splitters[i].out_sg = kmalloc_array(splitters[i].nents,
	171	+ sizeof(struct scatterlist),
	172	+ gfp_mask);
	173	+ if (!splitters[i].out_sg)
	174	+ goto err;
	175	+ }
	176	+
	177	+ /*
	178	+ * The order of these 3 calls is important and should be kept.
	179	+ */
	180	+ sg_split_phys(splitters, nb_splits);
	181	+ ret = sg_calculate_split(in, in_mapped_nents, nb_splits, skip,
	182	+ split_sizes, splitters, true);
	183	+ if (ret < 0)
	184	+ goto err;
	185	+ sg_split_mapped(splitters, nb_splits);
	186	+
	187	+ for (i = 0; i < nb_splits; i++) {
	188	+ out[i] = splitters[i].out_sg;
	189	+ if (out_mapped_nents)
	190	+ out_mapped_nents[i] = splitters[i].nents;
	191	+ }
	192	+
	193	+ kfree(splitters);
	194	+ return 0;
	195	+
	196	+err:
	197	+ for (i = 0; i < nb_splits; i++)
	198	+ kfree(splitters[i].out_sg);
	199	+ kfree(splitters);
	200	+ return ret;
	201	+}
	202	+EXPORT_SYMBOL(sg_split);