[PATCH] dm crypt: restructure for workqueue change

Restructure part of the dm-crypt code in preparation for workqueue changes. Use 'base_bio' or 'clone' variable names consistently throughout. No functional changes are included in this patch. Signed-off-by: Milan Broz <mbroz@redhat.com> Signed-off-by: Alasdair G Kergon <agk@redhat.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] dm crypt: restructure for workqueue change
Restructure part of the dm-crypt code in preparation for workqueue changes. Use 'base_bio' or 'clone' variable names consistently throughout. No functional changes are included in this patch. Signed-off-by: Milan Broz <mbroz@redhat.com> Signed-off-by: Alasdair G Kergon <agk@redhat.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Milan Broz · Linus Torvalds
1 parent e48d4bbf96
Showing 1 changed file with 156 additions and 123 deletions Side-by-side Diff
drivers/md/dm-crypt.c
@@ -30,7 +30,7 @@
  */
 struct crypt_io {
 	struct dm_target *target;
-	struct bio *bio;
+	struct bio *base_bio;
 	struct bio *first_clone;
 	struct work_struct work;
 	atomic_t pending;
@@ -319,7 +319,7 @@
 crypt_alloc_buffer(struct crypt_config *cc, unsigned int size,
                    struct bio *base_bio, unsigned int *bio_vec_idx)
 {
-	struct bio *bio;
+	struct bio *clone;
 	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
 	unsigned int i;
  
  
  
  
@@ -330,23 +330,23 @@
 	 * FIXME: Is this really intelligent?
 	 */
 	if (base_bio)
-		bio = bio_clone(base_bio, GFP_NOIO|__GFP_NOMEMALLOC);
+		clone = bio_clone(base_bio, GFP_NOIO|__GFP_NOMEMALLOC);
 	else
-		bio = bio_alloc(GFP_NOIO|__GFP_NOMEMALLOC, nr_iovecs);
-	if (!bio)
+		clone = bio_alloc(GFP_NOIO|__GFP_NOMEMALLOC, nr_iovecs);
+	if (!clone)
 		return NULL;
  
 	/* if the last bio was not complete, continue where that one ended */
-	bio->bi_idx = *bio_vec_idx;
-	bio->bi_vcnt = *bio_vec_idx;
-	bio->bi_size = 0;
-	bio->bi_flags &= ~(1 << BIO_SEG_VALID);
+	clone->bi_idx = *bio_vec_idx;
+	clone->bi_vcnt = *bio_vec_idx;
+	clone->bi_size = 0;
+	clone->bi_flags &= ~(1 << BIO_SEG_VALID);
  
-	/* bio->bi_idx pages have already been allocated */
-	size -= bio->bi_idx * PAGE_SIZE;
+	/* clone->bi_idx pages have already been allocated */
+	size -= clone->bi_idx * PAGE_SIZE;
  
-	for(i = bio->bi_idx; i < nr_iovecs; i++) {
-		struct bio_vec *bv = bio_iovec_idx(bio, i);
+	for (i = clone->bi_idx; i < nr_iovecs; i++) {
+		struct bio_vec *bv = bio_iovec_idx(clone, i);
  
 		bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
 		if (!bv->bv_page)
@@ -357,7 +357,7 @@
 		 * return a partially allocated bio, the caller will then try
 		 * to allocate additional bios while submitting this partial bio
 		 */
-		if ((i - bio->bi_idx) == (MIN_BIO_PAGES - 1))
+		if ((i - clone->bi_idx) == (MIN_BIO_PAGES - 1))
 			gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
  
 		bv->bv_offset = 0;
  
@@ -366,13 +366,13 @@
 		else
 			bv->bv_len = size;
  
-		bio->bi_size += bv->bv_len;
-		bio->bi_vcnt++;
+		clone->bi_size += bv->bv_len;
+		clone->bi_vcnt++;
 		size -= bv->bv_len;
 	}
  
-	if (!bio->bi_size) {
-		bio_put(bio);
+	if (!clone->bi_size) {
+		bio_put(clone);
 		return NULL;
 	}
  
  
  
@@ -380,13 +380,13 @@
 	 * Remember the last bio_vec allocated to be able
 	 * to correctly continue after the splitting.
 	 */
-	*bio_vec_idx = bio->bi_vcnt;
+	*bio_vec_idx = clone->bi_vcnt;
  
-	return bio;
+	return clone;
 }
  
 static void crypt_free_buffer_pages(struct crypt_config *cc,
-                                    struct bio *bio, unsigned int bytes)
+                                    struct bio *clone, unsigned int bytes)
 {
 	unsigned int i, start, end;
 	struct bio_vec *bv;
  
  
@@ -400,19 +400,19 @@
 	 * A fix to the bi_idx issue in the kernel is in the works, so
 	 * we will hopefully be able to revert to the cleaner solution soon.
 	 */
-	i = bio->bi_vcnt - 1;
-	bv = bio_iovec_idx(bio, i);
-	end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - bio->bi_size;
+	i = clone->bi_vcnt - 1;
+	bv = bio_iovec_idx(clone, i);
+	end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - clone->bi_size;
 	start = end - bytes;
  
 	start >>= PAGE_SHIFT;
-	if (!bio->bi_size)
-		end = bio->bi_vcnt;
+	if (!clone->bi_size)
+		end = clone->bi_vcnt;
 	else
 		end >>= PAGE_SHIFT;
  
-	for(i = start; i < end; i++) {
-		bv = bio_iovec_idx(bio, i);
+	for (i = start; i < end; i++) {
+		bv = bio_iovec_idx(clone, i);
 		BUG_ON(!bv->bv_page);
 		mempool_free(bv->bv_page, cc->page_pool);
 		bv->bv_page = NULL;
@@ -436,7 +436,7 @@
 	if (io->first_clone)
 		bio_put(io->first_clone);
  
-	bio_endio(io->bio, io->bio->bi_size, io->error);
+	bio_endio(io->base_bio, io->base_bio->bi_size, io->error);
  
 	mempool_free(io, cc->io_pool);
 }
  
  
  
  
  
  
  
@@ -449,25 +449,133 @@
  * queued here.
  */
 static struct workqueue_struct *_kcryptd_workqueue;
+static void kcryptd_do_work(void *data);
  
-static void kcryptd_do_work(void *data)
+static void kcryptd_queue_io(struct crypt_io *io)
 {
-	struct crypt_io *io = (struct crypt_io *) data;
-	struct crypt_config *cc = (struct crypt_config *) io->target->private;
+	INIT_WORK(&io->work, kcryptd_do_work, io);
+	queue_work(_kcryptd_workqueue, &io->work);
+}
+
+static int crypt_endio(struct bio *clone, unsigned int done, int error)
+{
+	struct crypt_io *io = clone->bi_private;
+	struct crypt_config *cc = io->target->private;
+	unsigned read_io = bio_data_dir(clone) == READ;
+
+	/*
+	 * free the processed pages, even if
+	 * it's only a partially completed write
+	 */
+	if (!read_io)
+		crypt_free_buffer_pages(cc, clone, done);
+
+	if (unlikely(clone->bi_size))
+		return 1;
+
+	/*
+	 * successful reads are decrypted by the worker thread
+	 */
+	if (!read_io)
+		goto out;
+
+	if (unlikely(!bio_flagged(clone, BIO_UPTODATE))) {
+		error = -EIO;
+		goto out;
+	}
+
+	bio_put(clone);
+	kcryptd_queue_io(io);
+	return 0;
+
+out:
+	bio_put(clone);
+	dec_pending(io, error);
+	return error;
+}
+
+static void clone_init(struct crypt_io *io, struct bio *clone)
+{
+	struct crypt_config *cc = io->target->private;
+
+	clone->bi_private = io;
+	clone->bi_end_io  = crypt_endio;
+	clone->bi_bdev    = cc->dev->bdev;
+	clone->bi_rw      = io->base_bio->bi_rw;
+}
+
+static struct bio *clone_read(struct crypt_io *io,
+			      sector_t sector)
+{
+	struct crypt_config *cc = io->target->private;
+	struct bio *base_bio = io->base_bio;
+	struct bio *clone;
+
+	/*
+	 * The block layer might modify the bvec array, so always
+	 * copy the required bvecs because we need the original
+	 * one in order to decrypt the whole bio data *afterwards*.
+	 */
+	clone = bio_alloc(GFP_NOIO, bio_segments(base_bio));
+	if (unlikely(!clone))
+		return NULL;
+
+	clone_init(io, clone);
+	clone->bi_idx = 0;
+	clone->bi_vcnt = bio_segments(base_bio);
+	clone->bi_size = base_bio->bi_size;
+	memcpy(clone->bi_io_vec, bio_iovec(base_bio),
+	       sizeof(struct bio_vec) * clone->bi_vcnt);
+	clone->bi_sector = cc->start + sector;
+
+	return clone;
+}
+
+static struct bio *clone_write(struct crypt_io *io,
+			       sector_t sector,
+			       unsigned *bvec_idx,
+			       struct convert_context *ctx)
+{
+	struct crypt_config *cc = io->target->private;
+	struct bio *base_bio = io->base_bio;
+	struct bio *clone;
+
+	clone = crypt_alloc_buffer(cc, base_bio->bi_size,
+				   io->first_clone, bvec_idx);
+	if (!clone)
+		return NULL;
+
+	ctx->bio_out = clone;
+
+	if (unlikely(crypt_convert(cc, ctx) < 0)) {
+		crypt_free_buffer_pages(cc, clone,
+		                        clone->bi_size);
+		bio_put(clone);
+		return NULL;
+	}
+
+	clone_init(io, clone);
+	clone->bi_sector = cc->start + sector;
+
+	return clone;
+}
+
+static void process_read_endio(struct crypt_io *io)
+{
+	struct crypt_config *cc = io->target->private;
 	struct convert_context ctx;
-	int r;
  
-	crypt_convert_init(cc, &ctx, io->bio, io->bio,
-	                   io->bio->bi_sector - io->target->begin, 0);
-	r = crypt_convert(cc, &ctx);
+	crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio,
+			   io->base_bio->bi_sector - io->target->begin, 0);
  
-	dec_pending(io, r);
+	dec_pending(io, crypt_convert(cc, &ctx));
 }
  
-static void kcryptd_queue_io(struct crypt_io *io)
+static void kcryptd_do_work(void *data)
 {
-	INIT_WORK(&io->work, kcryptd_do_work, io);
-	queue_work(_kcryptd_workqueue, &io->work);
+	struct crypt_io *io = data;
+
+	process_read_endio(io);
 }
  
 /*
@@ -481,7 +589,7 @@
  
 	buffer[2] = '\0';
  
-	for(i = 0; i < size; i++) {
+	for (i = 0; i < size; i++) {
 		buffer[0] = *hex++;
 		buffer[1] = *hex++;
  
@@ -504,7 +612,7 @@
 {
 	unsigned int i;
  
-	for(i = 0; i < size; i++) {
+	for (i = 0; i < size; i++) {
 		sprintf(hex, "%02x", *key);
 		hex += 2;
 		key++;
  
@@ -725,88 +833,10 @@
 	kfree(cc);
 }
  
-static int crypt_endio(struct bio *bio, unsigned int done, int error)
-{
-	struct crypt_io *io = (struct crypt_io *) bio->bi_private;
-	struct crypt_config *cc = (struct crypt_config *) io->target->private;
-
-	if (bio_data_dir(bio) == WRITE) {
-		/*
-		 * free the processed pages, even if
-		 * it's only a partially completed write
-		 */
-		crypt_free_buffer_pages(cc, bio, done);
-	}
-
-	if (bio->bi_size)
-		return 1;
-
-	bio_put(bio);
-
-	/*
-	 * successful reads are decrypted by the worker thread
-	 */
-	if ((bio_data_dir(bio) == READ)
-	    && bio_flagged(bio, BIO_UPTODATE)) {
-		kcryptd_queue_io(io);
-		return 0;
-	}
-
-	dec_pending(io, error);
-	return error;
-}
-
-static inline struct bio *
-crypt_clone(struct crypt_config *cc, struct crypt_io *io, struct bio *bio,
-            sector_t sector, unsigned int *bvec_idx,
-            struct convert_context *ctx)
-{
-	struct bio *clone;
-
-	if (bio_data_dir(bio) == WRITE) {
-		clone = crypt_alloc_buffer(cc, bio->bi_size,
-                                 io->first_clone, bvec_idx);
-		if (clone) {
-			ctx->bio_out = clone;
-			if (crypt_convert(cc, ctx) < 0) {
-				crypt_free_buffer_pages(cc, clone,
-				                        clone->bi_size);
-				bio_put(clone);
-				return NULL;
-			}
-		}
-	} else {
-		/*
-		 * The block layer might modify the bvec array, so always
-		 * copy the required bvecs because we need the original
-		 * one in order to decrypt the whole bio data *afterwards*.
-		 */
-		clone = bio_alloc(GFP_NOIO, bio_segments(bio));
-		if (clone) {
-			clone->bi_idx = 0;
-			clone->bi_vcnt = bio_segments(bio);
-			clone->bi_size = bio->bi_size;
-			memcpy(clone->bi_io_vec, bio_iovec(bio),
-			       sizeof(struct bio_vec) * clone->bi_vcnt);
-		}
-	}
-
-	if (!clone)
-		return NULL;
-
-	clone->bi_private = io;
-	clone->bi_end_io = crypt_endio;
-	clone->bi_bdev = cc->dev->bdev;
-	clone->bi_sector = cc->start + sector;
-	clone->bi_rw = bio->bi_rw;
-
-	return clone;
-}
-
 static int crypt_map(struct dm_target *ti, struct bio *bio,
 		     union map_info *map_context)
 {
-	struct crypt_config *cc = (struct crypt_config *) ti->private;
+	struct crypt_config *cc = ti->private;
 	struct crypt_io *io;
 	struct convert_context ctx;
 	struct bio *clone;
@@ -816,7 +846,7 @@
  
 	io = mempool_alloc(cc->io_pool, GFP_NOIO);
 	io->target = ti;
-	io->bio = bio;
+	io->base_bio = bio;
 	io->first_clone = NULL;
 	io->error = 0;
 	atomic_set(&io->pending, 1); /* hold a reference */
@@ -829,7 +859,10 @@
 	 * so repeat the whole process until all the data can be handled.
 	 */
 	while (remaining) {
-		clone = crypt_clone(cc, io, bio, sector, &bvec_idx, &ctx);
+		if (bio_data_dir(bio) == WRITE)
+			clone = clone_write(io, sector, &bvec_idx, &ctx);
+		else
+			clone = clone_read(io, sector);
 		if (!clone)
 			goto cleanup;
...	...	@@ -30,7 +30,7 @@
30	30	*/
31	31	struct crypt_io {
32	32	struct dm_target *target;
33		- struct bio *bio;
	33	+ struct bio *base_bio;
34	34	struct bio *first_clone;
35	35	struct work_struct work;
36	36	atomic_t pending;
...	...	@@ -319,7 +319,7 @@
319	319	crypt_alloc_buffer(struct crypt_config *cc, unsigned int size,
320	320	struct bio base_bio, unsigned int bio_vec_idx)
321	321	{
322		- struct bio *bio;
	322	+ struct bio *clone;
323	323	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
324	324	gfp_t gfp_mask = GFP_NOIO \| __GFP_HIGHMEM;
325	325	unsigned int i;
326	326
327	327
328	328
329	329
...	...	@@ -330,23 +330,23 @@
330	330	* FIXME: Is this really intelligent?
331	331	*/
332	332	if (base_bio)
333		- bio = bio_clone(base_bio, GFP_NOIO\|__GFP_NOMEMALLOC);
	333	+ clone = bio_clone(base_bio, GFP_NOIO\|__GFP_NOMEMALLOC);
334	334	else
335		- bio = bio_alloc(GFP_NOIO\|__GFP_NOMEMALLOC, nr_iovecs);
336		- if (!bio)
	335	+ clone = bio_alloc(GFP_NOIO\|__GFP_NOMEMALLOC, nr_iovecs);
	336	+ if (!clone)
337	337	return NULL;
338	338
339	339	/* if the last bio was not complete, continue where that one ended */
340		- bio->bi_idx = *bio_vec_idx;
341		- bio->bi_vcnt = *bio_vec_idx;
342		- bio->bi_size = 0;
343		- bio->bi_flags &= ~(1 << BIO_SEG_VALID);
	340	+ clone->bi_idx = *bio_vec_idx;
	341	+ clone->bi_vcnt = *bio_vec_idx;
	342	+ clone->bi_size = 0;
	343	+ clone->bi_flags &= ~(1 << BIO_SEG_VALID);
344	344
345		- /* bio->bi_idx pages have already been allocated */
346		- size -= bio->bi_idx * PAGE_SIZE;
	345	+ /* clone->bi_idx pages have already been allocated */
	346	+ size -= clone->bi_idx * PAGE_SIZE;
347	347
348		- for(i = bio->bi_idx; i < nr_iovecs; i++) {
349		- struct bio_vec *bv = bio_iovec_idx(bio, i);
	348	+ for (i = clone->bi_idx; i < nr_iovecs; i++) {
	349	+ struct bio_vec *bv = bio_iovec_idx(clone, i);
350	350
351	351	bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
352	352	if (!bv->bv_page)
...	...	@@ -357,7 +357,7 @@
357	357	* return a partially allocated bio, the caller will then try
358	358	* to allocate additional bios while submitting this partial bio
359	359	*/
360		- if ((i - bio->bi_idx) == (MIN_BIO_PAGES - 1))
	360	+ if ((i - clone->bi_idx) == (MIN_BIO_PAGES - 1))
361	361	gfp_mask = (gfp_mask \| __GFP_NOWARN) & ~__GFP_WAIT;
362	362
363	363	bv->bv_offset = 0;
364	364
...	...	@@ -366,13 +366,13 @@
366	366	else
367	367	bv->bv_len = size;
368	368
369		- bio->bi_size += bv->bv_len;
370		- bio->bi_vcnt++;
	369	+ clone->bi_size += bv->bv_len;
	370	+ clone->bi_vcnt++;
371	371	size -= bv->bv_len;
372	372	}
373	373
374		- if (!bio->bi_size) {
375		- bio_put(bio);
	374	+ if (!clone->bi_size) {
	375	+ bio_put(clone);
376	376	return NULL;
377	377	}
378	378
379	379
380	380
...	...	@@ -380,13 +380,13 @@
380	380	* Remember the last bio_vec allocated to be able
381	381	* to correctly continue after the splitting.
382	382	*/
383		- *bio_vec_idx = bio->bi_vcnt;
	383	+ *bio_vec_idx = clone->bi_vcnt;
384	384
385		- return bio;
	385	+ return clone;
386	386	}
387	387
388	388	static void crypt_free_buffer_pages(struct crypt_config *cc,
389		- struct bio *bio, unsigned int bytes)
	389	+ struct bio *clone, unsigned int bytes)
390	390	{
391	391	unsigned int i, start, end;
392	392	struct bio_vec *bv;
393	393
394	394
...	...	@@ -400,19 +400,19 @@
400	400	* A fix to the bi_idx issue in the kernel is in the works, so
401	401	* we will hopefully be able to revert to the cleaner solution soon.
402	402	*/
403		- i = bio->bi_vcnt - 1;
404		- bv = bio_iovec_idx(bio, i);
405		- end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - bio->bi_size;
	403	+ i = clone->bi_vcnt - 1;
	404	+ bv = bio_iovec_idx(clone, i);
	405	+ end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - clone->bi_size;
406	406	start = end - bytes;
407	407
408	408	start >>= PAGE_SHIFT;
409		- if (!bio->bi_size)
410		- end = bio->bi_vcnt;
	409	+ if (!clone->bi_size)
	410	+ end = clone->bi_vcnt;
411	411	else
412	412	end >>= PAGE_SHIFT;
413	413
414		- for(i = start; i < end; i++) {
415		- bv = bio_iovec_idx(bio, i);
	414	+ for (i = start; i < end; i++) {
	415	+ bv = bio_iovec_idx(clone, i);
416	416	BUG_ON(!bv->bv_page);
417	417	mempool_free(bv->bv_page, cc->page_pool);
418	418	bv->bv_page = NULL;
...	...	@@ -436,7 +436,7 @@
436	436	if (io->first_clone)
437	437	bio_put(io->first_clone);
438	438
439		- bio_endio(io->bio, io->bio->bi_size, io->error);
	439	+ bio_endio(io->base_bio, io->base_bio->bi_size, io->error);
440	440
441	441	mempool_free(io, cc->io_pool);
442	442	}
443	443
444	444
445	445
446	446
447	447
448	448
449	449
...	...	@@ -449,25 +449,133 @@
449	449	* queued here.
450	450	*/
451	451	static struct workqueue_struct *_kcryptd_workqueue;
	452	+static void kcryptd_do_work(void *data);
452	453
453		-static void kcryptd_do_work(void *data)
	454	+static void kcryptd_queue_io(struct crypt_io *io)
454	455	{
455		- struct crypt_io io = (struct crypt_io ) data;
456		- struct crypt_config cc = (struct crypt_config ) io->target->private;
	456	+ INIT_WORK(&io->work, kcryptd_do_work, io);
	457	+ queue_work(_kcryptd_workqueue, &io->work);
	458	+}
	459	+
	460	+static int crypt_endio(struct bio *clone, unsigned int done, int error)
	461	+{
	462	+ struct crypt_io *io = clone->bi_private;
	463	+ struct crypt_config *cc = io->target->private;
	464	+ unsigned read_io = bio_data_dir(clone) == READ;
	465	+
	466	+ /*
	467	+ * free the processed pages, even if
	468	+ * it's only a partially completed write
	469	+ */
	470	+ if (!read_io)
	471	+ crypt_free_buffer_pages(cc, clone, done);
	472	+
	473	+ if (unlikely(clone->bi_size))
	474	+ return 1;
	475	+
	476	+ /*
	477	+ * successful reads are decrypted by the worker thread
	478	+ */
	479	+ if (!read_io)
	480	+ goto out;
	481	+
	482	+ if (unlikely(!bio_flagged(clone, BIO_UPTODATE))) {
	483	+ error = -EIO;
	484	+ goto out;
	485	+ }
	486	+
	487	+ bio_put(clone);
	488	+ kcryptd_queue_io(io);
	489	+ return 0;
	490	+
	491	+out:
	492	+ bio_put(clone);
	493	+ dec_pending(io, error);
	494	+ return error;
	495	+}
	496	+
	497	+static void clone_init(struct crypt_io io, struct bio clone)
	498	+{
	499	+ struct crypt_config *cc = io->target->private;
	500	+
	501	+ clone->bi_private = io;
	502	+ clone->bi_end_io = crypt_endio;
	503	+ clone->bi_bdev = cc->dev->bdev;
	504	+ clone->bi_rw = io->base_bio->bi_rw;
	505	+}
	506	+
	507	+static struct bio clone_read(struct crypt_io io,
	508	+ sector_t sector)
	509	+{
	510	+ struct crypt_config *cc = io->target->private;
	511	+ struct bio *base_bio = io->base_bio;
	512	+ struct bio *clone;
	513	+
	514	+ /*
	515	+ * The block layer might modify the bvec array, so always
	516	+ * copy the required bvecs because we need the original
	517	+ * one in order to decrypt the whole bio data afterwards.
	518	+ */
	519	+ clone = bio_alloc(GFP_NOIO, bio_segments(base_bio));
	520	+ if (unlikely(!clone))
	521	+ return NULL;
	522	+
	523	+ clone_init(io, clone);
	524	+ clone->bi_idx = 0;
	525	+ clone->bi_vcnt = bio_segments(base_bio);
	526	+ clone->bi_size = base_bio->bi_size;
	527	+ memcpy(clone->bi_io_vec, bio_iovec(base_bio),
	528	+ sizeof(struct bio_vec) * clone->bi_vcnt);
	529	+ clone->bi_sector = cc->start + sector;
	530	+
	531	+ return clone;
	532	+}
	533	+
	534	+static struct bio clone_write(struct crypt_io io,
	535	+ sector_t sector,
	536	+ unsigned *bvec_idx,
	537	+ struct convert_context *ctx)
	538	+{
	539	+ struct crypt_config *cc = io->target->private;
	540	+ struct bio *base_bio = io->base_bio;
	541	+ struct bio *clone;
	542	+
	543	+ clone = crypt_alloc_buffer(cc, base_bio->bi_size,
	544	+ io->first_clone, bvec_idx);
	545	+ if (!clone)
	546	+ return NULL;
	547	+
	548	+ ctx->bio_out = clone;
	549	+
	550	+ if (unlikely(crypt_convert(cc, ctx) < 0)) {
	551	+ crypt_free_buffer_pages(cc, clone,
	552	+ clone->bi_size);
	553	+ bio_put(clone);
	554	+ return NULL;
	555	+ }
	556	+
	557	+ clone_init(io, clone);
	558	+ clone->bi_sector = cc->start + sector;
	559	+
	560	+ return clone;
	561	+}
	562	+
	563	+static void process_read_endio(struct crypt_io *io)
	564	+{
	565	+ struct crypt_config *cc = io->target->private;
457	566	struct convert_context ctx;
458		- int r;
459	567
460		- crypt_convert_init(cc, &ctx, io->bio, io->bio,
461		- io->bio->bi_sector - io->target->begin, 0);
462		- r = crypt_convert(cc, &ctx);
	568	+ crypt_convert_init(cc, &ctx, io->base_bio, io->base_bio,
	569	+ io->base_bio->bi_sector - io->target->begin, 0);
463	570
464		- dec_pending(io, r);
	571	+ dec_pending(io, crypt_convert(cc, &ctx));
465	572	}
466	573
467		-static void kcryptd_queue_io(struct crypt_io *io)
	574	+static void kcryptd_do_work(void *data)
468	575	{
469		- INIT_WORK(&io->work, kcryptd_do_work, io);
470		- queue_work(_kcryptd_workqueue, &io->work);
	576	+ struct crypt_io *io = data;
	577	+
	578	+ process_read_endio(io);
471	579	}
472	580
473	581	/*
...	...	@@ -481,7 +589,7 @@
481	589
482	590	buffer[2] = '\0';
483	591
484		- for(i = 0; i < size; i++) {
	592	+ for (i = 0; i < size; i++) {
485	593	buffer[0] = *hex++;
486	594	buffer[1] = *hex++;
487	595
...	...	@@ -504,7 +612,7 @@
504	612	{
505	613	unsigned int i;
506	614
507		- for(i = 0; i < size; i++) {
	615	+ for (i = 0; i < size; i++) {
508	616	sprintf(hex, "%02x", *key);
509	617	hex += 2;
510	618	key++;
511	619
...	...	@@ -725,88 +833,10 @@
725	833	kfree(cc);
726	834	}
727	835
728		-static int crypt_endio(struct bio *bio, unsigned int done, int error)
729		-{
730		- struct crypt_io io = (struct crypt_io ) bio->bi_private;
731		- struct crypt_config cc = (struct crypt_config ) io->target->private;
732		-
733		- if (bio_data_dir(bio) == WRITE) {
734		- /*
735		- * free the processed pages, even if
736		- * it's only a partially completed write
737		- */
738		- crypt_free_buffer_pages(cc, bio, done);
739		- }
740		-
741		- if (bio->bi_size)
742		- return 1;
743		-
744		- bio_put(bio);
745		-
746		- /*
747		- * successful reads are decrypted by the worker thread
748		- */
749		- if ((bio_data_dir(bio) == READ)
750		- && bio_flagged(bio, BIO_UPTODATE)) {
751		- kcryptd_queue_io(io);
752		- return 0;
753		- }
754		-
755		- dec_pending(io, error);
756		- return error;
757		-}
758		-
759		-static inline struct bio *
760		-crypt_clone(struct crypt_config cc, struct crypt_io io, struct bio *bio,
761		- sector_t sector, unsigned int *bvec_idx,
762		- struct convert_context *ctx)
763		-{
764		- struct bio *clone;
765		-
766		- if (bio_data_dir(bio) == WRITE) {
767		- clone = crypt_alloc_buffer(cc, bio->bi_size,
768		- io->first_clone, bvec_idx);
769		- if (clone) {
770		- ctx->bio_out = clone;
771		- if (crypt_convert(cc, ctx) < 0) {
772		- crypt_free_buffer_pages(cc, clone,
773		- clone->bi_size);
774		- bio_put(clone);
775		- return NULL;
776		- }
777		- }
778		- } else {
779		- /*
780		- * The block layer might modify the bvec array, so always
781		- * copy the required bvecs because we need the original
782		- * one in order to decrypt the whole bio data afterwards.
783		- */
784		- clone = bio_alloc(GFP_NOIO, bio_segments(bio));
785		- if (clone) {
786		- clone->bi_idx = 0;
787		- clone->bi_vcnt = bio_segments(bio);
788		- clone->bi_size = bio->bi_size;
789		- memcpy(clone->bi_io_vec, bio_iovec(bio),
790		- sizeof(struct bio_vec) * clone->bi_vcnt);
791		- }
792		- }
793		-
794		- if (!clone)
795		- return NULL;
796		-
797		- clone->bi_private = io;
798		- clone->bi_end_io = crypt_endio;
799		- clone->bi_bdev = cc->dev->bdev;
800		- clone->bi_sector = cc->start + sector;
801		- clone->bi_rw = bio->bi_rw;
802		-
803		- return clone;
804		-}
805		-
806	836	static int crypt_map(struct dm_target ti, struct bio bio,
807	837	union map_info *map_context)
808	838	{
809		- struct crypt_config cc = (struct crypt_config ) ti->private;
	839	+ struct crypt_config *cc = ti->private;
810	840	struct crypt_io *io;
811	841	struct convert_context ctx;
812	842	struct bio *clone;
...	...	@@ -816,7 +846,7 @@
816	846
817	847	io = mempool_alloc(cc->io_pool, GFP_NOIO);
818	848	io->target = ti;
819		- io->bio = bio;
	849	+ io->base_bio = bio;
820	850	io->first_clone = NULL;
821	851	io->error = 0;
822	852	atomic_set(&io->pending, 1); /* hold a reference */
...	...	@@ -829,7 +859,10 @@
829	859	* so repeat the whole process until all the data can be handled.
830	860	*/
831	861	while (remaining) {
832		- clone = crypt_clone(cc, io, bio, sector, &bvec_idx, &ctx);
	862	+ if (bio_data_dir(bio) == WRITE)
	863	+ clone = clone_write(io, sector, &bvec_idx, &ctx);
	864	+ else
	865	+ clone = clone_read(io, sector);
833	866	if (!clone)
834	867	goto cleanup;
835	868