[PATCH] BLOCK: Separate the bounce buffering code from the highmem code [try #6]

Move the bounce buffer code from mm/highmem.c to mm/bounce.c so that it can be more easily disabled when the block layer is disabled. !!!NOTE!!! There may be a bug in this code: Should init_emergency_pool() be contingent on CONFIG_HIGHMEM? Signed-Off-By: David Howells <dhowells@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>

[PATCH] BLOCK: Separate the bounce buffering code from the highmem code [try #6]
Move the bounce buffer code from mm/highmem.c to mm/bounce.c so that it can be more easily disabled when the block layer is disabled. !!!NOTE!!! There may be a bug in this code: Should init_emergency_pool() be contingent on CONFIG_HIGHMEM? Signed-Off-By: David Howells <dhowells@redhat.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
David Howells · Jens Axboe
1 parent b398f6bff9
Showing 3 changed files with 305 additions and 281 deletions Side-by-side Diff
mm/Makefile
mm/bounce.c
mm/highmem.c
@@ -12,6 +12,9 @@
 			   readahead.o swap.o truncate.o vmscan.o \
 			   prio_tree.o util.o mmzone.o vmstat.o $(mmu-y)
  
+ifeq ($(CONFIG_MMU),y)
+obj-y			+= bounce.o
+endif
 obj-$(CONFIG_SWAP)	+= page_io.o swap_state.o swapfile.o thrash.o
 obj-$(CONFIG_HUGETLBFS)	+= hugetlb.o
 obj-$(CONFIG_NUMA) 	+= mempolicy.o
+/* bounce buffer handling for block devices
+ *
+ * - Split from highmem.c
+ */
+
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/swap.h>
+#include <linux/bio.h>
+#include <linux/pagemap.h>
+#include <linux/mempool.h>
+#include <linux/blkdev.h>
+#include <linux/init.h>
+#include <linux/hash.h>
+#include <linux/highmem.h>
+#include <linux/blktrace_api.h>
+#include <asm/tlbflush.h>
+
+#define POOL_SIZE	64
+#define ISA_POOL_SIZE	16
+
+static mempool_t *page_pool, *isa_page_pool;
+
+#ifdef CONFIG_HIGHMEM
+static __init int init_emergency_pool(void)
+{
+	struct sysinfo i;
+	si_meminfo(&i);
+	si_swapinfo(&i);
+
+	if (!i.totalhigh)
+		return 0;
+
+	page_pool = mempool_create_page_pool(POOL_SIZE, 0);
+	BUG_ON(!page_pool);
+	printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
+
+	return 0;
+}
+
+__initcall(init_emergency_pool);
+
+/*
+ * highmem version, map in to vec
+ */
+static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
+{
+	unsigned long flags;
+	unsigned char *vto;
+
+	local_irq_save(flags);
+	vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
+	memcpy(vto + to->bv_offset, vfrom, to->bv_len);
+	kunmap_atomic(vto, KM_BOUNCE_READ);
+	local_irq_restore(flags);
+}
+
+#else /* CONFIG_HIGHMEM */
+
+#define bounce_copy_vec(to, vfrom)	\
+	memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
+
+#endif /* CONFIG_HIGHMEM */
+
+/*
+ * allocate pages in the DMA region for the ISA pool
+ */
+static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
+{
+	return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
+}
+
+/*
+ * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
+ * as the max address, so check if the pool has already been created.
+ */
+int init_emergency_isa_pool(void)
+{
+	if (isa_page_pool)
+		return 0;
+
+	isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
+				       mempool_free_pages, (void *) 0);
+	BUG_ON(!isa_page_pool);
+
+	printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
+	return 0;
+}
+
+/*
+ * Simple bounce buffer support for highmem pages. Depending on the
+ * queue gfp mask set, *to may or may not be a highmem page. kmap it
+ * always, it will do the Right Thing
+ */
+static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
+{
+	unsigned char *vfrom;
+	struct bio_vec *tovec, *fromvec;
+	int i;
+
+	__bio_for_each_segment(tovec, to, i, 0) {
+		fromvec = from->bi_io_vec + i;
+
+		/*
+		 * not bounced
+		 */
+		if (tovec->bv_page == fromvec->bv_page)
+			continue;
+
+		/*
+		 * fromvec->bv_offset and fromvec->bv_len might have been
+		 * modified by the block layer, so use the original copy,
+		 * bounce_copy_vec already uses tovec->bv_len
+		 */
+		vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
+
+		flush_dcache_page(tovec->bv_page);
+		bounce_copy_vec(tovec, vfrom);
+	}
+}
+
+static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
+{
+	struct bio *bio_orig = bio->bi_private;
+	struct bio_vec *bvec, *org_vec;
+	int i;
+
+	if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
+		set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
+
+	/*
+	 * free up bounce indirect pages used
+	 */
+	__bio_for_each_segment(bvec, bio, i, 0) {
+		org_vec = bio_orig->bi_io_vec + i;
+		if (bvec->bv_page == org_vec->bv_page)
+			continue;
+
+		dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
+		mempool_free(bvec->bv_page, pool);
+	}
+
+	bio_endio(bio_orig, bio_orig->bi_size, err);
+	bio_put(bio);
+}
+
+static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
+{
+	if (bio->bi_size)
+		return 1;
+
+	bounce_end_io(bio, page_pool, err);
+	return 0;
+}
+
+static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
+{
+	if (bio->bi_size)
+		return 1;
+
+	bounce_end_io(bio, isa_page_pool, err);
+	return 0;
+}
+
+static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err)
+{
+	struct bio *bio_orig = bio->bi_private;
+
+	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
+		copy_to_high_bio_irq(bio_orig, bio);
+
+	bounce_end_io(bio, pool, err);
+}
+
+static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
+{
+	if (bio->bi_size)
+		return 1;
+
+	__bounce_end_io_read(bio, page_pool, err);
+	return 0;
+}
+
+static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
+{
+	if (bio->bi_size)
+		return 1;
+
+	__bounce_end_io_read(bio, isa_page_pool, err);
+	return 0;
+}
+
+static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig,
+			       mempool_t *pool)
+{
+	struct page *page;
+	struct bio *bio = NULL;
+	int i, rw = bio_data_dir(*bio_orig);
+	struct bio_vec *to, *from;
+
+	bio_for_each_segment(from, *bio_orig, i) {
+		page = from->bv_page;
+
+		/*
+		 * is destination page below bounce pfn?
+		 */
+		if (page_to_pfn(page) < q->bounce_pfn)
+			continue;
+
+		/*
+		 * irk, bounce it
+		 */
+		if (!bio)
+			bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
+
+		to = bio->bi_io_vec + i;
+
+		to->bv_page = mempool_alloc(pool, q->bounce_gfp);
+		to->bv_len = from->bv_len;
+		to->bv_offset = from->bv_offset;
+		inc_zone_page_state(to->bv_page, NR_BOUNCE);
+
+		if (rw == WRITE) {
+			char *vto, *vfrom;
+
+			flush_dcache_page(from->bv_page);
+			vto = page_address(to->bv_page) + to->bv_offset;
+			vfrom = kmap(from->bv_page) + from->bv_offset;
+			memcpy(vto, vfrom, to->bv_len);
+			kunmap(from->bv_page);
+		}
+	}
+
+	/*
+	 * no pages bounced
+	 */
+	if (!bio)
+		return;
+
+	/*
+	 * at least one page was bounced, fill in possible non-highmem
+	 * pages
+	 */
+	__bio_for_each_segment(from, *bio_orig, i, 0) {
+		to = bio_iovec_idx(bio, i);
+		if (!to->bv_page) {
+			to->bv_page = from->bv_page;
+			to->bv_len = from->bv_len;
+			to->bv_offset = from->bv_offset;
+		}
+	}
+
+	bio->bi_bdev = (*bio_orig)->bi_bdev;
+	bio->bi_flags |= (1 << BIO_BOUNCED);
+	bio->bi_sector = (*bio_orig)->bi_sector;
+	bio->bi_rw = (*bio_orig)->bi_rw;
+
+	bio->bi_vcnt = (*bio_orig)->bi_vcnt;
+	bio->bi_idx = (*bio_orig)->bi_idx;
+	bio->bi_size = (*bio_orig)->bi_size;
+
+	if (pool == page_pool) {
+		bio->bi_end_io = bounce_end_io_write;
+		if (rw == READ)
+			bio->bi_end_io = bounce_end_io_read;
+	} else {
+		bio->bi_end_io = bounce_end_io_write_isa;
+		if (rw == READ)
+			bio->bi_end_io = bounce_end_io_read_isa;
+	}
+
+	bio->bi_private = *bio_orig;
+	*bio_orig = bio;
+}
+
+void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)
+{
+	mempool_t *pool;
+
+	/*
+	 * for non-isa bounce case, just check if the bounce pfn is equal
+	 * to or bigger than the highest pfn in the system -- in that case,
+	 * don't waste time iterating over bio segments
+	 */
+	if (!(q->bounce_gfp & GFP_DMA)) {
+		if (q->bounce_pfn >= blk_max_pfn)
+			return;
+		pool = page_pool;
+	} else {
+		BUG_ON(!isa_page_pool);
+		pool = isa_page_pool;
+	}
+
+	blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
+
+	/*
+	 * slow path
+	 */
+	__blk_queue_bounce(q, bio_orig, pool);
+}
+
+EXPORT_SYMBOL(blk_queue_bounce);
@@ -29,13 +29,6 @@
 #include <linux/blktrace_api.h>
 #include <asm/tlbflush.h>
  
-static mempool_t *page_pool, *isa_page_pool;
-
-static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
-{
-	return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
-}
-
 /*
  * Virtual_count is not a pure "count".
  *  0 means that it is not mapped, and has not been mapped
  
@@ -217,281 +210,7 @@
 }
  
 EXPORT_SYMBOL(kunmap_high);
-
-#define POOL_SIZE	64
-
-static __init int init_emergency_pool(void)
-{
-	struct sysinfo i;
-	si_meminfo(&i);
-	si_swapinfo(&i);
-        
-	if (!i.totalhigh)
-		return 0;
-
-	page_pool = mempool_create_page_pool(POOL_SIZE, 0);
-	BUG_ON(!page_pool);
-	printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
-
-	return 0;
-}
-
-__initcall(init_emergency_pool);
-
-/*
- * highmem version, map in to vec
- */
-static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
-{
-	unsigned long flags;
-	unsigned char *vto;
-
-	local_irq_save(flags);
-	vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
-	memcpy(vto + to->bv_offset, vfrom, to->bv_len);
-	kunmap_atomic(vto, KM_BOUNCE_READ);
-	local_irq_restore(flags);
-}
-
-#else /* CONFIG_HIGHMEM */
-
-#define bounce_copy_vec(to, vfrom)	\
-	memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
-
 #endif
-
-#define ISA_POOL_SIZE	16
-
-/*
- * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
- * as the max address, so check if the pool has already been created.
- */
-int init_emergency_isa_pool(void)
-{
-	if (isa_page_pool)
-		return 0;
-
-	isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
-				       mempool_free_pages, (void *) 0);
-	BUG_ON(!isa_page_pool);
-
-	printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
-	return 0;
-}
-
-/*
- * Simple bounce buffer support for highmem pages. Depending on the
- * queue gfp mask set, *to may or may not be a highmem page. kmap it
- * always, it will do the Right Thing
- */
-static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
-{
-	unsigned char *vfrom;
-	struct bio_vec *tovec, *fromvec;
-	int i;
-
-	__bio_for_each_segment(tovec, to, i, 0) {
-		fromvec = from->bi_io_vec + i;
-
-		/*
-		 * not bounced
-		 */
-		if (tovec->bv_page == fromvec->bv_page)
-			continue;
-
-		/*
-		 * fromvec->bv_offset and fromvec->bv_len might have been
-		 * modified by the block layer, so use the original copy,
-		 * bounce_copy_vec already uses tovec->bv_len
-		 */
-		vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
-
-		flush_dcache_page(tovec->bv_page);
-		bounce_copy_vec(tovec, vfrom);
-	}
-}
-
-static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
-{
-	struct bio *bio_orig = bio->bi_private;
-	struct bio_vec *bvec, *org_vec;
-	int i;
-
-	if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
-		set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
-
-	/*
-	 * free up bounce indirect pages used
-	 */
-	__bio_for_each_segment(bvec, bio, i, 0) {
-		org_vec = bio_orig->bi_io_vec + i;
-		if (bvec->bv_page == org_vec->bv_page)
-			continue;
-
-		dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
-		mempool_free(bvec->bv_page, pool);
-	}
-
-	bio_endio(bio_orig, bio_orig->bi_size, err);
-	bio_put(bio);
-}
-
-static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
-{
-	if (bio->bi_size)
-		return 1;
-
-	bounce_end_io(bio, page_pool, err);
-	return 0;
-}
-
-static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
-{
-	if (bio->bi_size)
-		return 1;
-
-	bounce_end_io(bio, isa_page_pool, err);
-	return 0;
-}
-
-static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err)
-{
-	struct bio *bio_orig = bio->bi_private;
-
-	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
-		copy_to_high_bio_irq(bio_orig, bio);
-
-	bounce_end_io(bio, pool, err);
-}
-
-static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
-{
-	if (bio->bi_size)
-		return 1;
-
-	__bounce_end_io_read(bio, page_pool, err);
-	return 0;
-}
-
-static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
-{
-	if (bio->bi_size)
-		return 1;
-
-	__bounce_end_io_read(bio, isa_page_pool, err);
-	return 0;
-}
-
-static void __blk_queue_bounce(request_queue_t *q, struct bio **bio_orig,
-			       mempool_t *pool)
-{
-	struct page *page;
-	struct bio *bio = NULL;
-	int i, rw = bio_data_dir(*bio_orig);
-	struct bio_vec *to, *from;
-
-	bio_for_each_segment(from, *bio_orig, i) {
-		page = from->bv_page;
-
-		/*
-		 * is destination page below bounce pfn?
-		 */
-		if (page_to_pfn(page) < q->bounce_pfn)
-			continue;
-
-		/*
-		 * irk, bounce it
-		 */
-		if (!bio)
-			bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
-
-		to = bio->bi_io_vec + i;
-
-		to->bv_page = mempool_alloc(pool, q->bounce_gfp);
-		to->bv_len = from->bv_len;
-		to->bv_offset = from->bv_offset;
-		inc_zone_page_state(to->bv_page, NR_BOUNCE);
-
-		if (rw == WRITE) {
-			char *vto, *vfrom;
-
-			flush_dcache_page(from->bv_page);
-			vto = page_address(to->bv_page) + to->bv_offset;
-			vfrom = kmap(from->bv_page) + from->bv_offset;
-			memcpy(vto, vfrom, to->bv_len);
-			kunmap(from->bv_page);
-		}
-	}
-
-	/*
-	 * no pages bounced
-	 */
-	if (!bio)
-		return;
-
-	/*
-	 * at least one page was bounced, fill in possible non-highmem
-	 * pages
-	 */
-	__bio_for_each_segment(from, *bio_orig, i, 0) {
-		to = bio_iovec_idx(bio, i);
-		if (!to->bv_page) {
-			to->bv_page = from->bv_page;
-			to->bv_len = from->bv_len;
-			to->bv_offset = from->bv_offset;
-		}
-	}
-
-	bio->bi_bdev = (*bio_orig)->bi_bdev;
-	bio->bi_flags |= (1 << BIO_BOUNCED);
-	bio->bi_sector = (*bio_orig)->bi_sector;
-	bio->bi_rw = (*bio_orig)->bi_rw;
-
-	bio->bi_vcnt = (*bio_orig)->bi_vcnt;
-	bio->bi_idx = (*bio_orig)->bi_idx;
-	bio->bi_size = (*bio_orig)->bi_size;
-
-	if (pool == page_pool) {
-		bio->bi_end_io = bounce_end_io_write;
-		if (rw == READ)
-			bio->bi_end_io = bounce_end_io_read;
-	} else {
-		bio->bi_end_io = bounce_end_io_write_isa;
-		if (rw == READ)
-			bio->bi_end_io = bounce_end_io_read_isa;
-	}
-
-	bio->bi_private = *bio_orig;
-	*bio_orig = bio;
-}
-
-void blk_queue_bounce(request_queue_t *q, struct bio **bio_orig)
-{
-	mempool_t *pool;
-
-	/*
-	 * for non-isa bounce case, just check if the bounce pfn is equal
-	 * to or bigger than the highest pfn in the system -- in that case,
-	 * don't waste time iterating over bio segments
-	 */
-	if (!(q->bounce_gfp & GFP_DMA)) {
-		if (q->bounce_pfn >= blk_max_pfn)
-			return;
-		pool = page_pool;
-	} else {
-		BUG_ON(!isa_page_pool);
-		pool = isa_page_pool;
-	}
-
-	blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
-
-	/*
-	 * slow path
-	 */
-	__blk_queue_bounce(q, bio_orig, pool);
-}
-
-EXPORT_SYMBOL(blk_queue_bounce);
  
 #if defined(HASHED_PAGE_VIRTUAL)
...	...	@@ -12,6 +12,9 @@
12	12	readahead.o swap.o truncate.o vmscan.o \
13	13	prio_tree.o util.o mmzone.o vmstat.o $(mmu-y)
14	14
	15	+ifeq ($(CONFIG_MMU),y)
	16	+obj-y += bounce.o
	17	+endif
15	18	obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o thrash.o
16	19	obj-$(CONFIG_HUGETLBFS) += hugetlb.o
17	20	obj-$(CONFIG_NUMA) += mempolicy.o
	1	+/* bounce buffer handling for block devices
	2	+ *
	3	+ * - Split from highmem.c
	4	+ */
	5	+
	6	+#include <linux/mm.h>
	7	+#include <linux/module.h>
	8	+#include <linux/swap.h>
	9	+#include <linux/bio.h>
	10	+#include <linux/pagemap.h>
	11	+#include <linux/mempool.h>
	12	+#include <linux/blkdev.h>
	13	+#include <linux/init.h>
	14	+#include <linux/hash.h>
	15	+#include <linux/highmem.h>
	16	+#include <linux/blktrace_api.h>
	17	+#include <asm/tlbflush.h>
	18	+
	19	+#define POOL_SIZE 64
	20	+#define ISA_POOL_SIZE 16
	21	+
	22	+static mempool_t page_pool, isa_page_pool;
	23	+
	24	+#ifdef CONFIG_HIGHMEM
	25	+static __init int init_emergency_pool(void)
	26	+{
	27	+ struct sysinfo i;
	28	+ si_meminfo(&i);
	29	+ si_swapinfo(&i);
	30	+
	31	+ if (!i.totalhigh)
	32	+ return 0;
	33	+
	34	+ page_pool = mempool_create_page_pool(POOL_SIZE, 0);
	35	+ BUG_ON(!page_pool);
	36	+ printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
	37	+
	38	+ return 0;
	39	+}
	40	+
	41	+__initcall(init_emergency_pool);
	42	+
	43	+/*
	44	+ * highmem version, map in to vec
	45	+ */
	46	+static void bounce_copy_vec(struct bio_vec to, unsigned char vfrom)
	47	+{
	48	+ unsigned long flags;
	49	+ unsigned char *vto;
	50	+
	51	+ local_irq_save(flags);
	52	+ vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
	53	+ memcpy(vto + to->bv_offset, vfrom, to->bv_len);
	54	+ kunmap_atomic(vto, KM_BOUNCE_READ);
	55	+ local_irq_restore(flags);
	56	+}
	57	+
	58	+#else /* CONFIG_HIGHMEM */
	59	+
	60	+#define bounce_copy_vec(to, vfrom) \
	61	+ memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
	62	+
	63	+#endif /* CONFIG_HIGHMEM */
	64	+
	65	+/*
	66	+ * allocate pages in the DMA region for the ISA pool
	67	+ */
	68	+static void mempool_alloc_pages_isa(gfp_t gfp_mask, void data)
	69	+{
	70	+ return mempool_alloc_pages(gfp_mask \| GFP_DMA, data);
	71	+}
	72	+
	73	+/*
	74	+ * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
	75	+ * as the max address, so check if the pool has already been created.
	76	+ */
	77	+int init_emergency_isa_pool(void)
	78	+{
	79	+ if (isa_page_pool)
	80	+ return 0;
	81	+
	82	+ isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
	83	+ mempool_free_pages, (void *) 0);
	84	+ BUG_ON(!isa_page_pool);
	85	+
	86	+ printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
	87	+ return 0;
	88	+}
	89	+
	90	+/*
	91	+ * Simple bounce buffer support for highmem pages. Depending on the
	92	+ * queue gfp mask set, *to may or may not be a highmem page. kmap it
	93	+ * always, it will do the Right Thing
	94	+ */
	95	+static void copy_to_high_bio_irq(struct bio to, struct bio from)
	96	+{
	97	+ unsigned char *vfrom;
	98	+ struct bio_vec tovec, fromvec;
	99	+ int i;
	100	+
	101	+ __bio_for_each_segment(tovec, to, i, 0) {
	102	+ fromvec = from->bi_io_vec + i;
	103	+
	104	+ /*
	105	+ * not bounced
	106	+ */
	107	+ if (tovec->bv_page == fromvec->bv_page)
	108	+ continue;
	109	+
	110	+ /*
	111	+ * fromvec->bv_offset and fromvec->bv_len might have been
	112	+ * modified by the block layer, so use the original copy,
	113	+ * bounce_copy_vec already uses tovec->bv_len
	114	+ */
	115	+ vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
	116	+
	117	+ flush_dcache_page(tovec->bv_page);
	118	+ bounce_copy_vec(tovec, vfrom);
	119	+ }
	120	+}
	121	+
	122	+static void bounce_end_io(struct bio bio, mempool_t pool, int err)
	123	+{
	124	+ struct bio *bio_orig = bio->bi_private;
	125	+ struct bio_vec bvec, org_vec;
	126	+ int i;
	127	+
	128	+ if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
	129	+ set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
	130	+
	131	+ /*
	132	+ * free up bounce indirect pages used
	133	+ */
	134	+ __bio_for_each_segment(bvec, bio, i, 0) {
	135	+ org_vec = bio_orig->bi_io_vec + i;
	136	+ if (bvec->bv_page == org_vec->bv_page)
	137	+ continue;
	138	+
	139	+ dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
	140	+ mempool_free(bvec->bv_page, pool);
	141	+ }
	142	+
	143	+ bio_endio(bio_orig, bio_orig->bi_size, err);
	144	+ bio_put(bio);
	145	+}
	146	+
	147	+static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
	148	+{
	149	+ if (bio->bi_size)
	150	+ return 1;
	151	+
	152	+ bounce_end_io(bio, page_pool, err);
	153	+ return 0;
	154	+}
	155	+
	156	+static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
	157	+{
	158	+ if (bio->bi_size)
	159	+ return 1;
	160	+
	161	+ bounce_end_io(bio, isa_page_pool, err);
	162	+ return 0;
	163	+}
	164	+
	165	+static void __bounce_end_io_read(struct bio bio, mempool_t pool, int err)
	166	+{
	167	+ struct bio *bio_orig = bio->bi_private;
	168	+
	169	+ if (test_bit(BIO_UPTODATE, &bio->bi_flags))
	170	+ copy_to_high_bio_irq(bio_orig, bio);
	171	+
	172	+ bounce_end_io(bio, pool, err);
	173	+}
	174	+
	175	+static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
	176	+{
	177	+ if (bio->bi_size)
	178	+ return 1;
	179	+
	180	+ __bounce_end_io_read(bio, page_pool, err);
	181	+ return 0;
	182	+}
	183	+
	184	+static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
	185	+{
	186	+ if (bio->bi_size)
	187	+ return 1;
	188	+
	189	+ __bounce_end_io_read(bio, isa_page_pool, err);
	190	+ return 0;
	191	+}
	192	+
	193	+static void __blk_queue_bounce(request_queue_t q, struct bio *bio_orig,
	194	+ mempool_t *pool)
	195	+{
	196	+ struct page *page;
	197	+ struct bio *bio = NULL;
	198	+ int i, rw = bio_data_dir(*bio_orig);
	199	+ struct bio_vec to, from;
	200	+
	201	+ bio_for_each_segment(from, *bio_orig, i) {
	202	+ page = from->bv_page;
	203	+
	204	+ /*
	205	+ * is destination page below bounce pfn?
	206	+ */
	207	+ if (page_to_pfn(page) < q->bounce_pfn)
	208	+ continue;
	209	+
	210	+ /*
	211	+ * irk, bounce it
	212	+ */
	213	+ if (!bio)
	214	+ bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
	215	+
	216	+ to = bio->bi_io_vec + i;
	217	+
	218	+ to->bv_page = mempool_alloc(pool, q->bounce_gfp);
	219	+ to->bv_len = from->bv_len;
	220	+ to->bv_offset = from->bv_offset;
	221	+ inc_zone_page_state(to->bv_page, NR_BOUNCE);
	222	+
	223	+ if (rw == WRITE) {
	224	+ char vto, vfrom;
	225	+
	226	+ flush_dcache_page(from->bv_page);
	227	+ vto = page_address(to->bv_page) + to->bv_offset;
	228	+ vfrom = kmap(from->bv_page) + from->bv_offset;
	229	+ memcpy(vto, vfrom, to->bv_len);
	230	+ kunmap(from->bv_page);
	231	+ }
	232	+ }
	233	+
	234	+ /*
	235	+ * no pages bounced
	236	+ */
	237	+ if (!bio)
	238	+ return;
	239	+
	240	+ /*
	241	+ * at least one page was bounced, fill in possible non-highmem
	242	+ * pages
	243	+ */
	244	+ __bio_for_each_segment(from, *bio_orig, i, 0) {
	245	+ to = bio_iovec_idx(bio, i);
	246	+ if (!to->bv_page) {
	247	+ to->bv_page = from->bv_page;
	248	+ to->bv_len = from->bv_len;
	249	+ to->bv_offset = from->bv_offset;
	250	+ }
	251	+ }
	252	+
	253	+ bio->bi_bdev = (*bio_orig)->bi_bdev;
	254	+ bio->bi_flags \|= (1 << BIO_BOUNCED);
	255	+ bio->bi_sector = (*bio_orig)->bi_sector;
	256	+ bio->bi_rw = (*bio_orig)->bi_rw;
	257	+
	258	+ bio->bi_vcnt = (*bio_orig)->bi_vcnt;
	259	+ bio->bi_idx = (*bio_orig)->bi_idx;
	260	+ bio->bi_size = (*bio_orig)->bi_size;
	261	+
	262	+ if (pool == page_pool) {
	263	+ bio->bi_end_io = bounce_end_io_write;
	264	+ if (rw == READ)
	265	+ bio->bi_end_io = bounce_end_io_read;
	266	+ } else {
	267	+ bio->bi_end_io = bounce_end_io_write_isa;
	268	+ if (rw == READ)
	269	+ bio->bi_end_io = bounce_end_io_read_isa;
	270	+ }
	271	+
	272	+ bio->bi_private = *bio_orig;
	273	+ *bio_orig = bio;
	274	+}
	275	+
	276	+void blk_queue_bounce(request_queue_t q, struct bio *bio_orig)
	277	+{
	278	+ mempool_t *pool;
	279	+
	280	+ /*
	281	+ * for non-isa bounce case, just check if the bounce pfn is equal
	282	+ * to or bigger than the highest pfn in the system -- in that case,
	283	+ * don't waste time iterating over bio segments
	284	+ */
	285	+ if (!(q->bounce_gfp & GFP_DMA)) {
	286	+ if (q->bounce_pfn >= blk_max_pfn)
	287	+ return;
	288	+ pool = page_pool;
	289	+ } else {
	290	+ BUG_ON(!isa_page_pool);
	291	+ pool = isa_page_pool;
	292	+ }
	293	+
	294	+ blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
	295	+
	296	+ /*
	297	+ * slow path
	298	+ */
	299	+ __blk_queue_bounce(q, bio_orig, pool);
	300	+}
	301	+
	302	+EXPORT_SYMBOL(blk_queue_bounce);
...	...	@@ -29,13 +29,6 @@
29	29	#include <linux/blktrace_api.h>
30	30	#include <asm/tlbflush.h>
31	31
32		-static mempool_t page_pool, isa_page_pool;
33		-
34		-static void mempool_alloc_pages_isa(gfp_t gfp_mask, void data)
35		-{
36		- return mempool_alloc_pages(gfp_mask \| GFP_DMA, data);
37		-}
38		-
39	32	/*
40	33	* Virtual_count is not a pure "count".
41	34	* 0 means that it is not mapped, and has not been mapped
42	35
...	...	@@ -217,281 +210,7 @@
217	210	}
218	211
219	212	EXPORT_SYMBOL(kunmap_high);
220		-
221		-#define POOL_SIZE 64
222		-
223		-static __init int init_emergency_pool(void)
224		-{
225		- struct sysinfo i;
226		- si_meminfo(&i);
227		- si_swapinfo(&i);
228		-
229		- if (!i.totalhigh)
230		- return 0;
231		-
232		- page_pool = mempool_create_page_pool(POOL_SIZE, 0);
233		- BUG_ON(!page_pool);
234		- printk("highmem bounce pool size: %d pages\n", POOL_SIZE);
235		-
236		- return 0;
237		-}
238		-
239		-__initcall(init_emergency_pool);
240		-
241		-/*
242		- * highmem version, map in to vec
243		- */
244		-static void bounce_copy_vec(struct bio_vec to, unsigned char vfrom)
245		-{
246		- unsigned long flags;
247		- unsigned char *vto;
248		-
249		- local_irq_save(flags);
250		- vto = kmap_atomic(to->bv_page, KM_BOUNCE_READ);
251		- memcpy(vto + to->bv_offset, vfrom, to->bv_len);
252		- kunmap_atomic(vto, KM_BOUNCE_READ);
253		- local_irq_restore(flags);
254		-}
255		-
256		-#else /* CONFIG_HIGHMEM */
257		-
258		-#define bounce_copy_vec(to, vfrom) \
259		- memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
260		-
261	213	#endif
262		-
263		-#define ISA_POOL_SIZE 16
264		-
265		-/*
266		- * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
267		- * as the max address, so check if the pool has already been created.
268		- */
269		-int init_emergency_isa_pool(void)
270		-{
271		- if (isa_page_pool)
272		- return 0;
273		-
274		- isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
275		- mempool_free_pages, (void *) 0);
276		- BUG_ON(!isa_page_pool);
277		-
278		- printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
279		- return 0;
280		-}
281		-
282		-/*
283		- * Simple bounce buffer support for highmem pages. Depending on the
284		- * queue gfp mask set, *to may or may not be a highmem page. kmap it
285		- * always, it will do the Right Thing
286		- */
287		-static void copy_to_high_bio_irq(struct bio to, struct bio from)
288		-{
289		- unsigned char *vfrom;
290		- struct bio_vec tovec, fromvec;
291		- int i;
292		-
293		- __bio_for_each_segment(tovec, to, i, 0) {
294		- fromvec = from->bi_io_vec + i;
295		-
296		- /*
297		- * not bounced
298		- */
299		- if (tovec->bv_page == fromvec->bv_page)
300		- continue;
301		-
302		- /*
303		- * fromvec->bv_offset and fromvec->bv_len might have been
304		- * modified by the block layer, so use the original copy,
305		- * bounce_copy_vec already uses tovec->bv_len
306		- */
307		- vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
308		-
309		- flush_dcache_page(tovec->bv_page);
310		- bounce_copy_vec(tovec, vfrom);
311		- }
312		-}
313		-
314		-static void bounce_end_io(struct bio bio, mempool_t pool, int err)
315		-{
316		- struct bio *bio_orig = bio->bi_private;
317		- struct bio_vec bvec, org_vec;
318		- int i;
319		-
320		- if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
321		- set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
322		-
323		- /*
324		- * free up bounce indirect pages used
325		- */
326		- __bio_for_each_segment(bvec, bio, i, 0) {
327		- org_vec = bio_orig->bi_io_vec + i;
328		- if (bvec->bv_page == org_vec->bv_page)
329		- continue;
330		-
331		- dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
332		- mempool_free(bvec->bv_page, pool);
333		- }
334		-
335		- bio_endio(bio_orig, bio_orig->bi_size, err);
336		- bio_put(bio);
337		-}
338		-
339		-static int bounce_end_io_write(struct bio *bio, unsigned int bytes_done, int err)
340		-{
341		- if (bio->bi_size)
342		- return 1;
343		-
344		- bounce_end_io(bio, page_pool, err);
345		- return 0;
346		-}
347		-
348		-static int bounce_end_io_write_isa(struct bio *bio, unsigned int bytes_done, int err)
349		-{
350		- if (bio->bi_size)
351		- return 1;
352		-
353		- bounce_end_io(bio, isa_page_pool, err);
354		- return 0;
355		-}
356		-
357		-static void __bounce_end_io_read(struct bio bio, mempool_t pool, int err)
358		-{
359		- struct bio *bio_orig = bio->bi_private;
360		-
361		- if (test_bit(BIO_UPTODATE, &bio->bi_flags))
362		- copy_to_high_bio_irq(bio_orig, bio);
363		-
364		- bounce_end_io(bio, pool, err);
365		-}
366		-
367		-static int bounce_end_io_read(struct bio *bio, unsigned int bytes_done, int err)
368		-{
369		- if (bio->bi_size)
370		- return 1;
371		-
372		- __bounce_end_io_read(bio, page_pool, err);
373		- return 0;
374		-}
375		-
376		-static int bounce_end_io_read_isa(struct bio *bio, unsigned int bytes_done, int err)
377		-{
378		- if (bio->bi_size)
379		- return 1;
380		-
381		- __bounce_end_io_read(bio, isa_page_pool, err);
382		- return 0;
383		-}
384		-
385		-static void __blk_queue_bounce(request_queue_t q, struct bio *bio_orig,
386		- mempool_t *pool)
387		-{
388		- struct page *page;
389		- struct bio *bio = NULL;
390		- int i, rw = bio_data_dir(*bio_orig);
391		- struct bio_vec to, from;
392		-
393		- bio_for_each_segment(from, *bio_orig, i) {
394		- page = from->bv_page;
395		-
396		- /*
397		- * is destination page below bounce pfn?
398		- */
399		- if (page_to_pfn(page) < q->bounce_pfn)
400		- continue;
401		-
402		- /*
403		- * irk, bounce it
404		- */
405		- if (!bio)
406		- bio = bio_alloc(GFP_NOIO, (*bio_orig)->bi_vcnt);
407		-
408		- to = bio->bi_io_vec + i;
409		-
410		- to->bv_page = mempool_alloc(pool, q->bounce_gfp);
411		- to->bv_len = from->bv_len;
412		- to->bv_offset = from->bv_offset;
413		- inc_zone_page_state(to->bv_page, NR_BOUNCE);
414		-
415		- if (rw == WRITE) {
416		- char vto, vfrom;
417		-
418		- flush_dcache_page(from->bv_page);
419		- vto = page_address(to->bv_page) + to->bv_offset;
420		- vfrom = kmap(from->bv_page) + from->bv_offset;
421		- memcpy(vto, vfrom, to->bv_len);
422		- kunmap(from->bv_page);
423		- }
424		- }
425		-
426		- /*
427		- * no pages bounced
428		- */
429		- if (!bio)
430		- return;
431		-
432		- /*
433		- * at least one page was bounced, fill in possible non-highmem
434		- * pages
435		- */
436		- __bio_for_each_segment(from, *bio_orig, i, 0) {
437		- to = bio_iovec_idx(bio, i);
438		- if (!to->bv_page) {
439		- to->bv_page = from->bv_page;
440		- to->bv_len = from->bv_len;
441		- to->bv_offset = from->bv_offset;
442		- }
443		- }
444		-
445		- bio->bi_bdev = (*bio_orig)->bi_bdev;
446		- bio->bi_flags \|= (1 << BIO_BOUNCED);
447		- bio->bi_sector = (*bio_orig)->bi_sector;
448		- bio->bi_rw = (*bio_orig)->bi_rw;
449		-
450		- bio->bi_vcnt = (*bio_orig)->bi_vcnt;
451		- bio->bi_idx = (*bio_orig)->bi_idx;
452		- bio->bi_size = (*bio_orig)->bi_size;
453		-
454		- if (pool == page_pool) {
455		- bio->bi_end_io = bounce_end_io_write;
456		- if (rw == READ)
457		- bio->bi_end_io = bounce_end_io_read;
458		- } else {
459		- bio->bi_end_io = bounce_end_io_write_isa;
460		- if (rw == READ)
461		- bio->bi_end_io = bounce_end_io_read_isa;
462		- }
463		-
464		- bio->bi_private = *bio_orig;
465		- *bio_orig = bio;
466		-}
467		-
468		-void blk_queue_bounce(request_queue_t q, struct bio *bio_orig)
469		-{
470		- mempool_t *pool;
471		-
472		- /*
473		- * for non-isa bounce case, just check if the bounce pfn is equal
474		- * to or bigger than the highest pfn in the system -- in that case,
475		- * don't waste time iterating over bio segments
476		- */
477		- if (!(q->bounce_gfp & GFP_DMA)) {
478		- if (q->bounce_pfn >= blk_max_pfn)
479		- return;
480		- pool = page_pool;
481		- } else {
482		- BUG_ON(!isa_page_pool);
483		- pool = isa_page_pool;
484		- }
485		-
486		- blk_add_trace_bio(q, *bio_orig, BLK_TA_BOUNCE);
487		-
488		- /*
489		- * slow path
490		- */
491		- __blk_queue_bounce(q, bio_orig, pool);
492		-}
493		-
494		-EXPORT_SYMBOL(blk_queue_bounce);
495	214
496	215	#if defined(HASHED_PAGE_VIRTUAL)
497	216