blkdev: add blkdev_issue_zeroout helper function

- Add bio_batch helper primitive. This is rather generic primitive for submitting/waiting a complex request which consists of several bios. - blkdev_issue_zeroout() generate number of zero filed write bios. Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

blkdev: add blkdev_issue_zeroout helper function
- Add bio_batch helper primitive. This is rather generic primitive for submitting/waiting a complex request which consists of several bios. - blkdev_issue_zeroout() generate number of zero filed write bios. Signed-off-by: Dmitry Monakhov <dmonakhov@openvz.org> Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
Dmitry Monakhov · Jens Axboe
1 parent f31e7e4022
Showing 2 changed files with 120 additions and 1 deletions Side-by-side Diff
block/blk-lib.c
include/linux/blkdev.h
@@ -112,4 +112,122 @@
 	return -ENOMEM;
 }
 EXPORT_SYMBOL(blkdev_issue_discard);
+
+struct bio_batch
+{
+	atomic_t 		done;
+	unsigned long 		flags;
+	struct completion 	*wait;
+	bio_end_io_t		*end_io;
+};
+
+static void bio_batch_end_io(struct bio *bio, int err)
+{
+	struct bio_batch *bb = bio->bi_private;
+	if (err) {
+		if (err == -EOPNOTSUPP)
+			set_bit(BIO_EOPNOTSUPP, &bb->flags);
+		else
+			clear_bit(BIO_UPTODATE, &bb->flags);
+	}
+	if (bb) {
+		if (bb->end_io)
+			bb->end_io(bio, err);
+		atomic_inc(&bb->done);
+		complete(bb->wait);
+	}
+	bio_put(bio);
+}
+
+/**
+ * blkdev_issue_zeroout generate number of zero filed write bios
+ * @bdev:	blockdev to issue
+ * @sector:	start sector
+ * @nr_sects:	number of sectors to write
+ * @gfp_mask:	memory allocation flags (for bio_alloc)
+ * @flags:	BLKDEV_IFL_* flags to control behaviour
+ *
+ * Description:
+ *  Generate and issue number of bios with zerofiled pages.
+ *  Send barrier at the beginning and at the end if requested. This guarantie
+ *  correct request ordering. Empty barrier allow us to avoid post queue flush.
+ */
+
+int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
+			sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
+{
+	int ret = 0;
+	struct bio *bio;
+	struct bio_batch bb;
+	unsigned int sz, issued = 0;
+	DECLARE_COMPLETION_ONSTACK(wait);
+
+	atomic_set(&bb.done, 0);
+	bb.flags = 1 << BIO_UPTODATE;
+	bb.wait = &wait;
+	bb.end_io = NULL;
+
+	if (flags & BLKDEV_IFL_BARRIER) {
+		/* issue async barrier before the data */
+		ret = blkdev_issue_flush(bdev, gfp_mask, NULL, 0);
+		if (ret)
+			return ret;
+	}
+submit:
+	while (nr_sects != 0) {
+		bio = bio_alloc(gfp_mask,
+				min(nr_sects, (sector_t)BIO_MAX_PAGES));
+		if (!bio)
+			break;
+
+		bio->bi_sector = sector;
+		bio->bi_bdev   = bdev;
+		bio->bi_end_io = bio_batch_end_io;
+		if (flags & BLKDEV_IFL_WAIT)
+			bio->bi_private = &bb;
+
+		while(nr_sects != 0) {
+			sz = min(PAGE_SIZE >> 9 , nr_sects);
+			if (sz == 0)
+				/* bio has maximum size possible */
+				break;
+			ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
+			nr_sects -= ret >> 9;
+			sector += ret >> 9;
+			if (ret < (sz << 9))
+				break;
+		}
+		issued++;
+		submit_bio(WRITE, bio);
+	}
+	/*
+	 * When all data bios are in flight. Send final barrier if requeted.
+	 */
+	if (nr_sects == 0 && flags & BLKDEV_IFL_BARRIER)
+		ret = blkdev_issue_flush(bdev, gfp_mask, NULL,
+					flags & BLKDEV_IFL_WAIT);
+
+
+	if (flags & BLKDEV_IFL_WAIT)
+		/* Wait for bios in-flight */
+		while ( issued != atomic_read(&bb.done))
+			wait_for_completion(&wait);
+
+	if (!test_bit(BIO_UPTODATE, &bb.flags))
+		/* One of bios in the batch was completed with error.*/
+		ret = -EIO;
+
+	if (ret)
+		goto out;
+
+	if (test_bit(BIO_EOPNOTSUPP, &bb.flags)) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
+	if (nr_sects != 0)
+		goto submit;
+out:
+	return ret;
+}
+EXPORT_SYMBOL(blkdev_issue_zeroout);
@@ -1008,7 +1008,8 @@
 			unsigned long);
 extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
-
+extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
+			sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
 static inline int sb_issue_discard(struct super_block *sb,
 				   sector_t block, sector_t nr_blocks)
 {
...	...	@@ -112,4 +112,122 @@
112	112	return -ENOMEM;
113	113	}
114	114	EXPORT_SYMBOL(blkdev_issue_discard);
	115	+
	116	+struct bio_batch
	117	+{
	118	+ atomic_t done;
	119	+ unsigned long flags;
	120	+ struct completion *wait;
	121	+ bio_end_io_t *end_io;
	122	+};
	123	+
	124	+static void bio_batch_end_io(struct bio *bio, int err)
	125	+{
	126	+ struct bio_batch *bb = bio->bi_private;
	127	+ if (err) {
	128	+ if (err == -EOPNOTSUPP)
	129	+ set_bit(BIO_EOPNOTSUPP, &bb->flags);
	130	+ else
	131	+ clear_bit(BIO_UPTODATE, &bb->flags);
	132	+ }
	133	+ if (bb) {
	134	+ if (bb->end_io)
	135	+ bb->end_io(bio, err);
	136	+ atomic_inc(&bb->done);
	137	+ complete(bb->wait);
	138	+ }
	139	+ bio_put(bio);
	140	+}
	141	+
	142	+/**
	143	+ * blkdev_issue_zeroout generate number of zero filed write bios
	144	+ * @bdev: blockdev to issue
	145	+ * @sector: start sector
	146	+ * @nr_sects: number of sectors to write
	147	+ * @gfp_mask: memory allocation flags (for bio_alloc)
	148	+ * @flags: BLKDEV_IFL_* flags to control behaviour
	149	+ *
	150	+ * Description:
	151	+ * Generate and issue number of bios with zerofiled pages.
	152	+ * Send barrier at the beginning and at the end if requested. This guarantie
	153	+ * correct request ordering. Empty barrier allow us to avoid post queue flush.
	154	+ */
	155	+
	156	+int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
	157	+ sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
	158	+{
	159	+ int ret = 0;
	160	+ struct bio *bio;
	161	+ struct bio_batch bb;
	162	+ unsigned int sz, issued = 0;
	163	+ DECLARE_COMPLETION_ONSTACK(wait);
	164	+
	165	+ atomic_set(&bb.done, 0);
	166	+ bb.flags = 1 << BIO_UPTODATE;
	167	+ bb.wait = &wait;
	168	+ bb.end_io = NULL;
	169	+
	170	+ if (flags & BLKDEV_IFL_BARRIER) {
	171	+ /* issue async barrier before the data */
	172	+ ret = blkdev_issue_flush(bdev, gfp_mask, NULL, 0);
	173	+ if (ret)
	174	+ return ret;
	175	+ }
	176	+submit:
	177	+ while (nr_sects != 0) {
	178	+ bio = bio_alloc(gfp_mask,
	179	+ min(nr_sects, (sector_t)BIO_MAX_PAGES));
	180	+ if (!bio)
	181	+ break;
	182	+
	183	+ bio->bi_sector = sector;
	184	+ bio->bi_bdev = bdev;
	185	+ bio->bi_end_io = bio_batch_end_io;
	186	+ if (flags & BLKDEV_IFL_WAIT)
	187	+ bio->bi_private = &bb;
	188	+
	189	+ while(nr_sects != 0) {
	190	+ sz = min(PAGE_SIZE >> 9 , nr_sects);
	191	+ if (sz == 0)
	192	+ /* bio has maximum size possible */
	193	+ break;
	194	+ ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
	195	+ nr_sects -= ret >> 9;
	196	+ sector += ret >> 9;
	197	+ if (ret < (sz << 9))
	198	+ break;
	199	+ }
	200	+ issued++;
	201	+ submit_bio(WRITE, bio);
	202	+ }
	203	+ /*
	204	+ * When all data bios are in flight. Send final barrier if requeted.
	205	+ */
	206	+ if (nr_sects == 0 && flags & BLKDEV_IFL_BARRIER)
	207	+ ret = blkdev_issue_flush(bdev, gfp_mask, NULL,
	208	+ flags & BLKDEV_IFL_WAIT);
	209	+
	210	+
	211	+ if (flags & BLKDEV_IFL_WAIT)
	212	+ /* Wait for bios in-flight */
	213	+ while ( issued != atomic_read(&bb.done))
	214	+ wait_for_completion(&wait);
	215	+
	216	+ if (!test_bit(BIO_UPTODATE, &bb.flags))
	217	+ /* One of bios in the batch was completed with error.*/
	218	+ ret = -EIO;
	219	+
	220	+ if (ret)
	221	+ goto out;
	222	+
	223	+ if (test_bit(BIO_EOPNOTSUPP, &bb.flags)) {
	224	+ ret = -EOPNOTSUPP;
	225	+ goto out;
	226	+ }
	227	+ if (nr_sects != 0)
	228	+ goto submit;
	229	+out:
	230	+ return ret;
	231	+}
	232	+EXPORT_SYMBOL(blkdev_issue_zeroout);
...	...	@@ -1008,7 +1008,8 @@
1008	1008	unsigned long);
1009	1009	extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1010	1010	sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1011		-
	1011	+extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
	1012	+ sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1012	1013	static inline int sb_issue_discard(struct super_block *sb,
1013	1014	sector_t block, sector_t nr_blocks)
1014	1015	{