dm bufio: prefetch

This patch introduces a new function dm_bufio_prefetch. It prefetches the specified range of blocks into dm-bufio cache without waiting for i/o completion. Signed-off-by: Mikulas Patocka <mpatocka@redhat.com> Signed-off-by: Alasdair G Kergon <agk@redhat.com>

dm bufio: prefetch
This patch introduces a new function dm_bufio_prefetch. It prefetches the specified range of blocks into dm-bufio cache without waiting for i/o completion. Signed-off-by: Mikulas Patocka <mpatocka@redhat.com> Signed-off-by: Alasdair G Kergon <agk@redhat.com>
Mikulas Patocka · Alasdair G Kergon
1 parent 67e2e2b281
Showing 2 changed files with 90 additions and 26 deletions Side-by-side Diff
drivers/md/dm-bufio.c
drivers/md/dm-bufio.h
@@ -578,7 +578,7 @@
 	struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
  
 	b->write_error = error;
-	if (error) {
+	if (unlikely(error)) {
 		struct dm_bufio_client *c = b->c;
 		(void)cmpxchg(&c->async_write_error, 0, error);
 	}
  
@@ -697,13 +697,20 @@
 	dm_bufio_lock(c);
 }
  
+enum new_flag {
+	NF_FRESH = 0,
+	NF_READ = 1,
+	NF_GET = 2,
+	NF_PREFETCH = 3
+};
+
 /*
  * Allocate a new buffer. If the allocation is not possible, wait until
  * some other thread frees a buffer.
  *
  * May drop the lock and regain it.
  */
-static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
 {
 	struct dm_buffer *b;
  
@@ -726,6 +733,9 @@
 				return b;
 		}
  
+		if (nf == NF_PREFETCH)
+			return NULL;
+
 		if (!list_empty(&c->reserved_buffers)) {
 			b = list_entry(c->reserved_buffers.next,
 				       struct dm_buffer, lru_list);
  
  
@@ -743,10 +753,13 @@
 	}
 }
  
-static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c)
+static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
 {
-	struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
+	struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
  
+	if (!b)
+		return NULL;
+
 	if (c->alloc_callback)
 		c->alloc_callback(b);
  
  
  
  
@@ -865,32 +878,23 @@
  * Getting a buffer
  *--------------------------------------------------------------*/
  
-enum new_flag {
-	NF_FRESH = 0,
-	NF_READ = 1,
-	NF_GET = 2
-};
-
 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
-				     enum new_flag nf, struct dm_buffer **bp,
-				     int *need_submit)
+				     enum new_flag nf, int *need_submit)
 {
 	struct dm_buffer *b, *new_b = NULL;
  
 	*need_submit = 0;
  
 	b = __find(c, block);
-	if (b) {
-		b->hold_count++;
-		__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
-			     test_bit(B_WRITING, &b->state));
-		return b;
-	}
+	if (b)
+		goto found_buffer;
  
 	if (nf == NF_GET)
 		return NULL;
  
-	new_b = __alloc_buffer_wait(c);
+	new_b = __alloc_buffer_wait(c, nf);
+	if (!new_b)
+		return NULL;
  
 	/*
 	 * We've had a period where the mutex was unlocked, so need to
@@ -899,10 +903,7 @@
 	b = __find(c, block);
 	if (b) {
 		__free_buffer_wake(new_b);
-		b->hold_count++;
-		__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
-			     test_bit(B_WRITING, &b->state));
-		return b;
+		goto found_buffer;
 	}
  
 	__check_watermark(c);
@@ -922,6 +923,24 @@
 	*need_submit = 1;
  
 	return b;
+
+found_buffer:
+	if (nf == NF_PREFETCH)
+		return NULL;
+	/*
+	 * Note: it is essential that we don't wait for the buffer to be
+	 * read if dm_bufio_get function is used. Both dm_bufio_get and
+	 * dm_bufio_prefetch can be used in the driver request routine.
+	 * If the user called both dm_bufio_prefetch and dm_bufio_get on
+	 * the same buffer, it would deadlock if we waited.
+	 */
+	if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
+		return NULL;
+
+	b->hold_count++;
+	__relink_lru(b, test_bit(B_DIRTY, &b->state) ||
+		     test_bit(B_WRITING, &b->state));
+	return b;
 }
  
 /*
  
@@ -956,10 +975,10 @@
 	struct dm_buffer *b;
  
 	dm_bufio_lock(c);
-	b = __bufio_new(c, block, nf, bp, &need_submit);
+	b = __bufio_new(c, block, nf, &need_submit);
 	dm_bufio_unlock(c);
  
-	if (!b || IS_ERR(b))
+	if (!b)
 		return b;
  
 	if (need_submit)
  
@@ -1005,13 +1024,47 @@
 }
 EXPORT_SYMBOL_GPL(dm_bufio_new);
  
+void dm_bufio_prefetch(struct dm_bufio_client *c,
+		       sector_t block, unsigned n_blocks)
+{
+	struct blk_plug plug;
+
+	blk_start_plug(&plug);
+	dm_bufio_lock(c);
+
+	for (; n_blocks--; block++) {
+		int need_submit;
+		struct dm_buffer *b;
+		b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
+		if (unlikely(b != NULL)) {
+			dm_bufio_unlock(c);
+
+			if (need_submit)
+				submit_io(b, READ, b->block, read_endio);
+			dm_bufio_release(b);
+
+			dm_bufio_cond_resched();
+
+			if (!n_blocks)
+				goto flush_plug;
+			dm_bufio_lock(c);
+		}
+
+	}
+
+	dm_bufio_unlock(c);
+
+flush_plug:
+	blk_finish_plug(&plug);
+}
+EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
+
 void dm_bufio_release(struct dm_buffer *b)
 {
 	struct dm_bufio_client *c = b->c;
  
 	dm_bufio_lock(c);
  
-	BUG_ON(test_bit(B_READING, &b->state));
 	BUG_ON(!b->hold_count);
  
 	b->hold_count--;
@@ -1024,6 +1077,7 @@
 		 * invalid buffer.
 		 */
 		if ((b->read_error || b->write_error) &&
+		    !test_bit(B_READING, &b->state) &&
 		    !test_bit(B_WRITING, &b->state) &&
 		    !test_bit(B_DIRTY, &b->state)) {
 			__unlink_buffer(b);
@@ -1040,6 +1094,8 @@
 	struct dm_bufio_client *c = b->c;
  
 	dm_bufio_lock(c);
+
+	BUG_ON(test_bit(B_READING, &b->state));
  
 	if (!test_and_set_bit(B_DIRTY, &b->state))
 		__relink_lru(b, LIST_DIRTY);
@@ -63,6 +63,14 @@
 		   struct dm_buffer **bp);
  
 /*
+ * Prefetch the specified blocks to the cache.
+ * The function starts to read the blocks and returns without waiting for
+ * I/O to finish.
+ */
+void dm_bufio_prefetch(struct dm_bufio_client *c,
+		       sector_t block, unsigned n_blocks);
+
+/*
  * Release a reference obtained with dm_bufio_{read,get,new}. The data
  * pointer and dm_buffer pointer is no longer valid after this call.
  */
...	...	@@ -578,7 +578,7 @@
578	578	struct dm_buffer *b = container_of(bio, struct dm_buffer, bio);
579	579
580	580	b->write_error = error;
581		- if (error) {
	581	+ if (unlikely(error)) {
582	582	struct dm_bufio_client *c = b->c;
583	583	(void)cmpxchg(&c->async_write_error, 0, error);
584	584	}
585	585
...	...	@@ -697,13 +697,20 @@
697	697	dm_bufio_lock(c);
698	698	}
699	699
	700	+enum new_flag {
	701	+ NF_FRESH = 0,
	702	+ NF_READ = 1,
	703	+ NF_GET = 2,
	704	+ NF_PREFETCH = 3
	705	+};
	706	+
700	707	/*
701	708	* Allocate a new buffer. If the allocation is not possible, wait until
702	709	* some other thread frees a buffer.
703	710	*
704	711	* May drop the lock and regain it.
705	712	*/
706		-static struct dm_buffer __alloc_buffer_wait_no_callback(struct dm_bufio_client c)
	713	+static struct dm_buffer __alloc_buffer_wait_no_callback(struct dm_bufio_client c, enum new_flag nf)
707	714	{
708	715	struct dm_buffer *b;
709	716
...	...	@@ -726,6 +733,9 @@
726	733	return b;
727	734	}
728	735
	736	+ if (nf == NF_PREFETCH)
	737	+ return NULL;
	738	+
729	739	if (!list_empty(&c->reserved_buffers)) {
730	740	b = list_entry(c->reserved_buffers.next,
731	741	struct dm_buffer, lru_list);
732	742
733	743
...	...	@@ -743,10 +753,13 @@
743	753	}
744	754	}
745	755
746		-static struct dm_buffer __alloc_buffer_wait(struct dm_bufio_client c)
	756	+static struct dm_buffer __alloc_buffer_wait(struct dm_bufio_client c, enum new_flag nf)
747	757	{
748		- struct dm_buffer *b = __alloc_buffer_wait_no_callback(c);
	758	+ struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
749	759
	760	+ if (!b)
	761	+ return NULL;
	762	+
750	763	if (c->alloc_callback)
751	764	c->alloc_callback(b);
752	765
753	766
754	767
755	768
...	...	@@ -865,32 +878,23 @@
865	878	* Getting a buffer
866	879	--------------------------------------------------------------/
867	880
868		-enum new_flag {
869		- NF_FRESH = 0,
870		- NF_READ = 1,
871		- NF_GET = 2
872		-};
873		-
874	881	static struct dm_buffer __bufio_new(struct dm_bufio_client c, sector_t block,
875		- enum new_flag nf, struct dm_buffer **bp,
876		- int *need_submit)
	882	+ enum new_flag nf, int *need_submit)
877	883	{
878	884	struct dm_buffer b, new_b = NULL;
879	885
880	886	*need_submit = 0;
881	887
882	888	b = __find(c, block);
883		- if (b) {
884		- b->hold_count++;
885		- __relink_lru(b, test_bit(B_DIRTY, &b->state) \|\|
886		- test_bit(B_WRITING, &b->state));
887		- return b;
888		- }
	889	+ if (b)
	890	+ goto found_buffer;
889	891
890	892	if (nf == NF_GET)
891	893	return NULL;
892	894
893		- new_b = __alloc_buffer_wait(c);
	895	+ new_b = __alloc_buffer_wait(c, nf);
	896	+ if (!new_b)
	897	+ return NULL;
894	898
895	899	/*
896	900	* We've had a period where the mutex was unlocked, so need to
...	...	@@ -899,10 +903,7 @@
899	903	b = __find(c, block);
900	904	if (b) {
901	905	__free_buffer_wake(new_b);
902		- b->hold_count++;
903		- __relink_lru(b, test_bit(B_DIRTY, &b->state) \|\|
904		- test_bit(B_WRITING, &b->state));
905		- return b;
	906	+ goto found_buffer;
906	907	}
907	908
908	909	__check_watermark(c);
...	...	@@ -922,6 +923,24 @@
922	923	*need_submit = 1;
923	924
924	925	return b;
	926	+
	927	+found_buffer:
	928	+ if (nf == NF_PREFETCH)
	929	+ return NULL;
	930	+ /*
	931	+ * Note: it is essential that we don't wait for the buffer to be
	932	+ * read if dm_bufio_get function is used. Both dm_bufio_get and
	933	+ * dm_bufio_prefetch can be used in the driver request routine.
	934	+ * If the user called both dm_bufio_prefetch and dm_bufio_get on
	935	+ * the same buffer, it would deadlock if we waited.
	936	+ */
	937	+ if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
	938	+ return NULL;
	939	+
	940	+ b->hold_count++;
	941	+ __relink_lru(b, test_bit(B_DIRTY, &b->state) \|\|
	942	+ test_bit(B_WRITING, &b->state));
	943	+ return b;
925	944	}
926	945
927	946	/*
928	947
...	...	@@ -956,10 +975,10 @@
956	975	struct dm_buffer *b;
957	976
958	977	dm_bufio_lock(c);
959		- b = __bufio_new(c, block, nf, bp, &need_submit);
	978	+ b = __bufio_new(c, block, nf, &need_submit);
960	979	dm_bufio_unlock(c);
961	980
962		- if (!b \|\| IS_ERR(b))
	981	+ if (!b)
963	982	return b;
964	983
965	984	if (need_submit)
966	985
...	...	@@ -1005,13 +1024,47 @@
1005	1024	}
1006	1025	EXPORT_SYMBOL_GPL(dm_bufio_new);
1007	1026
	1027	+void dm_bufio_prefetch(struct dm_bufio_client *c,
	1028	+ sector_t block, unsigned n_blocks)
	1029	+{
	1030	+ struct blk_plug plug;
	1031	+
	1032	+ blk_start_plug(&plug);
	1033	+ dm_bufio_lock(c);
	1034	+
	1035	+ for (; n_blocks--; block++) {
	1036	+ int need_submit;
	1037	+ struct dm_buffer *b;
	1038	+ b = __bufio_new(c, block, NF_PREFETCH, &need_submit);
	1039	+ if (unlikely(b != NULL)) {
	1040	+ dm_bufio_unlock(c);
	1041	+
	1042	+ if (need_submit)
	1043	+ submit_io(b, READ, b->block, read_endio);
	1044	+ dm_bufio_release(b);
	1045	+
	1046	+ dm_bufio_cond_resched();
	1047	+
	1048	+ if (!n_blocks)
	1049	+ goto flush_plug;
	1050	+ dm_bufio_lock(c);
	1051	+ }
	1052	+
	1053	+ }
	1054	+
	1055	+ dm_bufio_unlock(c);
	1056	+
	1057	+flush_plug:
	1058	+ blk_finish_plug(&plug);
	1059	+}
	1060	+EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
	1061	+
1008	1062	void dm_bufio_release(struct dm_buffer *b)
1009	1063	{
1010	1064	struct dm_bufio_client *c = b->c;
1011	1065
1012	1066	dm_bufio_lock(c);
1013	1067
1014		- BUG_ON(test_bit(B_READING, &b->state));
1015	1068	BUG_ON(!b->hold_count);
1016	1069
1017	1070	b->hold_count--;
...	...	@@ -1024,6 +1077,7 @@
1024	1077	* invalid buffer.
1025	1078	*/
1026	1079	if ((b->read_error \|\| b->write_error) &&
	1080	+ !test_bit(B_READING, &b->state) &&
1027	1081	!test_bit(B_WRITING, &b->state) &&
1028	1082	!test_bit(B_DIRTY, &b->state)) {
1029	1083	__unlink_buffer(b);
...	...	@@ -1040,6 +1094,8 @@
1040	1094	struct dm_bufio_client *c = b->c;
1041	1095
1042	1096	dm_bufio_lock(c);
	1097	+
	1098	+ BUG_ON(test_bit(B_READING, &b->state));
1043	1099
1044	1100	if (!test_and_set_bit(B_DIRTY, &b->state))
1045	1101	__relink_lru(b, LIST_DIRTY);
...	...	@@ -63,6 +63,14 @@
63	63	struct dm_buffer **bp);
64	64
65	65	/*
	66	+ * Prefetch the specified blocks to the cache.
	67	+ * The function starts to read the blocks and returns without waiting for
	68	+ * I/O to finish.
	69	+ */
	70	+void dm_bufio_prefetch(struct dm_bufio_client *c,
	71	+ sector_t block, unsigned n_blocks);
	72	+
	73	+/*
66	74	* Release a reference obtained with dm_bufio_{read,get,new}. The data
67	75	* pointer and dm_buffer pointer is no longer valid after this call.
68	76	*/