[PATCH] swsusp: write speedup

Switch the swsusp writeout code from 4k-at-a-time to 4MB-at-a-time. Crufty old PIII testbox: 12.9 MB/s -> 20.9 MB/s Sony Vaio: 14.7 MB/s -> 26.5 MB/s The implementation is crude. A better one would use larger BIOs, but wouldn't gain any performance. The memcpys will be mostly pipelined with the IO and basically come for free. The ENOMEM path has not been tested. It should be. Cc: Pavel Machek <pavel@ucw.cz> Cc: "Rafael J. Wysocki" <rjw@sisk.pl> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>

[PATCH] swsusp: write speedup
Switch the swsusp writeout code from 4k-at-a-time to 4MB-at-a-time. Crufty old PIII testbox: 12.9 MB/s -> 20.9 MB/s Sony Vaio: 14.7 MB/s -> 26.5 MB/s The implementation is crude. A better one would use larger BIOs, but wouldn't gain any performance. The memcpys will be mostly pipelined with the IO and basically come for free. The ENOMEM path has not been tested. It should be. Cc: Pavel Machek <pavel@ucw.cz> Cc: "Rafael J. Wysocki" <rjw@sisk.pl> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Andrew Morton · Linus Torvalds
1 parent 3a4f7577c9
Showing 3 changed files with 98 additions and 25 deletions Side-by-side Diff
include/linux/swap.h
kernel/power/swap.c
mm/page_io.c
@@ -12,6 +12,8 @@
  
 struct notifier_block;
  
+struct bio;
+
 #define SWAP_FLAG_PREFER	0x8000	/* set if swap priority specified */
 #define SWAP_FLAG_PRIO_MASK	0x7fff
 #define SWAP_FLAG_PRIO_SHIFT	0
@@ -216,7 +218,8 @@
 /* linux/mm/page_io.c */
 extern int swap_readpage(struct file *, struct page *);
 extern int swap_writepage(struct page *page, struct writeback_control *wbc);
-extern int rw_swap_page_sync(int, swp_entry_t, struct page *);
+extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
+				struct bio **bio_chain);
  
 /* linux/mm/swap_state.c */
 extern struct address_space swapper_space;
@@ -49,18 +49,16 @@
 {
 	int error;
  
-	rw_swap_page_sync(READ,
-			  swp_entry(root_swap, 0),
-			  virt_to_page((unsigned long)&swsusp_header));
+	rw_swap_page_sync(READ, swp_entry(root_swap, 0),
+			  virt_to_page((unsigned long)&swsusp_header), NULL);
 	if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) ||
 	    !memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
 		memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
 		memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
 		swsusp_header.image = start;
-		error = rw_swap_page_sync(WRITE,
-					  swp_entry(root_swap, 0),
-					  virt_to_page((unsigned long)
-						       &swsusp_header));
+		error = rw_swap_page_sync(WRITE, swp_entry(root_swap, 0),
+				virt_to_page((unsigned long)&swsusp_header),
+				NULL);
 	} else {
 		pr_debug("swsusp: Partition is not swap space.\n");
 		error = -ENODEV;
  
  
  
@@ -88,16 +86,37 @@
  *	write_page - Write one page to given swap location.
  *	@buf:		Address we're writing.
  *	@offset:	Offset of the swap page we're writing to.
+ *	@bio_chain:	Link the next write BIO here
  */
  
-static int write_page(void *buf, unsigned long offset)
+static int write_page(void *buf, unsigned long offset, struct bio **bio_chain)
 {
 	swp_entry_t entry;
 	int error = -ENOSPC;
  
 	if (offset) {
+		struct page *page = virt_to_page(buf);
+
+		if (bio_chain) {
+			/*
+			 * Whether or not we successfully allocated a copy page,
+			 * we take a ref on the page here.  It gets undone in
+			 * wait_on_bio_chain().
+			 */
+			struct page *page_copy;
+			page_copy = alloc_page(GFP_ATOMIC);
+			if (page_copy == NULL) {
+				WARN_ON_ONCE(1);
+				bio_chain = NULL;	/* Go synchronous */
+				get_page(page);
+			} else {
+				memcpy(page_address(page_copy),
+					page_address(page), PAGE_SIZE);
+				page = page_copy;
+			}
+		}
 		entry = swp_entry(root_swap, offset);
-		error = rw_swap_page_sync(WRITE, entry, virt_to_page(buf));
+		error = rw_swap_page_sync(WRITE, entry, page, bio_chain);
 	}
 	return error;
 }
  
  
  
  
  
  
  
@@ -185,37 +204,68 @@
 	return 0;
 }
  
-static int swap_write_page(struct swap_map_handle *handle, void *buf)
+static int wait_on_bio_chain(struct bio **bio_chain)
 {
-	int error;
+	struct bio *bio;
+	struct bio *next_bio;
+	int ret = 0;
+
+	if (bio_chain == NULL)
+		return 0;
+
+	bio = *bio_chain;
+	while (bio) {
+		struct page *page;
+
+		next_bio = bio->bi_private;
+		page = bio->bi_io_vec[0].bv_page;
+		wait_on_page_locked(page);
+		if (!PageUptodate(page) || PageError(page))
+			ret = -EIO;
+		put_page(page);
+		bio_put(bio);
+		bio = next_bio;
+	}
+	*bio_chain = NULL;
+	return ret;
+}
+
+static int swap_write_page(struct swap_map_handle *handle, void *buf,
+				struct bio **bio_chain)
+{
+	int error = 0;
 	unsigned long offset;
  
 	if (!handle->cur)
 		return -EINVAL;
 	offset = alloc_swap_page(root_swap, handle->bitmap);
-	error = write_page(buf, offset);
+	error = write_page(buf, offset, bio_chain);
 	if (error)
 		return error;
 	handle->cur->entries[handle->k++] = offset;
 	if (handle->k >= MAP_PAGE_ENTRIES) {
+		error = wait_on_bio_chain(bio_chain);
+		if (error)
+			goto out;
 		offset = alloc_swap_page(root_swap, handle->bitmap);
 		if (!offset)
 			return -ENOSPC;
 		handle->cur->next_swap = offset;
-		error = write_page(handle->cur, handle->cur_swap);
+		error = write_page(handle->cur, handle->cur_swap, NULL);
 		if (error)
-			return error;
+			goto out;
 		memset(handle->cur, 0, PAGE_SIZE);
 		handle->cur_swap = offset;
 		handle->k = 0;
 	}
-	return 0;
+out:
+	return error;
 }
  
 static int flush_swap_writer(struct swap_map_handle *handle)
 {
 	if (handle->cur && handle->cur_swap)
-		return write_page(handle->cur, handle->cur_swap);
+		return write_page(handle->cur, handle->cur_swap, NULL);
 	else
 		return -EINVAL;
 }
@@ -232,6 +282,8 @@
 	int ret;
 	int error = 0;
 	int nr_pages;
+	int err2;
+	struct bio *bio;
 	struct timeval start;
 	struct timeval stop;
  
  
@@ -240,11 +292,13 @@
 	if (!m)
 		m = 1;
 	nr_pages = 0;
+	bio = NULL;
 	do_gettimeofday(&start);
 	do {
 		ret = snapshot_read_next(snapshot, PAGE_SIZE);
 		if (ret > 0) {
-			error = swap_write_page(handle, data_of(*snapshot));
+			error = swap_write_page(handle, data_of(*snapshot),
+						&bio);
 			if (error)
 				break;
 			if (!(nr_pages % m))
  
@@ -252,8 +306,11 @@
 			nr_pages++;
 		}
 	} while (ret > 0);
+	err2 = wait_on_bio_chain(&bio);
 	do_gettimeofday(&stop);
 	if (!error)
+		error = err2;
+	if (!error)
 		printk("\b\b\b\bdone\n");
 	show_speed(&start, &stop, nr_to_write, "Wrote");
 	return error;
@@ -307,7 +364,7 @@
 	error = get_swap_writer(&handle);
 	if (!error) {
 		unsigned long start = handle.cur_swap;
-		error = swap_write_page(&handle, header);
+		error = swap_write_page(&handle, header, NULL);
 		if (!error)
 			error = save_image(&handle, &snapshot,
 					header->pages - 1);
@@ -156,10 +156,12 @@
  * We use end_swap_bio_read() even for writes, because it happens to do what
  * we want.
  */
-int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page)
+int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
+			struct bio **bio_chain)
 {
 	struct bio *bio;
 	int ret = 0;
+	int bio_rw;
  
 	lock_page(page);
  
  
@@ -170,11 +172,22 @@
 		goto out;
 	}
  
-	submit_bio(rw | (1 << BIO_RW_SYNC), bio);
-	wait_on_page_locked(page);
+	bio_rw = rw;
+	if (!bio_chain)
+		bio_rw |= (1 << BIO_RW_SYNC);
+	if (bio_chain)
+		bio_get(bio);
+	submit_bio(bio_rw, bio);
+	if (bio_chain == NULL) {
+		wait_on_page_locked(page);
  
-	if (!PageUptodate(page) || PageError(page))
-		ret = -EIO;
+		if (!PageUptodate(page) || PageError(page))
+			ret = -EIO;
+	}
+	if (bio_chain) {
+		bio->bi_private = *bio_chain;
+		*bio_chain = bio;
+	}
 out:
 	return ret;
 }
...	...	@@ -12,6 +12,8 @@
12	12
13	13	struct notifier_block;
14	14
	15	+struct bio;
	16	+
15	17	#define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */
16	18	#define SWAP_FLAG_PRIO_MASK 0x7fff
17	19	#define SWAP_FLAG_PRIO_SHIFT 0
...	...	@@ -216,7 +218,8 @@
216	218	/* linux/mm/page_io.c */
217	219	extern int swap_readpage(struct file , struct page );
218	220	extern int swap_writepage(struct page page, struct writeback_control wbc);
219		-extern int rw_swap_page_sync(int, swp_entry_t, struct page *);
	221	+extern int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
	222	+ struct bio **bio_chain);
220	223
221	224	/* linux/mm/swap_state.c */
222	225	extern struct address_space swapper_space;
...	...	@@ -49,18 +49,16 @@
49	49	{
50	50	int error;
51	51
52		- rw_swap_page_sync(READ,
53		- swp_entry(root_swap, 0),
54		- virt_to_page((unsigned long)&swsusp_header));
	52	+ rw_swap_page_sync(READ, swp_entry(root_swap, 0),
	53	+ virt_to_page((unsigned long)&swsusp_header), NULL);
55	54	if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) \|\|
56	55	!memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
57	56	memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
58	57	memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
59	58	swsusp_header.image = start;
60		- error = rw_swap_page_sync(WRITE,
61		- swp_entry(root_swap, 0),
62		- virt_to_page((unsigned long)
63		- &swsusp_header));
	59	+ error = rw_swap_page_sync(WRITE, swp_entry(root_swap, 0),
	60	+ virt_to_page((unsigned long)&swsusp_header),
	61	+ NULL);
64	62	} else {
65	63	pr_debug("swsusp: Partition is not swap space.\n");
66	64	error = -ENODEV;
67	65
68	66
69	67
...	...	@@ -88,16 +86,37 @@
88	86	* write_page - Write one page to given swap location.
89	87	* @buf: Address we're writing.
90	88	* @offset: Offset of the swap page we're writing to.
	89	+ * @bio_chain: Link the next write BIO here
91	90	*/
92	91
93		-static int write_page(void *buf, unsigned long offset)
	92	+static int write_page(void buf, unsigned long offset, struct bio *bio_chain)
94	93	{
95	94	swp_entry_t entry;
96	95	int error = -ENOSPC;
97	96
98	97	if (offset) {
	98	+ struct page *page = virt_to_page(buf);
	99	+
	100	+ if (bio_chain) {
	101	+ /*
	102	+ * Whether or not we successfully allocated a copy page,
	103	+ * we take a ref on the page here. It gets undone in
	104	+ * wait_on_bio_chain().
	105	+ */
	106	+ struct page *page_copy;
	107	+ page_copy = alloc_page(GFP_ATOMIC);
	108	+ if (page_copy == NULL) {
	109	+ WARN_ON_ONCE(1);
	110	+ bio_chain = NULL; /* Go synchronous */
	111	+ get_page(page);
	112	+ } else {
	113	+ memcpy(page_address(page_copy),
	114	+ page_address(page), PAGE_SIZE);
	115	+ page = page_copy;
	116	+ }
	117	+ }
99	118	entry = swp_entry(root_swap, offset);
100		- error = rw_swap_page_sync(WRITE, entry, virt_to_page(buf));
	119	+ error = rw_swap_page_sync(WRITE, entry, page, bio_chain);
101	120	}
102	121	return error;
103	122	}
104	123
105	124
106	125
107	126
108	127
109	128
110	129
...	...	@@ -185,37 +204,68 @@
185	204	return 0;
186	205	}
187	206
188		-static int swap_write_page(struct swap_map_handle handle, void buf)
	207	+static int wait_on_bio_chain(struct bio **bio_chain)
189	208	{
190		- int error;
	209	+ struct bio *bio;
	210	+ struct bio *next_bio;
	211	+ int ret = 0;
	212	+
	213	+ if (bio_chain == NULL)
	214	+ return 0;
	215	+
	216	+ bio = *bio_chain;
	217	+ while (bio) {
	218	+ struct page *page;
	219	+
	220	+ next_bio = bio->bi_private;
	221	+ page = bio->bi_io_vec[0].bv_page;
	222	+ wait_on_page_locked(page);
	223	+ if (!PageUptodate(page) \|\| PageError(page))
	224	+ ret = -EIO;
	225	+ put_page(page);
	226	+ bio_put(bio);
	227	+ bio = next_bio;
	228	+ }
	229	+ *bio_chain = NULL;
	230	+ return ret;
	231	+}
	232	+
	233	+static int swap_write_page(struct swap_map_handle handle, void buf,
	234	+ struct bio **bio_chain)
	235	+{
	236	+ int error = 0;
191	237	unsigned long offset;
192	238
193	239	if (!handle->cur)
194	240	return -EINVAL;
195	241	offset = alloc_swap_page(root_swap, handle->bitmap);
196		- error = write_page(buf, offset);
	242	+ error = write_page(buf, offset, bio_chain);
197	243	if (error)
198	244	return error;
199	245	handle->cur->entries[handle->k++] = offset;
200	246	if (handle->k >= MAP_PAGE_ENTRIES) {
	247	+ error = wait_on_bio_chain(bio_chain);
	248	+ if (error)
	249	+ goto out;
201	250	offset = alloc_swap_page(root_swap, handle->bitmap);
202	251	if (!offset)
203	252	return -ENOSPC;
204	253	handle->cur->next_swap = offset;
205		- error = write_page(handle->cur, handle->cur_swap);
	254	+ error = write_page(handle->cur, handle->cur_swap, NULL);
206	255	if (error)
207		- return error;
	256	+ goto out;
208	257	memset(handle->cur, 0, PAGE_SIZE);
209	258	handle->cur_swap = offset;
210	259	handle->k = 0;
211	260	}
212		- return 0;
	261	+out:
	262	+ return error;
213	263	}
214	264
215	265	static int flush_swap_writer(struct swap_map_handle *handle)
216	266	{
217	267	if (handle->cur && handle->cur_swap)
218		- return write_page(handle->cur, handle->cur_swap);
	268	+ return write_page(handle->cur, handle->cur_swap, NULL);
219	269	else
220	270	return -EINVAL;
221	271	}
...	...	@@ -232,6 +282,8 @@
232	282	int ret;
233	283	int error = 0;
234	284	int nr_pages;
	285	+ int err2;
	286	+ struct bio *bio;
235	287	struct timeval start;
236	288	struct timeval stop;
237	289
238	290
...	...	@@ -240,11 +292,13 @@
240	292	if (!m)
241	293	m = 1;
242	294	nr_pages = 0;
	295	+ bio = NULL;
243	296	do_gettimeofday(&start);
244	297	do {
245	298	ret = snapshot_read_next(snapshot, PAGE_SIZE);
246	299	if (ret > 0) {
247		- error = swap_write_page(handle, data_of(*snapshot));
	300	+ error = swap_write_page(handle, data_of(*snapshot),
	301	+ &bio);
248	302	if (error)
249	303	break;
250	304	if (!(nr_pages % m))
251	305
...	...	@@ -252,8 +306,11 @@
252	306	nr_pages++;
253	307	}
254	308	} while (ret > 0);
	309	+ err2 = wait_on_bio_chain(&bio);
255	310	do_gettimeofday(&stop);
256	311	if (!error)
	312	+ error = err2;
	313	+ if (!error)
257	314	printk("\b\b\b\bdone\n");
258	315	show_speed(&start, &stop, nr_to_write, "Wrote");
259	316	return error;
...	...	@@ -307,7 +364,7 @@
307	364	error = get_swap_writer(&handle);
308	365	if (!error) {
309	366	unsigned long start = handle.cur_swap;
310		- error = swap_write_page(&handle, header);
	367	+ error = swap_write_page(&handle, header, NULL);
311	368	if (!error)
312	369	error = save_image(&handle, &snapshot,
313	370	header->pages - 1);
...	...	@@ -156,10 +156,12 @@
156	156	* We use end_swap_bio_read() even for writes, because it happens to do what
157	157	* we want.
158	158	*/
159		-int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page)
	159	+int rw_swap_page_sync(int rw, swp_entry_t entry, struct page *page,
	160	+ struct bio **bio_chain)
160	161	{
161	162	struct bio *bio;
162	163	int ret = 0;
	164	+ int bio_rw;
163	165
164	166	lock_page(page);
165	167
166	168
...	...	@@ -170,11 +172,22 @@
170	172	goto out;
171	173	}
172	174
173		- submit_bio(rw \| (1 << BIO_RW_SYNC), bio);
174		- wait_on_page_locked(page);
	175	+ bio_rw = rw;
	176	+ if (!bio_chain)
	177	+ bio_rw \|= (1 << BIO_RW_SYNC);
	178	+ if (bio_chain)
	179	+ bio_get(bio);
	180	+ submit_bio(bio_rw, bio);
	181	+ if (bio_chain == NULL) {
	182	+ wait_on_page_locked(page);
175	183
176		- if (!PageUptodate(page) \|\| PageError(page))
177		- ret = -EIO;
	184	+ if (!PageUptodate(page) \|\| PageError(page))
	185	+ ret = -EIO;
	186	+ }
	187	+ if (bio_chain) {
	188	+ bio->bi_private = *bio_chain;
	189	+ *bio_chain = bio;
	190	+ }
178	191	out:
179	192	return ret;
180	193	}