nilfs2: separate wait function from nilfs_segctor_write

This separates wait function for submitted logs from the write function nilfs_segctor_write(). A new list of segment buffers "sc_write_logs" is added to hold logs under writing, and double buffering is partially applied to hide io latency. At this point, the double buffering is disabled for blocksize < pagesize because page dirty flag is turned off during write and dirty buffers are not properly collected for pages crossing over segments. To receive full benefit of the double buffering, further refinement is needed to move the io wait outside the lock section of log writer. Signed-off-by: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>

nilfs2: separate wait function from nilfs_segctor_write
This separates wait function for submitted logs from the write function nilfs_segctor_write(). A new list of segment buffers "sc_write_logs" is added to hold logs under writing, and double buffering is partially applied to hide io latency. At this point, the double buffering is disabled for blocksize < pagesize because page dirty flag is turned off during write and dirty buffers are not properly collected for pages crossing over segments. To receive full benefit of the double buffering, further refinement is needed to move the io wait outside the lock section of log writer. Signed-off-by: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>
Ryusuke Konishi
1 parent e29df395bc
Showing 4 changed files with 165 additions and 95 deletions Side-by-side Diff
fs/nilfs2/segbuf.c
fs/nilfs2/segbuf.h
fs/nilfs2/segment.c
fs/nilfs2/segment.h
@@ -100,6 +100,22 @@
 		segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1;
 }
  
+/**
+ * nilfs_segbuf_map_cont - map a new log behind a given log
+ * @segbuf: new segment buffer
+ * @prev: segment buffer containing a log to be continued
+ */
+void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf,
+			   struct nilfs_segment_buffer *prev)
+{
+	segbuf->sb_segnum = prev->sb_segnum;
+	segbuf->sb_fseg_start = prev->sb_fseg_start;
+	segbuf->sb_fseg_end = prev->sb_fseg_end;
+	segbuf->sb_pseg_start = prev->sb_pseg_start + prev->sb_sum.nblocks;
+	segbuf->sb_rest_blocks =
+		segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1;
+}
+
 void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *segbuf,
 				  __u64 nextnum, struct the_nilfs *nilfs)
 {
@@ -128,6 +128,8 @@
 void nilfs_segbuf_free(struct nilfs_segment_buffer *);
 void nilfs_segbuf_map(struct nilfs_segment_buffer *, __u64, unsigned long,
 		      struct the_nilfs *);
+void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf,
+			   struct nilfs_segment_buffer *prev);
 void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *, __u64,
 				  struct the_nilfs *);
 int nilfs_segbuf_reset(struct nilfs_segment_buffer *, unsigned, time_t);
@@ -1273,48 +1273,69 @@
 	return err;
 }
  
+/**
+ * nilfs_segctor_begin_construction - setup segment buffer to make a new log
+ * @sci: nilfs_sc_info
+ * @nilfs: nilfs object
+ */
 static int nilfs_segctor_begin_construction(struct nilfs_sc_info *sci,
 					    struct the_nilfs *nilfs)
 {
-	struct nilfs_segment_buffer *segbuf;
+	struct nilfs_segment_buffer *segbuf, *prev;
 	__u64 nextnum;
-	int err;
+	int err, alloc = 0;
  
-	if (list_empty(&sci->sc_segbufs)) {
-		segbuf = nilfs_segbuf_new(sci->sc_super);
-		if (unlikely(!segbuf))
-			return -ENOMEM;
-		list_add(&segbuf->sb_list, &sci->sc_segbufs);
-	} else
-		segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
+	segbuf = nilfs_segbuf_new(sci->sc_super);
+	if (unlikely(!segbuf))
+		return -ENOMEM;
  
-	nilfs_segbuf_map(segbuf, nilfs->ns_segnum, nilfs->ns_pseg_offset,
-			 nilfs);
+	if (list_empty(&sci->sc_write_logs)) {
+		nilfs_segbuf_map(segbuf, nilfs->ns_segnum,
+				 nilfs->ns_pseg_offset, nilfs);
+		if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
+			nilfs_shift_to_next_segment(nilfs);
+			nilfs_segbuf_map(segbuf, nilfs->ns_segnum, 0, nilfs);
+		}
  
-	if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
-		nilfs_shift_to_next_segment(nilfs);
-		nilfs_segbuf_map(segbuf, nilfs->ns_segnum, 0, nilfs);
+		segbuf->sb_sum.seg_seq = nilfs->ns_seg_seq;
+		nextnum = nilfs->ns_nextnum;
+
+		if (nilfs->ns_segnum == nilfs->ns_nextnum)
+			/* Start from the head of a new full segment */
+			alloc++;
+	} else {
+		/* Continue logs */
+		prev = NILFS_LAST_SEGBUF(&sci->sc_write_logs);
+		nilfs_segbuf_map_cont(segbuf, prev);
+		segbuf->sb_sum.seg_seq = prev->sb_sum.seg_seq;
+		nextnum = prev->sb_nextnum;
+
+		if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
+			nilfs_segbuf_map(segbuf, prev->sb_nextnum, 0, nilfs);
+			segbuf->sb_sum.seg_seq++;
+			alloc++;
+		}
 	}
-	sci->sc_segbuf_nblocks = segbuf->sb_rest_blocks;
  
 	err = nilfs_sufile_mark_dirty(nilfs->ns_sufile, segbuf->sb_segnum);
-	if (unlikely(err))
-		return err;
+	if (err)
+		goto failed;
  
-	if (nilfs->ns_segnum == nilfs->ns_nextnum) {
-		/* Start from the head of a new full segment */
+	if (alloc) {
 		err = nilfs_sufile_alloc(nilfs->ns_sufile, &nextnum);
-		if (unlikely(err))
-			return err;
-	} else
-		nextnum = nilfs->ns_nextnum;
-
-	segbuf->sb_sum.seg_seq = nilfs->ns_seg_seq;
+		if (err)
+			goto failed;
+	}
 	nilfs_segbuf_set_next_segnum(segbuf, nextnum, nilfs);
  
-	/* truncating segment buffers */
-	nilfs_truncate_logs(&sci->sc_segbufs, segbuf);
+	BUG_ON(!list_empty(&sci->sc_segbufs));
+	list_add_tail(&segbuf->sb_list, &sci->sc_segbufs);
+	sci->sc_segbuf_nblocks = segbuf->sb_rest_blocks;
 	return 0;
+
+ failed:
+	nilfs_segbuf_free(segbuf);
+	return err;
 }
  
 static int nilfs_segctor_extend_segments(struct nilfs_sc_info *sci,
  
  
  
@@ -1373,15 +1394,16 @@
 	return err;
 }
  
-static void nilfs_segctor_free_incomplete_segments(struct nilfs_sc_info *sci,
-						   struct the_nilfs *nilfs)
+static void nilfs_free_incomplete_logs(struct list_head *logs,
+				       struct the_nilfs *nilfs)
 {
-	struct nilfs_segment_buffer *segbuf;
+	struct nilfs_segment_buffer *segbuf, *prev;
+	struct inode *sufile = nilfs->ns_sufile;
 	int ret;
  
-	segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
+	segbuf = NILFS_FIRST_SEGBUF(logs);
 	if (nilfs->ns_nextnum != segbuf->sb_nextnum) {
-		ret = nilfs_sufile_free(nilfs->ns_sufile, segbuf->sb_nextnum);
+		ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum);
 		WARN_ON(ret); /* never fails */
 	}
 	if (atomic_read(&segbuf->sb_err)) {
  
  
@@ -1395,36 +1417,20 @@
 			set_nilfs_discontinued(nilfs);
 	}
  
-	list_for_each_entry_continue(segbuf, &sci->sc_segbufs, sb_list) {
-		ret = nilfs_sufile_free(nilfs->ns_sufile, segbuf->sb_nextnum);
-		WARN_ON(ret); /* never fails */
+	prev = segbuf;
+	list_for_each_entry_continue(segbuf, logs, sb_list) {
+		if (prev->sb_nextnum != segbuf->sb_nextnum) {
+			ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum);
+			WARN_ON(ret); /* never fails */
+		}
 		if (atomic_read(&segbuf->sb_err) &&
 		    segbuf->sb_segnum != nilfs->ns_nextnum)
 			/* Case 2: extended segment (!= next) failed */
-			nilfs_sufile_set_error(nilfs->ns_sufile,
-					       segbuf->sb_segnum);
+			nilfs_sufile_set_error(sufile, segbuf->sb_segnum);
+		prev = segbuf;
 	}
 }
  
-static void nilfs_segctor_end_construction(struct nilfs_sc_info *sci,
-					   struct the_nilfs *nilfs, int err)
-{
-	if (unlikely(err)) {
-		nilfs_segctor_free_incomplete_segments(sci, nilfs);
-		if (sci->sc_stage.flags & NILFS_CF_SUFREED) {
-			int ret;
-
-			ret = nilfs_sufile_cancel_freev(nilfs->ns_sufile,
-							sci->sc_freesegs,
-							sci->sc_nfreesegs,
-							NULL);
-			WARN_ON(ret); /* do not happen */
-		}
-	}
-	nilfs_clear_logs(&sci->sc_segbufs);
-	sci->sc_super_root = NULL;
-}
-
 static void nilfs_segctor_update_segusage(struct nilfs_sc_info *sci,
 					  struct inode *sufile)
 {
  
  
@@ -1442,19 +1448,18 @@
 	}
 }
  
-static void nilfs_segctor_cancel_segusage(struct nilfs_sc_info *sci,
-					  struct inode *sufile)
+static void nilfs_cancel_segusage(struct list_head *logs, struct inode *sufile)
 {
 	struct nilfs_segment_buffer *segbuf;
 	int ret;
  
-	segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
+	segbuf = NILFS_FIRST_SEGBUF(logs);
 	ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum,
 					     segbuf->sb_pseg_start -
 					     segbuf->sb_fseg_start, 0);
 	WARN_ON(ret); /* always succeed because the segusage is dirty */
  
-	list_for_each_entry_continue(segbuf, &sci->sc_segbufs, sb_list) {
+	list_for_each_entry_continue(segbuf, logs, sb_list) {
 		ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum,
 						     0, 0);
 		WARN_ON(ret); /* always succeed */
  
  
@@ -1760,17 +1765,15 @@
 			       struct the_nilfs *nilfs)
 {
 	struct nilfs_segment_buffer *segbuf;
-	int err, res;
+	int ret = 0;
  
 	list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
-		err = nilfs_segbuf_write(segbuf, nilfs);
-
-		res = nilfs_segbuf_wait(segbuf);
-		err = err ? : res;
-		if (err)
-			return err;
+		ret = nilfs_segbuf_write(segbuf, nilfs);
+		if (ret)
+			break;
 	}
-	return 0;
+	list_splice_tail_init(&sci->sc_segbufs, &sci->sc_write_logs);
+	return ret;
 }
  
 static void __nilfs_end_page_io(struct page *page, int err)
  
  
  
@@ -1848,15 +1851,17 @@
 	}
 }
  
-static void nilfs_segctor_abort_write(struct nilfs_sc_info *sci,
-				      struct page *failed_page, int err)
+static void nilfs_abort_logs(struct list_head *logs, struct page *failed_page,
+			     struct buffer_head *bh_sr, int err)
 {
 	struct nilfs_segment_buffer *segbuf;
 	struct page *bd_page = NULL, *fs_page = NULL;
+	struct buffer_head *bh;
  
-	list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
-		struct buffer_head *bh;
+	if (list_empty(logs))
+		return;
  
+	list_for_each_entry(segbuf, logs, sb_list) {
 		list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
 				    b_assoc_buffers) {
 			if (bh->b_page != bd_page) {
@@ -1868,7 +1873,7 @@
  
 		list_for_each_entry(bh, &segbuf->sb_payload_buffers,
 				    b_assoc_buffers) {
-			if (bh == sci->sc_super_root) {
+			if (bh == bh_sr) {
 				if (bh->b_page != bd_page) {
 					end_page_writeback(bd_page);
 					bd_page = bh->b_page;
@@ -1878,7 +1883,7 @@
 			if (bh->b_page != fs_page) {
 				nilfs_end_page_io(fs_page, err);
 				if (fs_page && fs_page == failed_page)
-					goto done;
+					return;
 				fs_page = bh->b_page;
 			}
 		}
  
@@ -1887,8 +1892,34 @@
 		end_page_writeback(bd_page);
  
 	nilfs_end_page_io(fs_page, err);
- done:
+}
+
+static void nilfs_segctor_abort_construction(struct nilfs_sc_info *sci,
+					     struct the_nilfs *nilfs, int err)
+{
+	LIST_HEAD(logs);
+	int ret;
+
+	list_splice_tail_init(&sci->sc_write_logs, &logs);
+	ret = nilfs_wait_on_logs(&logs);
+	if (ret)
+		nilfs_abort_logs(&logs, NULL, sci->sc_super_root, ret);
+
+	list_splice_tail_init(&sci->sc_segbufs, &logs);
+	nilfs_cancel_segusage(&logs, nilfs->ns_sufile);
+	nilfs_free_incomplete_logs(&logs, nilfs);
 	nilfs_clear_copied_buffers(&sci->sc_copied_buffers, err);
+
+	if (sci->sc_stage.flags & NILFS_CF_SUFREED) {
+		ret = nilfs_sufile_cancel_freev(nilfs->ns_sufile,
+						sci->sc_freesegs,
+						sci->sc_nfreesegs,
+						NULL);
+		WARN_ON(ret); /* do not happen */
+	}
+
+	nilfs_destroy_logs(&logs);
+	sci->sc_super_root = NULL;
 }
  
 static void nilfs_set_next_segment(struct the_nilfs *nilfs,
@@ -1910,7 +1941,7 @@
 	struct the_nilfs *nilfs = sbi->s_nilfs;
 	int update_sr = (sci->sc_super_root != NULL);
  
-	list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
+	list_for_each_entry(segbuf, &sci->sc_write_logs, sb_list) {
 		struct buffer_head *bh;
  
 		list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
@@ -1983,7 +2014,7 @@
  
 	sci->sc_nblk_inc += sci->sc_nblk_this_inc;
  
-	segbuf = NILFS_LAST_SEGBUF(&sci->sc_segbufs);
+	segbuf = NILFS_LAST_SEGBUF(&sci->sc_write_logs);
 	nilfs_set_next_segment(nilfs, segbuf);
  
 	if (update_sr) {
  
@@ -1994,10 +2025,23 @@
 		clear_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags);
 		clear_bit(NILFS_SC_DIRTY, &sci->sc_flags);
 		set_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags);
+		nilfs_segctor_clear_metadata_dirty(sci);
 	} else
 		clear_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags);
 }
  
+static int nilfs_segctor_wait(struct nilfs_sc_info *sci)
+{
+	int ret;
+
+	ret = nilfs_wait_on_logs(&sci->sc_write_logs);
+	if (!ret) {
+		nilfs_segctor_complete_write(sci);
+		nilfs_destroy_logs(&sci->sc_write_logs);
+	}
+	return ret;
+}
+
 static int nilfs_segctor_check_in_files(struct nilfs_sc_info *sci,
 					struct nilfs_sb_info *sbi)
 {
@@ -2110,7 +2154,7 @@
 		/* Avoid empty segment */
 		if (sci->sc_stage.scnt == NILFS_ST_DONE &&
 		    NILFS_SEG_EMPTY(&sci->sc_curseg->sb_sum)) {
-			nilfs_segctor_end_construction(sci, nilfs, 1);
+			nilfs_segctor_abort_construction(sci, nilfs, 1);
 			goto out;
 		}
  
@@ -2124,7 +2168,7 @@
 		if (has_sr) {
 			err = nilfs_segctor_fill_in_checkpoint(sci);
 			if (unlikely(err))
-				goto failed_to_make_up;
+				goto failed_to_write;
  
 			nilfs_segctor_fill_in_super_root(sci, nilfs);
 		}
  
  
  
  
  
  
@@ -2132,42 +2176,46 @@
  
 		/* Write partial segments */
 		err = nilfs_segctor_prepare_write(sci, &failed_page);
-		if (unlikely(err))
+		if (err) {
+			nilfs_abort_logs(&sci->sc_segbufs, failed_page,
+					 sci->sc_super_root, err);
 			goto failed_to_write;
-
+		}
 		nilfs_segctor_fill_in_checksums(sci, nilfs->ns_crc_seed);
  
 		err = nilfs_segctor_write(sci, nilfs);
 		if (unlikely(err))
 			goto failed_to_write;
  
-		nilfs_segctor_complete_write(sci);
-
-		/* Commit segments */
-		if (has_sr)
-			nilfs_segctor_clear_metadata_dirty(sci);
-
-		nilfs_segctor_end_construction(sci, nilfs, 0);
-
+		if (sci->sc_stage.scnt == NILFS_ST_DONE ||
+		    nilfs->ns_blocksize_bits != PAGE_CACHE_SHIFT) {
+			/*
+			 * At this point, we avoid double buffering
+			 * for blocksize < pagesize because page dirty
+			 * flag is turned off during write and dirty
+			 * buffers are not properly collected for
+			 * pages crossing over segments.
+			 */
+			err = nilfs_segctor_wait(sci);
+			if (err)
+				goto failed_to_write;
+		}
 	} while (sci->sc_stage.scnt != NILFS_ST_DONE);
  
+	sci->sc_super_root = NULL;
+
  out:
-	nilfs_destroy_logs(&sci->sc_segbufs);
 	nilfs_segctor_check_out_files(sci, sbi);
 	return err;
  
  failed_to_write:
-	nilfs_segctor_abort_write(sci, failed_page, err);
-	nilfs_segctor_cancel_segusage(sci, nilfs->ns_sufile);
-
- failed_to_make_up:
 	if (sci->sc_stage.flags & NILFS_CF_IFILE_STARTED)
 		nilfs_redirty_inodes(&sci->sc_dirty_files);
  
  failed:
 	if (nilfs_doing_gc())
 		nilfs_redirty_inodes(&sci->sc_gc_inodes);
-	nilfs_segctor_end_construction(sci, nilfs, err);
+	nilfs_segctor_abort_construction(sci, nilfs, err);
 	goto out;
 }
  
@@ -2725,6 +2773,7 @@
 	spin_lock_init(&sci->sc_state_lock);
 	INIT_LIST_HEAD(&sci->sc_dirty_files);
 	INIT_LIST_HEAD(&sci->sc_segbufs);
+	INIT_LIST_HEAD(&sci->sc_write_logs);
 	INIT_LIST_HEAD(&sci->sc_gc_inodes);
 	INIT_LIST_HEAD(&sci->sc_copied_buffers);
  
@@ -2792,6 +2841,7 @@
 	}
  
 	WARN_ON(!list_empty(&sci->sc_segbufs));
+	WARN_ON(!list_empty(&sci->sc_write_logs));
  
 	down_write(&sbi->s_nilfs->ns_segctor_sem);
  
@@ -97,6 +97,7 @@
  * @sc_dsync_start: start byte offset of data pages
  * @sc_dsync_end: end byte offset of data pages (inclusive)
  * @sc_segbufs: List of segment buffers
+ * @sc_write_logs: List of segment buffers to hold logs under writing
  * @sc_segbuf_nblocks: Number of available blocks in segment buffers.
  * @sc_curseg: Current segment buffer
  * @sc_super_root: Pointer to the super root buffer
@@ -143,6 +144,7 @@
  
 	/* Segment buffers */
 	struct list_head	sc_segbufs;
+	struct list_head	sc_write_logs;
 	unsigned long		sc_segbuf_nblocks;
 	struct nilfs_segment_buffer *sc_curseg;
 	struct buffer_head     *sc_super_root;
...	...	@@ -100,6 +100,22 @@
100	100	segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1;
101	101	}
102	102
	103	+/**
	104	+ * nilfs_segbuf_map_cont - map a new log behind a given log
	105	+ * @segbuf: new segment buffer
	106	+ * @prev: segment buffer containing a log to be continued
	107	+ */
	108	+void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf,
	109	+ struct nilfs_segment_buffer *prev)
	110	+{
	111	+ segbuf->sb_segnum = prev->sb_segnum;
	112	+ segbuf->sb_fseg_start = prev->sb_fseg_start;
	113	+ segbuf->sb_fseg_end = prev->sb_fseg_end;
	114	+ segbuf->sb_pseg_start = prev->sb_pseg_start + prev->sb_sum.nblocks;
	115	+ segbuf->sb_rest_blocks =
	116	+ segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1;
	117	+}
	118	+
103	119	void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *segbuf,
104	120	__u64 nextnum, struct the_nilfs *nilfs)
105	121	{
...	...	@@ -128,6 +128,8 @@
128	128	void nilfs_segbuf_free(struct nilfs_segment_buffer *);
129	129	void nilfs_segbuf_map(struct nilfs_segment_buffer *, __u64, unsigned long,
130	130	struct the_nilfs *);
	131	+void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf,
	132	+ struct nilfs_segment_buffer *prev);
131	133	void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *, __u64,
132	134	struct the_nilfs *);
133	135	int nilfs_segbuf_reset(struct nilfs_segment_buffer *, unsigned, time_t);
...	...	@@ -1273,48 +1273,69 @@
1273	1273	return err;
1274	1274	}
1275	1275
	1276	+/**
	1277	+ * nilfs_segctor_begin_construction - setup segment buffer to make a new log
	1278	+ * @sci: nilfs_sc_info
	1279	+ * @nilfs: nilfs object
	1280	+ */
1276	1281	static int nilfs_segctor_begin_construction(struct nilfs_sc_info *sci,
1277	1282	struct the_nilfs *nilfs)
1278	1283	{
1279		- struct nilfs_segment_buffer *segbuf;
	1284	+ struct nilfs_segment_buffer segbuf, prev;
1280	1285	__u64 nextnum;
1281		- int err;
	1286	+ int err, alloc = 0;
1282	1287
1283		- if (list_empty(&sci->sc_segbufs)) {
1284		- segbuf = nilfs_segbuf_new(sci->sc_super);
1285		- if (unlikely(!segbuf))
1286		- return -ENOMEM;
1287		- list_add(&segbuf->sb_list, &sci->sc_segbufs);
1288		- } else
1289		- segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
	1288	+ segbuf = nilfs_segbuf_new(sci->sc_super);
	1289	+ if (unlikely(!segbuf))
	1290	+ return -ENOMEM;
1290	1291
1291		- nilfs_segbuf_map(segbuf, nilfs->ns_segnum, nilfs->ns_pseg_offset,
1292		- nilfs);
	1292	+ if (list_empty(&sci->sc_write_logs)) {
	1293	+ nilfs_segbuf_map(segbuf, nilfs->ns_segnum,
	1294	+ nilfs->ns_pseg_offset, nilfs);
	1295	+ if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
	1296	+ nilfs_shift_to_next_segment(nilfs);
	1297	+ nilfs_segbuf_map(segbuf, nilfs->ns_segnum, 0, nilfs);
	1298	+ }
1293	1299
1294		- if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
1295		- nilfs_shift_to_next_segment(nilfs);
1296		- nilfs_segbuf_map(segbuf, nilfs->ns_segnum, 0, nilfs);
	1300	+ segbuf->sb_sum.seg_seq = nilfs->ns_seg_seq;
	1301	+ nextnum = nilfs->ns_nextnum;
	1302	+
	1303	+ if (nilfs->ns_segnum == nilfs->ns_nextnum)
	1304	+ /* Start from the head of a new full segment */
	1305	+ alloc++;
	1306	+ } else {
	1307	+ /* Continue logs */
	1308	+ prev = NILFS_LAST_SEGBUF(&sci->sc_write_logs);
	1309	+ nilfs_segbuf_map_cont(segbuf, prev);
	1310	+ segbuf->sb_sum.seg_seq = prev->sb_sum.seg_seq;
	1311	+ nextnum = prev->sb_nextnum;
	1312	+
	1313	+ if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) {
	1314	+ nilfs_segbuf_map(segbuf, prev->sb_nextnum, 0, nilfs);
	1315	+ segbuf->sb_sum.seg_seq++;
	1316	+ alloc++;
	1317	+ }
1297	1318	}
1298		- sci->sc_segbuf_nblocks = segbuf->sb_rest_blocks;
1299	1319
1300	1320	err = nilfs_sufile_mark_dirty(nilfs->ns_sufile, segbuf->sb_segnum);
1301		- if (unlikely(err))
1302		- return err;
	1321	+ if (err)
	1322	+ goto failed;
1303	1323
1304		- if (nilfs->ns_segnum == nilfs->ns_nextnum) {
1305		- /* Start from the head of a new full segment */
	1324	+ if (alloc) {
1306	1325	err = nilfs_sufile_alloc(nilfs->ns_sufile, &nextnum);
1307		- if (unlikely(err))
1308		- return err;
1309		- } else
1310		- nextnum = nilfs->ns_nextnum;
1311		-
1312		- segbuf->sb_sum.seg_seq = nilfs->ns_seg_seq;
	1326	+ if (err)
	1327	+ goto failed;
	1328	+ }
1313	1329	nilfs_segbuf_set_next_segnum(segbuf, nextnum, nilfs);
1314	1330
1315		- /* truncating segment buffers */
1316		- nilfs_truncate_logs(&sci->sc_segbufs, segbuf);
	1331	+ BUG_ON(!list_empty(&sci->sc_segbufs));
	1332	+ list_add_tail(&segbuf->sb_list, &sci->sc_segbufs);
	1333	+ sci->sc_segbuf_nblocks = segbuf->sb_rest_blocks;
1317	1334	return 0;
	1335	+
	1336	+ failed:
	1337	+ nilfs_segbuf_free(segbuf);
	1338	+ return err;
1318	1339	}
1319	1340
1320	1341	static int nilfs_segctor_extend_segments(struct nilfs_sc_info *sci,
1321	1342
1322	1343
1323	1344
...	...	@@ -1373,15 +1394,16 @@
1373	1394	return err;
1374	1395	}
1375	1396
1376		-static void nilfs_segctor_free_incomplete_segments(struct nilfs_sc_info *sci,
1377		- struct the_nilfs *nilfs)
	1397	+static void nilfs_free_incomplete_logs(struct list_head *logs,
	1398	+ struct the_nilfs *nilfs)
1378	1399	{
1379		- struct nilfs_segment_buffer *segbuf;
	1400	+ struct nilfs_segment_buffer segbuf, prev;
	1401	+ struct inode *sufile = nilfs->ns_sufile;
1380	1402	int ret;
1381	1403
1382		- segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
	1404	+ segbuf = NILFS_FIRST_SEGBUF(logs);
1383	1405	if (nilfs->ns_nextnum != segbuf->sb_nextnum) {
1384		- ret = nilfs_sufile_free(nilfs->ns_sufile, segbuf->sb_nextnum);
	1406	+ ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum);
1385	1407	WARN_ON(ret); /* never fails */
1386	1408	}
1387	1409	if (atomic_read(&segbuf->sb_err)) {
1388	1410
1389	1411
...	...	@@ -1395,36 +1417,20 @@
1395	1417	set_nilfs_discontinued(nilfs);
1396	1418	}
1397	1419
1398		- list_for_each_entry_continue(segbuf, &sci->sc_segbufs, sb_list) {
1399		- ret = nilfs_sufile_free(nilfs->ns_sufile, segbuf->sb_nextnum);
1400		- WARN_ON(ret); /* never fails */
	1420	+ prev = segbuf;
	1421	+ list_for_each_entry_continue(segbuf, logs, sb_list) {
	1422	+ if (prev->sb_nextnum != segbuf->sb_nextnum) {
	1423	+ ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum);
	1424	+ WARN_ON(ret); /* never fails */
	1425	+ }
1401	1426	if (atomic_read(&segbuf->sb_err) &&
1402	1427	segbuf->sb_segnum != nilfs->ns_nextnum)
1403	1428	/* Case 2: extended segment (!= next) failed */
1404		- nilfs_sufile_set_error(nilfs->ns_sufile,
1405		- segbuf->sb_segnum);
	1429	+ nilfs_sufile_set_error(sufile, segbuf->sb_segnum);
	1430	+ prev = segbuf;
1406	1431	}
1407	1432	}
1408	1433
1409		-static void nilfs_segctor_end_construction(struct nilfs_sc_info *sci,
1410		- struct the_nilfs *nilfs, int err)
1411		-{
1412		- if (unlikely(err)) {
1413		- nilfs_segctor_free_incomplete_segments(sci, nilfs);
1414		- if (sci->sc_stage.flags & NILFS_CF_SUFREED) {
1415		- int ret;
1416		-
1417		- ret = nilfs_sufile_cancel_freev(nilfs->ns_sufile,
1418		- sci->sc_freesegs,
1419		- sci->sc_nfreesegs,
1420		- NULL);
1421		- WARN_ON(ret); /* do not happen */
1422		- }
1423		- }
1424		- nilfs_clear_logs(&sci->sc_segbufs);
1425		- sci->sc_super_root = NULL;
1426		-}
1427		-
1428	1434	static void nilfs_segctor_update_segusage(struct nilfs_sc_info *sci,
1429	1435	struct inode *sufile)
1430	1436	{
1431	1437
1432	1438
...	...	@@ -1442,19 +1448,18 @@
1442	1448	}
1443	1449	}
1444	1450
1445		-static void nilfs_segctor_cancel_segusage(struct nilfs_sc_info *sci,
1446		- struct inode *sufile)
	1451	+static void nilfs_cancel_segusage(struct list_head logs, struct inode sufile)
1447	1452	{
1448	1453	struct nilfs_segment_buffer *segbuf;
1449	1454	int ret;
1450	1455
1451		- segbuf = NILFS_FIRST_SEGBUF(&sci->sc_segbufs);
	1456	+ segbuf = NILFS_FIRST_SEGBUF(logs);
1452	1457	ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum,
1453	1458	segbuf->sb_pseg_start -
1454	1459	segbuf->sb_fseg_start, 0);
1455	1460	WARN_ON(ret); /* always succeed because the segusage is dirty */
1456	1461
1457		- list_for_each_entry_continue(segbuf, &sci->sc_segbufs, sb_list) {
	1462	+ list_for_each_entry_continue(segbuf, logs, sb_list) {
1458	1463	ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum,
1459	1464	0, 0);
1460	1465	WARN_ON(ret); /* always succeed */
1461	1466
1462	1467
...	...	@@ -1760,17 +1765,15 @@
1760	1765	struct the_nilfs *nilfs)
1761	1766	{
1762	1767	struct nilfs_segment_buffer *segbuf;
1763		- int err, res;
	1768	+ int ret = 0;
1764	1769
1765	1770	list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
1766		- err = nilfs_segbuf_write(segbuf, nilfs);
1767		-
1768		- res = nilfs_segbuf_wait(segbuf);
1769		- err = err ? : res;
1770		- if (err)
1771		- return err;
	1771	+ ret = nilfs_segbuf_write(segbuf, nilfs);
	1772	+ if (ret)
	1773	+ break;
1772	1774	}
1773		- return 0;
	1775	+ list_splice_tail_init(&sci->sc_segbufs, &sci->sc_write_logs);
	1776	+ return ret;
1774	1777	}
1775	1778
1776	1779	static void __nilfs_end_page_io(struct page *page, int err)
1777	1780
1778	1781
1779	1782
...	...	@@ -1848,15 +1851,17 @@
1848	1851	}
1849	1852	}
1850	1853
1851		-static void nilfs_segctor_abort_write(struct nilfs_sc_info *sci,
1852		- struct page *failed_page, int err)
	1854	+static void nilfs_abort_logs(struct list_head logs, struct page failed_page,
	1855	+ struct buffer_head *bh_sr, int err)
1853	1856	{
1854	1857	struct nilfs_segment_buffer *segbuf;
1855	1858	struct page bd_page = NULL, fs_page = NULL;
	1859	+ struct buffer_head *bh;
1856	1860
1857		- list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
1858		- struct buffer_head *bh;
	1861	+ if (list_empty(logs))
	1862	+ return;
1859	1863
	1864	+ list_for_each_entry(segbuf, logs, sb_list) {
1860	1865	list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
1861	1866	b_assoc_buffers) {
1862	1867	if (bh->b_page != bd_page) {
...	...	@@ -1868,7 +1873,7 @@
1868	1873
1869	1874	list_for_each_entry(bh, &segbuf->sb_payload_buffers,
1870	1875	b_assoc_buffers) {
1871		- if (bh == sci->sc_super_root) {
	1876	+ if (bh == bh_sr) {
1872	1877	if (bh->b_page != bd_page) {
1873	1878	end_page_writeback(bd_page);
1874	1879	bd_page = bh->b_page;
...	...	@@ -1878,7 +1883,7 @@
1878	1883	if (bh->b_page != fs_page) {
1879	1884	nilfs_end_page_io(fs_page, err);
1880	1885	if (fs_page && fs_page == failed_page)
1881		- goto done;
	1886	+ return;
1882	1887	fs_page = bh->b_page;
1883	1888	}
1884	1889	}
1885	1890
...	...	@@ -1887,8 +1892,34 @@
1887	1892	end_page_writeback(bd_page);
1888	1893
1889	1894	nilfs_end_page_io(fs_page, err);
1890		- done:
	1895	+}
	1896	+
	1897	+static void nilfs_segctor_abort_construction(struct nilfs_sc_info *sci,
	1898	+ struct the_nilfs *nilfs, int err)
	1899	+{
	1900	+ LIST_HEAD(logs);
	1901	+ int ret;
	1902	+
	1903	+ list_splice_tail_init(&sci->sc_write_logs, &logs);
	1904	+ ret = nilfs_wait_on_logs(&logs);
	1905	+ if (ret)
	1906	+ nilfs_abort_logs(&logs, NULL, sci->sc_super_root, ret);
	1907	+
	1908	+ list_splice_tail_init(&sci->sc_segbufs, &logs);
	1909	+ nilfs_cancel_segusage(&logs, nilfs->ns_sufile);
	1910	+ nilfs_free_incomplete_logs(&logs, nilfs);
1891	1911	nilfs_clear_copied_buffers(&sci->sc_copied_buffers, err);
	1912	+
	1913	+ if (sci->sc_stage.flags & NILFS_CF_SUFREED) {
	1914	+ ret = nilfs_sufile_cancel_freev(nilfs->ns_sufile,
	1915	+ sci->sc_freesegs,
	1916	+ sci->sc_nfreesegs,
	1917	+ NULL);
	1918	+ WARN_ON(ret); /* do not happen */
	1919	+ }
	1920	+
	1921	+ nilfs_destroy_logs(&logs);
	1922	+ sci->sc_super_root = NULL;
1892	1923	}
1893	1924
1894	1925	static void nilfs_set_next_segment(struct the_nilfs *nilfs,
...	...	@@ -1910,7 +1941,7 @@
1910	1941	struct the_nilfs *nilfs = sbi->s_nilfs;
1911	1942	int update_sr = (sci->sc_super_root != NULL);
1912	1943
1913		- list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
	1944	+ list_for_each_entry(segbuf, &sci->sc_write_logs, sb_list) {
1914	1945	struct buffer_head *bh;
1915	1946
1916	1947	list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
...	...	@@ -1983,7 +2014,7 @@
1983	2014
1984	2015	sci->sc_nblk_inc += sci->sc_nblk_this_inc;
1985	2016
1986		- segbuf = NILFS_LAST_SEGBUF(&sci->sc_segbufs);
	2017	+ segbuf = NILFS_LAST_SEGBUF(&sci->sc_write_logs);
1987	2018	nilfs_set_next_segment(nilfs, segbuf);
1988	2019
1989	2020	if (update_sr) {
1990	2021
...	...	@@ -1994,10 +2025,23 @@
1994	2025	clear_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags);
1995	2026	clear_bit(NILFS_SC_DIRTY, &sci->sc_flags);
1996	2027	set_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags);
	2028	+ nilfs_segctor_clear_metadata_dirty(sci);
1997	2029	} else
1998	2030	clear_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags);
1999	2031	}
2000	2032
	2033	+static int nilfs_segctor_wait(struct nilfs_sc_info *sci)
	2034	+{
	2035	+ int ret;
	2036	+
	2037	+ ret = nilfs_wait_on_logs(&sci->sc_write_logs);
	2038	+ if (!ret) {
	2039	+ nilfs_segctor_complete_write(sci);
	2040	+ nilfs_destroy_logs(&sci->sc_write_logs);
	2041	+ }
	2042	+ return ret;
	2043	+}
	2044	+
2001	2045	static int nilfs_segctor_check_in_files(struct nilfs_sc_info *sci,
2002	2046	struct nilfs_sb_info *sbi)
2003	2047	{
...	...	@@ -2110,7 +2154,7 @@
2110	2154	/* Avoid empty segment */
2111	2155	if (sci->sc_stage.scnt == NILFS_ST_DONE &&
2112	2156	NILFS_SEG_EMPTY(&sci->sc_curseg->sb_sum)) {
2113		- nilfs_segctor_end_construction(sci, nilfs, 1);
	2157	+ nilfs_segctor_abort_construction(sci, nilfs, 1);
2114	2158	goto out;
2115	2159	}
2116	2160
...	...	@@ -2124,7 +2168,7 @@
2124	2168	if (has_sr) {
2125	2169	err = nilfs_segctor_fill_in_checkpoint(sci);
2126	2170	if (unlikely(err))
2127		- goto failed_to_make_up;
	2171	+ goto failed_to_write;
2128	2172
2129	2173	nilfs_segctor_fill_in_super_root(sci, nilfs);
2130	2174	}
2131	2175
2132	2176
2133	2177
2134	2178
2135	2179
2136	2180
...	...	@@ -2132,42 +2176,46 @@
2132	2176
2133	2177	/* Write partial segments */
2134	2178	err = nilfs_segctor_prepare_write(sci, &failed_page);
2135		- if (unlikely(err))
	2179	+ if (err) {
	2180	+ nilfs_abort_logs(&sci->sc_segbufs, failed_page,
	2181	+ sci->sc_super_root, err);
2136	2182	goto failed_to_write;
2137		-
	2183	+ }
2138	2184	nilfs_segctor_fill_in_checksums(sci, nilfs->ns_crc_seed);
2139	2185
2140	2186	err = nilfs_segctor_write(sci, nilfs);
2141	2187	if (unlikely(err))
2142	2188	goto failed_to_write;
2143	2189
2144		- nilfs_segctor_complete_write(sci);
2145		-
2146		- /* Commit segments */
2147		- if (has_sr)
2148		- nilfs_segctor_clear_metadata_dirty(sci);
2149		-
2150		- nilfs_segctor_end_construction(sci, nilfs, 0);
2151		-
	2190	+ if (sci->sc_stage.scnt == NILFS_ST_DONE \|\|
	2191	+ nilfs->ns_blocksize_bits != PAGE_CACHE_SHIFT) {
	2192	+ /*
	2193	+ * At this point, we avoid double buffering
	2194	+ * for blocksize < pagesize because page dirty
	2195	+ * flag is turned off during write and dirty
	2196	+ * buffers are not properly collected for
	2197	+ * pages crossing over segments.
	2198	+ */
	2199	+ err = nilfs_segctor_wait(sci);
	2200	+ if (err)
	2201	+ goto failed_to_write;
	2202	+ }
2152	2203	} while (sci->sc_stage.scnt != NILFS_ST_DONE);
2153	2204
	2205	+ sci->sc_super_root = NULL;
	2206	+
2154	2207	out:
2155		- nilfs_destroy_logs(&sci->sc_segbufs);
2156	2208	nilfs_segctor_check_out_files(sci, sbi);
2157	2209	return err;
2158	2210
2159	2211	failed_to_write:
2160		- nilfs_segctor_abort_write(sci, failed_page, err);
2161		- nilfs_segctor_cancel_segusage(sci, nilfs->ns_sufile);
2162		-
2163		- failed_to_make_up:
2164	2212	if (sci->sc_stage.flags & NILFS_CF_IFILE_STARTED)
2165	2213	nilfs_redirty_inodes(&sci->sc_dirty_files);
2166	2214
2167	2215	failed:
2168	2216	if (nilfs_doing_gc())
2169	2217	nilfs_redirty_inodes(&sci->sc_gc_inodes);
2170		- nilfs_segctor_end_construction(sci, nilfs, err);
	2218	+ nilfs_segctor_abort_construction(sci, nilfs, err);
2171	2219	goto out;
2172	2220	}
2173	2221
...	...	@@ -2725,6 +2773,7 @@
2725	2773	spin_lock_init(&sci->sc_state_lock);
2726	2774	INIT_LIST_HEAD(&sci->sc_dirty_files);
2727	2775	INIT_LIST_HEAD(&sci->sc_segbufs);
	2776	+ INIT_LIST_HEAD(&sci->sc_write_logs);
2728	2777	INIT_LIST_HEAD(&sci->sc_gc_inodes);
2729	2778	INIT_LIST_HEAD(&sci->sc_copied_buffers);
2730	2779
...	...	@@ -2792,6 +2841,7 @@
2792	2841	}
2793	2842
2794	2843	WARN_ON(!list_empty(&sci->sc_segbufs));
	2844	+ WARN_ON(!list_empty(&sci->sc_write_logs));
2795	2845
2796	2846	down_write(&sbi->s_nilfs->ns_segctor_sem);
2797	2847
...	...	@@ -97,6 +97,7 @@
97	97	* @sc_dsync_start: start byte offset of data pages
98	98	* @sc_dsync_end: end byte offset of data pages (inclusive)
99	99	* @sc_segbufs: List of segment buffers
	100	+ * @sc_write_logs: List of segment buffers to hold logs under writing
100	101	* @sc_segbuf_nblocks: Number of available blocks in segment buffers.
101	102	* @sc_curseg: Current segment buffer
102	103	* @sc_super_root: Pointer to the super root buffer
...	...	@@ -143,6 +144,7 @@
143	144
144	145	/* Segment buffers */
145	146	struct list_head sc_segbufs;
	147	+ struct list_head sc_write_logs;
146	148	unsigned long sc_segbuf_nblocks;
147	149	struct nilfs_segment_buffer *sc_curseg;
148	150	struct buffer_head *sc_super_root;