Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit ac5d156c78a68b39955ee9b09498ba93831c77d7

Authored by Jaegeuk Kim 2013-04-29 15:58:39 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

f2fs: modify the number of issued pages to merge IOs

When testing f2fs on an SSD, I found some 128 page IOs followed by 1 page IO
were issued by f2fs_write_node_pages.
This means that there were some mishandling flows which degrades performance.

Previous f2fs_write_node_pages determines the number of pages to be written,
nr_to_write, as follows.

1. The bio_get_nr_vecs returns 129 pages.
2. The bio_alloc makes a room for 128 pages.
3. The initial 128 pages go into one bio.
4. The existing bio is submitted, and a new bio is prepared for the last 1 page.
5. Finally, sync_node_pages submits the last 1 page bio.

The problem is from the use of bio_get_nr_vecs, so this patch replace it
with max_hw_blocks using queue_max_sectors.

Signed-off-by: Jaegeuk Kim <jaegeuk.kim@samsung.com>

Showing 3 changed files with 14 additions and 5 deletions Inline Diff

fs/f2fs/node.c
fs/f2fs/segment.c
fs/f2fs/segment.h

fs/f2fs/node.c

Diff comments View file @ ac5d156

 /*
  * fs/f2fs/node.c
  *
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *             http://www.samsung.com/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/mpage.h>
 #include <linux/backing-dev.h>
 #include <linux/blkdev.h>
 #include <linux/pagevec.h>
 #include <linux/swap.h>
 #include "f2fs.h"
 #include "node.h"
 #include "segment.h"
 #include <trace/events/f2fs.h>
 static struct kmem_cache *nat_entry_slab;
 static struct kmem_cache *free_nid_slab;
 static void clear_node_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
 	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
 	unsigned int long flags;
 	if (PageDirty(page)) {
 		spin_lock_irqsave(&mapping->tree_lock, flags);
 		radix_tree_tag_clear(&mapping->page_tree,
 				page_index(page),
 				PAGECACHE_TAG_DIRTY);
 		spin_unlock_irqrestore(&mapping->tree_lock, flags);
 		clear_page_dirty_for_io(page);
 		dec_page_count(sbi, F2FS_DIRTY_NODES);
 	}
 	ClearPageUptodate(page);
 }
 static struct page *get_current_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	pgoff_t index = current_nat_addr(sbi, nid);
 	return get_meta_page(sbi, index);
 }
 static struct page *get_next_nat_page(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	struct page *src_page;
 	struct page *dst_page;
 	pgoff_t src_off;
 	pgoff_t dst_off;
 	void *src_addr;
 	void *dst_addr;
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	src_off = current_nat_addr(sbi, nid);
 	dst_off = next_nat_addr(sbi, src_off);
 	/* get current nat block page with lock */
 	src_page = get_meta_page(sbi, src_off);
 	/* Dirty src_page means that it is already the new target NAT page. */
 	if (PageDirty(src_page))
 		return src_page;
 	dst_page = grab_meta_page(sbi, dst_off);
 	src_addr = page_address(src_page);
 	dst_addr = page_address(dst_page);
 	memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
 	set_page_dirty(dst_page);
 	f2fs_put_page(src_page, 1);
 	set_to_next_nat(nm_i, nid);
 	return dst_page;
 }
 /*
  * Readahead NAT pages
  */
 static void ra_nat_pages(struct f2fs_sb_info *sbi, int nid)
 {
 	struct address_space *mapping = sbi->meta_inode->i_mapping;
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct blk_plug plug;
 	struct page *page;
 	pgoff_t index;
 	int i;
 	blk_start_plug(&plug);
 	for (i = 0; i < FREE_NID_PAGES; i++, nid += NAT_ENTRY_PER_BLOCK) {
 		if (nid >= nm_i->max_nid)
 			nid = 0;
 		index = current_nat_addr(sbi, nid);
 		page = grab_cache_page(mapping, index);
 		if (!page)
 			continue;
 		if (PageUptodate(page)) {
 			f2fs_put_page(page, 1);
 			continue;
 		}
 		if (f2fs_readpage(sbi, page, index, READ))
 			continue;
 		f2fs_put_page(page, 0);
 	}
 	blk_finish_plug(&plug);
 }
 static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n)
 {
 	return radix_tree_lookup(&nm_i->nat_root, n);
 }
 static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i,
 		nid_t start, unsigned int nr, struct nat_entry **ep)
 {
 	return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr);
 }
 static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e)
 {
 	list_del(&e->list);
 	radix_tree_delete(&nm_i->nat_root, nat_get_nid(e));
 	nm_i->nat_cnt--;
 	kmem_cache_free(nat_entry_slab, e);
 }
 int is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct nat_entry *e;
 	int is_cp = 1;
 	read_lock(&nm_i->nat_tree_lock);
 	e = __lookup_nat_cache(nm_i, nid);
 	if (e && !e->checkpointed)
 		is_cp = 0;
 	read_unlock(&nm_i->nat_tree_lock);
 	return is_cp;
 }
 static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)
 {
 	struct nat_entry *new;
 	new = kmem_cache_alloc(nat_entry_slab, GFP_ATOMIC);
 	if (!new)
 		return NULL;
 	if (radix_tree_insert(&nm_i->nat_root, nid, new)) {
 		kmem_cache_free(nat_entry_slab, new);
 		return NULL;
 	}
 	memset(new, 0, sizeof(struct nat_entry));
 	nat_set_nid(new, nid);
 	list_add_tail(&new->list, &nm_i->nat_entries);
 	nm_i->nat_cnt++;
 	return new;
 }
 static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,
 						struct f2fs_nat_entry *ne)
 {
 	struct nat_entry *e;
 retry:
 	write_lock(&nm_i->nat_tree_lock);
 	e = __lookup_nat_cache(nm_i, nid);
 	if (!e) {
 		e = grab_nat_entry(nm_i, nid);
 		if (!e) {
 			write_unlock(&nm_i->nat_tree_lock);
 			goto retry;
 		}
 		nat_set_blkaddr(e, le32_to_cpu(ne->block_addr));
 		nat_set_ino(e, le32_to_cpu(ne->ino));
 		nat_set_version(e, ne->version);
 		e->checkpointed = true;
 	}
 	write_unlock(&nm_i->nat_tree_lock);
 }
 static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
 			block_t new_blkaddr)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct nat_entry *e;
 retry:
 	write_lock(&nm_i->nat_tree_lock);
 	e = __lookup_nat_cache(nm_i, ni->nid);
 	if (!e) {
 		e = grab_nat_entry(nm_i, ni->nid);
 		if (!e) {
 			write_unlock(&nm_i->nat_tree_lock);
 			goto retry;
 		}
 		e->ni = *ni;
 		e->checkpointed = true;
 		BUG_ON(ni->blk_addr == NEW_ADDR);
 	} else if (new_blkaddr == NEW_ADDR) {
 		/*
 		 * when nid is reallocated,
 		 * previous nat entry can be remained in nat cache.
 		 * So, reinitialize it with new information.
 		 */
 		e->ni = *ni;
 		BUG_ON(ni->blk_addr != NULL_ADDR);
 	}
 	if (new_blkaddr == NEW_ADDR)
 		e->checkpointed = false;
 	/* sanity check */
 	BUG_ON(nat_get_blkaddr(e) != ni->blk_addr);
 	BUG_ON(nat_get_blkaddr(e) == NULL_ADDR &&
 			new_blkaddr == NULL_ADDR);
 	BUG_ON(nat_get_blkaddr(e) == NEW_ADDR &&
 			new_blkaddr == NEW_ADDR);
 	BUG_ON(nat_get_blkaddr(e) != NEW_ADDR &&
 			nat_get_blkaddr(e) != NULL_ADDR &&
 			new_blkaddr == NEW_ADDR);
 	/* increament version no as node is removed */
 	if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) {
 		unsigned char version = nat_get_version(e);
 		nat_set_version(e, inc_node_version(version));
 	}
 	/* change address */
 	nat_set_blkaddr(e, new_blkaddr);
 	__set_nat_cache_dirty(nm_i, e);
 	write_unlock(&nm_i->nat_tree_lock);
 }
 static int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	if (nm_i->nat_cnt <= NM_WOUT_THRESHOLD)
 		return 0;
 	write_lock(&nm_i->nat_tree_lock);
 	while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
 		struct nat_entry *ne;
 		ne = list_first_entry(&nm_i->nat_entries,
 					struct nat_entry, list);
 		__del_from_nat_cache(nm_i, ne);
 		nr_shrink--;
 	}
 	write_unlock(&nm_i->nat_tree_lock);
 	return nr_shrink;
 }
 /*
  * This function returns always success
  */
 void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_summary_block *sum = curseg->sum_blk;
 	nid_t start_nid = START_NID(nid);
 	struct f2fs_nat_block *nat_blk;
 	struct page *page = NULL;
 	struct f2fs_nat_entry ne;
 	struct nat_entry *e;
 	int i;
 	memset(&ne, 0, sizeof(struct f2fs_nat_entry));
 	ni->nid = nid;
 	/* Check nat cache */
 	read_lock(&nm_i->nat_tree_lock);
 	e = __lookup_nat_cache(nm_i, nid);
 	if (e) {
 		ni->ino = nat_get_ino(e);
 		ni->blk_addr = nat_get_blkaddr(e);
 		ni->version = nat_get_version(e);
 	}
 	read_unlock(&nm_i->nat_tree_lock);
 	if (e)
 		return;
 	/* Check current segment summary */
 	mutex_lock(&curseg->curseg_mutex);
 	i = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 0);
 	if (i >= 0) {
 		ne = nat_in_journal(sum, i);
 		node_info_from_raw_nat(ni, &ne);
 	}
 	mutex_unlock(&curseg->curseg_mutex);
 	if (i >= 0)
 		goto cache;
 	/* Fill node_info from nat page */
 	page = get_current_nat_page(sbi, start_nid);
 	nat_blk = (struct f2fs_nat_block *)page_address(page);
 	ne = nat_blk->entries[nid - start_nid];
 	node_info_from_raw_nat(ni, &ne);
 	f2fs_put_page(page, 1);
 cache:
 	/* cache nat entry */
 	cache_nat_entry(NM_I(sbi), nid, &ne);
 }
 /*
  * The maximum depth is four.
  * Offset[0] will have raw inode offset.
  */
 static int get_node_path(long block, int offset[4], unsigned int noffset[4])
 {
 	const long direct_index = ADDRS_PER_INODE;
 	const long direct_blks = ADDRS_PER_BLOCK;
 	const long dptrs_per_blk = NIDS_PER_BLOCK;
 	const long indirect_blks = ADDRS_PER_BLOCK * NIDS_PER_BLOCK;
 	const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK;
 	int n = 0;
 	int level = 0;
 	noffset[0] = 0;
 	if (block < direct_index) {
 		offset[n] = block;
 		goto got;
 	}
 	block -= direct_index;
 	if (block < direct_blks) {
 		offset[n++] = NODE_DIR1_BLOCK;
 		noffset[n] = 1;
 		offset[n] = block;
 		level = 1;
 		goto got;
 	}
 	block -= direct_blks;
 	if (block < direct_blks) {
 		offset[n++] = NODE_DIR2_BLOCK;
 		noffset[n] = 2;
 		offset[n] = block;
 		level = 1;
 		goto got;
 	}
 	block -= direct_blks;
 	if (block < indirect_blks) {
 		offset[n++] = NODE_IND1_BLOCK;
 		noffset[n] = 3;
 		offset[n++] = block / direct_blks;
 		noffset[n] = 4 + offset[n - 1];
 		offset[n] = block % direct_blks;
 		level = 2;
 		goto got;
 	}
 	block -= indirect_blks;
 	if (block < indirect_blks) {
 		offset[n++] = NODE_IND2_BLOCK;
 		noffset[n] = 4 + dptrs_per_blk;
 		offset[n++] = block / direct_blks;
 		noffset[n] = 5 + dptrs_per_blk + offset[n - 1];
 		offset[n] = block % direct_blks;
 		level = 2;
 		goto got;
 	}
 	block -= indirect_blks;
 	if (block < dindirect_blks) {
 		offset[n++] = NODE_DIND_BLOCK;
 		noffset[n] = 5 + (dptrs_per_blk * 2);
 		offset[n++] = block / indirect_blks;
 		noffset[n] = 6 + (dptrs_per_blk * 2) +
 			      offset[n - 1] * (dptrs_per_blk + 1);
 		offset[n++] = (block / direct_blks) % dptrs_per_blk;
 		noffset[n] = 7 + (dptrs_per_blk * 2) +
 			      offset[n - 2] * (dptrs_per_blk + 1) +
 			      offset[n - 1];
 		offset[n] = block % direct_blks;
 		level = 3;
 		goto got;
 	} else {
 		BUG();
 	}
 got:
 	return level;
 }
 /*
  * Caller should call f2fs_put_dnode(dn).
  * Also, it should grab and release a mutex by calling mutex_lock_op() and
  * mutex_unlock_op() only if ro is not set RDONLY_NODE.
  * In the case of RDONLY_NODE, we don't need to care about mutex.
  */
 int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct page *npage[4];
 	struct page *parent;
 	int offset[4];
 	unsigned int noffset[4];
 	nid_t nids[4];
 	int level, i;
 	int err = 0;
 	level = get_node_path(index, offset, noffset);
 	nids[0] = dn->inode->i_ino;
 	npage[0] = get_node_page(sbi, nids[0]);
 	if (IS_ERR(npage[0]))
 		return PTR_ERR(npage[0]);
 	parent = npage[0];
 	if (level != 0)
 		nids[1] = get_nid(parent, offset[0], true);
 	dn->inode_page = npage[0];
 	dn->inode_page_locked = true;
 	/* get indirect or direct nodes */
 	for (i = 1; i <= level; i++) {
 		bool done = false;
 		if (!nids[i] && mode == ALLOC_NODE) {
 			/* alloc new node */
 			if (!alloc_nid(sbi, &(nids[i]))) {
 				err = -ENOSPC;
 				goto release_pages;
 			}
 			dn->nid = nids[i];
 			npage[i] = new_node_page(dn, noffset[i]);
 			if (IS_ERR(npage[i])) {
 				alloc_nid_failed(sbi, nids[i]);
 				err = PTR_ERR(npage[i]);
 				goto release_pages;
 			}
 			set_nid(parent, offset[i - 1], nids[i], i == 1);
 			alloc_nid_done(sbi, nids[i]);
 			done = true;
 		} else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
 			npage[i] = get_node_page_ra(parent, offset[i - 1]);
 			if (IS_ERR(npage[i])) {
 				err = PTR_ERR(npage[i]);
 				goto release_pages;
 			}
 			done = true;
 		}
 		if (i == 1) {
 			dn->inode_page_locked = false;
 			unlock_page(parent);
 		} else {
 			f2fs_put_page(parent, 1);
 		}
 		if (!done) {
 			npage[i] = get_node_page(sbi, nids[i]);
 			if (IS_ERR(npage[i])) {
 				err = PTR_ERR(npage[i]);
 				f2fs_put_page(npage[0], 0);
 				goto release_out;
 			}
 		}
 		if (i < level) {
 			parent = npage[i];
 			nids[i + 1] = get_nid(parent, offset[i], false);
 		}
 	}
 	dn->nid = nids[level];
 	dn->ofs_in_node = offset[level];
 	dn->node_page = npage[level];
 	dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
 	return 0;
 release_pages:
 	f2fs_put_page(parent, 1);
 	if (i > 1)
 		f2fs_put_page(npage[0], 0);
 release_out:
 	dn->inode_page = NULL;
 	dn->node_page = NULL;
 	return err;
 }
 static void truncate_node(struct dnode_of_data *dn)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct node_info ni;
 	get_node_info(sbi, dn->nid, &ni);
 	if (dn->inode->i_blocks == 0) {
 		BUG_ON(ni.blk_addr != NULL_ADDR);
 		goto invalidate;
 	}
 	BUG_ON(ni.blk_addr == NULL_ADDR);
 	/* Deallocate node address */
 	invalidate_blocks(sbi, ni.blk_addr);
 	dec_valid_node_count(sbi, dn->inode, 1);
 	set_node_addr(sbi, &ni, NULL_ADDR);
 	if (dn->nid == dn->inode->i_ino) {
 		remove_orphan_inode(sbi, dn->nid);
 		dec_valid_inode_count(sbi);
 	} else {
 		sync_inode_page(dn);
 	}
 invalidate:
 	clear_node_page_dirty(dn->node_page);
 	F2FS_SET_SB_DIRT(sbi);
 	f2fs_put_page(dn->node_page, 1);
 	dn->node_page = NULL;
 	trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr);
 }
 static int truncate_dnode(struct dnode_of_data *dn)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct page *page;
 	if (dn->nid == 0)
 		return 1;
 	/* get direct node */
 	page = get_node_page(sbi, dn->nid);
 	if (IS_ERR(page) && PTR_ERR(page) == -ENOENT)
 		return 1;
 	else if (IS_ERR(page))
 		return PTR_ERR(page);
 	/* Make dnode_of_data for parameter */
 	dn->node_page = page;
 	dn->ofs_in_node = 0;
 	truncate_data_blocks(dn);
 	truncate_node(dn);
 	return 1;
 }
 static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs,
 						int ofs, int depth)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct dnode_of_data rdn = *dn;
 	struct page *page;
 	struct f2fs_node *rn;
 	nid_t child_nid;
 	unsigned int child_nofs;
 	int freed = 0;
 	int i, ret;
 	if (dn->nid == 0)
 		return NIDS_PER_BLOCK + 1;
 	trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr);
 	page = get_node_page(sbi, dn->nid);
 	if (IS_ERR(page)) {
 		trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(page));
 		return PTR_ERR(page);
 	}
 	rn = (struct f2fs_node *)page_address(page);
 	if (depth < 3) {
 		for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) {
 			child_nid = le32_to_cpu(rn->in.nid[i]);
 			if (child_nid == 0)
 				continue;
 			rdn.nid = child_nid;
 			ret = truncate_dnode(&rdn);
 			if (ret < 0)
 				goto out_err;
 			set_nid(page, i, 0, false);
 		}
 	} else {
 		child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1;
 		for (i = ofs; i < NIDS_PER_BLOCK; i++) {
 			child_nid = le32_to_cpu(rn->in.nid[i]);
 			if (child_nid == 0) {
 				child_nofs += NIDS_PER_BLOCK + 1;
 				continue;
 			}
 			rdn.nid = child_nid;
 			ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1);
 			if (ret == (NIDS_PER_BLOCK + 1)) {
 				set_nid(page, i, 0, false);
 				child_nofs += ret;
 			} else if (ret < 0 && ret != -ENOENT) {
 				goto out_err;
 			}
 		}
 		freed = child_nofs;
 	}
 	if (!ofs) {
 		/* remove current indirect node */
 		dn->node_page = page;
 		truncate_node(dn);
 		freed++;
 	} else {
 		f2fs_put_page(page, 1);
 	}
 	trace_f2fs_truncate_nodes_exit(dn->inode, freed);
 	return freed;
 out_err:
 	f2fs_put_page(page, 1);
 	trace_f2fs_truncate_nodes_exit(dn->inode, ret);
 	return ret;
 }
 static int truncate_partial_nodes(struct dnode_of_data *dn,
 			struct f2fs_inode *ri, int *offset, int depth)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct page *pages[2];
 	nid_t nid[3];
 	nid_t child_nid;
 	int err = 0;
 	int i;
 	int idx = depth - 2;
 	nid[0] = le32_to_cpu(ri->i_nid[offset[0] - NODE_DIR1_BLOCK]);
 	if (!nid[0])
 		return 0;
 	/* get indirect nodes in the path */
 	for (i = 0; i < depth - 1; i++) {
 		/* refernece count'll be increased */
 		pages[i] = get_node_page(sbi, nid[i]);
 		if (IS_ERR(pages[i])) {
 			depth = i + 1;
 			err = PTR_ERR(pages[i]);
 			goto fail;
 		}
 		nid[i + 1] = get_nid(pages[i], offset[i + 1], false);
 	}
 	/* free direct nodes linked to a partial indirect node */
 	for (i = offset[depth - 1]; i < NIDS_PER_BLOCK; i++) {
 		child_nid = get_nid(pages[idx], i, false);
 		if (!child_nid)
 			continue;
 		dn->nid = child_nid;
 		err = truncate_dnode(dn);
 		if (err < 0)
 			goto fail;
 		set_nid(pages[idx], i, 0, false);
 	}
 	if (offset[depth - 1] == 0) {
 		dn->node_page = pages[idx];
 		dn->nid = nid[idx];
 		truncate_node(dn);
 	} else {
 		f2fs_put_page(pages[idx], 1);
 	}
 	offset[idx]++;
 	offset[depth - 1] = 0;
 fail:
 	for (i = depth - 3; i >= 0; i--)
 		f2fs_put_page(pages[i], 1);
 	trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err);
 	return err;
 }
 /*
  * All the block addresses of data and nodes should be nullified.
  */
 int truncate_inode_blocks(struct inode *inode, pgoff_t from)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 	struct address_space *node_mapping = sbi->node_inode->i_mapping;
 	int err = 0, cont = 1;
 	int level, offset[4], noffset[4];
 	unsigned int nofs = 0;
 	struct f2fs_node *rn;
 	struct dnode_of_data dn;
 	struct page *page;
 	trace_f2fs_truncate_inode_blocks_enter(inode, from);
 	level = get_node_path(from, offset, noffset);
 restart:
 	page = get_node_page(sbi, inode->i_ino);
 	if (IS_ERR(page)) {
 		trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(page));
 		return PTR_ERR(page);
 	}
 	set_new_dnode(&dn, inode, page, NULL, 0);
 	unlock_page(page);
 	rn = page_address(page);
 	switch (level) {
 	case 0:
 	case 1:
 		nofs = noffset[1];
 		break;
 	case 2:
 		nofs = noffset[1];
 		if (!offset[level - 1])
 			goto skip_partial;
 		err = truncate_partial_nodes(&dn, &rn->i, offset, level);
 		if (err < 0 && err != -ENOENT)
 			goto fail;
 		nofs += 1 + NIDS_PER_BLOCK;
 		break;
 	case 3:
 		nofs = 5 + 2 * NIDS_PER_BLOCK;
 		if (!offset[level - 1])
 			goto skip_partial;
 		err = truncate_partial_nodes(&dn, &rn->i, offset, level);
 		if (err < 0 && err != -ENOENT)
 			goto fail;
 		break;
 	default:
 		BUG();
 	}
 skip_partial:
 	while (cont) {
 		dn.nid = le32_to_cpu(rn->i.i_nid[offset[0] - NODE_DIR1_BLOCK]);
 		switch (offset[0]) {
 		case NODE_DIR1_BLOCK:
 		case NODE_DIR2_BLOCK:
 			err = truncate_dnode(&dn);
 			break;
 		case NODE_IND1_BLOCK:
 		case NODE_IND2_BLOCK:
 			err = truncate_nodes(&dn, nofs, offset[1], 2);
 			break;
 		case NODE_DIND_BLOCK:
 			err = truncate_nodes(&dn, nofs, offset[1], 3);
 			cont = 0;
 			break;
 		default:
 			BUG();
 		}
 		if (err < 0 && err != -ENOENT)
 			goto fail;
 		if (offset[1] == 0 &&
 				rn->i.i_nid[offset[0] - NODE_DIR1_BLOCK]) {
 			lock_page(page);
 			if (page->mapping != node_mapping) {
 				f2fs_put_page(page, 1);
 				goto restart;
 			}
 			wait_on_page_writeback(page);
 			rn->i.i_nid[offset[0] - NODE_DIR1_BLOCK] = 0;
 			set_page_dirty(page);
 			unlock_page(page);
 		}
 		offset[1] = 0;
 		offset[0]++;
 		nofs += err;
 	}
 fail:
 	f2fs_put_page(page, 0);
 	trace_f2fs_truncate_inode_blocks_exit(inode, err);
 	return err > 0 ? 0 : err;
 }
 /*
  * Caller should grab and release a mutex by calling mutex_lock_op() and
  * mutex_unlock_op().
  */
 int remove_inode_page(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 	struct page *page;
 	nid_t ino = inode->i_ino;
 	struct dnode_of_data dn;
 	page = get_node_page(sbi, ino);
 	if (IS_ERR(page))
 		return PTR_ERR(page);
 	if (F2FS_I(inode)->i_xattr_nid) {
 		nid_t nid = F2FS_I(inode)->i_xattr_nid;
 		struct page *npage = get_node_page(sbi, nid);
 		if (IS_ERR(npage))
 			return PTR_ERR(npage);
 		F2FS_I(inode)->i_xattr_nid = 0;
 		set_new_dnode(&dn, inode, page, npage, nid);
 		dn.inode_page_locked = 1;
 		truncate_node(&dn);
 	}
 	/* 0 is possible, after f2fs_new_inode() is failed */
 	BUG_ON(inode->i_blocks != 0 && inode->i_blocks != 1);
 	set_new_dnode(&dn, inode, page, page, ino);
 	truncate_node(&dn);
 	return 0;
 }
 int new_inode_page(struct inode *inode, const struct qstr *name)
 {
 	struct page *page;
 	struct dnode_of_data dn;
 	/* allocate inode page for new inode */
 	set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
 	page = new_node_page(&dn, 0);
 	init_dent_inode(name, page);
 	if (IS_ERR(page))
 		return PTR_ERR(page);
 	f2fs_put_page(page, 1);
 	return 0;
 }
 struct page *new_node_page(struct dnode_of_data *dn, unsigned int ofs)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(dn->inode->i_sb);
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	struct node_info old_ni, new_ni;
 	struct page *page;
 	int err;
 	if (is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC))
 		return ERR_PTR(-EPERM);
 	page = grab_cache_page(mapping, dn->nid);
 	if (!page)
 		return ERR_PTR(-ENOMEM);
 	get_node_info(sbi, dn->nid, &old_ni);
 	SetPageUptodate(page);
 	fill_node_footer(page, dn->nid, dn->inode->i_ino, ofs, true);
 	/* Reinitialize old_ni with new node page */
 	BUG_ON(old_ni.blk_addr != NULL_ADDR);
 	new_ni = old_ni;
 	new_ni.ino = dn->inode->i_ino;
 	if (!inc_valid_node_count(sbi, dn->inode, 1)) {
 		err = -ENOSPC;
 		goto fail;
 	}
 	set_node_addr(sbi, &new_ni, NEW_ADDR);
 	set_cold_node(dn->inode, page);
 	dn->node_page = page;
 	sync_inode_page(dn);
 	set_page_dirty(page);
 	if (ofs == 0)
 		inc_valid_inode_count(sbi);
 	return page;
 fail:
 	clear_node_page_dirty(page);
 	f2fs_put_page(page, 1);
 	return ERR_PTR(err);
 }
 /*
  * Caller should do after getting the following values.
  * 0: f2fs_put_page(page, 0)
  * LOCKED_PAGE: f2fs_put_page(page, 1)
  * error: nothing
  */
 static int read_node_page(struct page *page, int type)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
 	struct node_info ni;
 	get_node_info(sbi, page->index, &ni);
 	if (ni.blk_addr == NULL_ADDR) {
 		f2fs_put_page(page, 1);
 		return -ENOENT;
 	}
 	if (PageUptodate(page))
 		return LOCKED_PAGE;
 	return f2fs_readpage(sbi, page, ni.blk_addr, type);
 }
 /*
  * Readahead a node page
  */
 void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	struct page *apage;
 	int err;
 	apage = find_get_page(mapping, nid);
 	if (apage && PageUptodate(apage)) {
 		f2fs_put_page(apage, 0);
 		return;
 	}
 	f2fs_put_page(apage, 0);
 	apage = grab_cache_page(mapping, nid);
 	if (!apage)
 		return;
 	err = read_node_page(apage, READA);
 	if (err == 0)
 		f2fs_put_page(apage, 0);
 	else if (err == LOCKED_PAGE)
 		f2fs_put_page(apage, 1);
 	return;
 }
 struct page *get_node_page(struct f2fs_sb_info *sbi, pgoff_t nid)
 {
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	struct page *page;
 	int err;
 repeat:
 	page = grab_cache_page(mapping, nid);
 	if (!page)
 		return ERR_PTR(-ENOMEM);
 	err = read_node_page(page, READ_SYNC);
 	if (err < 0)
 		return ERR_PTR(err);
 	else if (err == LOCKED_PAGE)
 		goto got_it;
 	lock_page(page);
 	if (!PageUptodate(page)) {
 		f2fs_put_page(page, 1);
 		return ERR_PTR(-EIO);
 	}
 	if (page->mapping != mapping) {
 		f2fs_put_page(page, 1);
 		goto repeat;
 	}
 got_it:
 	BUG_ON(nid != nid_of_node(page));
 	mark_page_accessed(page);
 	return page;
 }
 /*
  * Return a locked page for the desired node page.
  * And, readahead MAX_RA_NODE number of node pages.
  */
 struct page *get_node_page_ra(struct page *parent, int start)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(parent->mapping->host->i_sb);
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	struct blk_plug plug;
 	struct page *page;
 	int err, i, end;
 	nid_t nid;
 	/* First, try getting the desired direct node. */
 	nid = get_nid(parent, start, false);
 	if (!nid)
 		return ERR_PTR(-ENOENT);
 repeat:
 	page = grab_cache_page(mapping, nid);
 	if (!page)
 		return ERR_PTR(-ENOMEM);
 	err = read_node_page(page, READ_SYNC);
 	if (err < 0)
 		return ERR_PTR(err);
 	else if (err == LOCKED_PAGE)
 		goto page_hit;
 	blk_start_plug(&plug);
 	/* Then, try readahead for siblings of the desired node */
 	end = start + MAX_RA_NODE;
 	end = min(end, NIDS_PER_BLOCK);
 	for (i = start + 1; i < end; i++) {
 		nid = get_nid(parent, i, false);
 		if (!nid)
 			continue;
 		ra_node_page(sbi, nid);
 	}
 	blk_finish_plug(&plug);
 	lock_page(page);
 	if (page->mapping != mapping) {
 		f2fs_put_page(page, 1);
 		goto repeat;
 	}
 page_hit:
 	if (!PageUptodate(page)) {
 		f2fs_put_page(page, 1);
 		return ERR_PTR(-EIO);
 	}
 	mark_page_accessed(page);
 	return page;
 }
 void sync_inode_page(struct dnode_of_data *dn)
 {
 	if (IS_INODE(dn->node_page) || dn->inode_page == dn->node_page) {
 		update_inode(dn->inode, dn->node_page);
 	} else if (dn->inode_page) {
 		if (!dn->inode_page_locked)
 			lock_page(dn->inode_page);
 		update_inode(dn->inode, dn->inode_page);
 		if (!dn->inode_page_locked)
 			unlock_page(dn->inode_page);
 	} else {
 		update_inode_page(dn->inode);
 	}
 }
 int sync_node_pages(struct f2fs_sb_info *sbi, nid_t ino,
 					struct writeback_control *wbc)
 {
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	pgoff_t index, end;
 	struct pagevec pvec;
 	int step = ino ? 2 : 0;
 	int nwritten = 0, wrote = 0;
 	pagevec_init(&pvec, 0);
 next_step:
 	index = 0;
 	end = LONG_MAX;
 	while (index <= end) {
 		int i, nr_pages;
 		nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
 				PAGECACHE_TAG_DIRTY,
 				min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1);
 		if (nr_pages == 0)
 			break;
 		for (i = 0; i < nr_pages; i++) {
 			struct page *page = pvec.pages[i];
 			/*
 			 * flushing sequence with step:
 			 * 0. indirect nodes
 			 * 1. dentry dnodes
 			 * 2. file dnodes
 			 */
 			if (step == 0 && IS_DNODE(page))
 				continue;
 			if (step == 1 && (!IS_DNODE(page) ||
 						is_cold_node(page)))
 				continue;
 			if (step == 2 && (!IS_DNODE(page) ||
 						!is_cold_node(page)))
 				continue;
 			/*
 			 * If an fsync mode,
 			 * we should not skip writing node pages.
 			 */
 			if (ino && ino_of_node(page) == ino)
 				lock_page(page);
 			else if (!trylock_page(page))
 				continue;
 			if (unlikely(page->mapping != mapping)) {
 continue_unlock:
 				unlock_page(page);
 				continue;
 			}
 			if (ino && ino_of_node(page) != ino)
 				goto continue_unlock;
 			if (!PageDirty(page)) {
 				/* someone wrote it for us */
 				goto continue_unlock;
 			}
 			if (!clear_page_dirty_for_io(page))
 				goto continue_unlock;
 			/* called by fsync() */
 			if (ino && IS_DNODE(page)) {
 				int mark = !is_checkpointed_node(sbi, ino);
 				set_fsync_mark(page, 1);
 				if (IS_INODE(page))
 					set_dentry_mark(page, mark);
 				nwritten++;
 			} else {
 				set_fsync_mark(page, 0);
 				set_dentry_mark(page, 0);
 			}
 			mapping->a_ops->writepage(page, wbc);
 			wrote++;
 			if (--wbc->nr_to_write == 0)
 				break;
 		}
 		pagevec_release(&pvec);
 		cond_resched();
 		if (wbc->nr_to_write == 0) {
 			step = 2;
 			break;
 		}
 	}
 	if (step < 2) {
 		step++;
 		goto next_step;
 	}
 	if (wrote)
 		f2fs_submit_bio(sbi, NODE, wbc->sync_mode == WB_SYNC_ALL);
 	return nwritten;
 }
 static int f2fs_write_node_page(struct page *page,
 				struct writeback_control *wbc)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
 	nid_t nid;
 	block_t new_addr;
 	struct node_info ni;
 	wait_on_page_writeback(page);
 	/* get old block addr of this node page */
 	nid = nid_of_node(page);
 	BUG_ON(page->index != nid);
 	get_node_info(sbi, nid, &ni);
 	/* This page is already truncated */
 	if (ni.blk_addr == NULL_ADDR) {
 		dec_page_count(sbi, F2FS_DIRTY_NODES);
 		unlock_page(page);
 		return 0;
 	}
 	if (wbc->for_reclaim) {
 		dec_page_count(sbi, F2FS_DIRTY_NODES);
 		wbc->pages_skipped++;
 		set_page_dirty(page);
 		return AOP_WRITEPAGE_ACTIVATE;
 	}
 	mutex_lock(&sbi->node_write);
 	set_page_writeback(page);
 	write_node_page(sbi, page, nid, ni.blk_addr, &new_addr);
 	set_node_addr(sbi, &ni, new_addr);
 	dec_page_count(sbi, F2FS_DIRTY_NODES);
 	mutex_unlock(&sbi->node_write);
 	unlock_page(page);
 	return 0;
 }
 /*
  * It is very important to gather dirty pages and write at once, so that we can
  * submit a big bio without interfering other data writes.
  * Be default, 512 pages (2MB), a segment size, is quite reasonable.
  */
 #define COLLECT_DIRTY_NODES	512
 static int f2fs_write_node_pages(struct address_space *mapping,
 			    struct writeback_control *wbc)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
-	struct block_device *bdev = sbi->sb->s_bdev;
 	long nr_to_write = wbc->nr_to_write;
 	/* First check balancing cached NAT entries */
 	if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK)) {
 		f2fs_sync_fs(sbi->sb, true);
 		return 0;
 	}
 	/* collect a number of dirty node pages and write together */
 	if (get_pages(sbi, F2FS_DIRTY_NODES) < COLLECT_DIRTY_NODES)
 		return 0;
 	/* if mounting is failed, skip writing node pages */
-	wbc->nr_to_write = bio_get_nr_vecs(bdev);
+	wbc->nr_to_write = max_hw_blocks(sbi);
 	sync_node_pages(sbi, 0, wbc);
-	wbc->nr_to_write = nr_to_write -
+	wbc->nr_to_write = nr_to_write - (max_hw_blocks(sbi) - wbc->nr_to_write);
-		(bio_get_nr_vecs(bdev) - wbc->nr_to_write);
 	return 0;
 }
 static int f2fs_set_node_page_dirty(struct page *page)
 {
 	struct address_space *mapping = page->mapping;
 	struct f2fs_sb_info *sbi = F2FS_SB(mapping->host->i_sb);
 	SetPageUptodate(page);
 	if (!PageDirty(page)) {
 		__set_page_dirty_nobuffers(page);
 		inc_page_count(sbi, F2FS_DIRTY_NODES);
 		SetPagePrivate(page);
 		return 1;
 	}
 	return 0;
 }
 static void f2fs_invalidate_node_page(struct page *page, unsigned long offset)
 {
 	struct inode *inode = page->mapping->host;
 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 	if (PageDirty(page))
 		dec_page_count(sbi, F2FS_DIRTY_NODES);
 	ClearPagePrivate(page);
 }
 static int f2fs_release_node_page(struct page *page, gfp_t wait)
 {
 	ClearPagePrivate(page);
 	return 1;
 }
 /*
  * Structure of the f2fs node operations
  */
 const struct address_space_operations f2fs_node_aops = {
 	.writepage	= f2fs_write_node_page,
 	.writepages	= f2fs_write_node_pages,
 	.set_page_dirty	= f2fs_set_node_page_dirty,
 	.invalidatepage	= f2fs_invalidate_node_page,
 	.releasepage	= f2fs_release_node_page,
 };
 static struct free_nid *__lookup_free_nid_list(nid_t n, struct list_head *head)
 {
 	struct list_head *this;
 	struct free_nid *i;
 	list_for_each(this, head) {
 		i = list_entry(this, struct free_nid, list);
 		if (i->nid == n)
 			return i;
 	}
 	return NULL;
 }
 static void __del_from_free_nid_list(struct free_nid *i)
 {
 	list_del(&i->list);
 	kmem_cache_free(free_nid_slab, i);
 }
 static int add_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
 {
 	struct free_nid *i;
 	if (nm_i->fcnt > 2 * MAX_FREE_NIDS)
 		return 0;
 	/* 0 nid should not be used */
 	if (nid == 0)
 		return 0;
 retry:
 	i = kmem_cache_alloc(free_nid_slab, GFP_NOFS);
 	if (!i) {
 		cond_resched();
 		goto retry;
 	}
 	i->nid = nid;
 	i->state = NID_NEW;
 	spin_lock(&nm_i->free_nid_list_lock);
 	if (__lookup_free_nid_list(nid, &nm_i->free_nid_list)) {
 		spin_unlock(&nm_i->free_nid_list_lock);
 		kmem_cache_free(free_nid_slab, i);
 		return 0;
 	}
 	list_add_tail(&i->list, &nm_i->free_nid_list);
 	nm_i->fcnt++;
 	spin_unlock(&nm_i->free_nid_list_lock);
 	return 1;
 }
 static void remove_free_nid(struct f2fs_nm_info *nm_i, nid_t nid)
 {
 	struct free_nid *i;
 	spin_lock(&nm_i->free_nid_list_lock);
 	i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
 	if (i && i->state == NID_NEW) {
 		__del_from_free_nid_list(i);
 		nm_i->fcnt--;
 	}
 	spin_unlock(&nm_i->free_nid_list_lock);
 }
 static int scan_nat_page(struct f2fs_nm_info *nm_i,
 			struct page *nat_page, nid_t start_nid)
 {
 	struct f2fs_nat_block *nat_blk = page_address(nat_page);
 	block_t blk_addr;
 	int fcnt = 0;
 	int i;
 	i = start_nid % NAT_ENTRY_PER_BLOCK;
 	for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) {
 		if (start_nid >= nm_i->max_nid)
 			break;
 		blk_addr  = le32_to_cpu(nat_blk->entries[i].block_addr);
 		BUG_ON(blk_addr == NEW_ADDR);
 		if (blk_addr == NULL_ADDR)
 			fcnt += add_free_nid(nm_i, start_nid);
 	}
 	return fcnt;
 }
 static void build_free_nids(struct f2fs_sb_info *sbi)
 {
 	struct free_nid *fnid, *next_fnid;
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_summary_block *sum = curseg->sum_blk;
 	int fcnt = 0, i = 0;
 	nid_t nid = nm_i->next_scan_nid;
 	/* Enough entries */
 	if (nm_i->fcnt > NAT_ENTRY_PER_BLOCK)
 		return;
 	/* readahead nat pages to be scanned */
 	ra_nat_pages(sbi, nid);
 	while (1) {
 		struct page *page = get_current_nat_page(sbi, nid);
 		fcnt += scan_nat_page(nm_i, page, nid);
 		f2fs_put_page(page, 1);
 		nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK));
 		if (nid >= nm_i->max_nid)
 			nid = 0;
 		if (i++ == FREE_NID_PAGES)
 			break;
 	}
 	/* go to the next free nat pages to find free nids abundantly */
 	nm_i->next_scan_nid = nid;
 	/* find free nids from current sum_pages */
 	mutex_lock(&curseg->curseg_mutex);
 	for (i = 0; i < nats_in_cursum(sum); i++) {
 		block_t addr = le32_to_cpu(nat_in_journal(sum, i).block_addr);
 		nid = le32_to_cpu(nid_in_journal(sum, i));
 		if (addr == NULL_ADDR)
 			add_free_nid(nm_i, nid);
 		else
 			remove_free_nid(nm_i, nid);
 	}
 	mutex_unlock(&curseg->curseg_mutex);
 	/* remove the free nids from current allocated nids */
 	list_for_each_entry_safe(fnid, next_fnid, &nm_i->free_nid_list, list) {
 		struct nat_entry *ne;
 		read_lock(&nm_i->nat_tree_lock);
 		ne = __lookup_nat_cache(nm_i, fnid->nid);
 		if (ne && nat_get_blkaddr(ne) != NULL_ADDR)
 			remove_free_nid(nm_i, fnid->nid);
 		read_unlock(&nm_i->nat_tree_lock);
 	}
 }
 /*
  * If this function returns success, caller can obtain a new nid
  * from second parameter of this function.
  * The returned nid could be used ino as well as nid when inode is created.
  */
 bool alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i = NULL;
 	struct list_head *this;
 retry:
 	if (sbi->total_valid_node_count + 1 >= nm_i->max_nid)
 		return false;
 	spin_lock(&nm_i->free_nid_list_lock);
 	/* We should not use stale free nids created by build_free_nids */
 	if (nm_i->fcnt && !sbi->on_build_free_nids) {
 		BUG_ON(list_empty(&nm_i->free_nid_list));
 		list_for_each(this, &nm_i->free_nid_list) {
 			i = list_entry(this, struct free_nid, list);
 			if (i->state == NID_NEW)
 				break;
 		}
 		BUG_ON(i->state != NID_NEW);
 		*nid = i->nid;
 		i->state = NID_ALLOC;
 		nm_i->fcnt--;
 		spin_unlock(&nm_i->free_nid_list_lock);
 		return true;
 	}
 	spin_unlock(&nm_i->free_nid_list_lock);
 	/* Let's scan nat pages and its caches to get free nids */
 	mutex_lock(&nm_i->build_lock);
 	sbi->on_build_free_nids = 1;
 	build_free_nids(sbi);
 	sbi->on_build_free_nids = 0;
 	mutex_unlock(&nm_i->build_lock);
 	goto retry;
 }
 /*
  * alloc_nid() should be called prior to this function.
  */
 void alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i;
 	spin_lock(&nm_i->free_nid_list_lock);
 	i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
 	BUG_ON(!i || i->state != NID_ALLOC);
 	__del_from_free_nid_list(i);
 	spin_unlock(&nm_i->free_nid_list_lock);
 }
 /*
  * alloc_nid() should be called prior to this function.
  */
 void alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i;
 	spin_lock(&nm_i->free_nid_list_lock);
 	i = __lookup_free_nid_list(nid, &nm_i->free_nid_list);
 	BUG_ON(!i || i->state != NID_ALLOC);
 	i->state = NID_NEW;
 	nm_i->fcnt++;
 	spin_unlock(&nm_i->free_nid_list_lock);
 }
 void recover_node_page(struct f2fs_sb_info *sbi, struct page *page,
 		struct f2fs_summary *sum, struct node_info *ni,
 		block_t new_blkaddr)
 {
 	rewrite_node_page(sbi, page, sum, ni->blk_addr, new_blkaddr);
 	set_node_addr(sbi, ni, new_blkaddr);
 	clear_node_page_dirty(page);
 }
 int recover_inode_page(struct f2fs_sb_info *sbi, struct page *page)
 {
 	struct address_space *mapping = sbi->node_inode->i_mapping;
 	struct f2fs_node *src, *dst;
 	nid_t ino = ino_of_node(page);
 	struct node_info old_ni, new_ni;
 	struct page *ipage;
 	ipage = grab_cache_page(mapping, ino);
 	if (!ipage)
 		return -ENOMEM;
 	/* Should not use this inode  from free nid list */
 	remove_free_nid(NM_I(sbi), ino);
 	get_node_info(sbi, ino, &old_ni);
 	SetPageUptodate(ipage);
 	fill_node_footer(ipage, ino, ino, 0, true);
 	src = (struct f2fs_node *)page_address(page);
 	dst = (struct f2fs_node *)page_address(ipage);
 	memcpy(dst, src, (unsigned long)&src->i.i_ext - (unsigned long)&src->i);
 	dst->i.i_size = 0;
 	dst->i.i_blocks = cpu_to_le64(1);
 	dst->i.i_links = cpu_to_le32(1);
 	dst->i.i_xattr_nid = 0;
 	new_ni = old_ni;
 	new_ni.ino = ino;
 	set_node_addr(sbi, &new_ni, NEW_ADDR);
 	inc_valid_inode_count(sbi);
 	f2fs_put_page(ipage, 1);
 	return 0;
 }
 int restore_node_summary(struct f2fs_sb_info *sbi,
 			unsigned int segno, struct f2fs_summary_block *sum)
 {
 	struct f2fs_node *rn;
 	struct f2fs_summary *sum_entry;
 	struct page *page;
 	block_t addr;
 	int i, last_offset;
 	/* alloc temporal page for read node */
 	page = alloc_page(GFP_NOFS | __GFP_ZERO);
 	if (IS_ERR(page))
 		return PTR_ERR(page);
 	lock_page(page);
 	/* scan the node segment */
 	last_offset = sbi->blocks_per_seg;
 	addr = START_BLOCK(sbi, segno);
 	sum_entry = &sum->entries[0];
 	for (i = 0; i < last_offset; i++, sum_entry++) {
 		/*
 		 * In order to read next node page,
 		 * we must clear PageUptodate flag.
 		 */
 		ClearPageUptodate(page);
 		if (f2fs_readpage(sbi, page, addr, READ_SYNC))
 			goto out;
 		lock_page(page);
 		rn = (struct f2fs_node *)page_address(page);
 		sum_entry->nid = rn->footer.nid;
 		sum_entry->version = 0;
 		sum_entry->ofs_in_node = 0;
 		addr++;
 	}
 	unlock_page(page);
 out:
 	__free_pages(page, 0);
 	return 0;
 }
 static bool flush_nats_in_journal(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_summary_block *sum = curseg->sum_blk;
 	int i;
 	mutex_lock(&curseg->curseg_mutex);
 	if (nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES) {
 		mutex_unlock(&curseg->curseg_mutex);
 		return false;
 	}
 	for (i = 0; i < nats_in_cursum(sum); i++) {
 		struct nat_entry *ne;
 		struct f2fs_nat_entry raw_ne;
 		nid_t nid = le32_to_cpu(nid_in_journal(sum, i));
 		raw_ne = nat_in_journal(sum, i);
 retry:
 		write_lock(&nm_i->nat_tree_lock);
 		ne = __lookup_nat_cache(nm_i, nid);
 		if (ne) {
 			__set_nat_cache_dirty(nm_i, ne);
 			write_unlock(&nm_i->nat_tree_lock);
 			continue;
 		}
 		ne = grab_nat_entry(nm_i, nid);
 		if (!ne) {
 			write_unlock(&nm_i->nat_tree_lock);
 			goto retry;
 		}
 		nat_set_blkaddr(ne, le32_to_cpu(raw_ne.block_addr));
 		nat_set_ino(ne, le32_to_cpu(raw_ne.ino));
 		nat_set_version(ne, raw_ne.version);
 		__set_nat_cache_dirty(nm_i, ne);
 		write_unlock(&nm_i->nat_tree_lock);
 	}
 	update_nats_in_cursum(sum, -i);
 	mutex_unlock(&curseg->curseg_mutex);
 	return true;
 }
 /*
  * This function is called during the checkpointing process.
  */
 void flush_nat_entries(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
 	struct f2fs_summary_block *sum = curseg->sum_blk;
 	struct list_head *cur, *n;
 	struct page *page = NULL;
 	struct f2fs_nat_block *nat_blk = NULL;
 	nid_t start_nid = 0, end_nid = 0;
 	bool flushed;
 	flushed = flush_nats_in_journal(sbi);
 	if (!flushed)
 		mutex_lock(&curseg->curseg_mutex);
 	/* 1) flush dirty nat caches */
 	list_for_each_safe(cur, n, &nm_i->dirty_nat_entries) {
 		struct nat_entry *ne;
 		nid_t nid;
 		struct f2fs_nat_entry raw_ne;
 		int offset = -1;
 		block_t new_blkaddr;
 		ne = list_entry(cur, struct nat_entry, list);
 		nid = nat_get_nid(ne);
 		if (nat_get_blkaddr(ne) == NEW_ADDR)
 			continue;
 		if (flushed)
 			goto to_nat_page;
 		/* if there is room for nat enries in curseg->sumpage */
 		offset = lookup_journal_in_cursum(sum, NAT_JOURNAL, nid, 1);
 		if (offset >= 0) {
 			raw_ne = nat_in_journal(sum, offset);
 			goto flush_now;
 		}
 to_nat_page:
 		if (!page || (start_nid > nid || nid > end_nid)) {
 			if (page) {
 				f2fs_put_page(page, 1);
 				page = NULL;
 			}
 			start_nid = START_NID(nid);
 			end_nid = start_nid + NAT_ENTRY_PER_BLOCK - 1;
 			/*
 			 * get nat block with dirty flag, increased reference
 			 * count, mapped and lock
 			 */
 			page = get_next_nat_page(sbi, start_nid);
 			nat_blk = page_address(page);
 		}
 		BUG_ON(!nat_blk);
 		raw_ne = nat_blk->entries[nid - start_nid];
 flush_now:
 		new_blkaddr = nat_get_blkaddr(ne);
 		raw_ne.ino = cpu_to_le32(nat_get_ino(ne));
 		raw_ne.block_addr = cpu_to_le32(new_blkaddr);
 		raw_ne.version = nat_get_version(ne);
 		if (offset < 0) {
 			nat_blk->entries[nid - start_nid] = raw_ne;
 		} else {
 			nat_in_journal(sum, offset) = raw_ne;
 			nid_in_journal(sum, offset) = cpu_to_le32(nid);
 		}
 		if (nat_get_blkaddr(ne) == NULL_ADDR &&
 					!add_free_nid(NM_I(sbi), nid)) {
 			write_lock(&nm_i->nat_tree_lock);
 			__del_from_nat_cache(nm_i, ne);
 			write_unlock(&nm_i->nat_tree_lock);
 		} else {
 			write_lock(&nm_i->nat_tree_lock);
 			__clear_nat_cache_dirty(nm_i, ne);
 			ne->checkpointed = true;
 			write_unlock(&nm_i->nat_tree_lock);
 		}
 	}
 	if (!flushed)
 		mutex_unlock(&curseg->curseg_mutex);
 	f2fs_put_page(page, 1);
 	/* 2) shrink nat caches if necessary */
 	try_to_free_nats(sbi, nm_i->nat_cnt - NM_WOUT_THRESHOLD);
 }
 static int init_node_manager(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi);
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	unsigned char *version_bitmap;
 	unsigned int nat_segs, nat_blocks;
 	nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr);
 	/* segment_count_nat includes pair segment so divide to 2. */
 	nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1;
 	nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg);
 	nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nat_blocks;
 	nm_i->fcnt = 0;
 	nm_i->nat_cnt = 0;
 	INIT_LIST_HEAD(&nm_i->free_nid_list);
 	INIT_RADIX_TREE(&nm_i->nat_root, GFP_ATOMIC);
 	INIT_LIST_HEAD(&nm_i->nat_entries);
 	INIT_LIST_HEAD(&nm_i->dirty_nat_entries);
 	mutex_init(&nm_i->build_lock);
 	spin_lock_init(&nm_i->free_nid_list_lock);
 	rwlock_init(&nm_i->nat_tree_lock);
 	nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
 	nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
 	version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP);
 	if (!version_bitmap)
 		return -EFAULT;
 	nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size,
 					GFP_KERNEL);
 	if (!nm_i->nat_bitmap)
 		return -ENOMEM;
 	return 0;
 }
 int build_node_manager(struct f2fs_sb_info *sbi)
 {
 	int err;
 	sbi->nm_info = kzalloc(sizeof(struct f2fs_nm_info), GFP_KERNEL);
 	if (!sbi->nm_info)
 		return -ENOMEM;
 	err = init_node_manager(sbi);
 	if (err)
 		return err;
 	build_free_nids(sbi);
 	return 0;
 }
 void destroy_node_manager(struct f2fs_sb_info *sbi)
 {
 	struct f2fs_nm_info *nm_i = NM_I(sbi);
 	struct free_nid *i, *next_i;
 	struct nat_entry *natvec[NATVEC_SIZE];
 	nid_t nid = 0;
 	unsigned int found;
 	if (!nm_i)
 		return;
 	/* destroy free nid list */
 	spin_lock(&nm_i->free_nid_list_lock);
 	list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) {
 		BUG_ON(i->state == NID_ALLOC);
 		__del_from_free_nid_list(i);
 		nm_i->fcnt--;
 	}
 	BUG_ON(nm_i->fcnt);
 	spin_unlock(&nm_i->free_nid_list_lock);
 	/* destroy nat cache */
 	write_lock(&nm_i->nat_tree_lock);
 	while ((found = __gang_lookup_nat_cache(nm_i,
 					nid, NATVEC_SIZE, natvec))) {
 		unsigned idx;
 		for (idx = 0; idx < found; idx++) {
 			struct nat_entry *e = natvec[idx];
 			nid = nat_get_nid(e) + 1;
 			__del_from_nat_cache(nm_i, e);
 		}
 	}
 	BUG_ON(nm_i->nat_cnt);
 	write_unlock(&nm_i->nat_tree_lock);
 	kfree(nm_i->nat_bitmap);
 	sbi->nm_info = NULL;
 	kfree(nm_i);
 }
 int __init create_node_manager_caches(void)
 {
 	nat_entry_slab = f2fs_kmem_cache_create("nat_entry",
 			sizeof(struct nat_entry), NULL);
 	if (!nat_entry_slab)
 		return -ENOMEM;
 	free_nid_slab = f2fs_kmem_cache_create("free_nid",
 			sizeof(struct free_nid), NULL);
 	if (!free_nid_slab) {
 		kmem_cache_destroy(nat_entry_slab);
 		return -ENOMEM;
 	}
 	return 0;
 }
 void destroy_node_manager_caches(void)
 {
 	kmem_cache_destroy(free_nid_slab);
 	kmem_cache_destroy(nat_entry_slab);
 }

fs/f2fs/segment.c

Diff comments View file @ ac5d156

1	/*	1	/*
2	* fs/f2fs/segment.c	2	* fs/f2fs/segment.c
3	*	3	*
4	* Copyright (c) 2012 Samsung Electronics Co., Ltd.	4	* Copyright (c) 2012 Samsung Electronics Co., Ltd.
5	* http://www.samsung.com/	5	* http://www.samsung.com/
6	*	6	*
7	* This program is free software; you can redistribute it and/or modify	7	* This program is free software; you can redistribute it and/or modify
8	* it under the terms of the GNU General Public License version 2 as	8	* it under the terms of the GNU General Public License version 2 as
9	* published by the Free Software Foundation.	9	* published by the Free Software Foundation.
10	*/	10	*/
11	#include <linux/fs.h>	11	#include <linux/fs.h>
12	#include <linux/f2fs_fs.h>	12	#include <linux/f2fs_fs.h>
13	#include <linux/bio.h>	13	#include <linux/bio.h>
14	#include <linux/blkdev.h>	14	#include <linux/blkdev.h>
15	#include <linux/prefetch.h>	15	#include <linux/prefetch.h>
16	#include <linux/vmalloc.h>	16	#include <linux/vmalloc.h>
17		17
18	#include "f2fs.h"	18	#include "f2fs.h"
19	#include "segment.h"	19	#include "segment.h"
20	#include "node.h"	20	#include "node.h"
21	#include <trace/events/f2fs.h>	21	#include <trace/events/f2fs.h>
22		22
23	/*	23	/*
24	* This function balances dirty node and dentry pages.	24	* This function balances dirty node and dentry pages.
25	* In addition, it controls garbage collection.	25	* In addition, it controls garbage collection.
26	*/	26	*/
27	void f2fs_balance_fs(struct f2fs_sb_info *sbi)	27	void f2fs_balance_fs(struct f2fs_sb_info *sbi)
28	{	28	{
29	/*	29	/*
30	* We should do GC or end up with checkpoint, if there are so many dirty	30	* We should do GC or end up with checkpoint, if there are so many dirty
31	* dir/node pages without enough free segments.	31	* dir/node pages without enough free segments.
32	*/	32	*/
33	if (has_not_enough_free_secs(sbi, 0)) {	33	if (has_not_enough_free_secs(sbi, 0)) {
34	mutex_lock(&sbi->gc_mutex);	34	mutex_lock(&sbi->gc_mutex);
35	f2fs_gc(sbi);	35	f2fs_gc(sbi);
36	}	36	}
37	}	37	}
38		38
39	static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,	39	static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
40	enum dirty_type dirty_type)	40	enum dirty_type dirty_type)
41	{	41	{
42	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	42	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
43		43
44	/* need not be added */	44	/* need not be added */
45	if (IS_CURSEG(sbi, segno))	45	if (IS_CURSEG(sbi, segno))
46	return;	46	return;
47		47
48	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))	48	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
49	dirty_i->nr_dirty[dirty_type]++;	49	dirty_i->nr_dirty[dirty_type]++;
50		50
51	if (dirty_type == DIRTY) {	51	if (dirty_type == DIRTY) {
52	struct seg_entry *sentry = get_seg_entry(sbi, segno);	52	struct seg_entry *sentry = get_seg_entry(sbi, segno);
53	enum dirty_type t = DIRTY_HOT_DATA;	53	enum dirty_type t = DIRTY_HOT_DATA;
54		54
55	dirty_type = sentry->type;	55	dirty_type = sentry->type;
56		56
57	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))	57	if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
58	dirty_i->nr_dirty[dirty_type]++;	58	dirty_i->nr_dirty[dirty_type]++;
59		59
60	/* Only one bitmap should be set */	60	/* Only one bitmap should be set */
61	for (; t <= DIRTY_COLD_NODE; t++) {	61	for (; t <= DIRTY_COLD_NODE; t++) {
62	if (t == dirty_type)	62	if (t == dirty_type)
63	continue;	63	continue;
64	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))	64	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
65	dirty_i->nr_dirty[t]--;	65	dirty_i->nr_dirty[t]--;
66	}	66	}
67	}	67	}
68	}	68	}
69		69
70	static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,	70	static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
71	enum dirty_type dirty_type)	71	enum dirty_type dirty_type)
72	{	72	{
73	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	73	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
74		74
75	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))	75	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
76	dirty_i->nr_dirty[dirty_type]--;	76	dirty_i->nr_dirty[dirty_type]--;
77		77
78	if (dirty_type == DIRTY) {	78	if (dirty_type == DIRTY) {
79	enum dirty_type t = DIRTY_HOT_DATA;	79	enum dirty_type t = DIRTY_HOT_DATA;
80		80
81	/* clear all the bitmaps */	81	/* clear all the bitmaps */
82	for (; t <= DIRTY_COLD_NODE; t++)	82	for (; t <= DIRTY_COLD_NODE; t++)
83	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))	83	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
84	dirty_i->nr_dirty[t]--;	84	dirty_i->nr_dirty[t]--;
85		85
86	if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)	86	if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
87	clear_bit(GET_SECNO(sbi, segno),	87	clear_bit(GET_SECNO(sbi, segno),
88	dirty_i->victim_secmap);	88	dirty_i->victim_secmap);
89	}	89	}
90	}	90	}
91		91
92	/*	92	/*
93	* Should not occur error such as -ENOMEM.	93	* Should not occur error such as -ENOMEM.
94	* Adding dirty entry into seglist is not critical operation.	94	* Adding dirty entry into seglist is not critical operation.
95	* If a given segment is one of current working segments, it won't be added.	95	* If a given segment is one of current working segments, it won't be added.
96	*/	96	*/
97	void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)	97	void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
98	{	98	{
99	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	99	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
100	unsigned short valid_blocks;	100	unsigned short valid_blocks;
101		101
102	if (segno == NULL_SEGNO \|\| IS_CURSEG(sbi, segno))	102	if (segno == NULL_SEGNO \|\| IS_CURSEG(sbi, segno))
103	return;	103	return;
104		104
105	mutex_lock(&dirty_i->seglist_lock);	105	mutex_lock(&dirty_i->seglist_lock);
106		106
107	valid_blocks = get_valid_blocks(sbi, segno, 0);	107	valid_blocks = get_valid_blocks(sbi, segno, 0);
108		108
109	if (valid_blocks == 0) {	109	if (valid_blocks == 0) {
110	__locate_dirty_segment(sbi, segno, PRE);	110	__locate_dirty_segment(sbi, segno, PRE);
111	__remove_dirty_segment(sbi, segno, DIRTY);	111	__remove_dirty_segment(sbi, segno, DIRTY);
112	} else if (valid_blocks < sbi->blocks_per_seg) {	112	} else if (valid_blocks < sbi->blocks_per_seg) {
113	__locate_dirty_segment(sbi, segno, DIRTY);	113	__locate_dirty_segment(sbi, segno, DIRTY);
114	} else {	114	} else {
115	/* Recovery routine with SSR needs this */	115	/* Recovery routine with SSR needs this */
116	__remove_dirty_segment(sbi, segno, DIRTY);	116	__remove_dirty_segment(sbi, segno, DIRTY);
117	}	117	}
118		118
119	mutex_unlock(&dirty_i->seglist_lock);	119	mutex_unlock(&dirty_i->seglist_lock);
120	return;	120	return;
121	}	121	}
122		122
123	/*	123	/*
124	* Should call clear_prefree_segments after checkpoint is done.	124	* Should call clear_prefree_segments after checkpoint is done.
125	*/	125	*/
126	static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)	126	static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
127	{	127	{
128	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	128	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
129	unsigned int segno, offset = 0;	129	unsigned int segno, offset = 0;
130	unsigned int total_segs = TOTAL_SEGS(sbi);	130	unsigned int total_segs = TOTAL_SEGS(sbi);
131		131
132	mutex_lock(&dirty_i->seglist_lock);	132	mutex_lock(&dirty_i->seglist_lock);
133	while (1) {	133	while (1) {
134	segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs,	134	segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs,
135	offset);	135	offset);
136	if (segno >= total_segs)	136	if (segno >= total_segs)
137	break;	137	break;
138	__set_test_and_free(sbi, segno);	138	__set_test_and_free(sbi, segno);
139	offset = segno + 1;	139	offset = segno + 1;
140	}	140	}
141	mutex_unlock(&dirty_i->seglist_lock);	141	mutex_unlock(&dirty_i->seglist_lock);
142	}	142	}
143		143
144	void clear_prefree_segments(struct f2fs_sb_info *sbi)	144	void clear_prefree_segments(struct f2fs_sb_info *sbi)
145	{	145	{
146	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	146	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
147	unsigned int segno, offset = 0;	147	unsigned int segno, offset = 0;
148	unsigned int total_segs = TOTAL_SEGS(sbi);	148	unsigned int total_segs = TOTAL_SEGS(sbi);
149		149
150	mutex_lock(&dirty_i->seglist_lock);	150	mutex_lock(&dirty_i->seglist_lock);
151	while (1) {	151	while (1) {
152	segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs,	152	segno = find_next_bit(dirty_i->dirty_segmap[PRE], total_segs,
153	offset);	153	offset);
154	if (segno >= total_segs)	154	if (segno >= total_segs)
155	break;	155	break;
156		156
157	offset = segno + 1;	157	offset = segno + 1;
158	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[PRE]))	158	if (test_and_clear_bit(segno, dirty_i->dirty_segmap[PRE]))
159	dirty_i->nr_dirty[PRE]--;	159	dirty_i->nr_dirty[PRE]--;
160		160
161	/* Let's use trim */	161	/* Let's use trim */
162	if (test_opt(sbi, DISCARD))	162	if (test_opt(sbi, DISCARD))
163	blkdev_issue_discard(sbi->sb->s_bdev,	163	blkdev_issue_discard(sbi->sb->s_bdev,
164	START_BLOCK(sbi, segno) <<	164	START_BLOCK(sbi, segno) <<
165	sbi->log_sectors_per_block,	165	sbi->log_sectors_per_block,
166	1 << (sbi->log_sectors_per_block +	166	1 << (sbi->log_sectors_per_block +
167	sbi->log_blocks_per_seg),	167	sbi->log_blocks_per_seg),
168	GFP_NOFS, 0);	168	GFP_NOFS, 0);
169	}	169	}
170	mutex_unlock(&dirty_i->seglist_lock);	170	mutex_unlock(&dirty_i->seglist_lock);
171	}	171	}
172		172
173	static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)	173	static void __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
174	{	174	{
175	struct sit_info *sit_i = SIT_I(sbi);	175	struct sit_info *sit_i = SIT_I(sbi);
176	if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap))	176	if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap))
177	sit_i->dirty_sentries++;	177	sit_i->dirty_sentries++;
178	}	178	}
179		179
180	static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,	180	static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
181	unsigned int segno, int modified)	181	unsigned int segno, int modified)
182	{	182	{
183	struct seg_entry *se = get_seg_entry(sbi, segno);	183	struct seg_entry *se = get_seg_entry(sbi, segno);
184	se->type = type;	184	se->type = type;
185	if (modified)	185	if (modified)
186	__mark_sit_entry_dirty(sbi, segno);	186	__mark_sit_entry_dirty(sbi, segno);
187	}	187	}
188		188
189	static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)	189	static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
190	{	190	{
191	struct seg_entry *se;	191	struct seg_entry *se;
192	unsigned int segno, offset;	192	unsigned int segno, offset;
193	long int new_vblocks;	193	long int new_vblocks;
194		194
195	segno = GET_SEGNO(sbi, blkaddr);	195	segno = GET_SEGNO(sbi, blkaddr);
196		196
197	se = get_seg_entry(sbi, segno);	197	se = get_seg_entry(sbi, segno);
198	new_vblocks = se->valid_blocks + del;	198	new_vblocks = se->valid_blocks + del;
199	offset = GET_SEGOFF_FROM_SEG0(sbi, blkaddr) & (sbi->blocks_per_seg - 1);	199	offset = GET_SEGOFF_FROM_SEG0(sbi, blkaddr) & (sbi->blocks_per_seg - 1);
200		200
201	BUG_ON((new_vblocks >> (sizeof(unsigned short) << 3) \|\|	201	BUG_ON((new_vblocks >> (sizeof(unsigned short) << 3) \|\|
202	(new_vblocks > sbi->blocks_per_seg)));	202	(new_vblocks > sbi->blocks_per_seg)));
203		203
204	se->valid_blocks = new_vblocks;	204	se->valid_blocks = new_vblocks;
205	se->mtime = get_mtime(sbi);	205	se->mtime = get_mtime(sbi);
206	SIT_I(sbi)->max_mtime = se->mtime;	206	SIT_I(sbi)->max_mtime = se->mtime;
207		207
208	/* Update valid block bitmap */	208	/* Update valid block bitmap */
209	if (del > 0) {	209	if (del > 0) {
210	if (f2fs_set_bit(offset, se->cur_valid_map))	210	if (f2fs_set_bit(offset, se->cur_valid_map))
211	BUG();	211	BUG();
212	} else {	212	} else {
213	if (!f2fs_clear_bit(offset, se->cur_valid_map))	213	if (!f2fs_clear_bit(offset, se->cur_valid_map))
214	BUG();	214	BUG();
215	}	215	}
216	if (!f2fs_test_bit(offset, se->ckpt_valid_map))	216	if (!f2fs_test_bit(offset, se->ckpt_valid_map))
217	se->ckpt_valid_blocks += del;	217	se->ckpt_valid_blocks += del;
218		218
219	__mark_sit_entry_dirty(sbi, segno);	219	__mark_sit_entry_dirty(sbi, segno);
220		220
221	/* update total number of valid blocks to be written in ckpt area */	221	/* update total number of valid blocks to be written in ckpt area */
222	SIT_I(sbi)->written_valid_blocks += del;	222	SIT_I(sbi)->written_valid_blocks += del;
223		223
224	if (sbi->segs_per_sec > 1)	224	if (sbi->segs_per_sec > 1)
225	get_sec_entry(sbi, segno)->valid_blocks += del;	225	get_sec_entry(sbi, segno)->valid_blocks += del;
226	}	226	}
227		227
228	static void refresh_sit_entry(struct f2fs_sb_info *sbi,	228	static void refresh_sit_entry(struct f2fs_sb_info *sbi,
229	block_t old_blkaddr, block_t new_blkaddr)	229	block_t old_blkaddr, block_t new_blkaddr)
230	{	230	{
231	update_sit_entry(sbi, new_blkaddr, 1);	231	update_sit_entry(sbi, new_blkaddr, 1);
232	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)	232	if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
233	update_sit_entry(sbi, old_blkaddr, -1);	233	update_sit_entry(sbi, old_blkaddr, -1);
234	}	234	}
235		235
236	void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)	236	void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
237	{	237	{
238	unsigned int segno = GET_SEGNO(sbi, addr);	238	unsigned int segno = GET_SEGNO(sbi, addr);
239	struct sit_info *sit_i = SIT_I(sbi);	239	struct sit_info *sit_i = SIT_I(sbi);
240		240
241	BUG_ON(addr == NULL_ADDR);	241	BUG_ON(addr == NULL_ADDR);
242	if (addr == NEW_ADDR)	242	if (addr == NEW_ADDR)
243	return;	243	return;
244		244
245	/* add it into sit main buffer */	245	/* add it into sit main buffer */
246	mutex_lock(&sit_i->sentry_lock);	246	mutex_lock(&sit_i->sentry_lock);
247		247
248	update_sit_entry(sbi, addr, -1);	248	update_sit_entry(sbi, addr, -1);
249		249
250	/* add it into dirty seglist */	250	/* add it into dirty seglist */
251	locate_dirty_segment(sbi, segno);	251	locate_dirty_segment(sbi, segno);
252		252
253	mutex_unlock(&sit_i->sentry_lock);	253	mutex_unlock(&sit_i->sentry_lock);
254	}	254	}
255		255
256	/*	256	/*
257	* This function should be resided under the curseg_mutex lock	257	* This function should be resided under the curseg_mutex lock
258	*/	258	*/
259	static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,	259	static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
260	struct f2fs_summary *sum, unsigned short offset)	260	struct f2fs_summary *sum, unsigned short offset)
261	{	261	{
262	struct curseg_info *curseg = CURSEG_I(sbi, type);	262	struct curseg_info *curseg = CURSEG_I(sbi, type);
263	void *addr = curseg->sum_blk;	263	void *addr = curseg->sum_blk;
264	addr += offset * sizeof(struct f2fs_summary);	264	addr += offset * sizeof(struct f2fs_summary);
265	memcpy(addr, sum, sizeof(struct f2fs_summary));	265	memcpy(addr, sum, sizeof(struct f2fs_summary));
266	return;	266	return;
267	}	267	}
268		268
269	/*	269	/*
270	* Calculate the number of current summary pages for writing	270	* Calculate the number of current summary pages for writing
271	*/	271	*/
272	int npages_for_summary_flush(struct f2fs_sb_info *sbi)	272	int npages_for_summary_flush(struct f2fs_sb_info *sbi)
273	{	273	{
274	int total_size_bytes = 0;	274	int total_size_bytes = 0;
275	int valid_sum_count = 0;	275	int valid_sum_count = 0;
276	int i, sum_space;	276	int i, sum_space;
277		277
278	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {	278	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
279	if (sbi->ckpt->alloc_type[i] == SSR)	279	if (sbi->ckpt->alloc_type[i] == SSR)
280	valid_sum_count += sbi->blocks_per_seg;	280	valid_sum_count += sbi->blocks_per_seg;
281	else	281	else
282	valid_sum_count += curseg_blkoff(sbi, i);	282	valid_sum_count += curseg_blkoff(sbi, i);
283	}	283	}
284		284
285	total_size_bytes = valid_sum_count * (SUMMARY_SIZE + 1)	285	total_size_bytes = valid_sum_count * (SUMMARY_SIZE + 1)
286	+ sizeof(struct nat_journal) + 2	286	+ sizeof(struct nat_journal) + 2
287	+ sizeof(struct sit_journal) + 2;	287	+ sizeof(struct sit_journal) + 2;
288	sum_space = PAGE_CACHE_SIZE - SUM_FOOTER_SIZE;	288	sum_space = PAGE_CACHE_SIZE - SUM_FOOTER_SIZE;
289	if (total_size_bytes < sum_space)	289	if (total_size_bytes < sum_space)
290	return 1;	290	return 1;
291	else if (total_size_bytes < 2 * sum_space)	291	else if (total_size_bytes < 2 * sum_space)
292	return 2;	292	return 2;
293	return 3;	293	return 3;
294	}	294	}
295		295
296	/*	296	/*
297	* Caller should put this summary page	297	* Caller should put this summary page
298	*/	298	*/
299	struct page get_sum_page(struct f2fs_sb_info sbi, unsigned int segno)	299	struct page get_sum_page(struct f2fs_sb_info sbi, unsigned int segno)
300	{	300	{
301	return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));	301	return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
302	}	302	}
303		303
304	static void write_sum_page(struct f2fs_sb_info *sbi,	304	static void write_sum_page(struct f2fs_sb_info *sbi,
305	struct f2fs_summary_block *sum_blk, block_t blk_addr)	305	struct f2fs_summary_block *sum_blk, block_t blk_addr)
306	{	306	{
307	struct page *page = grab_meta_page(sbi, blk_addr);	307	struct page *page = grab_meta_page(sbi, blk_addr);
308	void *kaddr = page_address(page);	308	void *kaddr = page_address(page);
309	memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE);	309	memcpy(kaddr, sum_blk, PAGE_CACHE_SIZE);
310	set_page_dirty(page);	310	set_page_dirty(page);
311	f2fs_put_page(page, 1);	311	f2fs_put_page(page, 1);
312	}	312	}
313		313
314	static unsigned int check_prefree_segments(struct f2fs_sb_info *sbi, int type)	314	static unsigned int check_prefree_segments(struct f2fs_sb_info *sbi, int type)
315	{	315	{
316	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	316	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
317	unsigned long *prefree_segmap = dirty_i->dirty_segmap[PRE];	317	unsigned long *prefree_segmap = dirty_i->dirty_segmap[PRE];
318	unsigned int segno;	318	unsigned int segno;
319	unsigned int ofs = 0;	319	unsigned int ofs = 0;
320		320
321	/*	321	/*
322	* If there is not enough reserved sections,	322	* If there is not enough reserved sections,
323	* we should not reuse prefree segments.	323	* we should not reuse prefree segments.
324	*/	324	*/
325	if (has_not_enough_free_secs(sbi, 0))	325	if (has_not_enough_free_secs(sbi, 0))
326	return NULL_SEGNO;	326	return NULL_SEGNO;
327		327
328	/*	328	/*
329	* NODE page should not reuse prefree segment,	329	* NODE page should not reuse prefree segment,
330	* since those information is used for SPOR.	330	* since those information is used for SPOR.
331	*/	331	*/
332	if (IS_NODESEG(type))	332	if (IS_NODESEG(type))
333	return NULL_SEGNO;	333	return NULL_SEGNO;
334	next:	334	next:
335	segno = find_next_bit(prefree_segmap, TOTAL_SEGS(sbi), ofs);	335	segno = find_next_bit(prefree_segmap, TOTAL_SEGS(sbi), ofs);
336	ofs += sbi->segs_per_sec;	336	ofs += sbi->segs_per_sec;
337		337
338	if (segno < TOTAL_SEGS(sbi)) {	338	if (segno < TOTAL_SEGS(sbi)) {
339	int i;	339	int i;
340		340
341	/* skip intermediate segments in a section */	341	/* skip intermediate segments in a section */
342	if (segno % sbi->segs_per_sec)	342	if (segno % sbi->segs_per_sec)
343	goto next;	343	goto next;
344		344
345	/* skip if the section is currently used */	345	/* skip if the section is currently used */
346	if (sec_usage_check(sbi, GET_SECNO(sbi, segno)))	346	if (sec_usage_check(sbi, GET_SECNO(sbi, segno)))
347	goto next;	347	goto next;
348		348
349	/* skip if whole section is not prefree */	349	/* skip if whole section is not prefree */
350	for (i = 1; i < sbi->segs_per_sec; i++)	350	for (i = 1; i < sbi->segs_per_sec; i++)
351	if (!test_bit(segno + i, prefree_segmap))	351	if (!test_bit(segno + i, prefree_segmap))
352	goto next;	352	goto next;
353		353
354	/* skip if whole section was not free at the last checkpoint */	354	/* skip if whole section was not free at the last checkpoint */
355	for (i = 0; i < sbi->segs_per_sec; i++)	355	for (i = 0; i < sbi->segs_per_sec; i++)
356	if (get_seg_entry(sbi, segno + i)->ckpt_valid_blocks)	356	if (get_seg_entry(sbi, segno + i)->ckpt_valid_blocks)
357	goto next;	357	goto next;
358		358
359	return segno;	359	return segno;
360	}	360	}
361	return NULL_SEGNO;	361	return NULL_SEGNO;
362	}	362	}
363		363
364	static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)	364	static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
365	{	365	{
366	struct curseg_info *curseg = CURSEG_I(sbi, type);	366	struct curseg_info *curseg = CURSEG_I(sbi, type);
367	unsigned int segno = curseg->segno;	367	unsigned int segno = curseg->segno;
368	struct free_segmap_info *free_i = FREE_I(sbi);	368	struct free_segmap_info *free_i = FREE_I(sbi);
369		369
370	if (segno + 1 < TOTAL_SEGS(sbi) && (segno + 1) % sbi->segs_per_sec)	370	if (segno + 1 < TOTAL_SEGS(sbi) && (segno + 1) % sbi->segs_per_sec)
371	return !test_bit(segno + 1, free_i->free_segmap);	371	return !test_bit(segno + 1, free_i->free_segmap);
372	return 0;	372	return 0;
373	}	373	}
374		374
375	/*	375	/*
376	* Find a new segment from the free segments bitmap to right order	376	* Find a new segment from the free segments bitmap to right order
377	* This function should be returned with success, otherwise BUG	377	* This function should be returned with success, otherwise BUG
378	*/	378	*/
379	static void get_new_segment(struct f2fs_sb_info *sbi,	379	static void get_new_segment(struct f2fs_sb_info *sbi,
380	unsigned int *newseg, bool new_sec, int dir)	380	unsigned int *newseg, bool new_sec, int dir)
381	{	381	{
382	struct free_segmap_info *free_i = FREE_I(sbi);	382	struct free_segmap_info *free_i = FREE_I(sbi);
383	unsigned int segno, secno, zoneno;	383	unsigned int segno, secno, zoneno;
384	unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone;	384	unsigned int total_zones = TOTAL_SECS(sbi) / sbi->secs_per_zone;
385	unsigned int hint = *newseg / sbi->segs_per_sec;	385	unsigned int hint = *newseg / sbi->segs_per_sec;
386	unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);	386	unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
387	unsigned int left_start = hint;	387	unsigned int left_start = hint;
388	bool init = true;	388	bool init = true;
389	int go_left = 0;	389	int go_left = 0;
390	int i;	390	int i;
391		391
392	write_lock(&free_i->segmap_lock);	392	write_lock(&free_i->segmap_lock);
393		393
394	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {	394	if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
395	segno = find_next_zero_bit(free_i->free_segmap,	395	segno = find_next_zero_bit(free_i->free_segmap,
396	TOTAL_SEGS(sbi), *newseg + 1);	396	TOTAL_SEGS(sbi), *newseg + 1);
397	if (segno - *newseg < sbi->segs_per_sec -	397	if (segno - *newseg < sbi->segs_per_sec -
398	(*newseg % sbi->segs_per_sec))	398	(*newseg % sbi->segs_per_sec))
399	goto got_it;	399	goto got_it;
400	}	400	}
401	find_other_zone:	401	find_other_zone:
402	secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint);	402	secno = find_next_zero_bit(free_i->free_secmap, TOTAL_SECS(sbi), hint);
403	if (secno >= TOTAL_SECS(sbi)) {	403	if (secno >= TOTAL_SECS(sbi)) {
404	if (dir == ALLOC_RIGHT) {	404	if (dir == ALLOC_RIGHT) {
405	secno = find_next_zero_bit(free_i->free_secmap,	405	secno = find_next_zero_bit(free_i->free_secmap,
406	TOTAL_SECS(sbi), 0);	406	TOTAL_SECS(sbi), 0);
407	BUG_ON(secno >= TOTAL_SECS(sbi));	407	BUG_ON(secno >= TOTAL_SECS(sbi));
408	} else {	408	} else {
409	go_left = 1;	409	go_left = 1;
410	left_start = hint - 1;	410	left_start = hint - 1;
411	}	411	}
412	}	412	}
413	if (go_left == 0)	413	if (go_left == 0)
414	goto skip_left;	414	goto skip_left;
415		415
416	while (test_bit(left_start, free_i->free_secmap)) {	416	while (test_bit(left_start, free_i->free_secmap)) {
417	if (left_start > 0) {	417	if (left_start > 0) {
418	left_start--;	418	left_start--;
419	continue;	419	continue;
420	}	420	}
421	left_start = find_next_zero_bit(free_i->free_secmap,	421	left_start = find_next_zero_bit(free_i->free_secmap,
422	TOTAL_SECS(sbi), 0);	422	TOTAL_SECS(sbi), 0);
423	BUG_ON(left_start >= TOTAL_SECS(sbi));	423	BUG_ON(left_start >= TOTAL_SECS(sbi));
424	break;	424	break;
425	}	425	}
426	secno = left_start;	426	secno = left_start;
427	skip_left:	427	skip_left:
428	hint = secno;	428	hint = secno;
429	segno = secno * sbi->segs_per_sec;	429	segno = secno * sbi->segs_per_sec;
430	zoneno = secno / sbi->secs_per_zone;	430	zoneno = secno / sbi->secs_per_zone;
431		431
432	/* give up on finding another zone */	432	/* give up on finding another zone */
433	if (!init)	433	if (!init)
434	goto got_it;	434	goto got_it;
435	if (sbi->secs_per_zone == 1)	435	if (sbi->secs_per_zone == 1)
436	goto got_it;	436	goto got_it;
437	if (zoneno == old_zoneno)	437	if (zoneno == old_zoneno)
438	goto got_it;	438	goto got_it;
439	if (dir == ALLOC_LEFT) {	439	if (dir == ALLOC_LEFT) {
440	if (!go_left && zoneno + 1 >= total_zones)	440	if (!go_left && zoneno + 1 >= total_zones)
441	goto got_it;	441	goto got_it;
442	if (go_left && zoneno == 0)	442	if (go_left && zoneno == 0)
443	goto got_it;	443	goto got_it;
444	}	444	}
445	for (i = 0; i < NR_CURSEG_TYPE; i++)	445	for (i = 0; i < NR_CURSEG_TYPE; i++)
446	if (CURSEG_I(sbi, i)->zone == zoneno)	446	if (CURSEG_I(sbi, i)->zone == zoneno)
447	break;	447	break;
448		448
449	if (i < NR_CURSEG_TYPE) {	449	if (i < NR_CURSEG_TYPE) {
450	/* zone is in user, try another */	450	/* zone is in user, try another */
451	if (go_left)	451	if (go_left)
452	hint = zoneno * sbi->secs_per_zone - 1;	452	hint = zoneno * sbi->secs_per_zone - 1;
453	else if (zoneno + 1 >= total_zones)	453	else if (zoneno + 1 >= total_zones)
454	hint = 0;	454	hint = 0;
455	else	455	else
456	hint = (zoneno + 1) * sbi->secs_per_zone;	456	hint = (zoneno + 1) * sbi->secs_per_zone;
457	init = false;	457	init = false;
458	goto find_other_zone;	458	goto find_other_zone;
459	}	459	}
460	got_it:	460	got_it:
461	/* set it as dirty segment in free segmap */	461	/* set it as dirty segment in free segmap */
462	BUG_ON(test_bit(segno, free_i->free_segmap));	462	BUG_ON(test_bit(segno, free_i->free_segmap));
463	__set_inuse(sbi, segno);	463	__set_inuse(sbi, segno);
464	*newseg = segno;	464	*newseg = segno;
465	write_unlock(&free_i->segmap_lock);	465	write_unlock(&free_i->segmap_lock);
466	}	466	}
467		467
468	static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)	468	static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
469	{	469	{
470	struct curseg_info *curseg = CURSEG_I(sbi, type);	470	struct curseg_info *curseg = CURSEG_I(sbi, type);
471	struct summary_footer *sum_footer;	471	struct summary_footer *sum_footer;
472		472
473	curseg->segno = curseg->next_segno;	473	curseg->segno = curseg->next_segno;
474	curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);	474	curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
475	curseg->next_blkoff = 0;	475	curseg->next_blkoff = 0;
476	curseg->next_segno = NULL_SEGNO;	476	curseg->next_segno = NULL_SEGNO;
477		477
478	sum_footer = &(curseg->sum_blk->footer);	478	sum_footer = &(curseg->sum_blk->footer);
479	memset(sum_footer, 0, sizeof(struct summary_footer));	479	memset(sum_footer, 0, sizeof(struct summary_footer));
480	if (IS_DATASEG(type))	480	if (IS_DATASEG(type))
481	SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);	481	SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
482	if (IS_NODESEG(type))	482	if (IS_NODESEG(type))
483	SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);	483	SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
484	__set_sit_entry_type(sbi, type, curseg->segno, modified);	484	__set_sit_entry_type(sbi, type, curseg->segno, modified);
485	}	485	}
486		486
487	/*	487	/*
488	* Allocate a current working segment.	488	* Allocate a current working segment.
489	* This function always allocates a free segment in LFS manner.	489	* This function always allocates a free segment in LFS manner.
490	*/	490	*/
491	static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)	491	static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
492	{	492	{
493	struct curseg_info *curseg = CURSEG_I(sbi, type);	493	struct curseg_info *curseg = CURSEG_I(sbi, type);
494	unsigned int segno = curseg->segno;	494	unsigned int segno = curseg->segno;
495	int dir = ALLOC_LEFT;	495	int dir = ALLOC_LEFT;
496		496
497	write_sum_page(sbi, curseg->sum_blk,	497	write_sum_page(sbi, curseg->sum_blk,
498	GET_SUM_BLOCK(sbi, curseg->segno));	498	GET_SUM_BLOCK(sbi, curseg->segno));
499	if (type == CURSEG_WARM_DATA \|\| type == CURSEG_COLD_DATA)	499	if (type == CURSEG_WARM_DATA \|\| type == CURSEG_COLD_DATA)
500	dir = ALLOC_RIGHT;	500	dir = ALLOC_RIGHT;
501		501
502	if (test_opt(sbi, NOHEAP))	502	if (test_opt(sbi, NOHEAP))
503	dir = ALLOC_RIGHT;	503	dir = ALLOC_RIGHT;
504		504
505	get_new_segment(sbi, &segno, new_sec, dir);	505	get_new_segment(sbi, &segno, new_sec, dir);
506	curseg->next_segno = segno;	506	curseg->next_segno = segno;
507	reset_curseg(sbi, type, 1);	507	reset_curseg(sbi, type, 1);
508	curseg->alloc_type = LFS;	508	curseg->alloc_type = LFS;
509	}	509	}
510		510
511	static void __next_free_blkoff(struct f2fs_sb_info *sbi,	511	static void __next_free_blkoff(struct f2fs_sb_info *sbi,
512	struct curseg_info *seg, block_t start)	512	struct curseg_info *seg, block_t start)
513	{	513	{
514	struct seg_entry *se = get_seg_entry(sbi, seg->segno);	514	struct seg_entry *se = get_seg_entry(sbi, seg->segno);
515	block_t ofs;	515	block_t ofs;
516	for (ofs = start; ofs < sbi->blocks_per_seg; ofs++) {	516	for (ofs = start; ofs < sbi->blocks_per_seg; ofs++) {
517	if (!f2fs_test_bit(ofs, se->ckpt_valid_map)	517	if (!f2fs_test_bit(ofs, se->ckpt_valid_map)
518	&& !f2fs_test_bit(ofs, se->cur_valid_map))	518	&& !f2fs_test_bit(ofs, se->cur_valid_map))
519	break;	519	break;
520	}	520	}
521	seg->next_blkoff = ofs;	521	seg->next_blkoff = ofs;
522	}	522	}
523		523
524	/*	524	/*
525	* If a segment is written by LFS manner, next block offset is just obtained	525	* If a segment is written by LFS manner, next block offset is just obtained
526	* by increasing the current block offset. However, if a segment is written by	526	* by increasing the current block offset. However, if a segment is written by
527	* SSR manner, next block offset obtained by calling __next_free_blkoff	527	* SSR manner, next block offset obtained by calling __next_free_blkoff
528	*/	528	*/
529	static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,	529	static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
530	struct curseg_info *seg)	530	struct curseg_info *seg)
531	{	531	{
532	if (seg->alloc_type == SSR)	532	if (seg->alloc_type == SSR)
533	__next_free_blkoff(sbi, seg, seg->next_blkoff + 1);	533	__next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
534	else	534	else
535	seg->next_blkoff++;	535	seg->next_blkoff++;
536	}	536	}
537		537
538	/*	538	/*
539	* This function always allocates a used segment (from dirty seglist) by SSR	539	* This function always allocates a used segment (from dirty seglist) by SSR
540	* manner, so it should recover the existing segment information of valid blocks	540	* manner, so it should recover the existing segment information of valid blocks
541	*/	541	*/
542	static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)	542	static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
543	{	543	{
544	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	544	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
545	struct curseg_info *curseg = CURSEG_I(sbi, type);	545	struct curseg_info *curseg = CURSEG_I(sbi, type);
546	unsigned int new_segno = curseg->next_segno;	546	unsigned int new_segno = curseg->next_segno;
547	struct f2fs_summary_block *sum_node;	547	struct f2fs_summary_block *sum_node;
548	struct page *sum_page;	548	struct page *sum_page;
549		549
550	write_sum_page(sbi, curseg->sum_blk,	550	write_sum_page(sbi, curseg->sum_blk,
551	GET_SUM_BLOCK(sbi, curseg->segno));	551	GET_SUM_BLOCK(sbi, curseg->segno));
552	__set_test_and_inuse(sbi, new_segno);	552	__set_test_and_inuse(sbi, new_segno);
553		553
554	mutex_lock(&dirty_i->seglist_lock);	554	mutex_lock(&dirty_i->seglist_lock);
555	__remove_dirty_segment(sbi, new_segno, PRE);	555	__remove_dirty_segment(sbi, new_segno, PRE);
556	__remove_dirty_segment(sbi, new_segno, DIRTY);	556	__remove_dirty_segment(sbi, new_segno, DIRTY);
557	mutex_unlock(&dirty_i->seglist_lock);	557	mutex_unlock(&dirty_i->seglist_lock);
558		558
559	reset_curseg(sbi, type, 1);	559	reset_curseg(sbi, type, 1);
560	curseg->alloc_type = SSR;	560	curseg->alloc_type = SSR;
561	__next_free_blkoff(sbi, curseg, 0);	561	__next_free_blkoff(sbi, curseg, 0);
562		562
563	if (reuse) {	563	if (reuse) {
564	sum_page = get_sum_page(sbi, new_segno);	564	sum_page = get_sum_page(sbi, new_segno);
565	sum_node = (struct f2fs_summary_block *)page_address(sum_page);	565	sum_node = (struct f2fs_summary_block *)page_address(sum_page);
566	memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);	566	memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
567	f2fs_put_page(sum_page, 1);	567	f2fs_put_page(sum_page, 1);
568	}	568	}
569	}	569	}
570		570
571	static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)	571	static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
572	{	572	{
573	struct curseg_info *curseg = CURSEG_I(sbi, type);	573	struct curseg_info *curseg = CURSEG_I(sbi, type);
574	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;	574	const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
575		575
576	if (IS_NODESEG(type) \|\| !has_not_enough_free_secs(sbi, 0))	576	if (IS_NODESEG(type) \|\| !has_not_enough_free_secs(sbi, 0))
577	return v_ops->get_victim(sbi,	577	return v_ops->get_victim(sbi,
578	&(curseg)->next_segno, BG_GC, type, SSR);	578	&(curseg)->next_segno, BG_GC, type, SSR);
579		579
580	/* For data segments, let's do SSR more intensively */	580	/* For data segments, let's do SSR more intensively */
581	for (; type >= CURSEG_HOT_DATA; type--)	581	for (; type >= CURSEG_HOT_DATA; type--)
582	if (v_ops->get_victim(sbi, &(curseg)->next_segno,	582	if (v_ops->get_victim(sbi, &(curseg)->next_segno,
583	BG_GC, type, SSR))	583	BG_GC, type, SSR))
584	return 1;	584	return 1;
585	return 0;	585	return 0;
586	}	586	}
587		587
588	/*	588	/*
589	* flush out current segment and replace it with new segment	589	* flush out current segment and replace it with new segment
590	* This function should be returned with success, otherwise BUG	590	* This function should be returned with success, otherwise BUG
591	*/	591	*/
592	static void allocate_segment_by_default(struct f2fs_sb_info *sbi,	592	static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
593	int type, bool force)	593	int type, bool force)
594	{	594	{
595	struct curseg_info *curseg = CURSEG_I(sbi, type);	595	struct curseg_info *curseg = CURSEG_I(sbi, type);
596		596
597	if (force) {	597	if (force) {
598	new_curseg(sbi, type, true);	598	new_curseg(sbi, type, true);
599	goto out;	599	goto out;
600	}	600	}
601		601
602	curseg->next_segno = check_prefree_segments(sbi, type);	602	curseg->next_segno = check_prefree_segments(sbi, type);
603		603
604	if (curseg->next_segno != NULL_SEGNO)	604	if (curseg->next_segno != NULL_SEGNO)
605	change_curseg(sbi, type, false);	605	change_curseg(sbi, type, false);
606	else if (type == CURSEG_WARM_NODE)	606	else if (type == CURSEG_WARM_NODE)
607	new_curseg(sbi, type, false);	607	new_curseg(sbi, type, false);
608	else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))	608	else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
609	new_curseg(sbi, type, false);	609	new_curseg(sbi, type, false);
610	else if (need_SSR(sbi) && get_ssr_segment(sbi, type))	610	else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
611	change_curseg(sbi, type, true);	611	change_curseg(sbi, type, true);
612	else	612	else
613	new_curseg(sbi, type, false);	613	new_curseg(sbi, type, false);
614	out:	614	out:
615	sbi->segment_count[curseg->alloc_type]++;	615	sbi->segment_count[curseg->alloc_type]++;
616	}	616	}
617		617
618	void allocate_new_segments(struct f2fs_sb_info *sbi)	618	void allocate_new_segments(struct f2fs_sb_info *sbi)
619	{	619	{
620	struct curseg_info *curseg;	620	struct curseg_info *curseg;
621	unsigned int old_curseg;	621	unsigned int old_curseg;
622	int i;	622	int i;
623		623
624	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {	624	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
625	curseg = CURSEG_I(sbi, i);	625	curseg = CURSEG_I(sbi, i);
626	old_curseg = curseg->segno;	626	old_curseg = curseg->segno;
627	SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);	627	SIT_I(sbi)->s_ops->allocate_segment(sbi, i, true);
628	locate_dirty_segment(sbi, old_curseg);	628	locate_dirty_segment(sbi, old_curseg);
629	}	629	}
630	}	630	}
631		631
632	static const struct segment_allocation default_salloc_ops = {	632	static const struct segment_allocation default_salloc_ops = {
633	.allocate_segment = allocate_segment_by_default,	633	.allocate_segment = allocate_segment_by_default,
634	};	634	};
635		635
636	static void f2fs_end_io_write(struct bio *bio, int err)	636	static void f2fs_end_io_write(struct bio *bio, int err)
637	{	637	{
638	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);	638	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
639	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;	639	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
640	struct bio_private *p = bio->bi_private;	640	struct bio_private *p = bio->bi_private;
641		641
642	do {	642	do {
643	struct page *page = bvec->bv_page;	643	struct page *page = bvec->bv_page;
644		644
645	if (--bvec >= bio->bi_io_vec)	645	if (--bvec >= bio->bi_io_vec)
646	prefetchw(&bvec->bv_page->flags);	646	prefetchw(&bvec->bv_page->flags);
647	if (!uptodate) {	647	if (!uptodate) {
648	SetPageError(page);	648	SetPageError(page);
649	if (page->mapping)	649	if (page->mapping)
650	set_bit(AS_EIO, &page->mapping->flags);	650	set_bit(AS_EIO, &page->mapping->flags);
651	set_ckpt_flags(p->sbi->ckpt, CP_ERROR_FLAG);	651	set_ckpt_flags(p->sbi->ckpt, CP_ERROR_FLAG);
652	p->sbi->sb->s_flags \|= MS_RDONLY;	652	p->sbi->sb->s_flags \|= MS_RDONLY;
653	}	653	}
654	end_page_writeback(page);	654	end_page_writeback(page);
655	dec_page_count(p->sbi, F2FS_WRITEBACK);	655	dec_page_count(p->sbi, F2FS_WRITEBACK);
656	} while (bvec >= bio->bi_io_vec);	656	} while (bvec >= bio->bi_io_vec);
657		657
658	if (p->is_sync)	658	if (p->is_sync)
659	complete(p->wait);	659	complete(p->wait);
660	kfree(p);	660	kfree(p);
661	bio_put(bio);	661	bio_put(bio);
662	}	662	}
663		663
664	struct bio f2fs_bio_alloc(struct block_device bdev, int npages)	664	struct bio f2fs_bio_alloc(struct block_device bdev, int npages)
665	{	665	{
666	struct bio *bio;	666	struct bio *bio;
667	struct bio_private *priv;	667	struct bio_private *priv;
668	retry:	668	retry:
669	priv = kmalloc(sizeof(struct bio_private), GFP_NOFS);	669	priv = kmalloc(sizeof(struct bio_private), GFP_NOFS);
670	if (!priv) {	670	if (!priv) {
671	cond_resched();	671	cond_resched();
672	goto retry;	672	goto retry;
673	}	673	}
674		674
675	/* No failure on bio allocation */	675	/* No failure on bio allocation */
676	bio = bio_alloc(GFP_NOIO, npages);	676	bio = bio_alloc(GFP_NOIO, npages);
677	bio->bi_bdev = bdev;	677	bio->bi_bdev = bdev;
678	bio->bi_private = priv;	678	bio->bi_private = priv;
679	return bio;	679	return bio;
680	}	680	}
681		681
682	static void do_submit_bio(struct f2fs_sb_info *sbi,	682	static void do_submit_bio(struct f2fs_sb_info *sbi,
683	enum page_type type, bool sync)	683	enum page_type type, bool sync)
684	{	684	{
685	int rw = sync ? WRITE_SYNC : WRITE;	685	int rw = sync ? WRITE_SYNC : WRITE;
686	enum page_type btype = type > META ? META : type;	686	enum page_type btype = type > META ? META : type;
687		687
688	if (type >= META_FLUSH)	688	if (type >= META_FLUSH)
689	rw = WRITE_FLUSH_FUA;	689	rw = WRITE_FLUSH_FUA;
690		690
691	if (btype == META)	691	if (btype == META)
692	rw \|= REQ_META;	692	rw \|= REQ_META;
693		693
694	if (sbi->bio[btype]) {	694	if (sbi->bio[btype]) {
695	struct bio_private *p = sbi->bio[btype]->bi_private;	695	struct bio_private *p = sbi->bio[btype]->bi_private;
696	p->sbi = sbi;	696	p->sbi = sbi;
697	sbi->bio[btype]->bi_end_io = f2fs_end_io_write;	697	sbi->bio[btype]->bi_end_io = f2fs_end_io_write;
698		698
699	trace_f2fs_do_submit_bio(sbi->sb, btype, sync, sbi->bio[btype]);	699	trace_f2fs_do_submit_bio(sbi->sb, btype, sync, sbi->bio[btype]);
700		700
701	if (type == META_FLUSH) {	701	if (type == META_FLUSH) {
702	DECLARE_COMPLETION_ONSTACK(wait);	702	DECLARE_COMPLETION_ONSTACK(wait);
703	p->is_sync = true;	703	p->is_sync = true;
704	p->wait = &wait;	704	p->wait = &wait;
705	submit_bio(rw, sbi->bio[btype]);	705	submit_bio(rw, sbi->bio[btype]);
706	wait_for_completion(&wait);	706	wait_for_completion(&wait);
707	} else {	707	} else {
708	p->is_sync = false;	708	p->is_sync = false;
709	submit_bio(rw, sbi->bio[btype]);	709	submit_bio(rw, sbi->bio[btype]);
710	}	710	}
711	sbi->bio[btype] = NULL;	711	sbi->bio[btype] = NULL;
712	}	712	}
713	}	713	}
714		714
715	void f2fs_submit_bio(struct f2fs_sb_info *sbi, enum page_type type, bool sync)	715	void f2fs_submit_bio(struct f2fs_sb_info *sbi, enum page_type type, bool sync)
716	{	716	{
717	down_write(&sbi->bio_sem);	717	down_write(&sbi->bio_sem);
718	do_submit_bio(sbi, type, sync);	718	do_submit_bio(sbi, type, sync);
719	up_write(&sbi->bio_sem);	719	up_write(&sbi->bio_sem);
720	}	720	}
721		721
722	static void submit_write_page(struct f2fs_sb_info sbi, struct page page,	722	static void submit_write_page(struct f2fs_sb_info sbi, struct page page,
723	block_t blk_addr, enum page_type type)	723	block_t blk_addr, enum page_type type)
724	{	724	{
725	struct block_device *bdev = sbi->sb->s_bdev;	725	struct block_device *bdev = sbi->sb->s_bdev;
726		726
727	verify_block_addr(sbi, blk_addr);	727	verify_block_addr(sbi, blk_addr);
728		728
729	down_write(&sbi->bio_sem);	729	down_write(&sbi->bio_sem);
730		730
731	inc_page_count(sbi, F2FS_WRITEBACK);	731	inc_page_count(sbi, F2FS_WRITEBACK);
732		732
733	if (sbi->bio[type] && sbi->last_block_in_bio[type] != blk_addr - 1)	733	if (sbi->bio[type] && sbi->last_block_in_bio[type] != blk_addr - 1)
734	do_submit_bio(sbi, type, false);	734	do_submit_bio(sbi, type, false);
735	alloc_new:	735	alloc_new:
736	if (sbi->bio[type] == NULL) {	736	if (sbi->bio[type] == NULL) {
737	sbi->bio[type] = f2fs_bio_alloc(bdev, bio_get_nr_vecs(bdev));	737	sbi->bio[type] = f2fs_bio_alloc(bdev, max_hw_blocks(sbi));
738	sbi->bio[type]->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);	738	sbi->bio[type]->bi_sector = SECTOR_FROM_BLOCK(sbi, blk_addr);
739	/*	739	/*
740	* The end_io will be assigned at the sumbission phase.	740	* The end_io will be assigned at the sumbission phase.
741	* Until then, let bio_add_page() merge consecutive IOs as much	741	* Until then, let bio_add_page() merge consecutive IOs as much
742	* as possible.	742	* as possible.
743	*/	743	*/
744	}	744	}
745		745
746	if (bio_add_page(sbi->bio[type], page, PAGE_CACHE_SIZE, 0) <	746	if (bio_add_page(sbi->bio[type], page, PAGE_CACHE_SIZE, 0) <
747	PAGE_CACHE_SIZE) {	747	PAGE_CACHE_SIZE) {
748	do_submit_bio(sbi, type, false);	748	do_submit_bio(sbi, type, false);
749	goto alloc_new;	749	goto alloc_new;
750	}	750	}
751		751
752	sbi->last_block_in_bio[type] = blk_addr;	752	sbi->last_block_in_bio[type] = blk_addr;
753		753
754	up_write(&sbi->bio_sem);	754	up_write(&sbi->bio_sem);
755	trace_f2fs_submit_write_page(page, blk_addr, type);	755	trace_f2fs_submit_write_page(page, blk_addr, type);
756	}	756	}
757		757
758	static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)	758	static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
759	{	759	{
760	struct curseg_info *curseg = CURSEG_I(sbi, type);	760	struct curseg_info *curseg = CURSEG_I(sbi, type);
761	if (curseg->next_blkoff < sbi->blocks_per_seg)	761	if (curseg->next_blkoff < sbi->blocks_per_seg)
762	return true;	762	return true;
763	return false;	763	return false;
764	}	764	}
765		765
766	static int __get_segment_type_2(struct page *page, enum page_type p_type)	766	static int __get_segment_type_2(struct page *page, enum page_type p_type)
767	{	767	{
768	if (p_type == DATA)	768	if (p_type == DATA)
769	return CURSEG_HOT_DATA;	769	return CURSEG_HOT_DATA;
770	else	770	else
771	return CURSEG_HOT_NODE;	771	return CURSEG_HOT_NODE;
772	}	772	}
773		773
774	static int __get_segment_type_4(struct page *page, enum page_type p_type)	774	static int __get_segment_type_4(struct page *page, enum page_type p_type)
775	{	775	{
776	if (p_type == DATA) {	776	if (p_type == DATA) {
777	struct inode *inode = page->mapping->host;	777	struct inode *inode = page->mapping->host;
778		778
779	if (S_ISDIR(inode->i_mode))	779	if (S_ISDIR(inode->i_mode))
780	return CURSEG_HOT_DATA;	780	return CURSEG_HOT_DATA;
781	else	781	else
782	return CURSEG_COLD_DATA;	782	return CURSEG_COLD_DATA;
783	} else {	783	} else {
784	if (IS_DNODE(page) && !is_cold_node(page))	784	if (IS_DNODE(page) && !is_cold_node(page))
785	return CURSEG_HOT_NODE;	785	return CURSEG_HOT_NODE;
786	else	786	else
787	return CURSEG_COLD_NODE;	787	return CURSEG_COLD_NODE;
788	}	788	}
789	}	789	}
790		790
791	static int __get_segment_type_6(struct page *page, enum page_type p_type)	791	static int __get_segment_type_6(struct page *page, enum page_type p_type)
792	{	792	{
793	if (p_type == DATA) {	793	if (p_type == DATA) {
794	struct inode *inode = page->mapping->host;	794	struct inode *inode = page->mapping->host;
795		795
796	if (S_ISDIR(inode->i_mode))	796	if (S_ISDIR(inode->i_mode))
797	return CURSEG_HOT_DATA;	797	return CURSEG_HOT_DATA;
798	else if (is_cold_data(page) \|\| is_cold_file(inode))	798	else if (is_cold_data(page) \|\| is_cold_file(inode))
799	return CURSEG_COLD_DATA;	799	return CURSEG_COLD_DATA;
800	else	800	else
801	return CURSEG_WARM_DATA;	801	return CURSEG_WARM_DATA;
802	} else {	802	} else {
803	if (IS_DNODE(page))	803	if (IS_DNODE(page))
804	return is_cold_node(page) ? CURSEG_WARM_NODE :	804	return is_cold_node(page) ? CURSEG_WARM_NODE :
805	CURSEG_HOT_NODE;	805	CURSEG_HOT_NODE;
806	else	806	else
807	return CURSEG_COLD_NODE;	807	return CURSEG_COLD_NODE;
808	}	808	}
809	}	809	}
810		810
811	static int __get_segment_type(struct page *page, enum page_type p_type)	811	static int __get_segment_type(struct page *page, enum page_type p_type)
812	{	812	{
813	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);	813	struct f2fs_sb_info *sbi = F2FS_SB(page->mapping->host->i_sb);
814	switch (sbi->active_logs) {	814	switch (sbi->active_logs) {
815	case 2:	815	case 2:
816	return __get_segment_type_2(page, p_type);	816	return __get_segment_type_2(page, p_type);
817	case 4:	817	case 4:
818	return __get_segment_type_4(page, p_type);	818	return __get_segment_type_4(page, p_type);
819	}	819	}
820	/* NR_CURSEG_TYPE(6) logs by default */	820	/* NR_CURSEG_TYPE(6) logs by default */
821	BUG_ON(sbi->active_logs != NR_CURSEG_TYPE);	821	BUG_ON(sbi->active_logs != NR_CURSEG_TYPE);
822	return __get_segment_type_6(page, p_type);	822	return __get_segment_type_6(page, p_type);
823	}	823	}
824		824
825	static void do_write_page(struct f2fs_sb_info sbi, struct page page,	825	static void do_write_page(struct f2fs_sb_info sbi, struct page page,
826	block_t old_blkaddr, block_t *new_blkaddr,	826	block_t old_blkaddr, block_t *new_blkaddr,
827	struct f2fs_summary *sum, enum page_type p_type)	827	struct f2fs_summary *sum, enum page_type p_type)
828	{	828	{
829	struct sit_info *sit_i = SIT_I(sbi);	829	struct sit_info *sit_i = SIT_I(sbi);
830	struct curseg_info *curseg;	830	struct curseg_info *curseg;
831	unsigned int old_cursegno;	831	unsigned int old_cursegno;
832	int type;	832	int type;
833		833
834	type = __get_segment_type(page, p_type);	834	type = __get_segment_type(page, p_type);
835	curseg = CURSEG_I(sbi, type);	835	curseg = CURSEG_I(sbi, type);
836		836
837	mutex_lock(&curseg->curseg_mutex);	837	mutex_lock(&curseg->curseg_mutex);
838		838
839	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);	839	*new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
840	old_cursegno = curseg->segno;	840	old_cursegno = curseg->segno;
841		841
842	/*	842	/*
843	* __add_sum_entry should be resided under the curseg_mutex	843	* __add_sum_entry should be resided under the curseg_mutex
844	* because, this function updates a summary entry in the	844	* because, this function updates a summary entry in the
845	* current summary block.	845	* current summary block.
846	*/	846	*/
847	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);	847	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);
848		848
849	mutex_lock(&sit_i->sentry_lock);	849	mutex_lock(&sit_i->sentry_lock);
850	__refresh_next_blkoff(sbi, curseg);	850	__refresh_next_blkoff(sbi, curseg);
851	sbi->block_count[curseg->alloc_type]++;	851	sbi->block_count[curseg->alloc_type]++;
852		852
853	/*	853	/*
854	* SIT information should be updated before segment allocation,	854	* SIT information should be updated before segment allocation,
855	* since SSR needs latest valid block information.	855	* since SSR needs latest valid block information.
856	*/	856	*/
857	refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);	857	refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
858		858
859	if (!__has_curseg_space(sbi, type))	859	if (!__has_curseg_space(sbi, type))
860	sit_i->s_ops->allocate_segment(sbi, type, false);	860	sit_i->s_ops->allocate_segment(sbi, type, false);
861		861
862	locate_dirty_segment(sbi, old_cursegno);	862	locate_dirty_segment(sbi, old_cursegno);
863	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));	863	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
864	mutex_unlock(&sit_i->sentry_lock);	864	mutex_unlock(&sit_i->sentry_lock);
865		865
866	if (p_type == NODE)	866	if (p_type == NODE)
867	fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));	867	fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
868		868
869	/* writeout dirty page into bdev */	869	/* writeout dirty page into bdev */
870	submit_write_page(sbi, page, *new_blkaddr, p_type);	870	submit_write_page(sbi, page, *new_blkaddr, p_type);
871		871
872	mutex_unlock(&curseg->curseg_mutex);	872	mutex_unlock(&curseg->curseg_mutex);
873	}	873	}
874		874
875	void write_meta_page(struct f2fs_sb_info sbi, struct page page)	875	void write_meta_page(struct f2fs_sb_info sbi, struct page page)
876	{	876	{
877	set_page_writeback(page);	877	set_page_writeback(page);
878	submit_write_page(sbi, page, page->index, META);	878	submit_write_page(sbi, page, page->index, META);
879	}	879	}
880		880
881	void write_node_page(struct f2fs_sb_info sbi, struct page page,	881	void write_node_page(struct f2fs_sb_info sbi, struct page page,
882	unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr)	882	unsigned int nid, block_t old_blkaddr, block_t *new_blkaddr)
883	{	883	{
884	struct f2fs_summary sum;	884	struct f2fs_summary sum;
885	set_summary(&sum, nid, 0, 0);	885	set_summary(&sum, nid, 0, 0);
886	do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, NODE);	886	do_write_page(sbi, page, old_blkaddr, new_blkaddr, &sum, NODE);
887	}	887	}
888		888
889	void write_data_page(struct inode inode, struct page page,	889	void write_data_page(struct inode inode, struct page page,
890	struct dnode_of_data *dn, block_t old_blkaddr,	890	struct dnode_of_data *dn, block_t old_blkaddr,
891	block_t *new_blkaddr)	891	block_t *new_blkaddr)
892	{	892	{
893	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);	893	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
894	struct f2fs_summary sum;	894	struct f2fs_summary sum;
895	struct node_info ni;	895	struct node_info ni;
896		896
897	BUG_ON(old_blkaddr == NULL_ADDR);	897	BUG_ON(old_blkaddr == NULL_ADDR);
898	get_node_info(sbi, dn->nid, &ni);	898	get_node_info(sbi, dn->nid, &ni);
899	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);	899	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
900		900
901	do_write_page(sbi, page, old_blkaddr,	901	do_write_page(sbi, page, old_blkaddr,
902	new_blkaddr, &sum, DATA);	902	new_blkaddr, &sum, DATA);
903	}	903	}
904		904
905	void rewrite_data_page(struct f2fs_sb_info sbi, struct page page,	905	void rewrite_data_page(struct f2fs_sb_info sbi, struct page page,
906	block_t old_blk_addr)	906	block_t old_blk_addr)
907	{	907	{
908	submit_write_page(sbi, page, old_blk_addr, DATA);	908	submit_write_page(sbi, page, old_blk_addr, DATA);
909	}	909	}
910		910
911	void recover_data_page(struct f2fs_sb_info *sbi,	911	void recover_data_page(struct f2fs_sb_info *sbi,
912	struct page page, struct f2fs_summary sum,	912	struct page page, struct f2fs_summary sum,
913	block_t old_blkaddr, block_t new_blkaddr)	913	block_t old_blkaddr, block_t new_blkaddr)
914	{	914	{
915	struct sit_info *sit_i = SIT_I(sbi);	915	struct sit_info *sit_i = SIT_I(sbi);
916	struct curseg_info *curseg;	916	struct curseg_info *curseg;
917	unsigned int segno, old_cursegno;	917	unsigned int segno, old_cursegno;
918	struct seg_entry *se;	918	struct seg_entry *se;
919	int type;	919	int type;
920		920
921	segno = GET_SEGNO(sbi, new_blkaddr);	921	segno = GET_SEGNO(sbi, new_blkaddr);
922	se = get_seg_entry(sbi, segno);	922	se = get_seg_entry(sbi, segno);
923	type = se->type;	923	type = se->type;
924		924
925	if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {	925	if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
926	if (old_blkaddr == NULL_ADDR)	926	if (old_blkaddr == NULL_ADDR)
927	type = CURSEG_COLD_DATA;	927	type = CURSEG_COLD_DATA;
928	else	928	else
929	type = CURSEG_WARM_DATA;	929	type = CURSEG_WARM_DATA;
930	}	930	}
931	curseg = CURSEG_I(sbi, type);	931	curseg = CURSEG_I(sbi, type);
932		932
933	mutex_lock(&curseg->curseg_mutex);	933	mutex_lock(&curseg->curseg_mutex);
934	mutex_lock(&sit_i->sentry_lock);	934	mutex_lock(&sit_i->sentry_lock);
935		935
936	old_cursegno = curseg->segno;	936	old_cursegno = curseg->segno;
937		937
938	/* change the current segment */	938	/* change the current segment */
939	if (segno != curseg->segno) {	939	if (segno != curseg->segno) {
940	curseg->next_segno = segno;	940	curseg->next_segno = segno;
941	change_curseg(sbi, type, true);	941	change_curseg(sbi, type, true);
942	}	942	}
943		943
944	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) &	944	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) &
945	(sbi->blocks_per_seg - 1);	945	(sbi->blocks_per_seg - 1);
946	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);	946	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);
947		947
948	refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);	948	refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
949		949
950	locate_dirty_segment(sbi, old_cursegno);	950	locate_dirty_segment(sbi, old_cursegno);
951	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));	951	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
952		952
953	mutex_unlock(&sit_i->sentry_lock);	953	mutex_unlock(&sit_i->sentry_lock);
954	mutex_unlock(&curseg->curseg_mutex);	954	mutex_unlock(&curseg->curseg_mutex);
955	}	955	}
956		956
957	void rewrite_node_page(struct f2fs_sb_info *sbi,	957	void rewrite_node_page(struct f2fs_sb_info *sbi,
958	struct page page, struct f2fs_summary sum,	958	struct page page, struct f2fs_summary sum,
959	block_t old_blkaddr, block_t new_blkaddr)	959	block_t old_blkaddr, block_t new_blkaddr)
960	{	960	{
961	struct sit_info *sit_i = SIT_I(sbi);	961	struct sit_info *sit_i = SIT_I(sbi);
962	int type = CURSEG_WARM_NODE;	962	int type = CURSEG_WARM_NODE;
963	struct curseg_info *curseg;	963	struct curseg_info *curseg;
964	unsigned int segno, old_cursegno;	964	unsigned int segno, old_cursegno;
965	block_t next_blkaddr = next_blkaddr_of_node(page);	965	block_t next_blkaddr = next_blkaddr_of_node(page);
966	unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr);	966	unsigned int next_segno = GET_SEGNO(sbi, next_blkaddr);
967		967
968	curseg = CURSEG_I(sbi, type);	968	curseg = CURSEG_I(sbi, type);
969		969
970	mutex_lock(&curseg->curseg_mutex);	970	mutex_lock(&curseg->curseg_mutex);
971	mutex_lock(&sit_i->sentry_lock);	971	mutex_lock(&sit_i->sentry_lock);
972		972
973	segno = GET_SEGNO(sbi, new_blkaddr);	973	segno = GET_SEGNO(sbi, new_blkaddr);
974	old_cursegno = curseg->segno;	974	old_cursegno = curseg->segno;
975		975
976	/* change the current segment */	976	/* change the current segment */
977	if (segno != curseg->segno) {	977	if (segno != curseg->segno) {
978	curseg->next_segno = segno;	978	curseg->next_segno = segno;
979	change_curseg(sbi, type, true);	979	change_curseg(sbi, type, true);
980	}	980	}
981	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) &	981	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, new_blkaddr) &
982	(sbi->blocks_per_seg - 1);	982	(sbi->blocks_per_seg - 1);
983	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);	983	__add_sum_entry(sbi, type, sum, curseg->next_blkoff);
984		984
985	/* change the current log to the next block addr in advance */	985	/* change the current log to the next block addr in advance */
986	if (next_segno != segno) {	986	if (next_segno != segno) {
987	curseg->next_segno = next_segno;	987	curseg->next_segno = next_segno;
988	change_curseg(sbi, type, true);	988	change_curseg(sbi, type, true);
989	}	989	}
990	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, next_blkaddr) &	990	curseg->next_blkoff = GET_SEGOFF_FROM_SEG0(sbi, next_blkaddr) &
991	(sbi->blocks_per_seg - 1);	991	(sbi->blocks_per_seg - 1);
992		992
993	/* rewrite node page */	993	/* rewrite node page */
994	set_page_writeback(page);	994	set_page_writeback(page);
995	submit_write_page(sbi, page, new_blkaddr, NODE);	995	submit_write_page(sbi, page, new_blkaddr, NODE);
996	f2fs_submit_bio(sbi, NODE, true);	996	f2fs_submit_bio(sbi, NODE, true);
997	refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);	997	refresh_sit_entry(sbi, old_blkaddr, new_blkaddr);
998		998
999	locate_dirty_segment(sbi, old_cursegno);	999	locate_dirty_segment(sbi, old_cursegno);
1000	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));	1000	locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1001		1001
1002	mutex_unlock(&sit_i->sentry_lock);	1002	mutex_unlock(&sit_i->sentry_lock);
1003	mutex_unlock(&curseg->curseg_mutex);	1003	mutex_unlock(&curseg->curseg_mutex);
1004	}	1004	}
1005		1005
1006	static int read_compacted_summaries(struct f2fs_sb_info *sbi)	1006	static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1007	{	1007	{
1008	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);	1008	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1009	struct curseg_info *seg_i;	1009	struct curseg_info *seg_i;
1010	unsigned char *kaddr;	1010	unsigned char *kaddr;
1011	struct page *page;	1011	struct page *page;
1012	block_t start;	1012	block_t start;
1013	int i, j, offset;	1013	int i, j, offset;
1014		1014
1015	start = start_sum_block(sbi);	1015	start = start_sum_block(sbi);
1016		1016
1017	page = get_meta_page(sbi, start++);	1017	page = get_meta_page(sbi, start++);
1018	kaddr = (unsigned char *)page_address(page);	1018	kaddr = (unsigned char *)page_address(page);
1019		1019
1020	/* Step 1: restore nat cache */	1020	/* Step 1: restore nat cache */
1021	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);	1021	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1022	memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);	1022	memcpy(&seg_i->sum_blk->n_nats, kaddr, SUM_JOURNAL_SIZE);
1023		1023
1024	/* Step 2: restore sit cache */	1024	/* Step 2: restore sit cache */
1025	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);	1025	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1026	memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,	1026	memcpy(&seg_i->sum_blk->n_sits, kaddr + SUM_JOURNAL_SIZE,
1027	SUM_JOURNAL_SIZE);	1027	SUM_JOURNAL_SIZE);
1028	offset = 2 * SUM_JOURNAL_SIZE;	1028	offset = 2 * SUM_JOURNAL_SIZE;
1029		1029
1030	/* Step 3: restore summary entries */	1030	/* Step 3: restore summary entries */
1031	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {	1031	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1032	unsigned short blk_off;	1032	unsigned short blk_off;
1033	unsigned int segno;	1033	unsigned int segno;
1034		1034
1035	seg_i = CURSEG_I(sbi, i);	1035	seg_i = CURSEG_I(sbi, i);
1036	segno = le32_to_cpu(ckpt->cur_data_segno[i]);	1036	segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1037	blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);	1037	blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1038	seg_i->next_segno = segno;	1038	seg_i->next_segno = segno;
1039	reset_curseg(sbi, i, 0);	1039	reset_curseg(sbi, i, 0);
1040	seg_i->alloc_type = ckpt->alloc_type[i];	1040	seg_i->alloc_type = ckpt->alloc_type[i];
1041	seg_i->next_blkoff = blk_off;	1041	seg_i->next_blkoff = blk_off;
1042		1042
1043	if (seg_i->alloc_type == SSR)	1043	if (seg_i->alloc_type == SSR)
1044	blk_off = sbi->blocks_per_seg;	1044	blk_off = sbi->blocks_per_seg;
1045		1045
1046	for (j = 0; j < blk_off; j++) {	1046	for (j = 0; j < blk_off; j++) {
1047	struct f2fs_summary *s;	1047	struct f2fs_summary *s;
1048	s = (struct f2fs_summary *)(kaddr + offset);	1048	s = (struct f2fs_summary *)(kaddr + offset);
1049	seg_i->sum_blk->entries[j] = *s;	1049	seg_i->sum_blk->entries[j] = *s;
1050	offset += SUMMARY_SIZE;	1050	offset += SUMMARY_SIZE;
1051	if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -	1051	if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1052	SUM_FOOTER_SIZE)	1052	SUM_FOOTER_SIZE)
1053	continue;	1053	continue;
1054		1054
1055	f2fs_put_page(page, 1);	1055	f2fs_put_page(page, 1);
1056	page = NULL;	1056	page = NULL;
1057		1057
1058	page = get_meta_page(sbi, start++);	1058	page = get_meta_page(sbi, start++);
1059	kaddr = (unsigned char *)page_address(page);	1059	kaddr = (unsigned char *)page_address(page);
1060	offset = 0;	1060	offset = 0;
1061	}	1061	}
1062	}	1062	}
1063	f2fs_put_page(page, 1);	1063	f2fs_put_page(page, 1);
1064	return 0;	1064	return 0;
1065	}	1065	}
1066		1066
1067	static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)	1067	static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1068	{	1068	{
1069	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);	1069	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1070	struct f2fs_summary_block *sum;	1070	struct f2fs_summary_block *sum;
1071	struct curseg_info *curseg;	1071	struct curseg_info *curseg;
1072	struct page *new;	1072	struct page *new;
1073	unsigned short blk_off;	1073	unsigned short blk_off;
1074	unsigned int segno = 0;	1074	unsigned int segno = 0;
1075	block_t blk_addr = 0;	1075	block_t blk_addr = 0;
1076		1076
1077	/* get segment number and block addr */	1077	/* get segment number and block addr */
1078	if (IS_DATASEG(type)) {	1078	if (IS_DATASEG(type)) {
1079	segno = le32_to_cpu(ckpt->cur_data_segno[type]);	1079	segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1080	blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -	1080	blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1081	CURSEG_HOT_DATA]);	1081	CURSEG_HOT_DATA]);
1082	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))	1082	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1083	blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);	1083	blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1084	else	1084	else
1085	blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);	1085	blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1086	} else {	1086	} else {
1087	segno = le32_to_cpu(ckpt->cur_node_segno[type -	1087	segno = le32_to_cpu(ckpt->cur_node_segno[type -
1088	CURSEG_HOT_NODE]);	1088	CURSEG_HOT_NODE]);
1089	blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -	1089	blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1090	CURSEG_HOT_NODE]);	1090	CURSEG_HOT_NODE]);
1091	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))	1091	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG))
1092	blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,	1092	blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1093	type - CURSEG_HOT_NODE);	1093	type - CURSEG_HOT_NODE);
1094	else	1094	else
1095	blk_addr = GET_SUM_BLOCK(sbi, segno);	1095	blk_addr = GET_SUM_BLOCK(sbi, segno);
1096	}	1096	}
1097		1097
1098	new = get_meta_page(sbi, blk_addr);	1098	new = get_meta_page(sbi, blk_addr);
1099	sum = (struct f2fs_summary_block *)page_address(new);	1099	sum = (struct f2fs_summary_block *)page_address(new);
1100		1100
1101	if (IS_NODESEG(type)) {	1101	if (IS_NODESEG(type)) {
1102	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) {	1102	if (is_set_ckpt_flags(ckpt, CP_UMOUNT_FLAG)) {
1103	struct f2fs_summary *ns = &sum->entries[0];	1103	struct f2fs_summary *ns = &sum->entries[0];
1104	int i;	1104	int i;
1105	for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {	1105	for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1106	ns->version = 0;	1106	ns->version = 0;
1107	ns->ofs_in_node = 0;	1107	ns->ofs_in_node = 0;
1108	}	1108	}
1109	} else {	1109	} else {
1110	if (restore_node_summary(sbi, segno, sum)) {	1110	if (restore_node_summary(sbi, segno, sum)) {
1111	f2fs_put_page(new, 1);	1111	f2fs_put_page(new, 1);
1112	return -EINVAL;	1112	return -EINVAL;
1113	}	1113	}
1114	}	1114	}
1115	}	1115	}
1116		1116
1117	/* set uncompleted segment to curseg */	1117	/* set uncompleted segment to curseg */
1118	curseg = CURSEG_I(sbi, type);	1118	curseg = CURSEG_I(sbi, type);
1119	mutex_lock(&curseg->curseg_mutex);	1119	mutex_lock(&curseg->curseg_mutex);
1120	memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);	1120	memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1121	curseg->next_segno = segno;	1121	curseg->next_segno = segno;
1122	reset_curseg(sbi, type, 0);	1122	reset_curseg(sbi, type, 0);
1123	curseg->alloc_type = ckpt->alloc_type[type];	1123	curseg->alloc_type = ckpt->alloc_type[type];
1124	curseg->next_blkoff = blk_off;	1124	curseg->next_blkoff = blk_off;
1125	mutex_unlock(&curseg->curseg_mutex);	1125	mutex_unlock(&curseg->curseg_mutex);
1126	f2fs_put_page(new, 1);	1126	f2fs_put_page(new, 1);
1127	return 0;	1127	return 0;
1128	}	1128	}
1129		1129
1130	static int restore_curseg_summaries(struct f2fs_sb_info *sbi)	1130	static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1131	{	1131	{
1132	int type = CURSEG_HOT_DATA;	1132	int type = CURSEG_HOT_DATA;
1133		1133
1134	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {	1134	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1135	/* restore for compacted data summary */	1135	/* restore for compacted data summary */
1136	if (read_compacted_summaries(sbi))	1136	if (read_compacted_summaries(sbi))
1137	return -EINVAL;	1137	return -EINVAL;
1138	type = CURSEG_HOT_NODE;	1138	type = CURSEG_HOT_NODE;
1139	}	1139	}
1140		1140
1141	for (; type <= CURSEG_COLD_NODE; type++)	1141	for (; type <= CURSEG_COLD_NODE; type++)
1142	if (read_normal_summaries(sbi, type))	1142	if (read_normal_summaries(sbi, type))
1143	return -EINVAL;	1143	return -EINVAL;
1144	return 0;	1144	return 0;
1145	}	1145	}
1146		1146
1147	static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)	1147	static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1148	{	1148	{
1149	struct page *page;	1149	struct page *page;
1150	unsigned char *kaddr;	1150	unsigned char *kaddr;
1151	struct f2fs_summary *summary;	1151	struct f2fs_summary *summary;
1152	struct curseg_info *seg_i;	1152	struct curseg_info *seg_i;
1153	int written_size = 0;	1153	int written_size = 0;
1154	int i, j;	1154	int i, j;
1155		1155
1156	page = grab_meta_page(sbi, blkaddr++);	1156	page = grab_meta_page(sbi, blkaddr++);
1157	kaddr = (unsigned char *)page_address(page);	1157	kaddr = (unsigned char *)page_address(page);
1158		1158
1159	/* Step 1: write nat cache */	1159	/* Step 1: write nat cache */
1160	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);	1160	seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1161	memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);	1161	memcpy(kaddr, &seg_i->sum_blk->n_nats, SUM_JOURNAL_SIZE);
1162	written_size += SUM_JOURNAL_SIZE;	1162	written_size += SUM_JOURNAL_SIZE;
1163		1163
1164	/* Step 2: write sit cache */	1164	/* Step 2: write sit cache */
1165	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);	1165	seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1166	memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,	1166	memcpy(kaddr + written_size, &seg_i->sum_blk->n_sits,
1167	SUM_JOURNAL_SIZE);	1167	SUM_JOURNAL_SIZE);
1168	written_size += SUM_JOURNAL_SIZE;	1168	written_size += SUM_JOURNAL_SIZE;
1169		1169
1170	set_page_dirty(page);	1170	set_page_dirty(page);
1171		1171
1172	/* Step 3: write summary entries */	1172	/* Step 3: write summary entries */
1173	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {	1173	for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1174	unsigned short blkoff;	1174	unsigned short blkoff;
1175	seg_i = CURSEG_I(sbi, i);	1175	seg_i = CURSEG_I(sbi, i);
1176	if (sbi->ckpt->alloc_type[i] == SSR)	1176	if (sbi->ckpt->alloc_type[i] == SSR)
1177	blkoff = sbi->blocks_per_seg;	1177	blkoff = sbi->blocks_per_seg;
1178	else	1178	else
1179	blkoff = curseg_blkoff(sbi, i);	1179	blkoff = curseg_blkoff(sbi, i);
1180		1180
1181	for (j = 0; j < blkoff; j++) {	1181	for (j = 0; j < blkoff; j++) {
1182	if (!page) {	1182	if (!page) {
1183	page = grab_meta_page(sbi, blkaddr++);	1183	page = grab_meta_page(sbi, blkaddr++);
1184	kaddr = (unsigned char *)page_address(page);	1184	kaddr = (unsigned char *)page_address(page);
1185	written_size = 0;	1185	written_size = 0;
1186	}	1186	}
1187	summary = (struct f2fs_summary *)(kaddr + written_size);	1187	summary = (struct f2fs_summary *)(kaddr + written_size);
1188	*summary = seg_i->sum_blk->entries[j];	1188	*summary = seg_i->sum_blk->entries[j];
1189	written_size += SUMMARY_SIZE;	1189	written_size += SUMMARY_SIZE;
1190	set_page_dirty(page);	1190	set_page_dirty(page);
1191		1191
1192	if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -	1192	if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1193	SUM_FOOTER_SIZE)	1193	SUM_FOOTER_SIZE)
1194	continue;	1194	continue;
1195		1195
1196	f2fs_put_page(page, 1);	1196	f2fs_put_page(page, 1);
1197	page = NULL;	1197	page = NULL;
1198	}	1198	}
1199	}	1199	}
1200	if (page)	1200	if (page)
1201	f2fs_put_page(page, 1);	1201	f2fs_put_page(page, 1);
1202	}	1202	}
1203		1203
1204	static void write_normal_summaries(struct f2fs_sb_info *sbi,	1204	static void write_normal_summaries(struct f2fs_sb_info *sbi,
1205	block_t blkaddr, int type)	1205	block_t blkaddr, int type)
1206	{	1206	{
1207	int i, end;	1207	int i, end;
1208	if (IS_DATASEG(type))	1208	if (IS_DATASEG(type))
1209	end = type + NR_CURSEG_DATA_TYPE;	1209	end = type + NR_CURSEG_DATA_TYPE;
1210	else	1210	else
1211	end = type + NR_CURSEG_NODE_TYPE;	1211	end = type + NR_CURSEG_NODE_TYPE;
1212		1212
1213	for (i = type; i < end; i++) {	1213	for (i = type; i < end; i++) {
1214	struct curseg_info *sum = CURSEG_I(sbi, i);	1214	struct curseg_info *sum = CURSEG_I(sbi, i);
1215	mutex_lock(&sum->curseg_mutex);	1215	mutex_lock(&sum->curseg_mutex);
1216	write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));	1216	write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1217	mutex_unlock(&sum->curseg_mutex);	1217	mutex_unlock(&sum->curseg_mutex);
1218	}	1218	}
1219	}	1219	}
1220		1220
1221	void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)	1221	void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1222	{	1222	{
1223	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))	1223	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1224	write_compacted_summaries(sbi, start_blk);	1224	write_compacted_summaries(sbi, start_blk);
1225	else	1225	else
1226	write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);	1226	write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1227	}	1227	}
1228		1228
1229	void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)	1229	void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1230	{	1230	{
1231	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG))	1231	if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_UMOUNT_FLAG))
1232	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);	1232	write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1233	return;	1233	return;
1234	}	1234	}
1235		1235
1236	int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,	1236	int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1237	unsigned int val, int alloc)	1237	unsigned int val, int alloc)
1238	{	1238	{
1239	int i;	1239	int i;
1240		1240
1241	if (type == NAT_JOURNAL) {	1241	if (type == NAT_JOURNAL) {
1242	for (i = 0; i < nats_in_cursum(sum); i++) {	1242	for (i = 0; i < nats_in_cursum(sum); i++) {
1243	if (le32_to_cpu(nid_in_journal(sum, i)) == val)	1243	if (le32_to_cpu(nid_in_journal(sum, i)) == val)
1244	return i;	1244	return i;
1245	}	1245	}
1246	if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)	1246	if (alloc && nats_in_cursum(sum) < NAT_JOURNAL_ENTRIES)
1247	return update_nats_in_cursum(sum, 1);	1247	return update_nats_in_cursum(sum, 1);
1248	} else if (type == SIT_JOURNAL) {	1248	} else if (type == SIT_JOURNAL) {
1249	for (i = 0; i < sits_in_cursum(sum); i++)	1249	for (i = 0; i < sits_in_cursum(sum); i++)
1250	if (le32_to_cpu(segno_in_journal(sum, i)) == val)	1250	if (le32_to_cpu(segno_in_journal(sum, i)) == val)
1251	return i;	1251	return i;
1252	if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)	1252	if (alloc && sits_in_cursum(sum) < SIT_JOURNAL_ENTRIES)
1253	return update_sits_in_cursum(sum, 1);	1253	return update_sits_in_cursum(sum, 1);
1254	}	1254	}
1255	return -1;	1255	return -1;
1256	}	1256	}
1257		1257
1258	static struct page get_current_sit_page(struct f2fs_sb_info sbi,	1258	static struct page get_current_sit_page(struct f2fs_sb_info sbi,
1259	unsigned int segno)	1259	unsigned int segno)
1260	{	1260	{
1261	struct sit_info *sit_i = SIT_I(sbi);	1261	struct sit_info *sit_i = SIT_I(sbi);
1262	unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno);	1262	unsigned int offset = SIT_BLOCK_OFFSET(sit_i, segno);
1263	block_t blk_addr = sit_i->sit_base_addr + offset;	1263	block_t blk_addr = sit_i->sit_base_addr + offset;
1264		1264
1265	check_seg_range(sbi, segno);	1265	check_seg_range(sbi, segno);
1266		1266
1267	/* calculate sit block address */	1267	/* calculate sit block address */
1268	if (f2fs_test_bit(offset, sit_i->sit_bitmap))	1268	if (f2fs_test_bit(offset, sit_i->sit_bitmap))
1269	blk_addr += sit_i->sit_blocks;	1269	blk_addr += sit_i->sit_blocks;
1270		1270
1271	return get_meta_page(sbi, blk_addr);	1271	return get_meta_page(sbi, blk_addr);
1272	}	1272	}
1273		1273
1274	static struct page get_next_sit_page(struct f2fs_sb_info sbi,	1274	static struct page get_next_sit_page(struct f2fs_sb_info sbi,
1275	unsigned int start)	1275	unsigned int start)
1276	{	1276	{
1277	struct sit_info *sit_i = SIT_I(sbi);	1277	struct sit_info *sit_i = SIT_I(sbi);
1278	struct page src_page, dst_page;	1278	struct page src_page, dst_page;
1279	pgoff_t src_off, dst_off;	1279	pgoff_t src_off, dst_off;
1280	void src_addr, dst_addr;	1280	void src_addr, dst_addr;
1281		1281
1282	src_off = current_sit_addr(sbi, start);	1282	src_off = current_sit_addr(sbi, start);
1283	dst_off = next_sit_addr(sbi, src_off);	1283	dst_off = next_sit_addr(sbi, src_off);
1284		1284
1285	/* get current sit block page without lock */	1285	/* get current sit block page without lock */
1286	src_page = get_meta_page(sbi, src_off);	1286	src_page = get_meta_page(sbi, src_off);
1287	dst_page = grab_meta_page(sbi, dst_off);	1287	dst_page = grab_meta_page(sbi, dst_off);
1288	BUG_ON(PageDirty(src_page));	1288	BUG_ON(PageDirty(src_page));
1289		1289
1290	src_addr = page_address(src_page);	1290	src_addr = page_address(src_page);
1291	dst_addr = page_address(dst_page);	1291	dst_addr = page_address(dst_page);
1292	memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);	1292	memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1293		1293
1294	set_page_dirty(dst_page);	1294	set_page_dirty(dst_page);
1295	f2fs_put_page(src_page, 1);	1295	f2fs_put_page(src_page, 1);
1296		1296
1297	set_to_next_sit(sit_i, start);	1297	set_to_next_sit(sit_i, start);
1298		1298
1299	return dst_page;	1299	return dst_page;
1300	}	1300	}
1301		1301
1302	static bool flush_sits_in_journal(struct f2fs_sb_info *sbi)	1302	static bool flush_sits_in_journal(struct f2fs_sb_info *sbi)
1303	{	1303	{
1304	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);	1304	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1305	struct f2fs_summary_block *sum = curseg->sum_blk;	1305	struct f2fs_summary_block *sum = curseg->sum_blk;
1306	int i;	1306	int i;
1307		1307
1308	/*	1308	/*
1309	* If the journal area in the current summary is full of sit entries,	1309	* If the journal area in the current summary is full of sit entries,
1310	* all the sit entries will be flushed. Otherwise the sit entries	1310	* all the sit entries will be flushed. Otherwise the sit entries
1311	* are not able to replace with newly hot sit entries.	1311	* are not able to replace with newly hot sit entries.
1312	*/	1312	*/
1313	if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) {	1313	if (sits_in_cursum(sum) >= SIT_JOURNAL_ENTRIES) {
1314	for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {	1314	for (i = sits_in_cursum(sum) - 1; i >= 0; i--) {
1315	unsigned int segno;	1315	unsigned int segno;
1316	segno = le32_to_cpu(segno_in_journal(sum, i));	1316	segno = le32_to_cpu(segno_in_journal(sum, i));
1317	__mark_sit_entry_dirty(sbi, segno);	1317	__mark_sit_entry_dirty(sbi, segno);
1318	}	1318	}
1319	update_sits_in_cursum(sum, -sits_in_cursum(sum));	1319	update_sits_in_cursum(sum, -sits_in_cursum(sum));
1320	return 1;	1320	return 1;
1321	}	1321	}
1322	return 0;	1322	return 0;
1323	}	1323	}
1324		1324
1325	/*	1325	/*
1326	* CP calls this function, which flushes SIT entries including sit_journal,	1326	* CP calls this function, which flushes SIT entries including sit_journal,
1327	* and moves prefree segs to free segs.	1327	* and moves prefree segs to free segs.
1328	*/	1328	*/
1329	void flush_sit_entries(struct f2fs_sb_info *sbi)	1329	void flush_sit_entries(struct f2fs_sb_info *sbi)
1330	{	1330	{
1331	struct sit_info *sit_i = SIT_I(sbi);	1331	struct sit_info *sit_i = SIT_I(sbi);
1332	unsigned long *bitmap = sit_i->dirty_sentries_bitmap;	1332	unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1333	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);	1333	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1334	struct f2fs_summary_block *sum = curseg->sum_blk;	1334	struct f2fs_summary_block *sum = curseg->sum_blk;
1335	unsigned long nsegs = TOTAL_SEGS(sbi);	1335	unsigned long nsegs = TOTAL_SEGS(sbi);
1336	struct page *page = NULL;	1336	struct page *page = NULL;
1337	struct f2fs_sit_block *raw_sit = NULL;	1337	struct f2fs_sit_block *raw_sit = NULL;
1338	unsigned int start = 0, end = 0;	1338	unsigned int start = 0, end = 0;
1339	unsigned int segno = -1;	1339	unsigned int segno = -1;
1340	bool flushed;	1340	bool flushed;
1341		1341
1342	mutex_lock(&curseg->curseg_mutex);	1342	mutex_lock(&curseg->curseg_mutex);
1343	mutex_lock(&sit_i->sentry_lock);	1343	mutex_lock(&sit_i->sentry_lock);
1344		1344
1345	/*	1345	/*
1346	* "flushed" indicates whether sit entries in journal are flushed	1346	* "flushed" indicates whether sit entries in journal are flushed
1347	* to the SIT area or not.	1347	* to the SIT area or not.
1348	*/	1348	*/
1349	flushed = flush_sits_in_journal(sbi);	1349	flushed = flush_sits_in_journal(sbi);
1350		1350
1351	while ((segno = find_next_bit(bitmap, nsegs, segno + 1)) < nsegs) {	1351	while ((segno = find_next_bit(bitmap, nsegs, segno + 1)) < nsegs) {
1352	struct seg_entry *se = get_seg_entry(sbi, segno);	1352	struct seg_entry *se = get_seg_entry(sbi, segno);
1353	int sit_offset, offset;	1353	int sit_offset, offset;
1354		1354
1355	sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);	1355	sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1356		1356
1357	if (flushed)	1357	if (flushed)
1358	goto to_sit_page;	1358	goto to_sit_page;
1359		1359
1360	offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1);	1360	offset = lookup_journal_in_cursum(sum, SIT_JOURNAL, segno, 1);
1361	if (offset >= 0) {	1361	if (offset >= 0) {
1362	segno_in_journal(sum, offset) = cpu_to_le32(segno);	1362	segno_in_journal(sum, offset) = cpu_to_le32(segno);
1363	seg_info_to_raw_sit(se, &sit_in_journal(sum, offset));	1363	seg_info_to_raw_sit(se, &sit_in_journal(sum, offset));
1364	goto flush_done;	1364	goto flush_done;
1365	}	1365	}
1366	to_sit_page:	1366	to_sit_page:
1367	if (!page \|\| (start > segno) \|\| (segno > end)) {	1367	if (!page \|\| (start > segno) \|\| (segno > end)) {
1368	if (page) {	1368	if (page) {
1369	f2fs_put_page(page, 1);	1369	f2fs_put_page(page, 1);
1370	page = NULL;	1370	page = NULL;
1371	}	1371	}
1372		1372
1373	start = START_SEGNO(sit_i, segno);	1373	start = START_SEGNO(sit_i, segno);
1374	end = start + SIT_ENTRY_PER_BLOCK - 1;	1374	end = start + SIT_ENTRY_PER_BLOCK - 1;
1375		1375
1376	/* read sit block that will be updated */	1376	/* read sit block that will be updated */
1377	page = get_next_sit_page(sbi, start);	1377	page = get_next_sit_page(sbi, start);
1378	raw_sit = page_address(page);	1378	raw_sit = page_address(page);
1379	}	1379	}
1380		1380
1381	/* udpate entry in SIT block */	1381	/* udpate entry in SIT block */
1382	seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]);	1382	seg_info_to_raw_sit(se, &raw_sit->entries[sit_offset]);
1383	flush_done:	1383	flush_done:
1384	__clear_bit(segno, bitmap);	1384	__clear_bit(segno, bitmap);
1385	sit_i->dirty_sentries--;	1385	sit_i->dirty_sentries--;
1386	}	1386	}
1387	mutex_unlock(&sit_i->sentry_lock);	1387	mutex_unlock(&sit_i->sentry_lock);
1388	mutex_unlock(&curseg->curseg_mutex);	1388	mutex_unlock(&curseg->curseg_mutex);
1389		1389
1390	/* writeout last modified SIT block */	1390	/* writeout last modified SIT block */
1391	f2fs_put_page(page, 1);	1391	f2fs_put_page(page, 1);
1392		1392
1393	set_prefree_as_free_segments(sbi);	1393	set_prefree_as_free_segments(sbi);
1394	}	1394	}
1395		1395
1396	static int build_sit_info(struct f2fs_sb_info *sbi)	1396	static int build_sit_info(struct f2fs_sb_info *sbi)
1397	{	1397	{
1398	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);	1398	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1399	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);	1399	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1400	struct sit_info *sit_i;	1400	struct sit_info *sit_i;
1401	unsigned int sit_segs, start;	1401	unsigned int sit_segs, start;
1402	char src_bitmap, dst_bitmap;	1402	char src_bitmap, dst_bitmap;
1403	unsigned int bitmap_size;	1403	unsigned int bitmap_size;
1404		1404
1405	/* allocate memory for SIT information */	1405	/* allocate memory for SIT information */
1406	sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);	1406	sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
1407	if (!sit_i)	1407	if (!sit_i)
1408	return -ENOMEM;	1408	return -ENOMEM;
1409		1409
1410	SM_I(sbi)->sit_info = sit_i;	1410	SM_I(sbi)->sit_info = sit_i;
1411		1411
1412	sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry));	1412	sit_i->sentries = vzalloc(TOTAL_SEGS(sbi) * sizeof(struct seg_entry));
1413	if (!sit_i->sentries)	1413	if (!sit_i->sentries)
1414	return -ENOMEM;	1414	return -ENOMEM;
1415		1415
1416	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));	1416	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1417	sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL);	1417	sit_i->dirty_sentries_bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1418	if (!sit_i->dirty_sentries_bitmap)	1418	if (!sit_i->dirty_sentries_bitmap)
1419	return -ENOMEM;	1419	return -ENOMEM;
1420		1420
1421	for (start = 0; start < TOTAL_SEGS(sbi); start++) {	1421	for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1422	sit_i->sentries[start].cur_valid_map	1422	sit_i->sentries[start].cur_valid_map
1423	= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);	1423	= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1424	sit_i->sentries[start].ckpt_valid_map	1424	sit_i->sentries[start].ckpt_valid_map
1425	= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);	1425	= kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
1426	if (!sit_i->sentries[start].cur_valid_map	1426	if (!sit_i->sentries[start].cur_valid_map
1427	\|\| !sit_i->sentries[start].ckpt_valid_map)	1427	\|\| !sit_i->sentries[start].ckpt_valid_map)
1428	return -ENOMEM;	1428	return -ENOMEM;
1429	}	1429	}
1430		1430
1431	if (sbi->segs_per_sec > 1) {	1431	if (sbi->segs_per_sec > 1) {
1432	sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) *	1432	sit_i->sec_entries = vzalloc(TOTAL_SECS(sbi) *
1433	sizeof(struct sec_entry));	1433	sizeof(struct sec_entry));
1434	if (!sit_i->sec_entries)	1434	if (!sit_i->sec_entries)
1435	return -ENOMEM;	1435	return -ENOMEM;
1436	}	1436	}
1437		1437
1438	/* get information related with SIT */	1438	/* get information related with SIT */
1439	sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;	1439	sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
1440		1440
1441	/* setup SIT bitmap from ckeckpoint pack */	1441	/* setup SIT bitmap from ckeckpoint pack */
1442	bitmap_size = __bitmap_size(sbi, SIT_BITMAP);	1442	bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
1443	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);	1443	src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
1444		1444
1445	dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);	1445	dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
1446	if (!dst_bitmap)	1446	if (!dst_bitmap)
1447	return -ENOMEM;	1447	return -ENOMEM;
1448		1448
1449	/* init SIT information */	1449	/* init SIT information */
1450	sit_i->s_ops = &default_salloc_ops;	1450	sit_i->s_ops = &default_salloc_ops;
1451		1451
1452	sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);	1452	sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
1453	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;	1453	sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
1454	sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);	1454	sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
1455	sit_i->sit_bitmap = dst_bitmap;	1455	sit_i->sit_bitmap = dst_bitmap;
1456	sit_i->bitmap_size = bitmap_size;	1456	sit_i->bitmap_size = bitmap_size;
1457	sit_i->dirty_sentries = 0;	1457	sit_i->dirty_sentries = 0;
1458	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;	1458	sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
1459	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);	1459	sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
1460	sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;	1460	sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
1461	mutex_init(&sit_i->sentry_lock);	1461	mutex_init(&sit_i->sentry_lock);
1462	return 0;	1462	return 0;
1463	}	1463	}
1464		1464
1465	static int build_free_segmap(struct f2fs_sb_info *sbi)	1465	static int build_free_segmap(struct f2fs_sb_info *sbi)
1466	{	1466	{
1467	struct f2fs_sm_info *sm_info = SM_I(sbi);	1467	struct f2fs_sm_info *sm_info = SM_I(sbi);
1468	struct free_segmap_info *free_i;	1468	struct free_segmap_info *free_i;
1469	unsigned int bitmap_size, sec_bitmap_size;	1469	unsigned int bitmap_size, sec_bitmap_size;
1470		1470
1471	/* allocate memory for free segmap information */	1471	/* allocate memory for free segmap information */
1472	free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);	1472	free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
1473	if (!free_i)	1473	if (!free_i)
1474	return -ENOMEM;	1474	return -ENOMEM;
1475		1475
1476	SM_I(sbi)->free_info = free_i;	1476	SM_I(sbi)->free_info = free_i;
1477		1477
1478	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));	1478	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1479	free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL);	1479	free_i->free_segmap = kmalloc(bitmap_size, GFP_KERNEL);
1480	if (!free_i->free_segmap)	1480	if (!free_i->free_segmap)
1481	return -ENOMEM;	1481	return -ENOMEM;
1482		1482
1483	sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));	1483	sec_bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1484	free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL);	1484	free_i->free_secmap = kmalloc(sec_bitmap_size, GFP_KERNEL);
1485	if (!free_i->free_secmap)	1485	if (!free_i->free_secmap)
1486	return -ENOMEM;	1486	return -ENOMEM;
1487		1487
1488	/* set all segments as dirty temporarily */	1488	/* set all segments as dirty temporarily */
1489	memset(free_i->free_segmap, 0xff, bitmap_size);	1489	memset(free_i->free_segmap, 0xff, bitmap_size);
1490	memset(free_i->free_secmap, 0xff, sec_bitmap_size);	1490	memset(free_i->free_secmap, 0xff, sec_bitmap_size);
1491		1491
1492	/* init free segmap information */	1492	/* init free segmap information */
1493	free_i->start_segno =	1493	free_i->start_segno =
1494	(unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr);	1494	(unsigned int) GET_SEGNO_FROM_SEG0(sbi, sm_info->main_blkaddr);
1495	free_i->free_segments = 0;	1495	free_i->free_segments = 0;
1496	free_i->free_sections = 0;	1496	free_i->free_sections = 0;
1497	rwlock_init(&free_i->segmap_lock);	1497	rwlock_init(&free_i->segmap_lock);
1498	return 0;	1498	return 0;
1499	}	1499	}
1500		1500
1501	static int build_curseg(struct f2fs_sb_info *sbi)	1501	static int build_curseg(struct f2fs_sb_info *sbi)
1502	{	1502	{
1503	struct curseg_info *array;	1503	struct curseg_info *array;
1504	int i;	1504	int i;
1505		1505
1506	array = kzalloc(sizeof(array) NR_CURSEG_TYPE, GFP_KERNEL);	1506	array = kzalloc(sizeof(array) NR_CURSEG_TYPE, GFP_KERNEL);
1507	if (!array)	1507	if (!array)
1508	return -ENOMEM;	1508	return -ENOMEM;
1509		1509
1510	SM_I(sbi)->curseg_array = array;	1510	SM_I(sbi)->curseg_array = array;
1511		1511
1512	for (i = 0; i < NR_CURSEG_TYPE; i++) {	1512	for (i = 0; i < NR_CURSEG_TYPE; i++) {
1513	mutex_init(&array[i].curseg_mutex);	1513	mutex_init(&array[i].curseg_mutex);
1514	array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);	1514	array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
1515	if (!array[i].sum_blk)	1515	if (!array[i].sum_blk)
1516	return -ENOMEM;	1516	return -ENOMEM;
1517	array[i].segno = NULL_SEGNO;	1517	array[i].segno = NULL_SEGNO;
1518	array[i].next_blkoff = 0;	1518	array[i].next_blkoff = 0;
1519	}	1519	}
1520	return restore_curseg_summaries(sbi);	1520	return restore_curseg_summaries(sbi);
1521	}	1521	}
1522		1522
1523	static void build_sit_entries(struct f2fs_sb_info *sbi)	1523	static void build_sit_entries(struct f2fs_sb_info *sbi)
1524	{	1524	{
1525	struct sit_info *sit_i = SIT_I(sbi);	1525	struct sit_info *sit_i = SIT_I(sbi);
1526	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);	1526	struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1527	struct f2fs_summary_block *sum = curseg->sum_blk;	1527	struct f2fs_summary_block *sum = curseg->sum_blk;
1528	unsigned int start;	1528	unsigned int start;
1529		1529
1530	for (start = 0; start < TOTAL_SEGS(sbi); start++) {	1530	for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1531	struct seg_entry *se = &sit_i->sentries[start];	1531	struct seg_entry *se = &sit_i->sentries[start];
1532	struct f2fs_sit_block *sit_blk;	1532	struct f2fs_sit_block *sit_blk;
1533	struct f2fs_sit_entry sit;	1533	struct f2fs_sit_entry sit;
1534	struct page *page;	1534	struct page *page;
1535	int i;	1535	int i;
1536		1536
1537	mutex_lock(&curseg->curseg_mutex);	1537	mutex_lock(&curseg->curseg_mutex);
1538	for (i = 0; i < sits_in_cursum(sum); i++) {	1538	for (i = 0; i < sits_in_cursum(sum); i++) {
1539	if (le32_to_cpu(segno_in_journal(sum, i)) == start) {	1539	if (le32_to_cpu(segno_in_journal(sum, i)) == start) {
1540	sit = sit_in_journal(sum, i);	1540	sit = sit_in_journal(sum, i);
1541	mutex_unlock(&curseg->curseg_mutex);	1541	mutex_unlock(&curseg->curseg_mutex);
1542	goto got_it;	1542	goto got_it;
1543	}	1543	}
1544	}	1544	}
1545	mutex_unlock(&curseg->curseg_mutex);	1545	mutex_unlock(&curseg->curseg_mutex);
1546	page = get_current_sit_page(sbi, start);	1546	page = get_current_sit_page(sbi, start);
1547	sit_blk = (struct f2fs_sit_block *)page_address(page);	1547	sit_blk = (struct f2fs_sit_block *)page_address(page);
1548	sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];	1548	sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
1549	f2fs_put_page(page, 1);	1549	f2fs_put_page(page, 1);
1550	got_it:	1550	got_it:
1551	check_block_count(sbi, start, &sit);	1551	check_block_count(sbi, start, &sit);
1552	seg_info_from_raw_sit(se, &sit);	1552	seg_info_from_raw_sit(se, &sit);
1553	if (sbi->segs_per_sec > 1) {	1553	if (sbi->segs_per_sec > 1) {
1554	struct sec_entry *e = get_sec_entry(sbi, start);	1554	struct sec_entry *e = get_sec_entry(sbi, start);
1555	e->valid_blocks += se->valid_blocks;	1555	e->valid_blocks += se->valid_blocks;
1556	}	1556	}
1557	}	1557	}
1558	}	1558	}
1559		1559
1560	static void init_free_segmap(struct f2fs_sb_info *sbi)	1560	static void init_free_segmap(struct f2fs_sb_info *sbi)
1561	{	1561	{
1562	unsigned int start;	1562	unsigned int start;
1563	int type;	1563	int type;
1564		1564
1565	for (start = 0; start < TOTAL_SEGS(sbi); start++) {	1565	for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1566	struct seg_entry *sentry = get_seg_entry(sbi, start);	1566	struct seg_entry *sentry = get_seg_entry(sbi, start);
1567	if (!sentry->valid_blocks)	1567	if (!sentry->valid_blocks)
1568	__set_free(sbi, start);	1568	__set_free(sbi, start);
1569	}	1569	}
1570		1570
1571	/* set use the current segments */	1571	/* set use the current segments */
1572	for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {	1572	for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
1573	struct curseg_info *curseg_t = CURSEG_I(sbi, type);	1573	struct curseg_info *curseg_t = CURSEG_I(sbi, type);
1574	__set_test_and_inuse(sbi, curseg_t->segno);	1574	__set_test_and_inuse(sbi, curseg_t->segno);
1575	}	1575	}
1576	}	1576	}
1577		1577
1578	static void init_dirty_segmap(struct f2fs_sb_info *sbi)	1578	static void init_dirty_segmap(struct f2fs_sb_info *sbi)
1579	{	1579	{
1580	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	1580	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1581	struct free_segmap_info *free_i = FREE_I(sbi);	1581	struct free_segmap_info *free_i = FREE_I(sbi);
1582	unsigned int segno = 0, offset = 0;	1582	unsigned int segno = 0, offset = 0;
1583	unsigned short valid_blocks;	1583	unsigned short valid_blocks;
1584		1584
1585	while (segno < TOTAL_SEGS(sbi)) {	1585	while (segno < TOTAL_SEGS(sbi)) {
1586	/* find dirty segment based on free segmap */	1586	/* find dirty segment based on free segmap */
1587	segno = find_next_inuse(free_i, TOTAL_SEGS(sbi), offset);	1587	segno = find_next_inuse(free_i, TOTAL_SEGS(sbi), offset);
1588	if (segno >= TOTAL_SEGS(sbi))	1588	if (segno >= TOTAL_SEGS(sbi))
1589	break;	1589	break;
1590	offset = segno + 1;	1590	offset = segno + 1;
1591	valid_blocks = get_valid_blocks(sbi, segno, 0);	1591	valid_blocks = get_valid_blocks(sbi, segno, 0);
1592	if (valid_blocks >= sbi->blocks_per_seg \|\| !valid_blocks)	1592	if (valid_blocks >= sbi->blocks_per_seg \|\| !valid_blocks)
1593	continue;	1593	continue;
1594	mutex_lock(&dirty_i->seglist_lock);	1594	mutex_lock(&dirty_i->seglist_lock);
1595	__locate_dirty_segment(sbi, segno, DIRTY);	1595	__locate_dirty_segment(sbi, segno, DIRTY);
1596	mutex_unlock(&dirty_i->seglist_lock);	1596	mutex_unlock(&dirty_i->seglist_lock);
1597	}	1597	}
1598	}	1598	}
1599		1599
1600	static int init_victim_secmap(struct f2fs_sb_info *sbi)	1600	static int init_victim_secmap(struct f2fs_sb_info *sbi)
1601	{	1601	{
1602	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	1602	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1603	unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));	1603	unsigned int bitmap_size = f2fs_bitmap_size(TOTAL_SECS(sbi));
1604		1604
1605	dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL);	1605	dirty_i->victim_secmap = kzalloc(bitmap_size, GFP_KERNEL);
1606	if (!dirty_i->victim_secmap)	1606	if (!dirty_i->victim_secmap)
1607	return -ENOMEM;	1607	return -ENOMEM;
1608	return 0;	1608	return 0;
1609	}	1609	}
1610		1610
1611	static int build_dirty_segmap(struct f2fs_sb_info *sbi)	1611	static int build_dirty_segmap(struct f2fs_sb_info *sbi)
1612	{	1612	{
1613	struct dirty_seglist_info *dirty_i;	1613	struct dirty_seglist_info *dirty_i;
1614	unsigned int bitmap_size, i;	1614	unsigned int bitmap_size, i;
1615		1615
1616	/* allocate memory for dirty segments list information */	1616	/* allocate memory for dirty segments list information */
1617	dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);	1617	dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
1618	if (!dirty_i)	1618	if (!dirty_i)
1619	return -ENOMEM;	1619	return -ENOMEM;
1620		1620
1621	SM_I(sbi)->dirty_info = dirty_i;	1621	SM_I(sbi)->dirty_info = dirty_i;
1622	mutex_init(&dirty_i->seglist_lock);	1622	mutex_init(&dirty_i->seglist_lock);
1623		1623
1624	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));	1624	bitmap_size = f2fs_bitmap_size(TOTAL_SEGS(sbi));
1625		1625
1626	for (i = 0; i < NR_DIRTY_TYPE; i++) {	1626	for (i = 0; i < NR_DIRTY_TYPE; i++) {
1627	dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL);	1627	dirty_i->dirty_segmap[i] = kzalloc(bitmap_size, GFP_KERNEL);
1628	if (!dirty_i->dirty_segmap[i])	1628	if (!dirty_i->dirty_segmap[i])
1629	return -ENOMEM;	1629	return -ENOMEM;
1630	}	1630	}
1631		1631
1632	init_dirty_segmap(sbi);	1632	init_dirty_segmap(sbi);
1633	return init_victim_secmap(sbi);	1633	return init_victim_secmap(sbi);
1634	}	1634	}
1635		1635
1636	/*	1636	/*
1637	* Update min, max modified time for cost-benefit GC algorithm	1637	* Update min, max modified time for cost-benefit GC algorithm
1638	*/	1638	*/
1639	static void init_min_max_mtime(struct f2fs_sb_info *sbi)	1639	static void init_min_max_mtime(struct f2fs_sb_info *sbi)
1640	{	1640	{
1641	struct sit_info *sit_i = SIT_I(sbi);	1641	struct sit_info *sit_i = SIT_I(sbi);
1642	unsigned int segno;	1642	unsigned int segno;
1643		1643
1644	mutex_lock(&sit_i->sentry_lock);	1644	mutex_lock(&sit_i->sentry_lock);
1645		1645
1646	sit_i->min_mtime = LLONG_MAX;	1646	sit_i->min_mtime = LLONG_MAX;
1647		1647
1648	for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) {	1648	for (segno = 0; segno < TOTAL_SEGS(sbi); segno += sbi->segs_per_sec) {
1649	unsigned int i;	1649	unsigned int i;
1650	unsigned long long mtime = 0;	1650	unsigned long long mtime = 0;
1651		1651
1652	for (i = 0; i < sbi->segs_per_sec; i++)	1652	for (i = 0; i < sbi->segs_per_sec; i++)
1653	mtime += get_seg_entry(sbi, segno + i)->mtime;	1653	mtime += get_seg_entry(sbi, segno + i)->mtime;
1654		1654
1655	mtime = div_u64(mtime, sbi->segs_per_sec);	1655	mtime = div_u64(mtime, sbi->segs_per_sec);
1656		1656
1657	if (sit_i->min_mtime > mtime)	1657	if (sit_i->min_mtime > mtime)
1658	sit_i->min_mtime = mtime;	1658	sit_i->min_mtime = mtime;
1659	}	1659	}
1660	sit_i->max_mtime = get_mtime(sbi);	1660	sit_i->max_mtime = get_mtime(sbi);
1661	mutex_unlock(&sit_i->sentry_lock);	1661	mutex_unlock(&sit_i->sentry_lock);
1662	}	1662	}
1663		1663
1664	int build_segment_manager(struct f2fs_sb_info *sbi)	1664	int build_segment_manager(struct f2fs_sb_info *sbi)
1665	{	1665	{
1666	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);	1666	struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
1667	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);	1667	struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1668	struct f2fs_sm_info *sm_info;	1668	struct f2fs_sm_info *sm_info;
1669	int err;	1669	int err;
1670		1670
1671	sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);	1671	sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
1672	if (!sm_info)	1672	if (!sm_info)
1673	return -ENOMEM;	1673	return -ENOMEM;
1674		1674
1675	/* init sm info */	1675	/* init sm info */
1676	sbi->sm_info = sm_info;	1676	sbi->sm_info = sm_info;
1677	INIT_LIST_HEAD(&sm_info->wblist_head);	1677	INIT_LIST_HEAD(&sm_info->wblist_head);
1678	spin_lock_init(&sm_info->wblist_lock);	1678	spin_lock_init(&sm_info->wblist_lock);
1679	sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);	1679	sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
1680	sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);	1680	sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
1681	sm_info->segment_count = le32_to_cpu(raw_super->segment_count);	1681	sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
1682	sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);	1682	sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
1683	sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);	1683	sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
1684	sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);	1684	sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
1685	sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);	1685	sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
1686		1686
1687	err = build_sit_info(sbi);	1687	err = build_sit_info(sbi);
1688	if (err)	1688	if (err)
1689	return err;	1689	return err;
1690	err = build_free_segmap(sbi);	1690	err = build_free_segmap(sbi);
1691	if (err)	1691	if (err)
1692	return err;	1692	return err;
1693	err = build_curseg(sbi);	1693	err = build_curseg(sbi);
1694	if (err)	1694	if (err)
1695	return err;	1695	return err;
1696		1696
1697	/* reinit free segmap based on SIT */	1697	/* reinit free segmap based on SIT */
1698	build_sit_entries(sbi);	1698	build_sit_entries(sbi);
1699		1699
1700	init_free_segmap(sbi);	1700	init_free_segmap(sbi);
1701	err = build_dirty_segmap(sbi);	1701	err = build_dirty_segmap(sbi);
1702	if (err)	1702	if (err)
1703	return err;	1703	return err;
1704		1704
1705	init_min_max_mtime(sbi);	1705	init_min_max_mtime(sbi);
1706	return 0;	1706	return 0;
1707	}	1707	}
1708		1708
1709	static void discard_dirty_segmap(struct f2fs_sb_info *sbi,	1709	static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
1710	enum dirty_type dirty_type)	1710	enum dirty_type dirty_type)
1711	{	1711	{
1712	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	1712	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1713		1713
1714	mutex_lock(&dirty_i->seglist_lock);	1714	mutex_lock(&dirty_i->seglist_lock);
1715	kfree(dirty_i->dirty_segmap[dirty_type]);	1715	kfree(dirty_i->dirty_segmap[dirty_type]);
1716	dirty_i->nr_dirty[dirty_type] = 0;	1716	dirty_i->nr_dirty[dirty_type] = 0;
1717	mutex_unlock(&dirty_i->seglist_lock);	1717	mutex_unlock(&dirty_i->seglist_lock);
1718	}	1718	}
1719		1719
1720	static void destroy_victim_secmap(struct f2fs_sb_info *sbi)	1720	static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
1721	{	1721	{
1722	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	1722	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1723	kfree(dirty_i->victim_secmap);	1723	kfree(dirty_i->victim_secmap);
1724	}	1724	}
1725		1725
1726	static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)	1726	static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
1727	{	1727	{
1728	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);	1728	struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1729	int i;	1729	int i;
1730		1730
1731	if (!dirty_i)	1731	if (!dirty_i)
1732	return;	1732	return;
1733		1733
1734	/* discard pre-free/dirty segments list */	1734	/* discard pre-free/dirty segments list */
1735	for (i = 0; i < NR_DIRTY_TYPE; i++)	1735	for (i = 0; i < NR_DIRTY_TYPE; i++)
1736	discard_dirty_segmap(sbi, i);	1736	discard_dirty_segmap(sbi, i);
1737		1737
1738	destroy_victim_secmap(sbi);	1738	destroy_victim_secmap(sbi);
1739	SM_I(sbi)->dirty_info = NULL;	1739	SM_I(sbi)->dirty_info = NULL;
1740	kfree(dirty_i);	1740	kfree(dirty_i);
1741	}	1741	}
1742		1742
1743	static void destroy_curseg(struct f2fs_sb_info *sbi)	1743	static void destroy_curseg(struct f2fs_sb_info *sbi)
1744	{	1744	{
1745	struct curseg_info *array = SM_I(sbi)->curseg_array;	1745	struct curseg_info *array = SM_I(sbi)->curseg_array;
1746	int i;	1746	int i;
1747		1747
1748	if (!array)	1748	if (!array)
1749	return;	1749	return;
1750	SM_I(sbi)->curseg_array = NULL;	1750	SM_I(sbi)->curseg_array = NULL;
1751	for (i = 0; i < NR_CURSEG_TYPE; i++)	1751	for (i = 0; i < NR_CURSEG_TYPE; i++)
1752	kfree(array[i].sum_blk);	1752	kfree(array[i].sum_blk);
1753	kfree(array);	1753	kfree(array);
1754	}	1754	}
1755		1755
1756	static void destroy_free_segmap(struct f2fs_sb_info *sbi)	1756	static void destroy_free_segmap(struct f2fs_sb_info *sbi)
1757	{	1757	{
1758	struct free_segmap_info *free_i = SM_I(sbi)->free_info;	1758	struct free_segmap_info *free_i = SM_I(sbi)->free_info;
1759	if (!free_i)	1759	if (!free_i)
1760	return;	1760	return;
1761	SM_I(sbi)->free_info = NULL;	1761	SM_I(sbi)->free_info = NULL;
1762	kfree(free_i->free_segmap);	1762	kfree(free_i->free_segmap);
1763	kfree(free_i->free_secmap);	1763	kfree(free_i->free_secmap);
1764	kfree(free_i);	1764	kfree(free_i);
1765	}	1765	}
1766		1766
1767	static void destroy_sit_info(struct f2fs_sb_info *sbi)	1767	static void destroy_sit_info(struct f2fs_sb_info *sbi)
1768	{	1768	{
1769	struct sit_info *sit_i = SIT_I(sbi);	1769	struct sit_info *sit_i = SIT_I(sbi);
1770	unsigned int start;	1770	unsigned int start;
1771		1771
1772	if (!sit_i)	1772	if (!sit_i)
1773	return;	1773	return;
1774		1774
1775	if (sit_i->sentries) {	1775	if (sit_i->sentries) {
1776	for (start = 0; start < TOTAL_SEGS(sbi); start++) {	1776	for (start = 0; start < TOTAL_SEGS(sbi); start++) {
1777	kfree(sit_i->sentries[start].cur_valid_map);	1777	kfree(sit_i->sentries[start].cur_valid_map);
1778	kfree(sit_i->sentries[start].ckpt_valid_map);	1778	kfree(sit_i->sentries[start].ckpt_valid_map);
1779	}	1779	}
1780	}	1780	}
1781	vfree(sit_i->sentries);	1781	vfree(sit_i->sentries);
1782	vfree(sit_i->sec_entries);	1782	vfree(sit_i->sec_entries);
1783	kfree(sit_i->dirty_sentries_bitmap);	1783	kfree(sit_i->dirty_sentries_bitmap);
1784		1784
1785	SM_I(sbi)->sit_info = NULL;	1785	SM_I(sbi)->sit_info = NULL;
1786	kfree(sit_i->sit_bitmap);	1786	kfree(sit_i->sit_bitmap);
1787	kfree(sit_i);	1787	kfree(sit_i);
1788	}	1788	}
1789		1789
1790	void destroy_segment_manager(struct f2fs_sb_info *sbi)	1790	void destroy_segment_manager(struct f2fs_sb_info *sbi)
1791	{	1791	{
1792	struct f2fs_sm_info *sm_info = SM_I(sbi);	1792	struct f2fs_sm_info *sm_info = SM_I(sbi);
1793	destroy_dirty_segmap(sbi);	1793	destroy_dirty_segmap(sbi);
1794	destroy_curseg(sbi);	1794	destroy_curseg(sbi);
1795	destroy_free_segmap(sbi);	1795	destroy_free_segmap(sbi);
1796	destroy_sit_info(sbi);	1796	destroy_sit_info(sbi);
1797	sbi->sm_info = NULL;	1797	sbi->sm_info = NULL;
1798	kfree(sm_info);	1798	kfree(sm_info);
1799	}	1799	}
1800		1800

fs/f2fs/segment.h

Diff comments View file @ ac5d156

 /*
  * fs/f2fs/segment.h
  *
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *             http://www.samsung.com/
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
+#include <linux/blkdev.h>
 /* constant macro */
 #define NULL_SEGNO			((unsigned int)(~0))
 #define NULL_SECNO			((unsigned int)(~0))
 /* L: Logical segment # in volume, R: Relative segment # in main area */
 #define GET_L2R_SEGNO(free_i, segno)	(segno - free_i->start_segno)
 #define GET_R2L_SEGNO(free_i, segno)	(segno + free_i->start_segno)
 #define IS_DATASEG(t)							\
 	((t == CURSEG_HOT_DATA) || (t == CURSEG_COLD_DATA) ||		\
 	(t == CURSEG_WARM_DATA))
 #define IS_NODESEG(t)							\
 	((t == CURSEG_HOT_NODE) || (t == CURSEG_COLD_NODE) ||		\
 	(t == CURSEG_WARM_NODE))
 #define IS_CURSEG(sbi, seg)						\
 	((seg == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno) ||	\
 	 (seg == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno) ||	\
 	 (seg == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno) ||	\
 	 (seg == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno) ||	\
 	 (seg == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno) ||	\
 	 (seg == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno))
 #define IS_CURSEC(sbi, secno)						\
 	((secno == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno /		\
 	  sbi->segs_per_sec) ||	\
 	 (secno == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno /		\
 	  sbi->segs_per_sec) ||	\
 	 (secno == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno /		\
 	  sbi->segs_per_sec) ||	\
 	 (secno == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno /		\
 	  sbi->segs_per_sec) ||	\
 	 (secno == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno /		\
 	  sbi->segs_per_sec) ||	\
 	 (secno == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno /		\
 	  sbi->segs_per_sec))	\
 #define START_BLOCK(sbi, segno)						\
 	(SM_I(sbi)->seg0_blkaddr +					\
 	 (GET_R2L_SEGNO(FREE_I(sbi), segno) << sbi->log_blocks_per_seg))
 #define NEXT_FREE_BLKADDR(sbi, curseg)					\
 	(START_BLOCK(sbi, curseg->segno) + curseg->next_blkoff)
 #define MAIN_BASE_BLOCK(sbi)	(SM_I(sbi)->main_blkaddr)
 #define GET_SEGOFF_FROM_SEG0(sbi, blk_addr)				\
 	((blk_addr) - SM_I(sbi)->seg0_blkaddr)
 #define GET_SEGNO_FROM_SEG0(sbi, blk_addr)				\
 	(GET_SEGOFF_FROM_SEG0(sbi, blk_addr) >> sbi->log_blocks_per_seg)
 #define GET_SEGNO(sbi, blk_addr)					\
 	(((blk_addr == NULL_ADDR) || (blk_addr == NEW_ADDR)) ?		\
 	NULL_SEGNO : GET_L2R_SEGNO(FREE_I(sbi),			\
 		GET_SEGNO_FROM_SEG0(sbi, blk_addr)))
 #define GET_SECNO(sbi, segno)					\
 	((segno) / sbi->segs_per_sec)
 #define GET_ZONENO_FROM_SEGNO(sbi, segno)				\
 	((segno / sbi->segs_per_sec) / sbi->secs_per_zone)
 #define GET_SUM_BLOCK(sbi, segno)				\
 	((sbi->sm_info->ssa_blkaddr) + segno)
 #define GET_SUM_TYPE(footer) ((footer)->entry_type)
 #define SET_SUM_TYPE(footer, type) ((footer)->entry_type = type)
 #define SIT_ENTRY_OFFSET(sit_i, segno)					\
 	(segno % sit_i->sents_per_block)
 #define SIT_BLOCK_OFFSET(sit_i, segno)					\
 	(segno / SIT_ENTRY_PER_BLOCK)
 #define	START_SEGNO(sit_i, segno)		\
 	(SIT_BLOCK_OFFSET(sit_i, segno) * SIT_ENTRY_PER_BLOCK)
 #define f2fs_bitmap_size(nr)			\
 	(BITS_TO_LONGS(nr) * sizeof(unsigned long))
 #define TOTAL_SEGS(sbi)	(SM_I(sbi)->main_segments)
 #define TOTAL_SECS(sbi)	(sbi->total_sections)
 #define SECTOR_FROM_BLOCK(sbi, blk_addr)				\
 	(blk_addr << ((sbi)->log_blocksize - F2FS_LOG_SECTOR_SIZE))
+#define SECTOR_TO_BLOCK(sbi, sectors)					\
+	(sectors >> ((sbi)->log_blocksize - F2FS_LOG_SECTOR_SIZE))
 /* during checkpoint, bio_private is used to synchronize the last bio */
 struct bio_private {
 	struct f2fs_sb_info *sbi;
 	bool is_sync;
 	void *wait;
 };
 /*
  * indicate a block allocation direction: RIGHT and LEFT.
  * RIGHT means allocating new sections towards the end of volume.
  * LEFT means the opposite direction.
  */
 enum {
 	ALLOC_RIGHT = 0,
 	ALLOC_LEFT
 };
 /*
  * In the victim_sel_policy->alloc_mode, there are two block allocation modes.
  * LFS writes data sequentially with cleaning operations.
  * SSR (Slack Space Recycle) reuses obsolete space without cleaning operations.
  */
 enum {
 	LFS = 0,
 	SSR
 };
 /*
  * In the victim_sel_policy->gc_mode, there are two gc, aka cleaning, modes.
  * GC_CB is based on cost-benefit algorithm.
  * GC_GREEDY is based on greedy algorithm.
  */
 enum {
 	GC_CB = 0,
 	GC_GREEDY
 };
 /*
  * BG_GC means the background cleaning job.
  * FG_GC means the on-demand cleaning job.
  */
 enum {
 	BG_GC = 0,
 	FG_GC
 };
 /* for a function parameter to select a victim segment */
 struct victim_sel_policy {
 	int alloc_mode;			/* LFS or SSR */
 	int gc_mode;			/* GC_CB or GC_GREEDY */
 	unsigned long *dirty_segmap;	/* dirty segment bitmap */
 	unsigned int offset;		/* last scanned bitmap offset */
 	unsigned int ofs_unit;		/* bitmap search unit */
 	unsigned int min_cost;		/* minimum cost */
 	unsigned int min_segno;		/* segment # having min. cost */
 };
 struct seg_entry {
 	unsigned short valid_blocks;	/* # of valid blocks */
 	unsigned char *cur_valid_map;	/* validity bitmap of blocks */
 	/*
 	 * # of valid blocks and the validity bitmap stored in the the last
 	 * checkpoint pack. This information is used by the SSR mode.
 	 */
 	unsigned short ckpt_valid_blocks;
 	unsigned char *ckpt_valid_map;
 	unsigned char type;		/* segment type like CURSEG_XXX_TYPE */
 	unsigned long long mtime;	/* modification time of the segment */
 };
 struct sec_entry {
 	unsigned int valid_blocks;	/* # of valid blocks in a section */
 };
 struct segment_allocation {
 	void (*allocate_segment)(struct f2fs_sb_info *, int, bool);
 };
 struct sit_info {
 	const struct segment_allocation *s_ops;
 	block_t sit_base_addr;		/* start block address of SIT area */
 	block_t sit_blocks;		/* # of blocks used by SIT area */
 	block_t written_valid_blocks;	/* # of valid blocks in main area */
 	char *sit_bitmap;		/* SIT bitmap pointer */
 	unsigned int bitmap_size;	/* SIT bitmap size */
 	unsigned long *dirty_sentries_bitmap;	/* bitmap for dirty sentries */
 	unsigned int dirty_sentries;		/* # of dirty sentries */
 	unsigned int sents_per_block;		/* # of SIT entries per block */
 	struct mutex sentry_lock;		/* to protect SIT cache */
 	struct seg_entry *sentries;		/* SIT segment-level cache */
 	struct sec_entry *sec_entries;		/* SIT section-level cache */
 	/* for cost-benefit algorithm in cleaning procedure */
 	unsigned long long elapsed_time;	/* elapsed time after mount */
 	unsigned long long mounted_time;	/* mount time */
 	unsigned long long min_mtime;		/* min. modification time */
 	unsigned long long max_mtime;		/* max. modification time */
 };
 struct free_segmap_info {
 	unsigned int start_segno;	/* start segment number logically */
 	unsigned int free_segments;	/* # of free segments */
 	unsigned int free_sections;	/* # of free sections */
 	rwlock_t segmap_lock;		/* free segmap lock */
 	unsigned long *free_segmap;	/* free segment bitmap */
 	unsigned long *free_secmap;	/* free section bitmap */
 };
 /* Notice: The order of dirty type is same with CURSEG_XXX in f2fs.h */
 enum dirty_type {
 	DIRTY_HOT_DATA,		/* dirty segments assigned as hot data logs */
 	DIRTY_WARM_DATA,	/* dirty segments assigned as warm data logs */
 	DIRTY_COLD_DATA,	/* dirty segments assigned as cold data logs */
 	DIRTY_HOT_NODE,		/* dirty segments assigned as hot node logs */
 	DIRTY_WARM_NODE,	/* dirty segments assigned as warm node logs */
 	DIRTY_COLD_NODE,	/* dirty segments assigned as cold node logs */
 	DIRTY,			/* to count # of dirty segments */
 	PRE,			/* to count # of entirely obsolete segments */
 	NR_DIRTY_TYPE
 };
 struct dirty_seglist_info {
 	const struct victim_selection *v_ops;	/* victim selction operation */
 	unsigned long *dirty_segmap[NR_DIRTY_TYPE];
 	struct mutex seglist_lock;		/* lock for segment bitmaps */
 	int nr_dirty[NR_DIRTY_TYPE];		/* # of dirty segments */
 	unsigned long *victim_secmap;		/* background GC victims */
 };
 /* victim selection function for cleaning and SSR */
 struct victim_selection {
 	int (*get_victim)(struct f2fs_sb_info *, unsigned int *,
 							int, int, char);
 };
 /* for active log information */
 struct curseg_info {
 	struct mutex curseg_mutex;		/* lock for consistency */
 	struct f2fs_summary_block *sum_blk;	/* cached summary block */
 	unsigned char alloc_type;		/* current allocation type */
 	unsigned int segno;			/* current segment number */
 	unsigned short next_blkoff;		/* next block offset to write */
 	unsigned int zone;			/* current zone number */
 	unsigned int next_segno;		/* preallocated segment */
 };
 /*
  * inline functions
  */
 static inline struct curseg_info *CURSEG_I(struct f2fs_sb_info *sbi, int type)
 {
 	return (struct curseg_info *)(SM_I(sbi)->curseg_array + type);
 }
 static inline struct seg_entry *get_seg_entry(struct f2fs_sb_info *sbi,
 						unsigned int segno)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	return &sit_i->sentries[segno];
 }
 static inline struct sec_entry *get_sec_entry(struct f2fs_sb_info *sbi,
 						unsigned int segno)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	return &sit_i->sec_entries[GET_SECNO(sbi, segno)];
 }
 static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi,
 				unsigned int segno, int section)
 {
 	/*
 	 * In order to get # of valid blocks in a section instantly from many
 	 * segments, f2fs manages two counting structures separately.
 	 */
 	if (section > 1)
 		return get_sec_entry(sbi, segno)->valid_blocks;
 	else
 		return get_seg_entry(sbi, segno)->valid_blocks;
 }
 static inline void seg_info_from_raw_sit(struct seg_entry *se,
 					struct f2fs_sit_entry *rs)
 {
 	se->valid_blocks = GET_SIT_VBLOCKS(rs);
 	se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs);
 	memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
 	memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
 	se->type = GET_SIT_TYPE(rs);
 	se->mtime = le64_to_cpu(rs->mtime);
 }
 static inline void seg_info_to_raw_sit(struct seg_entry *se,
 					struct f2fs_sit_entry *rs)
 {
 	unsigned short raw_vblocks = (se->type << SIT_VBLOCKS_SHIFT) |
 					se->valid_blocks;
 	rs->vblocks = cpu_to_le16(raw_vblocks);
 	memcpy(rs->valid_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
 	memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
 	se->ckpt_valid_blocks = se->valid_blocks;
 	rs->mtime = cpu_to_le64(se->mtime);
 }
 static inline unsigned int find_next_inuse(struct free_segmap_info *free_i,
 		unsigned int max, unsigned int segno)
 {
 	unsigned int ret;
 	read_lock(&free_i->segmap_lock);
 	ret = find_next_bit(free_i->free_segmap, max, segno);
 	read_unlock(&free_i->segmap_lock);
 	return ret;
 }
 static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int secno = segno / sbi->segs_per_sec;
 	unsigned int start_segno = secno * sbi->segs_per_sec;
 	unsigned int next;
 	write_lock(&free_i->segmap_lock);
 	clear_bit(segno, free_i->free_segmap);
 	free_i->free_segments++;
 	next = find_next_bit(free_i->free_segmap, TOTAL_SEGS(sbi), start_segno);
 	if (next >= start_segno + sbi->segs_per_sec) {
 		clear_bit(secno, free_i->free_secmap);
 		free_i->free_sections++;
 	}
 	write_unlock(&free_i->segmap_lock);
 }
 static inline void __set_inuse(struct f2fs_sb_info *sbi,
 		unsigned int segno)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int secno = segno / sbi->segs_per_sec;
 	set_bit(segno, free_i->free_segmap);
 	free_i->free_segments--;
 	if (!test_and_set_bit(secno, free_i->free_secmap))
 		free_i->free_sections--;
 }
 static inline void __set_test_and_free(struct f2fs_sb_info *sbi,
 		unsigned int segno)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int secno = segno / sbi->segs_per_sec;
 	unsigned int start_segno = secno * sbi->segs_per_sec;
 	unsigned int next;
 	write_lock(&free_i->segmap_lock);
 	if (test_and_clear_bit(segno, free_i->free_segmap)) {
 		free_i->free_segments++;
 		next = find_next_bit(free_i->free_segmap, TOTAL_SEGS(sbi),
 								start_segno);
 		if (next >= start_segno + sbi->segs_per_sec) {
 			if (test_and_clear_bit(secno, free_i->free_secmap))
 				free_i->free_sections++;
 		}
 	}
 	write_unlock(&free_i->segmap_lock);
 }
 static inline void __set_test_and_inuse(struct f2fs_sb_info *sbi,
 		unsigned int segno)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int secno = segno / sbi->segs_per_sec;
 	write_lock(&free_i->segmap_lock);
 	if (!test_and_set_bit(segno, free_i->free_segmap)) {
 		free_i->free_segments--;
 		if (!test_and_set_bit(secno, free_i->free_secmap))
 			free_i->free_sections--;
 	}
 	write_unlock(&free_i->segmap_lock);
 }
 static inline void get_sit_bitmap(struct f2fs_sb_info *sbi,
 		void *dst_addr)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size);
 }
 static inline block_t written_block_count(struct f2fs_sb_info *sbi)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	block_t vblocks;
 	mutex_lock(&sit_i->sentry_lock);
 	vblocks = sit_i->written_valid_blocks;
 	mutex_unlock(&sit_i->sentry_lock);
 	return vblocks;
 }
 static inline unsigned int free_segments(struct f2fs_sb_info *sbi)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int free_segs;
 	read_lock(&free_i->segmap_lock);
 	free_segs = free_i->free_segments;
 	read_unlock(&free_i->segmap_lock);
 	return free_segs;
 }
 static inline int reserved_segments(struct f2fs_sb_info *sbi)
 {
 	return SM_I(sbi)->reserved_segments;
 }
 static inline unsigned int free_sections(struct f2fs_sb_info *sbi)
 {
 	struct free_segmap_info *free_i = FREE_I(sbi);
 	unsigned int free_secs;
 	read_lock(&free_i->segmap_lock);
 	free_secs = free_i->free_sections;
 	read_unlock(&free_i->segmap_lock);
 	return free_secs;
 }
 static inline unsigned int prefree_segments(struct f2fs_sb_info *sbi)
 {
 	return DIRTY_I(sbi)->nr_dirty[PRE];
 }
 static inline unsigned int dirty_segments(struct f2fs_sb_info *sbi)
 {
 	return DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_DATA] +
 		DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_DATA] +
 		DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_DATA] +
 		DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_NODE] +
 		DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_NODE] +
 		DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_NODE];
 }
 static inline int overprovision_segments(struct f2fs_sb_info *sbi)
 {
 	return SM_I(sbi)->ovp_segments;
 }
 static inline int overprovision_sections(struct f2fs_sb_info *sbi)
 {
 	return ((unsigned int) overprovision_segments(sbi)) / sbi->segs_per_sec;
 }
 static inline int reserved_sections(struct f2fs_sb_info *sbi)
 {
 	return ((unsigned int) reserved_segments(sbi)) / sbi->segs_per_sec;
 }
 static inline bool need_SSR(struct f2fs_sb_info *sbi)
 {
 	return (free_sections(sbi) < overprovision_sections(sbi));
 }
 static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, int freed)
 {
 	int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 	int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
 	if (sbi->por_doing)
 		return false;
 	return ((free_sections(sbi) + freed) <= (node_secs + 2 * dent_secs +
 						reserved_sections(sbi)));
 }
 static inline int utilization(struct f2fs_sb_info *sbi)
 {
 	return div_u64(valid_user_blocks(sbi) * 100, sbi->user_block_count);
 }
 /*
  * Sometimes f2fs may be better to drop out-of-place update policy.
  * So, if fs utilization is over MIN_IPU_UTIL, then f2fs tries to write
  * data in the original place likewise other traditional file systems.
  * But, currently set 100 in percentage, which means it is disabled.
  * See below need_inplace_update().
  */
 #define MIN_IPU_UTIL		100
 static inline bool need_inplace_update(struct inode *inode)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
 	if (S_ISDIR(inode->i_mode))
 		return false;
 	if (need_SSR(sbi) && utilization(sbi) > MIN_IPU_UTIL)
 		return true;
 	return false;
 }
 static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi,
 		int type)
 {
 	struct curseg_info *curseg = CURSEG_I(sbi, type);
 	return curseg->segno;
 }
 static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi,
 		int type)
 {
 	struct curseg_info *curseg = CURSEG_I(sbi, type);
 	return curseg->alloc_type;
 }
 static inline unsigned short curseg_blkoff(struct f2fs_sb_info *sbi, int type)
 {
 	struct curseg_info *curseg = CURSEG_I(sbi, type);
 	return curseg->next_blkoff;
 }
 static inline void check_seg_range(struct f2fs_sb_info *sbi, unsigned int segno)
 {
 	unsigned int end_segno = SM_I(sbi)->segment_count - 1;
 	BUG_ON(segno > end_segno);
 }
 /*
  * This function is used for only debugging.
  * NOTE: In future, we have to remove this function.
  */
 static inline void verify_block_addr(struct f2fs_sb_info *sbi, block_t blk_addr)
 {
 	struct f2fs_sm_info *sm_info = SM_I(sbi);
 	block_t total_blks = sm_info->segment_count << sbi->log_blocks_per_seg;
 	block_t start_addr = sm_info->seg0_blkaddr;
 	block_t end_addr = start_addr + total_blks - 1;
 	BUG_ON(blk_addr < start_addr);
 	BUG_ON(blk_addr > end_addr);
 }
 /*
  * Summary block is always treated as invalid block
  */
 static inline void check_block_count(struct f2fs_sb_info *sbi,
 		int segno, struct f2fs_sit_entry *raw_sit)
 {
 	struct f2fs_sm_info *sm_info = SM_I(sbi);
 	unsigned int end_segno = sm_info->segment_count - 1;
 	int valid_blocks = 0;
 	int i;
 	/* check segment usage */
 	BUG_ON(GET_SIT_VBLOCKS(raw_sit) > sbi->blocks_per_seg);
 	/* check boundary of a given segment number */
 	BUG_ON(segno > end_segno);
 	/* check bitmap with valid block count */
 	for (i = 0; i < sbi->blocks_per_seg; i++)
 		if (f2fs_test_bit(i, raw_sit->valid_map))
 			valid_blocks++;
 	BUG_ON(GET_SIT_VBLOCKS(raw_sit) != valid_blocks);
 }
 static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi,
 						unsigned int start)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	unsigned int offset = SIT_BLOCK_OFFSET(sit_i, start);
 	block_t blk_addr = sit_i->sit_base_addr + offset;
 	check_seg_range(sbi, start);
 	/* calculate sit block address */
 	if (f2fs_test_bit(offset, sit_i->sit_bitmap))
 		blk_addr += sit_i->sit_blocks;
 	return blk_addr;
 }
 static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi,
 						pgoff_t block_addr)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	block_addr -= sit_i->sit_base_addr;
 	if (block_addr < sit_i->sit_blocks)
 		block_addr += sit_i->sit_blocks;
 	else
 		block_addr -= sit_i->sit_blocks;
 	return block_addr + sit_i->sit_base_addr;
 }
 static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start)
 {
 	unsigned int block_off = SIT_BLOCK_OFFSET(sit_i, start);
 	if (f2fs_test_bit(block_off, sit_i->sit_bitmap))
 		f2fs_clear_bit(block_off, sit_i->sit_bitmap);
 	else
 		f2fs_set_bit(block_off, sit_i->sit_bitmap);
 }
 static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi)
 {
 	struct sit_info *sit_i = SIT_I(sbi);
 	return sit_i->elapsed_time + CURRENT_TIME_SEC.tv_sec -
 						sit_i->mounted_time;
 }
 static inline void set_summary(struct f2fs_summary *sum, nid_t nid,
 			unsigned int ofs_in_node, unsigned char version)
 {
 	sum->nid = cpu_to_le32(nid);
 	sum->ofs_in_node = cpu_to_le16(ofs_in_node);
 	sum->version = version;
 }
 static inline block_t start_sum_block(struct f2fs_sb_info *sbi)
 {
 	return __start_cp_addr(sbi) +
 		le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
 }
 static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type)
 {
 	return __start_cp_addr(sbi) +
 		le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_total_block_count)
 				- (base + 1) + type;
 }
 static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
 {
 	if (IS_CURSEC(sbi, secno) || (sbi->cur_victim_sec == secno))
 		return true;
 	return false;
+}
+static inline unsigned int max_hw_blocks(struct f2fs_sb_info *sbi)
+{
+	struct block_device *bdev = sbi->sb->s_bdev;
+	struct request_queue *q = bdev_get_queue(bdev);
+	return SECTOR_TO_BLOCK(sbi, queue_max_sectors(q));
 }