Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit a848c23fb1e59ddcde23b7715b08ad8bfef78ee0

Authored by Liu Bo
Committed by Greg Kroah-Hartman
1 parent bd7c57d3ca
Exists in ti-linux-3.14.y and in 2 other branches processor-sdk-linux-01.00.00, smarc-ti-linux-3.14.y

Btrfs: fix crash on endio of reading corrupted block 

commit 38c1c2e44bacb37efd68b90b3f70386a8ee370ee upstream.

The crash is

------------[ cut here ]------------
kernel BUG at fs/btrfs/extent_io.c:2124!
[...]
Workqueue: btrfs-endio normal_work_helper [btrfs]
RIP: 0010:[<ffffffffa02d6055>]  [<ffffffffa02d6055>] end_bio_extent_readpage+0xb45/0xcd0 [btrfs]

This is in fact a regression.

It is because we forgot to increase @offset properly in reading corrupted block,
so that the @offset remains, and this leads to checksum errors while reading
left blocks queued up in the same bio, and then ends up with hiting the above
BUG_ON.

Reported-by: Chris Murphy <lists@colorremedies.com>
Signed-off-by: Liu Bo <bo.li.liu@oracle.com>
Signed-off-by: Chris Mason <clm@fb.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

Showing 1 changed file with 1 additions and 0 deletions Inline Diff

fs/btrfs/extent_io.c
Diff comments   View file @ a848c23
1 #include <linux/bitops.h> 1 #include <linux/bitops.h>
2 #include <linux/slab.h> 2 #include <linux/slab.h>
3 #include <linux/bio.h> 3 #include <linux/bio.h>
4 #include <linux/mm.h> 4 #include <linux/mm.h>
5 #include <linux/pagemap.h> 5 #include <linux/pagemap.h>
6 #include <linux/page-flags.h> 6 #include <linux/page-flags.h>
7 #include <linux/spinlock.h> 7 #include <linux/spinlock.h>
8 #include <linux/blkdev.h> 8 #include <linux/blkdev.h>
9 #include <linux/swap.h> 9 #include <linux/swap.h>
10 #include <linux/writeback.h> 10 #include <linux/writeback.h>
11 #include <linux/pagevec.h> 11 #include <linux/pagevec.h>
12 #include <linux/prefetch.h> 12 #include <linux/prefetch.h>
13 #include <linux/cleancache.h> 13 #include <linux/cleancache.h>
14 #include "extent_io.h" 14 #include "extent_io.h"
15 #include "extent_map.h" 15 #include "extent_map.h"
16 #include "ctree.h" 16 #include "ctree.h"
17 #include "btrfs_inode.h" 17 #include "btrfs_inode.h"
18 #include "volumes.h" 18 #include "volumes.h"
19 #include "check-integrity.h" 19 #include "check-integrity.h"
20 #include "locking.h" 20 #include "locking.h"
21 #include "rcu-string.h" 21 #include "rcu-string.h"
22 #include "backref.h" 22 #include "backref.h"
23 23
24 static struct kmem_cache *extent_state_cache; 24 static struct kmem_cache *extent_state_cache;
25 static struct kmem_cache *extent_buffer_cache; 25 static struct kmem_cache *extent_buffer_cache;
26 static struct bio_set *btrfs_bioset; 26 static struct bio_set *btrfs_bioset;
27 27
28 #ifdef CONFIG_BTRFS_DEBUG 28 #ifdef CONFIG_BTRFS_DEBUG
29 static LIST_HEAD(buffers); 29 static LIST_HEAD(buffers);
30 static LIST_HEAD(states); 30 static LIST_HEAD(states);
31 31
32 static DEFINE_SPINLOCK(leak_lock); 32 static DEFINE_SPINLOCK(leak_lock);
33 33
34 static inline 34 static inline
35 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head) 35 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
36 { 36 {
37 unsigned long flags; 37 unsigned long flags;
38 38
39 spin_lock_irqsave(&leak_lock, flags); 39 spin_lock_irqsave(&leak_lock, flags);
40 list_add(new, head); 40 list_add(new, head);
41 spin_unlock_irqrestore(&leak_lock, flags); 41 spin_unlock_irqrestore(&leak_lock, flags);
42 } 42 }
43 43
44 static inline 44 static inline
45 void btrfs_leak_debug_del(struct list_head *entry) 45 void btrfs_leak_debug_del(struct list_head *entry)
46 { 46 {
47 unsigned long flags; 47 unsigned long flags;
48 48
49 spin_lock_irqsave(&leak_lock, flags); 49 spin_lock_irqsave(&leak_lock, flags);
50 list_del(entry); 50 list_del(entry);
51 spin_unlock_irqrestore(&leak_lock, flags); 51 spin_unlock_irqrestore(&leak_lock, flags);
52 } 52 }
53 53
54 static inline 54 static inline
55 void btrfs_leak_debug_check(void) 55 void btrfs_leak_debug_check(void)
56 { 56 {
57 struct extent_state *state; 57 struct extent_state *state;
58 struct extent_buffer *eb; 58 struct extent_buffer *eb;
59 59
60 while (!list_empty(&states)) { 60 while (!list_empty(&states)) {
61 state = list_entry(states.next, struct extent_state, leak_list); 61 state = list_entry(states.next, struct extent_state, leak_list);
62 printk(KERN_ERR "BTRFS: state leak: start %llu end %llu " 62 printk(KERN_ERR "BTRFS: state leak: start %llu end %llu "
63 "state %lu in tree %p refs %d\n", 63 "state %lu in tree %p refs %d\n",
64 state->start, state->end, state->state, state->tree, 64 state->start, state->end, state->state, state->tree,
65 atomic_read(&state->refs)); 65 atomic_read(&state->refs));
66 list_del(&state->leak_list); 66 list_del(&state->leak_list);
67 kmem_cache_free(extent_state_cache, state); 67 kmem_cache_free(extent_state_cache, state);
68 } 68 }
69 69
70 while (!list_empty(&buffers)) { 70 while (!list_empty(&buffers)) {
71 eb = list_entry(buffers.next, struct extent_buffer, leak_list); 71 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
72 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu " 72 printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
73 "refs %d\n", 73 "refs %d\n",
74 eb->start, eb->len, atomic_read(&eb->refs)); 74 eb->start, eb->len, atomic_read(&eb->refs));
75 list_del(&eb->leak_list); 75 list_del(&eb->leak_list);
76 kmem_cache_free(extent_buffer_cache, eb); 76 kmem_cache_free(extent_buffer_cache, eb);
77 } 77 }
78 } 78 }
79 79
80 #define btrfs_debug_check_extent_io_range(tree, start, end) \ 80 #define btrfs_debug_check_extent_io_range(tree, start, end) \
81 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end)) 81 __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
82 static inline void __btrfs_debug_check_extent_io_range(const char *caller, 82 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
83 struct extent_io_tree *tree, u64 start, u64 end) 83 struct extent_io_tree *tree, u64 start, u64 end)
84 { 84 {
85 struct inode *inode; 85 struct inode *inode;
86 u64 isize; 86 u64 isize;
87 87
88 if (!tree->mapping) 88 if (!tree->mapping)
89 return; 89 return;
90 90
91 inode = tree->mapping->host; 91 inode = tree->mapping->host;
92 isize = i_size_read(inode); 92 isize = i_size_read(inode);
93 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { 93 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
94 printk_ratelimited(KERN_DEBUG 94 printk_ratelimited(KERN_DEBUG
95 "BTRFS: %s: ino %llu isize %llu odd range [%llu,%llu]\n", 95 "BTRFS: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
96 caller, btrfs_ino(inode), isize, start, end); 96 caller, btrfs_ino(inode), isize, start, end);
97 } 97 }
98 } 98 }
99 #else 99 #else
100 #define btrfs_leak_debug_add(new, head) do {} while (0) 100 #define btrfs_leak_debug_add(new, head) do {} while (0)
101 #define btrfs_leak_debug_del(entry) do {} while (0) 101 #define btrfs_leak_debug_del(entry) do {} while (0)
102 #define btrfs_leak_debug_check() do {} while (0) 102 #define btrfs_leak_debug_check() do {} while (0)
103 #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0) 103 #define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
104 #endif 104 #endif
105 105
106 #define BUFFER_LRU_MAX 64 106 #define BUFFER_LRU_MAX 64
107 107
108 struct tree_entry { 108 struct tree_entry {
109 u64 start; 109 u64 start;
110 u64 end; 110 u64 end;
111 struct rb_node rb_node; 111 struct rb_node rb_node;
112 }; 112 };
113 113
114 struct extent_page_data { 114 struct extent_page_data {
115 struct bio *bio; 115 struct bio *bio;
116 struct extent_io_tree *tree; 116 struct extent_io_tree *tree;
117 get_extent_t *get_extent; 117 get_extent_t *get_extent;
118 unsigned long bio_flags; 118 unsigned long bio_flags;
119 119
120 /* tells writepage not to lock the state bits for this range 120 /* tells writepage not to lock the state bits for this range
121 * it still does the unlocking 121 * it still does the unlocking
122 */ 122 */
123 unsigned int extent_locked:1; 123 unsigned int extent_locked:1;
124 124
125 /* tells the submit_bio code to use a WRITE_SYNC */ 125 /* tells the submit_bio code to use a WRITE_SYNC */
126 unsigned int sync_io:1; 126 unsigned int sync_io:1;
127 }; 127 };
128 128
129 static noinline void flush_write_bio(void *data); 129 static noinline void flush_write_bio(void *data);
130 static inline struct btrfs_fs_info * 130 static inline struct btrfs_fs_info *
131 tree_fs_info(struct extent_io_tree *tree) 131 tree_fs_info(struct extent_io_tree *tree)
132 { 132 {
133 if (!tree->mapping) 133 if (!tree->mapping)
134 return NULL; 134 return NULL;
135 return btrfs_sb(tree->mapping->host->i_sb); 135 return btrfs_sb(tree->mapping->host->i_sb);
136 } 136 }
137 137
138 int __init extent_io_init(void) 138 int __init extent_io_init(void)
139 { 139 {
140 extent_state_cache = kmem_cache_create("btrfs_extent_state", 140 extent_state_cache = kmem_cache_create("btrfs_extent_state",
141 sizeof(struct extent_state), 0, 141 sizeof(struct extent_state), 0,
142 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 142 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
143 if (!extent_state_cache) 143 if (!extent_state_cache)
144 return -ENOMEM; 144 return -ENOMEM;
145 145
146 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer", 146 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
147 sizeof(struct extent_buffer), 0, 147 sizeof(struct extent_buffer), 0,
148 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL); 148 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
149 if (!extent_buffer_cache) 149 if (!extent_buffer_cache)
150 goto free_state_cache; 150 goto free_state_cache;
151 151
152 btrfs_bioset = bioset_create(BIO_POOL_SIZE, 152 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
153 offsetof(struct btrfs_io_bio, bio)); 153 offsetof(struct btrfs_io_bio, bio));
154 if (!btrfs_bioset) 154 if (!btrfs_bioset)
155 goto free_buffer_cache; 155 goto free_buffer_cache;
156 156
157 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE)) 157 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
158 goto free_bioset; 158 goto free_bioset;
159 159
160 return 0; 160 return 0;
161 161
162 free_bioset: 162 free_bioset:
163 bioset_free(btrfs_bioset); 163 bioset_free(btrfs_bioset);
164 btrfs_bioset = NULL; 164 btrfs_bioset = NULL;
165 165
166 free_buffer_cache: 166 free_buffer_cache:
167 kmem_cache_destroy(extent_buffer_cache); 167 kmem_cache_destroy(extent_buffer_cache);
168 extent_buffer_cache = NULL; 168 extent_buffer_cache = NULL;
169 169
170 free_state_cache: 170 free_state_cache:
171 kmem_cache_destroy(extent_state_cache); 171 kmem_cache_destroy(extent_state_cache);
172 extent_state_cache = NULL; 172 extent_state_cache = NULL;
173 return -ENOMEM; 173 return -ENOMEM;
174 } 174 }
175 175
176 void extent_io_exit(void) 176 void extent_io_exit(void)
177 { 177 {
178 btrfs_leak_debug_check(); 178 btrfs_leak_debug_check();
179 179
180 /* 180 /*
181 * Make sure all delayed rcu free are flushed before we 181 * Make sure all delayed rcu free are flushed before we
182 * destroy caches. 182 * destroy caches.
183 */ 183 */
184 rcu_barrier(); 184 rcu_barrier();
185 if (extent_state_cache) 185 if (extent_state_cache)
186 kmem_cache_destroy(extent_state_cache); 186 kmem_cache_destroy(extent_state_cache);
187 if (extent_buffer_cache) 187 if (extent_buffer_cache)
188 kmem_cache_destroy(extent_buffer_cache); 188 kmem_cache_destroy(extent_buffer_cache);
189 if (btrfs_bioset) 189 if (btrfs_bioset)
190 bioset_free(btrfs_bioset); 190 bioset_free(btrfs_bioset);
191 } 191 }
192 192
193 void extent_io_tree_init(struct extent_io_tree *tree, 193 void extent_io_tree_init(struct extent_io_tree *tree,
194 struct address_space *mapping) 194 struct address_space *mapping)
195 { 195 {
196 tree->state = RB_ROOT; 196 tree->state = RB_ROOT;
197 tree->ops = NULL; 197 tree->ops = NULL;
198 tree->dirty_bytes = 0; 198 tree->dirty_bytes = 0;
199 spin_lock_init(&tree->lock); 199 spin_lock_init(&tree->lock);
200 tree->mapping = mapping; 200 tree->mapping = mapping;
201 } 201 }
202 202
203 static struct extent_state *alloc_extent_state(gfp_t mask) 203 static struct extent_state *alloc_extent_state(gfp_t mask)
204 { 204 {
205 struct extent_state *state; 205 struct extent_state *state;
206 206
207 state = kmem_cache_alloc(extent_state_cache, mask); 207 state = kmem_cache_alloc(extent_state_cache, mask);
208 if (!state) 208 if (!state)
209 return state; 209 return state;
210 state->state = 0; 210 state->state = 0;
211 state->private = 0; 211 state->private = 0;
212 state->tree = NULL; 212 state->tree = NULL;
213 btrfs_leak_debug_add(&state->leak_list, &states); 213 btrfs_leak_debug_add(&state->leak_list, &states);
214 atomic_set(&state->refs, 1); 214 atomic_set(&state->refs, 1);
215 init_waitqueue_head(&state->wq); 215 init_waitqueue_head(&state->wq);
216 trace_alloc_extent_state(state, mask, _RET_IP_); 216 trace_alloc_extent_state(state, mask, _RET_IP_);
217 return state; 217 return state;
218 } 218 }
219 219
220 void free_extent_state(struct extent_state *state) 220 void free_extent_state(struct extent_state *state)
221 { 221 {
222 if (!state) 222 if (!state)
223 return; 223 return;
224 if (atomic_dec_and_test(&state->refs)) { 224 if (atomic_dec_and_test(&state->refs)) {
225 WARN_ON(state->tree); 225 WARN_ON(state->tree);
226 btrfs_leak_debug_del(&state->leak_list); 226 btrfs_leak_debug_del(&state->leak_list);
227 trace_free_extent_state(state, _RET_IP_); 227 trace_free_extent_state(state, _RET_IP_);
228 kmem_cache_free(extent_state_cache, state); 228 kmem_cache_free(extent_state_cache, state);
229 } 229 }
230 } 230 }
231 231
232 static struct rb_node *tree_insert(struct rb_root *root, u64 offset, 232 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
233 struct rb_node *node, 233 struct rb_node *node,
234 struct rb_node ***p_in, 234 struct rb_node ***p_in,
235 struct rb_node **parent_in) 235 struct rb_node **parent_in)
236 { 236 {
237 struct rb_node **p = &root->rb_node; 237 struct rb_node **p = &root->rb_node;
238 struct rb_node *parent = NULL; 238 struct rb_node *parent = NULL;
239 struct tree_entry *entry; 239 struct tree_entry *entry;
240 240
241 if (p_in && parent_in) { 241 if (p_in && parent_in) {
242 p = *p_in; 242 p = *p_in;
243 parent = *parent_in; 243 parent = *parent_in;
244 goto do_insert; 244 goto do_insert;
245 } 245 }
246 246
247 while (*p) { 247 while (*p) {
248 parent = *p; 248 parent = *p;
249 entry = rb_entry(parent, struct tree_entry, rb_node); 249 entry = rb_entry(parent, struct tree_entry, rb_node);
250 250
251 if (offset < entry->start) 251 if (offset < entry->start)
252 p = &(*p)->rb_left; 252 p = &(*p)->rb_left;
253 else if (offset > entry->end) 253 else if (offset > entry->end)
254 p = &(*p)->rb_right; 254 p = &(*p)->rb_right;
255 else 255 else
256 return parent; 256 return parent;
257 } 257 }
258 258
259 do_insert: 259 do_insert:
260 rb_link_node(node, parent, p); 260 rb_link_node(node, parent, p);
261 rb_insert_color(node, root); 261 rb_insert_color(node, root);
262 return NULL; 262 return NULL;
263 } 263 }
264 264
265 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset, 265 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
266 struct rb_node **prev_ret, 266 struct rb_node **prev_ret,
267 struct rb_node **next_ret, 267 struct rb_node **next_ret,
268 struct rb_node ***p_ret, 268 struct rb_node ***p_ret,
269 struct rb_node **parent_ret) 269 struct rb_node **parent_ret)
270 { 270 {
271 struct rb_root *root = &tree->state; 271 struct rb_root *root = &tree->state;
272 struct rb_node **n = &root->rb_node; 272 struct rb_node **n = &root->rb_node;
273 struct rb_node *prev = NULL; 273 struct rb_node *prev = NULL;
274 struct rb_node *orig_prev = NULL; 274 struct rb_node *orig_prev = NULL;
275 struct tree_entry *entry; 275 struct tree_entry *entry;
276 struct tree_entry *prev_entry = NULL; 276 struct tree_entry *prev_entry = NULL;
277 277
278 while (*n) { 278 while (*n) {
279 prev = *n; 279 prev = *n;
280 entry = rb_entry(prev, struct tree_entry, rb_node); 280 entry = rb_entry(prev, struct tree_entry, rb_node);
281 prev_entry = entry; 281 prev_entry = entry;
282 282
283 if (offset < entry->start) 283 if (offset < entry->start)
284 n = &(*n)->rb_left; 284 n = &(*n)->rb_left;
285 else if (offset > entry->end) 285 else if (offset > entry->end)
286 n = &(*n)->rb_right; 286 n = &(*n)->rb_right;
287 else 287 else
288 return *n; 288 return *n;
289 } 289 }
290 290
291 if (p_ret) 291 if (p_ret)
292 *p_ret = n; 292 *p_ret = n;
293 if (parent_ret) 293 if (parent_ret)
294 *parent_ret = prev; 294 *parent_ret = prev;
295 295
296 if (prev_ret) { 296 if (prev_ret) {
297 orig_prev = prev; 297 orig_prev = prev;
298 while (prev && offset > prev_entry->end) { 298 while (prev && offset > prev_entry->end) {
299 prev = rb_next(prev); 299 prev = rb_next(prev);
300 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 300 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
301 } 301 }
302 *prev_ret = prev; 302 *prev_ret = prev;
303 prev = orig_prev; 303 prev = orig_prev;
304 } 304 }
305 305
306 if (next_ret) { 306 if (next_ret) {
307 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 307 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
308 while (prev && offset < prev_entry->start) { 308 while (prev && offset < prev_entry->start) {
309 prev = rb_prev(prev); 309 prev = rb_prev(prev);
310 prev_entry = rb_entry(prev, struct tree_entry, rb_node); 310 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
311 } 311 }
312 *next_ret = prev; 312 *next_ret = prev;
313 } 313 }
314 return NULL; 314 return NULL;
315 } 315 }
316 316
317 static inline struct rb_node * 317 static inline struct rb_node *
318 tree_search_for_insert(struct extent_io_tree *tree, 318 tree_search_for_insert(struct extent_io_tree *tree,
319 u64 offset, 319 u64 offset,
320 struct rb_node ***p_ret, 320 struct rb_node ***p_ret,
321 struct rb_node **parent_ret) 321 struct rb_node **parent_ret)
322 { 322 {
323 struct rb_node *prev = NULL; 323 struct rb_node *prev = NULL;
324 struct rb_node *ret; 324 struct rb_node *ret;
325 325
326 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret); 326 ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
327 if (!ret) 327 if (!ret)
328 return prev; 328 return prev;
329 return ret; 329 return ret;
330 } 330 }
331 331
332 static inline struct rb_node *tree_search(struct extent_io_tree *tree, 332 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
333 u64 offset) 333 u64 offset)
334 { 334 {
335 return tree_search_for_insert(tree, offset, NULL, NULL); 335 return tree_search_for_insert(tree, offset, NULL, NULL);
336 } 336 }
337 337
338 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new, 338 static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
339 struct extent_state *other) 339 struct extent_state *other)
340 { 340 {
341 if (tree->ops && tree->ops->merge_extent_hook) 341 if (tree->ops && tree->ops->merge_extent_hook)
342 tree->ops->merge_extent_hook(tree->mapping->host, new, 342 tree->ops->merge_extent_hook(tree->mapping->host, new,
343 other); 343 other);
344 } 344 }
345 345
346 /* 346 /*
347 * utility function to look for merge candidates inside a given range. 347 * utility function to look for merge candidates inside a given range.
348 * Any extents with matching state are merged together into a single 348 * Any extents with matching state are merged together into a single
349 * extent in the tree. Extents with EXTENT_IO in their state field 349 * extent in the tree. Extents with EXTENT_IO in their state field
350 * are not merged because the end_io handlers need to be able to do 350 * are not merged because the end_io handlers need to be able to do
351 * operations on them without sleeping (or doing allocations/splits). 351 * operations on them without sleeping (or doing allocations/splits).
352 * 352 *
353 * This should be called with the tree lock held. 353 * This should be called with the tree lock held.
354 */ 354 */
355 static void merge_state(struct extent_io_tree *tree, 355 static void merge_state(struct extent_io_tree *tree,
356 struct extent_state *state) 356 struct extent_state *state)
357 { 357 {
358 struct extent_state *other; 358 struct extent_state *other;
359 struct rb_node *other_node; 359 struct rb_node *other_node;
360 360
361 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) 361 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
362 return; 362 return;
363 363
364 other_node = rb_prev(&state->rb_node); 364 other_node = rb_prev(&state->rb_node);
365 if (other_node) { 365 if (other_node) {
366 other = rb_entry(other_node, struct extent_state, rb_node); 366 other = rb_entry(other_node, struct extent_state, rb_node);
367 if (other->end == state->start - 1 && 367 if (other->end == state->start - 1 &&
368 other->state == state->state) { 368 other->state == state->state) {
369 merge_cb(tree, state, other); 369 merge_cb(tree, state, other);
370 state->start = other->start; 370 state->start = other->start;
371 other->tree = NULL; 371 other->tree = NULL;
372 rb_erase(&other->rb_node, &tree->state); 372 rb_erase(&other->rb_node, &tree->state);
373 free_extent_state(other); 373 free_extent_state(other);
374 } 374 }
375 } 375 }
376 other_node = rb_next(&state->rb_node); 376 other_node = rb_next(&state->rb_node);
377 if (other_node) { 377 if (other_node) {
378 other = rb_entry(other_node, struct extent_state, rb_node); 378 other = rb_entry(other_node, struct extent_state, rb_node);
379 if (other->start == state->end + 1 && 379 if (other->start == state->end + 1 &&
380 other->state == state->state) { 380 other->state == state->state) {
381 merge_cb(tree, state, other); 381 merge_cb(tree, state, other);
382 state->end = other->end; 382 state->end = other->end;
383 other->tree = NULL; 383 other->tree = NULL;
384 rb_erase(&other->rb_node, &tree->state); 384 rb_erase(&other->rb_node, &tree->state);
385 free_extent_state(other); 385 free_extent_state(other);
386 } 386 }
387 } 387 }
388 } 388 }
389 389
390 static void set_state_cb(struct extent_io_tree *tree, 390 static void set_state_cb(struct extent_io_tree *tree,
391 struct extent_state *state, unsigned long *bits) 391 struct extent_state *state, unsigned long *bits)
392 { 392 {
393 if (tree->ops && tree->ops->set_bit_hook) 393 if (tree->ops && tree->ops->set_bit_hook)
394 tree->ops->set_bit_hook(tree->mapping->host, state, bits); 394 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
395 } 395 }
396 396
397 static void clear_state_cb(struct extent_io_tree *tree, 397 static void clear_state_cb(struct extent_io_tree *tree,
398 struct extent_state *state, unsigned long *bits) 398 struct extent_state *state, unsigned long *bits)
399 { 399 {
400 if (tree->ops && tree->ops->clear_bit_hook) 400 if (tree->ops && tree->ops->clear_bit_hook)
401 tree->ops->clear_bit_hook(tree->mapping->host, state, bits); 401 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
402 } 402 }
403 403
404 static void set_state_bits(struct extent_io_tree *tree, 404 static void set_state_bits(struct extent_io_tree *tree,
405 struct extent_state *state, unsigned long *bits); 405 struct extent_state *state, unsigned long *bits);
406 406
407 /* 407 /*
408 * insert an extent_state struct into the tree. 'bits' are set on the 408 * insert an extent_state struct into the tree. 'bits' are set on the
409 * struct before it is inserted. 409 * struct before it is inserted.
410 * 410 *
411 * This may return -EEXIST if the extent is already there, in which case the 411 * This may return -EEXIST if the extent is already there, in which case the
412 * state struct is freed. 412 * state struct is freed.
413 * 413 *
414 * The tree lock is not taken internally. This is a utility function and 414 * The tree lock is not taken internally. This is a utility function and
415 * probably isn't what you want to call (see set/clear_extent_bit). 415 * probably isn't what you want to call (see set/clear_extent_bit).
416 */ 416 */
417 static int insert_state(struct extent_io_tree *tree, 417 static int insert_state(struct extent_io_tree *tree,
418 struct extent_state *state, u64 start, u64 end, 418 struct extent_state *state, u64 start, u64 end,
419 struct rb_node ***p, 419 struct rb_node ***p,
420 struct rb_node **parent, 420 struct rb_node **parent,
421 unsigned long *bits) 421 unsigned long *bits)
422 { 422 {
423 struct rb_node *node; 423 struct rb_node *node;
424 424
425 if (end < start) 425 if (end < start)
426 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n", 426 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
427 end, start); 427 end, start);
428 state->start = start; 428 state->start = start;
429 state->end = end; 429 state->end = end;
430 430
431 set_state_bits(tree, state, bits); 431 set_state_bits(tree, state, bits);
432 432
433 node = tree_insert(&tree->state, end, &state->rb_node, p, parent); 433 node = tree_insert(&tree->state, end, &state->rb_node, p, parent);
434 if (node) { 434 if (node) {
435 struct extent_state *found; 435 struct extent_state *found;
436 found = rb_entry(node, struct extent_state, rb_node); 436 found = rb_entry(node, struct extent_state, rb_node);
437 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of " 437 printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
438 "%llu %llu\n", 438 "%llu %llu\n",
439 found->start, found->end, start, end); 439 found->start, found->end, start, end);
440 return -EEXIST; 440 return -EEXIST;
441 } 441 }
442 state->tree = tree; 442 state->tree = tree;
443 merge_state(tree, state); 443 merge_state(tree, state);
444 return 0; 444 return 0;
445 } 445 }
446 446
447 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig, 447 static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
448 u64 split) 448 u64 split)
449 { 449 {
450 if (tree->ops && tree->ops->split_extent_hook) 450 if (tree->ops && tree->ops->split_extent_hook)
451 tree->ops->split_extent_hook(tree->mapping->host, orig, split); 451 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
452 } 452 }
453 453
454 /* 454 /*
455 * split a given extent state struct in two, inserting the preallocated 455 * split a given extent state struct in two, inserting the preallocated
456 * struct 'prealloc' as the newly created second half. 'split' indicates an 456 * struct 'prealloc' as the newly created second half. 'split' indicates an
457 * offset inside 'orig' where it should be split. 457 * offset inside 'orig' where it should be split.
458 * 458 *
459 * Before calling, 459 * Before calling,
460 * the tree has 'orig' at [orig->start, orig->end]. After calling, there 460 * the tree has 'orig' at [orig->start, orig->end]. After calling, there
461 * are two extent state structs in the tree: 461 * are two extent state structs in the tree:
462 * prealloc: [orig->start, split - 1] 462 * prealloc: [orig->start, split - 1]
463 * orig: [ split, orig->end ] 463 * orig: [ split, orig->end ]
464 * 464 *
465 * The tree locks are not taken by this function. They need to be held 465 * The tree locks are not taken by this function. They need to be held
466 * by the caller. 466 * by the caller.
467 */ 467 */
468 static int split_state(struct extent_io_tree *tree, struct extent_state *orig, 468 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
469 struct extent_state *prealloc, u64 split) 469 struct extent_state *prealloc, u64 split)
470 { 470 {
471 struct rb_node *node; 471 struct rb_node *node;
472 472
473 split_cb(tree, orig, split); 473 split_cb(tree, orig, split);
474 474
475 prealloc->start = orig->start; 475 prealloc->start = orig->start;
476 prealloc->end = split - 1; 476 prealloc->end = split - 1;
477 prealloc->state = orig->state; 477 prealloc->state = orig->state;
478 orig->start = split; 478 orig->start = split;
479 479
480 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node, 480 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node,
481 NULL, NULL); 481 NULL, NULL);
482 if (node) { 482 if (node) {
483 free_extent_state(prealloc); 483 free_extent_state(prealloc);
484 return -EEXIST; 484 return -EEXIST;
485 } 485 }
486 prealloc->tree = tree; 486 prealloc->tree = tree;
487 return 0; 487 return 0;
488 } 488 }
489 489
490 static struct extent_state *next_state(struct extent_state *state) 490 static struct extent_state *next_state(struct extent_state *state)
491 { 491 {
492 struct rb_node *next = rb_next(&state->rb_node); 492 struct rb_node *next = rb_next(&state->rb_node);
493 if (next) 493 if (next)
494 return rb_entry(next, struct extent_state, rb_node); 494 return rb_entry(next, struct extent_state, rb_node);
495 else 495 else
496 return NULL; 496 return NULL;
497 } 497 }
498 498
499 /* 499 /*
500 * utility function to clear some bits in an extent state struct. 500 * utility function to clear some bits in an extent state struct.
501 * it will optionally wake up any one waiting on this state (wake == 1). 501 * it will optionally wake up any one waiting on this state (wake == 1).
502 * 502 *
503 * If no bits are set on the state struct after clearing things, the 503 * If no bits are set on the state struct after clearing things, the
504 * struct is freed and removed from the tree 504 * struct is freed and removed from the tree
505 */ 505 */
506 static struct extent_state *clear_state_bit(struct extent_io_tree *tree, 506 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
507 struct extent_state *state, 507 struct extent_state *state,
508 unsigned long *bits, int wake) 508 unsigned long *bits, int wake)
509 { 509 {
510 struct extent_state *next; 510 struct extent_state *next;
511 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS; 511 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
512 512
513 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) { 513 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
514 u64 range = state->end - state->start + 1; 514 u64 range = state->end - state->start + 1;
515 WARN_ON(range > tree->dirty_bytes); 515 WARN_ON(range > tree->dirty_bytes);
516 tree->dirty_bytes -= range; 516 tree->dirty_bytes -= range;
517 } 517 }
518 clear_state_cb(tree, state, bits); 518 clear_state_cb(tree, state, bits);
519 state->state &= ~bits_to_clear; 519 state->state &= ~bits_to_clear;
520 if (wake) 520 if (wake)
521 wake_up(&state->wq); 521 wake_up(&state->wq);
522 if (state->state == 0) { 522 if (state->state == 0) {
523 next = next_state(state); 523 next = next_state(state);
524 if (state->tree) { 524 if (state->tree) {
525 rb_erase(&state->rb_node, &tree->state); 525 rb_erase(&state->rb_node, &tree->state);
526 state->tree = NULL; 526 state->tree = NULL;
527 free_extent_state(state); 527 free_extent_state(state);
528 } else { 528 } else {
529 WARN_ON(1); 529 WARN_ON(1);
530 } 530 }
531 } else { 531 } else {
532 merge_state(tree, state); 532 merge_state(tree, state);
533 next = next_state(state); 533 next = next_state(state);
534 } 534 }
535 return next; 535 return next;
536 } 536 }
537 537
538 static struct extent_state * 538 static struct extent_state *
539 alloc_extent_state_atomic(struct extent_state *prealloc) 539 alloc_extent_state_atomic(struct extent_state *prealloc)
540 { 540 {
541 if (!prealloc) 541 if (!prealloc)
542 prealloc = alloc_extent_state(GFP_ATOMIC); 542 prealloc = alloc_extent_state(GFP_ATOMIC);
543 543
544 return prealloc; 544 return prealloc;
545 } 545 }
546 546
547 static void extent_io_tree_panic(struct extent_io_tree *tree, int err) 547 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
548 { 548 {
549 btrfs_panic(tree_fs_info(tree), err, "Locking error: " 549 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
550 "Extent tree was modified by another " 550 "Extent tree was modified by another "
551 "thread while locked."); 551 "thread while locked.");
552 } 552 }
553 553
554 /* 554 /*
555 * clear some bits on a range in the tree. This may require splitting 555 * clear some bits on a range in the tree. This may require splitting
556 * or inserting elements in the tree, so the gfp mask is used to 556 * or inserting elements in the tree, so the gfp mask is used to
557 * indicate which allocations or sleeping are allowed. 557 * indicate which allocations or sleeping are allowed.
558 * 558 *
559 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove 559 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
560 * the given range from the tree regardless of state (ie for truncate). 560 * the given range from the tree regardless of state (ie for truncate).
561 * 561 *
562 * the range [start, end] is inclusive. 562 * the range [start, end] is inclusive.
563 * 563 *
564 * This takes the tree lock, and returns 0 on success and < 0 on error. 564 * This takes the tree lock, and returns 0 on success and < 0 on error.
565 */ 565 */
566 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 566 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
567 unsigned long bits, int wake, int delete, 567 unsigned long bits, int wake, int delete,
568 struct extent_state **cached_state, 568 struct extent_state **cached_state,
569 gfp_t mask) 569 gfp_t mask)
570 { 570 {
571 struct extent_state *state; 571 struct extent_state *state;
572 struct extent_state *cached; 572 struct extent_state *cached;
573 struct extent_state *prealloc = NULL; 573 struct extent_state *prealloc = NULL;
574 struct rb_node *node; 574 struct rb_node *node;
575 u64 last_end; 575 u64 last_end;
576 int err; 576 int err;
577 int clear = 0; 577 int clear = 0;
578 578
579 btrfs_debug_check_extent_io_range(tree, start, end); 579 btrfs_debug_check_extent_io_range(tree, start, end);
580 580
581 if (bits & EXTENT_DELALLOC) 581 if (bits & EXTENT_DELALLOC)
582 bits |= EXTENT_NORESERVE; 582 bits |= EXTENT_NORESERVE;
583 583
584 if (delete) 584 if (delete)
585 bits |= ~EXTENT_CTLBITS; 585 bits |= ~EXTENT_CTLBITS;
586 bits |= EXTENT_FIRST_DELALLOC; 586 bits |= EXTENT_FIRST_DELALLOC;
587 587
588 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY)) 588 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
589 clear = 1; 589 clear = 1;
590 again: 590 again:
591 if (!prealloc && (mask & __GFP_WAIT)) { 591 if (!prealloc && (mask & __GFP_WAIT)) {
592 prealloc = alloc_extent_state(mask); 592 prealloc = alloc_extent_state(mask);
593 if (!prealloc) 593 if (!prealloc)
594 return -ENOMEM; 594 return -ENOMEM;
595 } 595 }
596 596
597 spin_lock(&tree->lock); 597 spin_lock(&tree->lock);
598 if (cached_state) { 598 if (cached_state) {
599 cached = *cached_state; 599 cached = *cached_state;
600 600
601 if (clear) { 601 if (clear) {
602 *cached_state = NULL; 602 *cached_state = NULL;
603 cached_state = NULL; 603 cached_state = NULL;
604 } 604 }
605 605
606 if (cached && cached->tree && cached->start <= start && 606 if (cached && cached->tree && cached->start <= start &&
607 cached->end > start) { 607 cached->end > start) {
608 if (clear) 608 if (clear)
609 atomic_dec(&cached->refs); 609 atomic_dec(&cached->refs);
610 state = cached; 610 state = cached;
611 goto hit_next; 611 goto hit_next;
612 } 612 }
613 if (clear) 613 if (clear)
614 free_extent_state(cached); 614 free_extent_state(cached);
615 } 615 }
616 /* 616 /*
617 * this search will find the extents that end after 617 * this search will find the extents that end after
618 * our range starts 618 * our range starts
619 */ 619 */
620 node = tree_search(tree, start); 620 node = tree_search(tree, start);
621 if (!node) 621 if (!node)
622 goto out; 622 goto out;
623 state = rb_entry(node, struct extent_state, rb_node); 623 state = rb_entry(node, struct extent_state, rb_node);
624 hit_next: 624 hit_next:
625 if (state->start > end) 625 if (state->start > end)
626 goto out; 626 goto out;
627 WARN_ON(state->end < start); 627 WARN_ON(state->end < start);
628 last_end = state->end; 628 last_end = state->end;
629 629
630 /* the state doesn't have the wanted bits, go ahead */ 630 /* the state doesn't have the wanted bits, go ahead */
631 if (!(state->state & bits)) { 631 if (!(state->state & bits)) {
632 state = next_state(state); 632 state = next_state(state);
633 goto next; 633 goto next;
634 } 634 }
635 635
636 /* 636 /*
637 * | ---- desired range ---- | 637 * | ---- desired range ---- |
638 * | state | or 638 * | state | or
639 * | ------------- state -------------- | 639 * | ------------- state -------------- |
640 * 640 *
641 * We need to split the extent we found, and may flip 641 * We need to split the extent we found, and may flip
642 * bits on second half. 642 * bits on second half.
643 * 643 *
644 * If the extent we found extends past our range, we 644 * If the extent we found extends past our range, we
645 * just split and search again. It'll get split again 645 * just split and search again. It'll get split again
646 * the next time though. 646 * the next time though.
647 * 647 *
648 * If the extent we found is inside our range, we clear 648 * If the extent we found is inside our range, we clear
649 * the desired bit on it. 649 * the desired bit on it.
650 */ 650 */
651 651
652 if (state->start < start) { 652 if (state->start < start) {
653 prealloc = alloc_extent_state_atomic(prealloc); 653 prealloc = alloc_extent_state_atomic(prealloc);
654 BUG_ON(!prealloc); 654 BUG_ON(!prealloc);
655 err = split_state(tree, state, prealloc, start); 655 err = split_state(tree, state, prealloc, start);
656 if (err) 656 if (err)
657 extent_io_tree_panic(tree, err); 657 extent_io_tree_panic(tree, err);
658 658
659 prealloc = NULL; 659 prealloc = NULL;
660 if (err) 660 if (err)
661 goto out; 661 goto out;
662 if (state->end <= end) { 662 if (state->end <= end) {
663 state = clear_state_bit(tree, state, &bits, wake); 663 state = clear_state_bit(tree, state, &bits, wake);
664 goto next; 664 goto next;
665 } 665 }
666 goto search_again; 666 goto search_again;
667 } 667 }
668 /* 668 /*
669 * | ---- desired range ---- | 669 * | ---- desired range ---- |
670 * | state | 670 * | state |
671 * We need to split the extent, and clear the bit 671 * We need to split the extent, and clear the bit
672 * on the first half 672 * on the first half
673 */ 673 */
674 if (state->start <= end && state->end > end) { 674 if (state->start <= end && state->end > end) {
675 prealloc = alloc_extent_state_atomic(prealloc); 675 prealloc = alloc_extent_state_atomic(prealloc);
676 BUG_ON(!prealloc); 676 BUG_ON(!prealloc);
677 err = split_state(tree, state, prealloc, end + 1); 677 err = split_state(tree, state, prealloc, end + 1);
678 if (err) 678 if (err)
679 extent_io_tree_panic(tree, err); 679 extent_io_tree_panic(tree, err);
680 680
681 if (wake) 681 if (wake)
682 wake_up(&state->wq); 682 wake_up(&state->wq);
683 683
684 clear_state_bit(tree, prealloc, &bits, wake); 684 clear_state_bit(tree, prealloc, &bits, wake);
685 685
686 prealloc = NULL; 686 prealloc = NULL;
687 goto out; 687 goto out;
688 } 688 }
689 689
690 state = clear_state_bit(tree, state, &bits, wake); 690 state = clear_state_bit(tree, state, &bits, wake);
691 next: 691 next:
692 if (last_end == (u64)-1) 692 if (last_end == (u64)-1)
693 goto out; 693 goto out;
694 start = last_end + 1; 694 start = last_end + 1;
695 if (start <= end && state && !need_resched()) 695 if (start <= end && state && !need_resched())
696 goto hit_next; 696 goto hit_next;
697 goto search_again; 697 goto search_again;
698 698
699 out: 699 out:
700 spin_unlock(&tree->lock); 700 spin_unlock(&tree->lock);
701 if (prealloc) 701 if (prealloc)
702 free_extent_state(prealloc); 702 free_extent_state(prealloc);
703 703
704 return 0; 704 return 0;
705 705
706 search_again: 706 search_again:
707 if (start > end) 707 if (start > end)
708 goto out; 708 goto out;
709 spin_unlock(&tree->lock); 709 spin_unlock(&tree->lock);
710 if (mask & __GFP_WAIT) 710 if (mask & __GFP_WAIT)
711 cond_resched(); 711 cond_resched();
712 goto again; 712 goto again;
713 } 713 }
714 714
715 static void wait_on_state(struct extent_io_tree *tree, 715 static void wait_on_state(struct extent_io_tree *tree,
716 struct extent_state *state) 716 struct extent_state *state)
717 __releases(tree->lock) 717 __releases(tree->lock)
718 __acquires(tree->lock) 718 __acquires(tree->lock)
719 { 719 {
720 DEFINE_WAIT(wait); 720 DEFINE_WAIT(wait);
721 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); 721 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
722 spin_unlock(&tree->lock); 722 spin_unlock(&tree->lock);
723 schedule(); 723 schedule();
724 spin_lock(&tree->lock); 724 spin_lock(&tree->lock);
725 finish_wait(&state->wq, &wait); 725 finish_wait(&state->wq, &wait);
726 } 726 }
727 727
728 /* 728 /*
729 * waits for one or more bits to clear on a range in the state tree. 729 * waits for one or more bits to clear on a range in the state tree.
730 * The range [start, end] is inclusive. 730 * The range [start, end] is inclusive.
731 * The tree lock is taken by this function 731 * The tree lock is taken by this function
732 */ 732 */
733 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 733 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
734 unsigned long bits) 734 unsigned long bits)
735 { 735 {
736 struct extent_state *state; 736 struct extent_state *state;
737 struct rb_node *node; 737 struct rb_node *node;
738 738
739 btrfs_debug_check_extent_io_range(tree, start, end); 739 btrfs_debug_check_extent_io_range(tree, start, end);
740 740
741 spin_lock(&tree->lock); 741 spin_lock(&tree->lock);
742 again: 742 again:
743 while (1) { 743 while (1) {
744 /* 744 /*
745 * this search will find all the extents that end after 745 * this search will find all the extents that end after
746 * our range starts 746 * our range starts
747 */ 747 */
748 node = tree_search(tree, start); 748 node = tree_search(tree, start);
749 if (!node) 749 if (!node)
750 break; 750 break;
751 751
752 state = rb_entry(node, struct extent_state, rb_node); 752 state = rb_entry(node, struct extent_state, rb_node);
753 753
754 if (state->start > end) 754 if (state->start > end)
755 goto out; 755 goto out;
756 756
757 if (state->state & bits) { 757 if (state->state & bits) {
758 start = state->start; 758 start = state->start;
759 atomic_inc(&state->refs); 759 atomic_inc(&state->refs);
760 wait_on_state(tree, state); 760 wait_on_state(tree, state);
761 free_extent_state(state); 761 free_extent_state(state);
762 goto again; 762 goto again;
763 } 763 }
764 start = state->end + 1; 764 start = state->end + 1;
765 765
766 if (start > end) 766 if (start > end)
767 break; 767 break;
768 768
769 cond_resched_lock(&tree->lock); 769 cond_resched_lock(&tree->lock);
770 } 770 }
771 out: 771 out:
772 spin_unlock(&tree->lock); 772 spin_unlock(&tree->lock);
773 } 773 }
774 774
775 static void set_state_bits(struct extent_io_tree *tree, 775 static void set_state_bits(struct extent_io_tree *tree,
776 struct extent_state *state, 776 struct extent_state *state,
777 unsigned long *bits) 777 unsigned long *bits)
778 { 778 {
779 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS; 779 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
780 780
781 set_state_cb(tree, state, bits); 781 set_state_cb(tree, state, bits);
782 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) { 782 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
783 u64 range = state->end - state->start + 1; 783 u64 range = state->end - state->start + 1;
784 tree->dirty_bytes += range; 784 tree->dirty_bytes += range;
785 } 785 }
786 state->state |= bits_to_set; 786 state->state |= bits_to_set;
787 } 787 }
788 788
789 static void cache_state(struct extent_state *state, 789 static void cache_state(struct extent_state *state,
790 struct extent_state **cached_ptr) 790 struct extent_state **cached_ptr)
791 { 791 {
792 if (cached_ptr && !(*cached_ptr)) { 792 if (cached_ptr && !(*cached_ptr)) {
793 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) { 793 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
794 *cached_ptr = state; 794 *cached_ptr = state;
795 atomic_inc(&state->refs); 795 atomic_inc(&state->refs);
796 } 796 }
797 } 797 }
798 } 798 }
799 799
800 /* 800 /*
801 * set some bits on a range in the tree. This may require allocations or 801 * set some bits on a range in the tree. This may require allocations or
802 * sleeping, so the gfp mask is used to indicate what is allowed. 802 * sleeping, so the gfp mask is used to indicate what is allowed.
803 * 803 *
804 * If any of the exclusive bits are set, this will fail with -EEXIST if some 804 * If any of the exclusive bits are set, this will fail with -EEXIST if some
805 * part of the range already has the desired bits set. The start of the 805 * part of the range already has the desired bits set. The start of the
806 * existing range is returned in failed_start in this case. 806 * existing range is returned in failed_start in this case.
807 * 807 *
808 * [start, end] is inclusive This takes the tree lock. 808 * [start, end] is inclusive This takes the tree lock.
809 */ 809 */
810 810
811 static int __must_check 811 static int __must_check
812 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 812 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
813 unsigned long bits, unsigned long exclusive_bits, 813 unsigned long bits, unsigned long exclusive_bits,
814 u64 *failed_start, struct extent_state **cached_state, 814 u64 *failed_start, struct extent_state **cached_state,
815 gfp_t mask) 815 gfp_t mask)
816 { 816 {
817 struct extent_state *state; 817 struct extent_state *state;
818 struct extent_state *prealloc = NULL; 818 struct extent_state *prealloc = NULL;
819 struct rb_node *node; 819 struct rb_node *node;
820 struct rb_node **p; 820 struct rb_node **p;
821 struct rb_node *parent; 821 struct rb_node *parent;
822 int err = 0; 822 int err = 0;
823 u64 last_start; 823 u64 last_start;
824 u64 last_end; 824 u64 last_end;
825 825
826 btrfs_debug_check_extent_io_range(tree, start, end); 826 btrfs_debug_check_extent_io_range(tree, start, end);
827 827
828 bits |= EXTENT_FIRST_DELALLOC; 828 bits |= EXTENT_FIRST_DELALLOC;
829 again: 829 again:
830 if (!prealloc && (mask & __GFP_WAIT)) { 830 if (!prealloc && (mask & __GFP_WAIT)) {
831 prealloc = alloc_extent_state(mask); 831 prealloc = alloc_extent_state(mask);
832 BUG_ON(!prealloc); 832 BUG_ON(!prealloc);
833 } 833 }
834 834
835 spin_lock(&tree->lock); 835 spin_lock(&tree->lock);
836 if (cached_state && *cached_state) { 836 if (cached_state && *cached_state) {
837 state = *cached_state; 837 state = *cached_state;
838 if (state->start <= start && state->end > start && 838 if (state->start <= start && state->end > start &&
839 state->tree) { 839 state->tree) {
840 node = &state->rb_node; 840 node = &state->rb_node;
841 goto hit_next; 841 goto hit_next;
842 } 842 }
843 } 843 }
844 /* 844 /*
845 * this search will find all the extents that end after 845 * this search will find all the extents that end after
846 * our range starts. 846 * our range starts.
847 */ 847 */
848 node = tree_search_for_insert(tree, start, &p, &parent); 848 node = tree_search_for_insert(tree, start, &p, &parent);
849 if (!node) { 849 if (!node) {
850 prealloc = alloc_extent_state_atomic(prealloc); 850 prealloc = alloc_extent_state_atomic(prealloc);
851 BUG_ON(!prealloc); 851 BUG_ON(!prealloc);
852 err = insert_state(tree, prealloc, start, end, 852 err = insert_state(tree, prealloc, start, end,
853 &p, &parent, &bits); 853 &p, &parent, &bits);
854 if (err) 854 if (err)
855 extent_io_tree_panic(tree, err); 855 extent_io_tree_panic(tree, err);
856 856
857 cache_state(prealloc, cached_state); 857 cache_state(prealloc, cached_state);
858 prealloc = NULL; 858 prealloc = NULL;
859 goto out; 859 goto out;
860 } 860 }
861 state = rb_entry(node, struct extent_state, rb_node); 861 state = rb_entry(node, struct extent_state, rb_node);
862 hit_next: 862 hit_next:
863 last_start = state->start; 863 last_start = state->start;
864 last_end = state->end; 864 last_end = state->end;
865 865
866 /* 866 /*
867 * | ---- desired range ---- | 867 * | ---- desired range ---- |
868 * | state | 868 * | state |
869 * 869 *
870 * Just lock what we found and keep going 870 * Just lock what we found and keep going
871 */ 871 */
872 if (state->start == start && state->end <= end) { 872 if (state->start == start && state->end <= end) {
873 if (state->state & exclusive_bits) { 873 if (state->state & exclusive_bits) {
874 *failed_start = state->start; 874 *failed_start = state->start;
875 err = -EEXIST; 875 err = -EEXIST;
876 goto out; 876 goto out;
877 } 877 }
878 878
879 set_state_bits(tree, state, &bits); 879 set_state_bits(tree, state, &bits);
880 cache_state(state, cached_state); 880 cache_state(state, cached_state);
881 merge_state(tree, state); 881 merge_state(tree, state);
882 if (last_end == (u64)-1) 882 if (last_end == (u64)-1)
883 goto out; 883 goto out;
884 start = last_end + 1; 884 start = last_end + 1;
885 state = next_state(state); 885 state = next_state(state);
886 if (start < end && state && state->start == start && 886 if (start < end && state && state->start == start &&
887 !need_resched()) 887 !need_resched())
888 goto hit_next; 888 goto hit_next;
889 goto search_again; 889 goto search_again;
890 } 890 }
891 891
892 /* 892 /*
893 * | ---- desired range ---- | 893 * | ---- desired range ---- |
894 * | state | 894 * | state |
895 * or 895 * or
896 * | ------------- state -------------- | 896 * | ------------- state -------------- |
897 * 897 *
898 * We need to split the extent we found, and may flip bits on 898 * We need to split the extent we found, and may flip bits on
899 * second half. 899 * second half.
900 * 900 *
901 * If the extent we found extends past our 901 * If the extent we found extends past our
902 * range, we just split and search again. It'll get split 902 * range, we just split and search again. It'll get split
903 * again the next time though. 903 * again the next time though.
904 * 904 *
905 * If the extent we found is inside our range, we set the 905 * If the extent we found is inside our range, we set the
906 * desired bit on it. 906 * desired bit on it.
907 */ 907 */
908 if (state->start < start) { 908 if (state->start < start) {
909 if (state->state & exclusive_bits) { 909 if (state->state & exclusive_bits) {
910 *failed_start = start; 910 *failed_start = start;
911 err = -EEXIST; 911 err = -EEXIST;
912 goto out; 912 goto out;
913 } 913 }
914 914
915 prealloc = alloc_extent_state_atomic(prealloc); 915 prealloc = alloc_extent_state_atomic(prealloc);
916 BUG_ON(!prealloc); 916 BUG_ON(!prealloc);
917 err = split_state(tree, state, prealloc, start); 917 err = split_state(tree, state, prealloc, start);
918 if (err) 918 if (err)
919 extent_io_tree_panic(tree, err); 919 extent_io_tree_panic(tree, err);
920 920
921 prealloc = NULL; 921 prealloc = NULL;
922 if (err) 922 if (err)
923 goto out; 923 goto out;
924 if (state->end <= end) { 924 if (state->end <= end) {
925 set_state_bits(tree, state, &bits); 925 set_state_bits(tree, state, &bits);
926 cache_state(state, cached_state); 926 cache_state(state, cached_state);
927 merge_state(tree, state); 927 merge_state(tree, state);
928 if (last_end == (u64)-1) 928 if (last_end == (u64)-1)
929 goto out; 929 goto out;
930 start = last_end + 1; 930 start = last_end + 1;
931 state = next_state(state); 931 state = next_state(state);
932 if (start < end && state && state->start == start && 932 if (start < end && state && state->start == start &&
933 !need_resched()) 933 !need_resched())
934 goto hit_next; 934 goto hit_next;
935 } 935 }
936 goto search_again; 936 goto search_again;
937 } 937 }
938 /* 938 /*
939 * | ---- desired range ---- | 939 * | ---- desired range ---- |
940 * | state | or | state | 940 * | state | or | state |
941 * 941 *
942 * There's a hole, we need to insert something in it and 942 * There's a hole, we need to insert something in it and
943 * ignore the extent we found. 943 * ignore the extent we found.
944 */ 944 */
945 if (state->start > start) { 945 if (state->start > start) {
946 u64 this_end; 946 u64 this_end;
947 if (end < last_start) 947 if (end < last_start)
948 this_end = end; 948 this_end = end;
949 else 949 else
950 this_end = last_start - 1; 950 this_end = last_start - 1;
951 951
952 prealloc = alloc_extent_state_atomic(prealloc); 952 prealloc = alloc_extent_state_atomic(prealloc);
953 BUG_ON(!prealloc); 953 BUG_ON(!prealloc);
954 954
955 /* 955 /*
956 * Avoid to free 'prealloc' if it can be merged with 956 * Avoid to free 'prealloc' if it can be merged with
957 * the later extent. 957 * the later extent.
958 */ 958 */
959 err = insert_state(tree, prealloc, start, this_end, 959 err = insert_state(tree, prealloc, start, this_end,
960 NULL, NULL, &bits); 960 NULL, NULL, &bits);
961 if (err) 961 if (err)
962 extent_io_tree_panic(tree, err); 962 extent_io_tree_panic(tree, err);
963 963
964 cache_state(prealloc, cached_state); 964 cache_state(prealloc, cached_state);
965 prealloc = NULL; 965 prealloc = NULL;
966 start = this_end + 1; 966 start = this_end + 1;
967 goto search_again; 967 goto search_again;
968 } 968 }
969 /* 969 /*
970 * | ---- desired range ---- | 970 * | ---- desired range ---- |
971 * | state | 971 * | state |
972 * We need to split the extent, and set the bit 972 * We need to split the extent, and set the bit
973 * on the first half 973 * on the first half
974 */ 974 */
975 if (state->start <= end && state->end > end) { 975 if (state->start <= end && state->end > end) {
976 if (state->state & exclusive_bits) { 976 if (state->state & exclusive_bits) {
977 *failed_start = start; 977 *failed_start = start;
978 err = -EEXIST; 978 err = -EEXIST;
979 goto out; 979 goto out;
980 } 980 }
981 981
982 prealloc = alloc_extent_state_atomic(prealloc); 982 prealloc = alloc_extent_state_atomic(prealloc);
983 BUG_ON(!prealloc); 983 BUG_ON(!prealloc);
984 err = split_state(tree, state, prealloc, end + 1); 984 err = split_state(tree, state, prealloc, end + 1);
985 if (err) 985 if (err)
986 extent_io_tree_panic(tree, err); 986 extent_io_tree_panic(tree, err);
987 987
988 set_state_bits(tree, prealloc, &bits); 988 set_state_bits(tree, prealloc, &bits);
989 cache_state(prealloc, cached_state); 989 cache_state(prealloc, cached_state);
990 merge_state(tree, prealloc); 990 merge_state(tree, prealloc);
991 prealloc = NULL; 991 prealloc = NULL;
992 goto out; 992 goto out;
993 } 993 }
994 994
995 goto search_again; 995 goto search_again;
996 996
997 out: 997 out:
998 spin_unlock(&tree->lock); 998 spin_unlock(&tree->lock);
999 if (prealloc) 999 if (prealloc)
1000 free_extent_state(prealloc); 1000 free_extent_state(prealloc);
1001 1001
1002 return err; 1002 return err;
1003 1003
1004 search_again: 1004 search_again:
1005 if (start > end) 1005 if (start > end)
1006 goto out; 1006 goto out;
1007 spin_unlock(&tree->lock); 1007 spin_unlock(&tree->lock);
1008 if (mask & __GFP_WAIT) 1008 if (mask & __GFP_WAIT)
1009 cond_resched(); 1009 cond_resched();
1010 goto again; 1010 goto again;
1011 } 1011 }
1012 1012
1013 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 1013 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1014 unsigned long bits, u64 * failed_start, 1014 unsigned long bits, u64 * failed_start,
1015 struct extent_state **cached_state, gfp_t mask) 1015 struct extent_state **cached_state, gfp_t mask)
1016 { 1016 {
1017 return __set_extent_bit(tree, start, end, bits, 0, failed_start, 1017 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1018 cached_state, mask); 1018 cached_state, mask);
1019 } 1019 }
1020 1020
1021 1021
1022 /** 1022 /**
1023 * convert_extent_bit - convert all bits in a given range from one bit to 1023 * convert_extent_bit - convert all bits in a given range from one bit to
1024 * another 1024 * another
1025 * @tree: the io tree to search 1025 * @tree: the io tree to search
1026 * @start: the start offset in bytes 1026 * @start: the start offset in bytes
1027 * @end: the end offset in bytes (inclusive) 1027 * @end: the end offset in bytes (inclusive)
1028 * @bits: the bits to set in this range 1028 * @bits: the bits to set in this range
1029 * @clear_bits: the bits to clear in this range 1029 * @clear_bits: the bits to clear in this range
1030 * @cached_state: state that we're going to cache 1030 * @cached_state: state that we're going to cache
1031 * @mask: the allocation mask 1031 * @mask: the allocation mask
1032 * 1032 *
1033 * This will go through and set bits for the given range. If any states exist 1033 * This will go through and set bits for the given range. If any states exist
1034 * already in this range they are set with the given bit and cleared of the 1034 * already in this range they are set with the given bit and cleared of the
1035 * clear_bits. This is only meant to be used by things that are mergeable, ie 1035 * clear_bits. This is only meant to be used by things that are mergeable, ie
1036 * converting from say DELALLOC to DIRTY. This is not meant to be used with 1036 * converting from say DELALLOC to DIRTY. This is not meant to be used with
1037 * boundary bits like LOCK. 1037 * boundary bits like LOCK.
1038 */ 1038 */
1039 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 1039 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1040 unsigned long bits, unsigned long clear_bits, 1040 unsigned long bits, unsigned long clear_bits,
1041 struct extent_state **cached_state, gfp_t mask) 1041 struct extent_state **cached_state, gfp_t mask)
1042 { 1042 {
1043 struct extent_state *state; 1043 struct extent_state *state;
1044 struct extent_state *prealloc = NULL; 1044 struct extent_state *prealloc = NULL;
1045 struct rb_node *node; 1045 struct rb_node *node;
1046 struct rb_node **p; 1046 struct rb_node **p;
1047 struct rb_node *parent; 1047 struct rb_node *parent;
1048 int err = 0; 1048 int err = 0;
1049 u64 last_start; 1049 u64 last_start;
1050 u64 last_end; 1050 u64 last_end;
1051 1051
1052 btrfs_debug_check_extent_io_range(tree, start, end); 1052 btrfs_debug_check_extent_io_range(tree, start, end);
1053 1053
1054 again: 1054 again:
1055 if (!prealloc && (mask & __GFP_WAIT)) { 1055 if (!prealloc && (mask & __GFP_WAIT)) {
1056 prealloc = alloc_extent_state(mask); 1056 prealloc = alloc_extent_state(mask);
1057 if (!prealloc) 1057 if (!prealloc)
1058 return -ENOMEM; 1058 return -ENOMEM;
1059 } 1059 }
1060 1060
1061 spin_lock(&tree->lock); 1061 spin_lock(&tree->lock);
1062 if (cached_state && *cached_state) { 1062 if (cached_state && *cached_state) {
1063 state = *cached_state; 1063 state = *cached_state;
1064 if (state->start <= start && state->end > start && 1064 if (state->start <= start && state->end > start &&
1065 state->tree) { 1065 state->tree) {
1066 node = &state->rb_node; 1066 node = &state->rb_node;
1067 goto hit_next; 1067 goto hit_next;
1068 } 1068 }
1069 } 1069 }
1070 1070
1071 /* 1071 /*
1072 * this search will find all the extents that end after 1072 * this search will find all the extents that end after
1073 * our range starts. 1073 * our range starts.
1074 */ 1074 */
1075 node = tree_search_for_insert(tree, start, &p, &parent); 1075 node = tree_search_for_insert(tree, start, &p, &parent);
1076 if (!node) { 1076 if (!node) {
1077 prealloc = alloc_extent_state_atomic(prealloc); 1077 prealloc = alloc_extent_state_atomic(prealloc);
1078 if (!prealloc) { 1078 if (!prealloc) {
1079 err = -ENOMEM; 1079 err = -ENOMEM;
1080 goto out; 1080 goto out;
1081 } 1081 }
1082 err = insert_state(tree, prealloc, start, end, 1082 err = insert_state(tree, prealloc, start, end,
1083 &p, &parent, &bits); 1083 &p, &parent, &bits);
1084 if (err) 1084 if (err)
1085 extent_io_tree_panic(tree, err); 1085 extent_io_tree_panic(tree, err);
1086 cache_state(prealloc, cached_state); 1086 cache_state(prealloc, cached_state);
1087 prealloc = NULL; 1087 prealloc = NULL;
1088 goto out; 1088 goto out;
1089 } 1089 }
1090 state = rb_entry(node, struct extent_state, rb_node); 1090 state = rb_entry(node, struct extent_state, rb_node);
1091 hit_next: 1091 hit_next:
1092 last_start = state->start; 1092 last_start = state->start;
1093 last_end = state->end; 1093 last_end = state->end;
1094 1094
1095 /* 1095 /*
1096 * | ---- desired range ---- | 1096 * | ---- desired range ---- |
1097 * | state | 1097 * | state |
1098 * 1098 *
1099 * Just lock what we found and keep going 1099 * Just lock what we found and keep going
1100 */ 1100 */
1101 if (state->start == start && state->end <= end) { 1101 if (state->start == start && state->end <= end) {
1102 set_state_bits(tree, state, &bits); 1102 set_state_bits(tree, state, &bits);
1103 cache_state(state, cached_state); 1103 cache_state(state, cached_state);
1104 state = clear_state_bit(tree, state, &clear_bits, 0); 1104 state = clear_state_bit(tree, state, &clear_bits, 0);
1105 if (last_end == (u64)-1) 1105 if (last_end == (u64)-1)
1106 goto out; 1106 goto out;
1107 start = last_end + 1; 1107 start = last_end + 1;
1108 if (start < end && state && state->start == start && 1108 if (start < end && state && state->start == start &&
1109 !need_resched()) 1109 !need_resched())
1110 goto hit_next; 1110 goto hit_next;
1111 goto search_again; 1111 goto search_again;
1112 } 1112 }
1113 1113
1114 /* 1114 /*
1115 * | ---- desired range ---- | 1115 * | ---- desired range ---- |
1116 * | state | 1116 * | state |
1117 * or 1117 * or
1118 * | ------------- state -------------- | 1118 * | ------------- state -------------- |
1119 * 1119 *
1120 * We need to split the extent we found, and may flip bits on 1120 * We need to split the extent we found, and may flip bits on
1121 * second half. 1121 * second half.
1122 * 1122 *
1123 * If the extent we found extends past our 1123 * If the extent we found extends past our
1124 * range, we just split and search again. It'll get split 1124 * range, we just split and search again. It'll get split
1125 * again the next time though. 1125 * again the next time though.
1126 * 1126 *
1127 * If the extent we found is inside our range, we set the 1127 * If the extent we found is inside our range, we set the
1128 * desired bit on it. 1128 * desired bit on it.
1129 */ 1129 */
1130 if (state->start < start) { 1130 if (state->start < start) {
1131 prealloc = alloc_extent_state_atomic(prealloc); 1131 prealloc = alloc_extent_state_atomic(prealloc);
1132 if (!prealloc) { 1132 if (!prealloc) {
1133 err = -ENOMEM; 1133 err = -ENOMEM;
1134 goto out; 1134 goto out;
1135 } 1135 }
1136 err = split_state(tree, state, prealloc, start); 1136 err = split_state(tree, state, prealloc, start);
1137 if (err) 1137 if (err)
1138 extent_io_tree_panic(tree, err); 1138 extent_io_tree_panic(tree, err);
1139 prealloc = NULL; 1139 prealloc = NULL;
1140 if (err) 1140 if (err)
1141 goto out; 1141 goto out;
1142 if (state->end <= end) { 1142 if (state->end <= end) {
1143 set_state_bits(tree, state, &bits); 1143 set_state_bits(tree, state, &bits);
1144 cache_state(state, cached_state); 1144 cache_state(state, cached_state);
1145 state = clear_state_bit(tree, state, &clear_bits, 0); 1145 state = clear_state_bit(tree, state, &clear_bits, 0);
1146 if (last_end == (u64)-1) 1146 if (last_end == (u64)-1)
1147 goto out; 1147 goto out;
1148 start = last_end + 1; 1148 start = last_end + 1;
1149 if (start < end && state && state->start == start && 1149 if (start < end && state && state->start == start &&
1150 !need_resched()) 1150 !need_resched())
1151 goto hit_next; 1151 goto hit_next;
1152 } 1152 }
1153 goto search_again; 1153 goto search_again;
1154 } 1154 }
1155 /* 1155 /*
1156 * | ---- desired range ---- | 1156 * | ---- desired range ---- |
1157 * | state | or | state | 1157 * | state | or | state |
1158 * 1158 *
1159 * There's a hole, we need to insert something in it and 1159 * There's a hole, we need to insert something in it and
1160 * ignore the extent we found. 1160 * ignore the extent we found.
1161 */ 1161 */
1162 if (state->start > start) { 1162 if (state->start > start) {
1163 u64 this_end; 1163 u64 this_end;
1164 if (end < last_start) 1164 if (end < last_start)
1165 this_end = end; 1165 this_end = end;
1166 else 1166 else
1167 this_end = last_start - 1; 1167 this_end = last_start - 1;
1168 1168
1169 prealloc = alloc_extent_state_atomic(prealloc); 1169 prealloc = alloc_extent_state_atomic(prealloc);
1170 if (!prealloc) { 1170 if (!prealloc) {
1171 err = -ENOMEM; 1171 err = -ENOMEM;
1172 goto out; 1172 goto out;
1173 } 1173 }
1174 1174
1175 /* 1175 /*
1176 * Avoid to free 'prealloc' if it can be merged with 1176 * Avoid to free 'prealloc' if it can be merged with
1177 * the later extent. 1177 * the later extent.
1178 */ 1178 */
1179 err = insert_state(tree, prealloc, start, this_end, 1179 err = insert_state(tree, prealloc, start, this_end,
1180 NULL, NULL, &bits); 1180 NULL, NULL, &bits);
1181 if (err) 1181 if (err)
1182 extent_io_tree_panic(tree, err); 1182 extent_io_tree_panic(tree, err);
1183 cache_state(prealloc, cached_state); 1183 cache_state(prealloc, cached_state);
1184 prealloc = NULL; 1184 prealloc = NULL;
1185 start = this_end + 1; 1185 start = this_end + 1;
1186 goto search_again; 1186 goto search_again;
1187 } 1187 }
1188 /* 1188 /*
1189 * | ---- desired range ---- | 1189 * | ---- desired range ---- |
1190 * | state | 1190 * | state |
1191 * We need to split the extent, and set the bit 1191 * We need to split the extent, and set the bit
1192 * on the first half 1192 * on the first half
1193 */ 1193 */
1194 if (state->start <= end && state->end > end) { 1194 if (state->start <= end && state->end > end) {
1195 prealloc = alloc_extent_state_atomic(prealloc); 1195 prealloc = alloc_extent_state_atomic(prealloc);
1196 if (!prealloc) { 1196 if (!prealloc) {
1197 err = -ENOMEM; 1197 err = -ENOMEM;
1198 goto out; 1198 goto out;
1199 } 1199 }
1200 1200
1201 err = split_state(tree, state, prealloc, end + 1); 1201 err = split_state(tree, state, prealloc, end + 1);
1202 if (err) 1202 if (err)
1203 extent_io_tree_panic(tree, err); 1203 extent_io_tree_panic(tree, err);
1204 1204
1205 set_state_bits(tree, prealloc, &bits); 1205 set_state_bits(tree, prealloc, &bits);
1206 cache_state(prealloc, cached_state); 1206 cache_state(prealloc, cached_state);
1207 clear_state_bit(tree, prealloc, &clear_bits, 0); 1207 clear_state_bit(tree, prealloc, &clear_bits, 0);
1208 prealloc = NULL; 1208 prealloc = NULL;
1209 goto out; 1209 goto out;
1210 } 1210 }
1211 1211
1212 goto search_again; 1212 goto search_again;
1213 1213
1214 out: 1214 out:
1215 spin_unlock(&tree->lock); 1215 spin_unlock(&tree->lock);
1216 if (prealloc) 1216 if (prealloc)
1217 free_extent_state(prealloc); 1217 free_extent_state(prealloc);
1218 1218
1219 return err; 1219 return err;
1220 1220
1221 search_again: 1221 search_again:
1222 if (start > end) 1222 if (start > end)
1223 goto out; 1223 goto out;
1224 spin_unlock(&tree->lock); 1224 spin_unlock(&tree->lock);
1225 if (mask & __GFP_WAIT) 1225 if (mask & __GFP_WAIT)
1226 cond_resched(); 1226 cond_resched();
1227 goto again; 1227 goto again;
1228 } 1228 }
1229 1229
1230 /* wrappers around set/clear extent bit */ 1230 /* wrappers around set/clear extent bit */
1231 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 1231 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1232 gfp_t mask) 1232 gfp_t mask)
1233 { 1233 {
1234 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL, 1234 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
1235 NULL, mask); 1235 NULL, mask);
1236 } 1236 }
1237 1237
1238 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1238 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1239 unsigned long bits, gfp_t mask) 1239 unsigned long bits, gfp_t mask)
1240 { 1240 {
1241 return set_extent_bit(tree, start, end, bits, NULL, 1241 return set_extent_bit(tree, start, end, bits, NULL,
1242 NULL, mask); 1242 NULL, mask);
1243 } 1243 }
1244 1244
1245 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1245 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1246 unsigned long bits, gfp_t mask) 1246 unsigned long bits, gfp_t mask)
1247 { 1247 {
1248 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask); 1248 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
1249 } 1249 }
1250 1250
1251 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end, 1251 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
1252 struct extent_state **cached_state, gfp_t mask) 1252 struct extent_state **cached_state, gfp_t mask)
1253 { 1253 {
1254 return set_extent_bit(tree, start, end, 1254 return set_extent_bit(tree, start, end,
1255 EXTENT_DELALLOC | EXTENT_UPTODATE, 1255 EXTENT_DELALLOC | EXTENT_UPTODATE,
1256 NULL, cached_state, mask); 1256 NULL, cached_state, mask);
1257 } 1257 }
1258 1258
1259 int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end, 1259 int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1260 struct extent_state **cached_state, gfp_t mask) 1260 struct extent_state **cached_state, gfp_t mask)
1261 { 1261 {
1262 return set_extent_bit(tree, start, end, 1262 return set_extent_bit(tree, start, end,
1263 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG, 1263 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1264 NULL, cached_state, mask); 1264 NULL, cached_state, mask);
1265 } 1265 }
1266 1266
1267 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 1267 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1268 gfp_t mask) 1268 gfp_t mask)
1269 { 1269 {
1270 return clear_extent_bit(tree, start, end, 1270 return clear_extent_bit(tree, start, end,
1271 EXTENT_DIRTY | EXTENT_DELALLOC | 1271 EXTENT_DIRTY | EXTENT_DELALLOC |
1272 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask); 1272 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
1273 } 1273 }
1274 1274
1275 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 1275 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1276 gfp_t mask) 1276 gfp_t mask)
1277 { 1277 {
1278 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL, 1278 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
1279 NULL, mask); 1279 NULL, mask);
1280 } 1280 }
1281 1281
1282 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, 1282 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1283 struct extent_state **cached_state, gfp_t mask) 1283 struct extent_state **cached_state, gfp_t mask)
1284 { 1284 {
1285 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL, 1285 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
1286 cached_state, mask); 1286 cached_state, mask);
1287 } 1287 }
1288 1288
1289 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, 1289 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1290 struct extent_state **cached_state, gfp_t mask) 1290 struct extent_state **cached_state, gfp_t mask)
1291 { 1291 {
1292 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, 1292 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
1293 cached_state, mask); 1293 cached_state, mask);
1294 } 1294 }
1295 1295
1296 /* 1296 /*
1297 * either insert or lock state struct between start and end use mask to tell 1297 * either insert or lock state struct between start and end use mask to tell
1298 * us if waiting is desired. 1298 * us if waiting is desired.
1299 */ 1299 */
1300 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 1300 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1301 unsigned long bits, struct extent_state **cached_state) 1301 unsigned long bits, struct extent_state **cached_state)
1302 { 1302 {
1303 int err; 1303 int err;
1304 u64 failed_start; 1304 u64 failed_start;
1305 while (1) { 1305 while (1) {
1306 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits, 1306 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1307 EXTENT_LOCKED, &failed_start, 1307 EXTENT_LOCKED, &failed_start,
1308 cached_state, GFP_NOFS); 1308 cached_state, GFP_NOFS);
1309 if (err == -EEXIST) { 1309 if (err == -EEXIST) {
1310 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); 1310 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1311 start = failed_start; 1311 start = failed_start;
1312 } else 1312 } else
1313 break; 1313 break;
1314 WARN_ON(start > end); 1314 WARN_ON(start > end);
1315 } 1315 }
1316 return err; 1316 return err;
1317 } 1317 }
1318 1318
1319 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end) 1319 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1320 { 1320 {
1321 return lock_extent_bits(tree, start, end, 0, NULL); 1321 return lock_extent_bits(tree, start, end, 0, NULL);
1322 } 1322 }
1323 1323
1324 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end) 1324 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1325 { 1325 {
1326 int err; 1326 int err;
1327 u64 failed_start; 1327 u64 failed_start;
1328 1328
1329 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED, 1329 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1330 &failed_start, NULL, GFP_NOFS); 1330 &failed_start, NULL, GFP_NOFS);
1331 if (err == -EEXIST) { 1331 if (err == -EEXIST) {
1332 if (failed_start > start) 1332 if (failed_start > start)
1333 clear_extent_bit(tree, start, failed_start - 1, 1333 clear_extent_bit(tree, start, failed_start - 1,
1334 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS); 1334 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
1335 return 0; 1335 return 0;
1336 } 1336 }
1337 return 1; 1337 return 1;
1338 } 1338 }
1339 1339
1340 int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end, 1340 int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1341 struct extent_state **cached, gfp_t mask) 1341 struct extent_state **cached, gfp_t mask)
1342 { 1342 {
1343 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached, 1343 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1344 mask); 1344 mask);
1345 } 1345 }
1346 1346
1347 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end) 1347 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1348 { 1348 {
1349 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL, 1349 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1350 GFP_NOFS); 1350 GFP_NOFS);
1351 } 1351 }
1352 1352
1353 int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end) 1353 int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1354 { 1354 {
1355 unsigned long index = start >> PAGE_CACHE_SHIFT; 1355 unsigned long index = start >> PAGE_CACHE_SHIFT;
1356 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1356 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1357 struct page *page; 1357 struct page *page;
1358 1358
1359 while (index <= end_index) { 1359 while (index <= end_index) {
1360 page = find_get_page(inode->i_mapping, index); 1360 page = find_get_page(inode->i_mapping, index);
1361 BUG_ON(!page); /* Pages should be in the extent_io_tree */ 1361 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1362 clear_page_dirty_for_io(page); 1362 clear_page_dirty_for_io(page);
1363 page_cache_release(page); 1363 page_cache_release(page);
1364 index++; 1364 index++;
1365 } 1365 }
1366 return 0; 1366 return 0;
1367 } 1367 }
1368 1368
1369 int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end) 1369 int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1370 { 1370 {
1371 unsigned long index = start >> PAGE_CACHE_SHIFT; 1371 unsigned long index = start >> PAGE_CACHE_SHIFT;
1372 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1372 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1373 struct page *page; 1373 struct page *page;
1374 1374
1375 while (index <= end_index) { 1375 while (index <= end_index) {
1376 page = find_get_page(inode->i_mapping, index); 1376 page = find_get_page(inode->i_mapping, index);
1377 BUG_ON(!page); /* Pages should be in the extent_io_tree */ 1377 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1378 account_page_redirty(page); 1378 account_page_redirty(page);
1379 __set_page_dirty_nobuffers(page); 1379 __set_page_dirty_nobuffers(page);
1380 page_cache_release(page); 1380 page_cache_release(page);
1381 index++; 1381 index++;
1382 } 1382 }
1383 return 0; 1383 return 0;
1384 } 1384 }
1385 1385
1386 /* 1386 /*
1387 * helper function to set both pages and extents in the tree writeback 1387 * helper function to set both pages and extents in the tree writeback
1388 */ 1388 */
1389 static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end) 1389 static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1390 { 1390 {
1391 unsigned long index = start >> PAGE_CACHE_SHIFT; 1391 unsigned long index = start >> PAGE_CACHE_SHIFT;
1392 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1392 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1393 struct page *page; 1393 struct page *page;
1394 1394
1395 while (index <= end_index) { 1395 while (index <= end_index) {
1396 page = find_get_page(tree->mapping, index); 1396 page = find_get_page(tree->mapping, index);
1397 BUG_ON(!page); /* Pages should be in the extent_io_tree */ 1397 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1398 set_page_writeback(page); 1398 set_page_writeback(page);
1399 page_cache_release(page); 1399 page_cache_release(page);
1400 index++; 1400 index++;
1401 } 1401 }
1402 return 0; 1402 return 0;
1403 } 1403 }
1404 1404
1405 /* find the first state struct with 'bits' set after 'start', and 1405 /* find the first state struct with 'bits' set after 'start', and
1406 * return it. tree->lock must be held. NULL will returned if 1406 * return it. tree->lock must be held. NULL will returned if
1407 * nothing was found after 'start' 1407 * nothing was found after 'start'
1408 */ 1408 */
1409 static struct extent_state * 1409 static struct extent_state *
1410 find_first_extent_bit_state(struct extent_io_tree *tree, 1410 find_first_extent_bit_state(struct extent_io_tree *tree,
1411 u64 start, unsigned long bits) 1411 u64 start, unsigned long bits)
1412 { 1412 {
1413 struct rb_node *node; 1413 struct rb_node *node;
1414 struct extent_state *state; 1414 struct extent_state *state;
1415 1415
1416 /* 1416 /*
1417 * this search will find all the extents that end after 1417 * this search will find all the extents that end after
1418 * our range starts. 1418 * our range starts.
1419 */ 1419 */
1420 node = tree_search(tree, start); 1420 node = tree_search(tree, start);
1421 if (!node) 1421 if (!node)
1422 goto out; 1422 goto out;
1423 1423
1424 while (1) { 1424 while (1) {
1425 state = rb_entry(node, struct extent_state, rb_node); 1425 state = rb_entry(node, struct extent_state, rb_node);
1426 if (state->end >= start && (state->state & bits)) 1426 if (state->end >= start && (state->state & bits))
1427 return state; 1427 return state;
1428 1428
1429 node = rb_next(node); 1429 node = rb_next(node);
1430 if (!node) 1430 if (!node)
1431 break; 1431 break;
1432 } 1432 }
1433 out: 1433 out:
1434 return NULL; 1434 return NULL;
1435 } 1435 }
1436 1436
1437 /* 1437 /*
1438 * find the first offset in the io tree with 'bits' set. zero is 1438 * find the first offset in the io tree with 'bits' set. zero is
1439 * returned if we find something, and *start_ret and *end_ret are 1439 * returned if we find something, and *start_ret and *end_ret are
1440 * set to reflect the state struct that was found. 1440 * set to reflect the state struct that was found.
1441 * 1441 *
1442 * If nothing was found, 1 is returned. If found something, return 0. 1442 * If nothing was found, 1 is returned. If found something, return 0.
1443 */ 1443 */
1444 int find_first_extent_bit(struct extent_io_tree *tree, u64 start, 1444 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1445 u64 *start_ret, u64 *end_ret, unsigned long bits, 1445 u64 *start_ret, u64 *end_ret, unsigned long bits,
1446 struct extent_state **cached_state) 1446 struct extent_state **cached_state)
1447 { 1447 {
1448 struct extent_state *state; 1448 struct extent_state *state;
1449 struct rb_node *n; 1449 struct rb_node *n;
1450 int ret = 1; 1450 int ret = 1;
1451 1451
1452 spin_lock(&tree->lock); 1452 spin_lock(&tree->lock);
1453 if (cached_state && *cached_state) { 1453 if (cached_state && *cached_state) {
1454 state = *cached_state; 1454 state = *cached_state;
1455 if (state->end == start - 1 && state->tree) { 1455 if (state->end == start - 1 && state->tree) {
1456 n = rb_next(&state->rb_node); 1456 n = rb_next(&state->rb_node);
1457 while (n) { 1457 while (n) {
1458 state = rb_entry(n, struct extent_state, 1458 state = rb_entry(n, struct extent_state,
1459 rb_node); 1459 rb_node);
1460 if (state->state & bits) 1460 if (state->state & bits)
1461 goto got_it; 1461 goto got_it;
1462 n = rb_next(n); 1462 n = rb_next(n);
1463 } 1463 }
1464 free_extent_state(*cached_state); 1464 free_extent_state(*cached_state);
1465 *cached_state = NULL; 1465 *cached_state = NULL;
1466 goto out; 1466 goto out;
1467 } 1467 }
1468 free_extent_state(*cached_state); 1468 free_extent_state(*cached_state);
1469 *cached_state = NULL; 1469 *cached_state = NULL;
1470 } 1470 }
1471 1471
1472 state = find_first_extent_bit_state(tree, start, bits); 1472 state = find_first_extent_bit_state(tree, start, bits);
1473 got_it: 1473 got_it:
1474 if (state) { 1474 if (state) {
1475 cache_state(state, cached_state); 1475 cache_state(state, cached_state);
1476 *start_ret = state->start; 1476 *start_ret = state->start;
1477 *end_ret = state->end; 1477 *end_ret = state->end;
1478 ret = 0; 1478 ret = 0;
1479 } 1479 }
1480 out: 1480 out:
1481 spin_unlock(&tree->lock); 1481 spin_unlock(&tree->lock);
1482 return ret; 1482 return ret;
1483 } 1483 }
1484 1484
1485 /* 1485 /*
1486 * find a contiguous range of bytes in the file marked as delalloc, not 1486 * find a contiguous range of bytes in the file marked as delalloc, not
1487 * more than 'max_bytes'. start and end are used to return the range, 1487 * more than 'max_bytes'. start and end are used to return the range,
1488 * 1488 *
1489 * 1 is returned if we find something, 0 if nothing was in the tree 1489 * 1 is returned if we find something, 0 if nothing was in the tree
1490 */ 1490 */
1491 static noinline u64 find_delalloc_range(struct extent_io_tree *tree, 1491 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1492 u64 *start, u64 *end, u64 max_bytes, 1492 u64 *start, u64 *end, u64 max_bytes,
1493 struct extent_state **cached_state) 1493 struct extent_state **cached_state)
1494 { 1494 {
1495 struct rb_node *node; 1495 struct rb_node *node;
1496 struct extent_state *state; 1496 struct extent_state *state;
1497 u64 cur_start = *start; 1497 u64 cur_start = *start;
1498 u64 found = 0; 1498 u64 found = 0;
1499 u64 total_bytes = 0; 1499 u64 total_bytes = 0;
1500 1500
1501 spin_lock(&tree->lock); 1501 spin_lock(&tree->lock);
1502 1502
1503 /* 1503 /*
1504 * this search will find all the extents that end after 1504 * this search will find all the extents that end after
1505 * our range starts. 1505 * our range starts.
1506 */ 1506 */
1507 node = tree_search(tree, cur_start); 1507 node = tree_search(tree, cur_start);
1508 if (!node) { 1508 if (!node) {
1509 if (!found) 1509 if (!found)
1510 *end = (u64)-1; 1510 *end = (u64)-1;
1511 goto out; 1511 goto out;
1512 } 1512 }
1513 1513
1514 while (1) { 1514 while (1) {
1515 state = rb_entry(node, struct extent_state, rb_node); 1515 state = rb_entry(node, struct extent_state, rb_node);
1516 if (found && (state->start != cur_start || 1516 if (found && (state->start != cur_start ||
1517 (state->state & EXTENT_BOUNDARY))) { 1517 (state->state & EXTENT_BOUNDARY))) {
1518 goto out; 1518 goto out;
1519 } 1519 }
1520 if (!(state->state & EXTENT_DELALLOC)) { 1520 if (!(state->state & EXTENT_DELALLOC)) {
1521 if (!found) 1521 if (!found)
1522 *end = state->end; 1522 *end = state->end;
1523 goto out; 1523 goto out;
1524 } 1524 }
1525 if (!found) { 1525 if (!found) {
1526 *start = state->start; 1526 *start = state->start;
1527 *cached_state = state; 1527 *cached_state = state;
1528 atomic_inc(&state->refs); 1528 atomic_inc(&state->refs);
1529 } 1529 }
1530 found++; 1530 found++;
1531 *end = state->end; 1531 *end = state->end;
1532 cur_start = state->end + 1; 1532 cur_start = state->end + 1;
1533 node = rb_next(node); 1533 node = rb_next(node);
1534 total_bytes += state->end - state->start + 1; 1534 total_bytes += state->end - state->start + 1;
1535 if (total_bytes >= max_bytes) 1535 if (total_bytes >= max_bytes)
1536 break; 1536 break;
1537 if (!node) 1537 if (!node)
1538 break; 1538 break;
1539 } 1539 }
1540 out: 1540 out:
1541 spin_unlock(&tree->lock); 1541 spin_unlock(&tree->lock);
1542 return found; 1542 return found;
1543 } 1543 }
1544 1544
1545 static noinline void __unlock_for_delalloc(struct inode *inode, 1545 static noinline void __unlock_for_delalloc(struct inode *inode,
1546 struct page *locked_page, 1546 struct page *locked_page,
1547 u64 start, u64 end) 1547 u64 start, u64 end)
1548 { 1548 {
1549 int ret; 1549 int ret;
1550 struct page *pages[16]; 1550 struct page *pages[16];
1551 unsigned long index = start >> PAGE_CACHE_SHIFT; 1551 unsigned long index = start >> PAGE_CACHE_SHIFT;
1552 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1552 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1553 unsigned long nr_pages = end_index - index + 1; 1553 unsigned long nr_pages = end_index - index + 1;
1554 int i; 1554 int i;
1555 1555
1556 if (index == locked_page->index && end_index == index) 1556 if (index == locked_page->index && end_index == index)
1557 return; 1557 return;
1558 1558
1559 while (nr_pages > 0) { 1559 while (nr_pages > 0) {
1560 ret = find_get_pages_contig(inode->i_mapping, index, 1560 ret = find_get_pages_contig(inode->i_mapping, index,
1561 min_t(unsigned long, nr_pages, 1561 min_t(unsigned long, nr_pages,
1562 ARRAY_SIZE(pages)), pages); 1562 ARRAY_SIZE(pages)), pages);
1563 for (i = 0; i < ret; i++) { 1563 for (i = 0; i < ret; i++) {
1564 if (pages[i] != locked_page) 1564 if (pages[i] != locked_page)
1565 unlock_page(pages[i]); 1565 unlock_page(pages[i]);
1566 page_cache_release(pages[i]); 1566 page_cache_release(pages[i]);
1567 } 1567 }
1568 nr_pages -= ret; 1568 nr_pages -= ret;
1569 index += ret; 1569 index += ret;
1570 cond_resched(); 1570 cond_resched();
1571 } 1571 }
1572 } 1572 }
1573 1573
1574 static noinline int lock_delalloc_pages(struct inode *inode, 1574 static noinline int lock_delalloc_pages(struct inode *inode,
1575 struct page *locked_page, 1575 struct page *locked_page,
1576 u64 delalloc_start, 1576 u64 delalloc_start,
1577 u64 delalloc_end) 1577 u64 delalloc_end)
1578 { 1578 {
1579 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT; 1579 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1580 unsigned long start_index = index; 1580 unsigned long start_index = index;
1581 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT; 1581 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1582 unsigned long pages_locked = 0; 1582 unsigned long pages_locked = 0;
1583 struct page *pages[16]; 1583 struct page *pages[16];
1584 unsigned long nrpages; 1584 unsigned long nrpages;
1585 int ret; 1585 int ret;
1586 int i; 1586 int i;
1587 1587
1588 /* the caller is responsible for locking the start index */ 1588 /* the caller is responsible for locking the start index */
1589 if (index == locked_page->index && index == end_index) 1589 if (index == locked_page->index && index == end_index)
1590 return 0; 1590 return 0;
1591 1591
1592 /* skip the page at the start index */ 1592 /* skip the page at the start index */
1593 nrpages = end_index - index + 1; 1593 nrpages = end_index - index + 1;
1594 while (nrpages > 0) { 1594 while (nrpages > 0) {
1595 ret = find_get_pages_contig(inode->i_mapping, index, 1595 ret = find_get_pages_contig(inode->i_mapping, index,
1596 min_t(unsigned long, 1596 min_t(unsigned long,
1597 nrpages, ARRAY_SIZE(pages)), pages); 1597 nrpages, ARRAY_SIZE(pages)), pages);
1598 if (ret == 0) { 1598 if (ret == 0) {
1599 ret = -EAGAIN; 1599 ret = -EAGAIN;
1600 goto done; 1600 goto done;
1601 } 1601 }
1602 /* now we have an array of pages, lock them all */ 1602 /* now we have an array of pages, lock them all */
1603 for (i = 0; i < ret; i++) { 1603 for (i = 0; i < ret; i++) {
1604 /* 1604 /*
1605 * the caller is taking responsibility for 1605 * the caller is taking responsibility for
1606 * locked_page 1606 * locked_page
1607 */ 1607 */
1608 if (pages[i] != locked_page) { 1608 if (pages[i] != locked_page) {
1609 lock_page(pages[i]); 1609 lock_page(pages[i]);
1610 if (!PageDirty(pages[i]) || 1610 if (!PageDirty(pages[i]) ||
1611 pages[i]->mapping != inode->i_mapping) { 1611 pages[i]->mapping != inode->i_mapping) {
1612 ret = -EAGAIN; 1612 ret = -EAGAIN;
1613 unlock_page(pages[i]); 1613 unlock_page(pages[i]);
1614 page_cache_release(pages[i]); 1614 page_cache_release(pages[i]);
1615 goto done; 1615 goto done;
1616 } 1616 }
1617 } 1617 }
1618 page_cache_release(pages[i]); 1618 page_cache_release(pages[i]);
1619 pages_locked++; 1619 pages_locked++;
1620 } 1620 }
1621 nrpages -= ret; 1621 nrpages -= ret;
1622 index += ret; 1622 index += ret;
1623 cond_resched(); 1623 cond_resched();
1624 } 1624 }
1625 ret = 0; 1625 ret = 0;
1626 done: 1626 done:
1627 if (ret && pages_locked) { 1627 if (ret && pages_locked) {
1628 __unlock_for_delalloc(inode, locked_page, 1628 __unlock_for_delalloc(inode, locked_page,
1629 delalloc_start, 1629 delalloc_start,
1630 ((u64)(start_index + pages_locked - 1)) << 1630 ((u64)(start_index + pages_locked - 1)) <<
1631 PAGE_CACHE_SHIFT); 1631 PAGE_CACHE_SHIFT);
1632 } 1632 }
1633 return ret; 1633 return ret;
1634 } 1634 }
1635 1635
1636 /* 1636 /*
1637 * find a contiguous range of bytes in the file marked as delalloc, not 1637 * find a contiguous range of bytes in the file marked as delalloc, not
1638 * more than 'max_bytes'. start and end are used to return the range, 1638 * more than 'max_bytes'. start and end are used to return the range,
1639 * 1639 *
1640 * 1 is returned if we find something, 0 if nothing was in the tree 1640 * 1 is returned if we find something, 0 if nothing was in the tree
1641 */ 1641 */
1642 STATIC u64 find_lock_delalloc_range(struct inode *inode, 1642 STATIC u64 find_lock_delalloc_range(struct inode *inode,
1643 struct extent_io_tree *tree, 1643 struct extent_io_tree *tree,
1644 struct page *locked_page, u64 *start, 1644 struct page *locked_page, u64 *start,
1645 u64 *end, u64 max_bytes) 1645 u64 *end, u64 max_bytes)
1646 { 1646 {
1647 u64 delalloc_start; 1647 u64 delalloc_start;
1648 u64 delalloc_end; 1648 u64 delalloc_end;
1649 u64 found; 1649 u64 found;
1650 struct extent_state *cached_state = NULL; 1650 struct extent_state *cached_state = NULL;
1651 int ret; 1651 int ret;
1652 int loops = 0; 1652 int loops = 0;
1653 1653
1654 again: 1654 again:
1655 /* step one, find a bunch of delalloc bytes starting at start */ 1655 /* step one, find a bunch of delalloc bytes starting at start */
1656 delalloc_start = *start; 1656 delalloc_start = *start;
1657 delalloc_end = 0; 1657 delalloc_end = 0;
1658 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end, 1658 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1659 max_bytes, &cached_state); 1659 max_bytes, &cached_state);
1660 if (!found || delalloc_end <= *start) { 1660 if (!found || delalloc_end <= *start) {
1661 *start = delalloc_start; 1661 *start = delalloc_start;
1662 *end = delalloc_end; 1662 *end = delalloc_end;
1663 free_extent_state(cached_state); 1663 free_extent_state(cached_state);
1664 return 0; 1664 return 0;
1665 } 1665 }
1666 1666
1667 /* 1667 /*
1668 * start comes from the offset of locked_page. We have to lock 1668 * start comes from the offset of locked_page. We have to lock
1669 * pages in order, so we can't process delalloc bytes before 1669 * pages in order, so we can't process delalloc bytes before
1670 * locked_page 1670 * locked_page
1671 */ 1671 */
1672 if (delalloc_start < *start) 1672 if (delalloc_start < *start)
1673 delalloc_start = *start; 1673 delalloc_start = *start;
1674 1674
1675 /* 1675 /*
1676 * make sure to limit the number of pages we try to lock down 1676 * make sure to limit the number of pages we try to lock down
1677 */ 1677 */
1678 if (delalloc_end + 1 - delalloc_start > max_bytes) 1678 if (delalloc_end + 1 - delalloc_start > max_bytes)
1679 delalloc_end = delalloc_start + max_bytes - 1; 1679 delalloc_end = delalloc_start + max_bytes - 1;
1680 1680
1681 /* step two, lock all the pages after the page that has start */ 1681 /* step two, lock all the pages after the page that has start */
1682 ret = lock_delalloc_pages(inode, locked_page, 1682 ret = lock_delalloc_pages(inode, locked_page,
1683 delalloc_start, delalloc_end); 1683 delalloc_start, delalloc_end);
1684 if (ret == -EAGAIN) { 1684 if (ret == -EAGAIN) {
1685 /* some of the pages are gone, lets avoid looping by 1685 /* some of the pages are gone, lets avoid looping by
1686 * shortening the size of the delalloc range we're searching 1686 * shortening the size of the delalloc range we're searching
1687 */ 1687 */
1688 free_extent_state(cached_state); 1688 free_extent_state(cached_state);
1689 cached_state = NULL; 1689 cached_state = NULL;
1690 if (!loops) { 1690 if (!loops) {
1691 max_bytes = PAGE_CACHE_SIZE; 1691 max_bytes = PAGE_CACHE_SIZE;
1692 loops = 1; 1692 loops = 1;
1693 goto again; 1693 goto again;
1694 } else { 1694 } else {
1695 found = 0; 1695 found = 0;
1696 goto out_failed; 1696 goto out_failed;
1697 } 1697 }
1698 } 1698 }
1699 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */ 1699 BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1700 1700
1701 /* step three, lock the state bits for the whole range */ 1701 /* step three, lock the state bits for the whole range */
1702 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state); 1702 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
1703 1703
1704 /* then test to make sure it is all still delalloc */ 1704 /* then test to make sure it is all still delalloc */
1705 ret = test_range_bit(tree, delalloc_start, delalloc_end, 1705 ret = test_range_bit(tree, delalloc_start, delalloc_end,
1706 EXTENT_DELALLOC, 1, cached_state); 1706 EXTENT_DELALLOC, 1, cached_state);
1707 if (!ret) { 1707 if (!ret) {
1708 unlock_extent_cached(tree, delalloc_start, delalloc_end, 1708 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1709 &cached_state, GFP_NOFS); 1709 &cached_state, GFP_NOFS);
1710 __unlock_for_delalloc(inode, locked_page, 1710 __unlock_for_delalloc(inode, locked_page,
1711 delalloc_start, delalloc_end); 1711 delalloc_start, delalloc_end);
1712 cond_resched(); 1712 cond_resched();
1713 goto again; 1713 goto again;
1714 } 1714 }
1715 free_extent_state(cached_state); 1715 free_extent_state(cached_state);
1716 *start = delalloc_start; 1716 *start = delalloc_start;
1717 *end = delalloc_end; 1717 *end = delalloc_end;
1718 out_failed: 1718 out_failed:
1719 return found; 1719 return found;
1720 } 1720 }
1721 1721
1722 int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end, 1722 int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1723 struct page *locked_page, 1723 struct page *locked_page,
1724 unsigned long clear_bits, 1724 unsigned long clear_bits,
1725 unsigned long page_ops) 1725 unsigned long page_ops)
1726 { 1726 {
1727 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 1727 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1728 int ret; 1728 int ret;
1729 struct page *pages[16]; 1729 struct page *pages[16];
1730 unsigned long index = start >> PAGE_CACHE_SHIFT; 1730 unsigned long index = start >> PAGE_CACHE_SHIFT;
1731 unsigned long end_index = end >> PAGE_CACHE_SHIFT; 1731 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1732 unsigned long nr_pages = end_index - index + 1; 1732 unsigned long nr_pages = end_index - index + 1;
1733 int i; 1733 int i;
1734 1734
1735 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS); 1735 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1736 if (page_ops == 0) 1736 if (page_ops == 0)
1737 return 0; 1737 return 0;
1738 1738
1739 while (nr_pages > 0) { 1739 while (nr_pages > 0) {
1740 ret = find_get_pages_contig(inode->i_mapping, index, 1740 ret = find_get_pages_contig(inode->i_mapping, index,
1741 min_t(unsigned long, 1741 min_t(unsigned long,
1742 nr_pages, ARRAY_SIZE(pages)), pages); 1742 nr_pages, ARRAY_SIZE(pages)), pages);
1743 for (i = 0; i < ret; i++) { 1743 for (i = 0; i < ret; i++) {
1744 1744
1745 if (page_ops & PAGE_SET_PRIVATE2) 1745 if (page_ops & PAGE_SET_PRIVATE2)
1746 SetPagePrivate2(pages[i]); 1746 SetPagePrivate2(pages[i]);
1747 1747
1748 if (pages[i] == locked_page) { 1748 if (pages[i] == locked_page) {
1749 page_cache_release(pages[i]); 1749 page_cache_release(pages[i]);
1750 continue; 1750 continue;
1751 } 1751 }
1752 if (page_ops & PAGE_CLEAR_DIRTY) 1752 if (page_ops & PAGE_CLEAR_DIRTY)
1753 clear_page_dirty_for_io(pages[i]); 1753 clear_page_dirty_for_io(pages[i]);
1754 if (page_ops & PAGE_SET_WRITEBACK) 1754 if (page_ops & PAGE_SET_WRITEBACK)
1755 set_page_writeback(pages[i]); 1755 set_page_writeback(pages[i]);
1756 if (page_ops & PAGE_END_WRITEBACK) 1756 if (page_ops & PAGE_END_WRITEBACK)
1757 end_page_writeback(pages[i]); 1757 end_page_writeback(pages[i]);
1758 if (page_ops & PAGE_UNLOCK) 1758 if (page_ops & PAGE_UNLOCK)
1759 unlock_page(pages[i]); 1759 unlock_page(pages[i]);
1760 page_cache_release(pages[i]); 1760 page_cache_release(pages[i]);
1761 } 1761 }
1762 nr_pages -= ret; 1762 nr_pages -= ret;
1763 index += ret; 1763 index += ret;
1764 cond_resched(); 1764 cond_resched();
1765 } 1765 }
1766 return 0; 1766 return 0;
1767 } 1767 }
1768 1768
1769 /* 1769 /*
1770 * count the number of bytes in the tree that have a given bit(s) 1770 * count the number of bytes in the tree that have a given bit(s)
1771 * set. This can be fairly slow, except for EXTENT_DIRTY which is 1771 * set. This can be fairly slow, except for EXTENT_DIRTY which is
1772 * cached. The total number found is returned. 1772 * cached. The total number found is returned.
1773 */ 1773 */
1774 u64 count_range_bits(struct extent_io_tree *tree, 1774 u64 count_range_bits(struct extent_io_tree *tree,
1775 u64 *start, u64 search_end, u64 max_bytes, 1775 u64 *start, u64 search_end, u64 max_bytes,
1776 unsigned long bits, int contig) 1776 unsigned long bits, int contig)
1777 { 1777 {
1778 struct rb_node *node; 1778 struct rb_node *node;
1779 struct extent_state *state; 1779 struct extent_state *state;
1780 u64 cur_start = *start; 1780 u64 cur_start = *start;
1781 u64 total_bytes = 0; 1781 u64 total_bytes = 0;
1782 u64 last = 0; 1782 u64 last = 0;
1783 int found = 0; 1783 int found = 0;
1784 1784
1785 if (WARN_ON(search_end <= cur_start)) 1785 if (WARN_ON(search_end <= cur_start))
1786 return 0; 1786 return 0;
1787 1787
1788 spin_lock(&tree->lock); 1788 spin_lock(&tree->lock);
1789 if (cur_start == 0 && bits == EXTENT_DIRTY) { 1789 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1790 total_bytes = tree->dirty_bytes; 1790 total_bytes = tree->dirty_bytes;
1791 goto out; 1791 goto out;
1792 } 1792 }
1793 /* 1793 /*
1794 * this search will find all the extents that end after 1794 * this search will find all the extents that end after
1795 * our range starts. 1795 * our range starts.
1796 */ 1796 */
1797 node = tree_search(tree, cur_start); 1797 node = tree_search(tree, cur_start);
1798 if (!node) 1798 if (!node)
1799 goto out; 1799 goto out;
1800 1800
1801 while (1) { 1801 while (1) {
1802 state = rb_entry(node, struct extent_state, rb_node); 1802 state = rb_entry(node, struct extent_state, rb_node);
1803 if (state->start > search_end) 1803 if (state->start > search_end)
1804 break; 1804 break;
1805 if (contig && found && state->start > last + 1) 1805 if (contig && found && state->start > last + 1)
1806 break; 1806 break;
1807 if (state->end >= cur_start && (state->state & bits) == bits) { 1807 if (state->end >= cur_start && (state->state & bits) == bits) {
1808 total_bytes += min(search_end, state->end) + 1 - 1808 total_bytes += min(search_end, state->end) + 1 -
1809 max(cur_start, state->start); 1809 max(cur_start, state->start);
1810 if (total_bytes >= max_bytes) 1810 if (total_bytes >= max_bytes)
1811 break; 1811 break;
1812 if (!found) { 1812 if (!found) {
1813 *start = max(cur_start, state->start); 1813 *start = max(cur_start, state->start);
1814 found = 1; 1814 found = 1;
1815 } 1815 }
1816 last = state->end; 1816 last = state->end;
1817 } else if (contig && found) { 1817 } else if (contig && found) {
1818 break; 1818 break;
1819 } 1819 }
1820 node = rb_next(node); 1820 node = rb_next(node);
1821 if (!node) 1821 if (!node)
1822 break; 1822 break;
1823 } 1823 }
1824 out: 1824 out:
1825 spin_unlock(&tree->lock); 1825 spin_unlock(&tree->lock);
1826 return total_bytes; 1826 return total_bytes;
1827 } 1827 }
1828 1828
1829 /* 1829 /*
1830 * set the private field for a given byte offset in the tree. If there isn't 1830 * set the private field for a given byte offset in the tree. If there isn't
1831 * an extent_state there already, this does nothing. 1831 * an extent_state there already, this does nothing.
1832 */ 1832 */
1833 static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private) 1833 static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1834 { 1834 {
1835 struct rb_node *node; 1835 struct rb_node *node;
1836 struct extent_state *state; 1836 struct extent_state *state;
1837 int ret = 0; 1837 int ret = 0;
1838 1838
1839 spin_lock(&tree->lock); 1839 spin_lock(&tree->lock);
1840 /* 1840 /*
1841 * this search will find all the extents that end after 1841 * this search will find all the extents that end after
1842 * our range starts. 1842 * our range starts.
1843 */ 1843 */
1844 node = tree_search(tree, start); 1844 node = tree_search(tree, start);
1845 if (!node) { 1845 if (!node) {
1846 ret = -ENOENT; 1846 ret = -ENOENT;
1847 goto out; 1847 goto out;
1848 } 1848 }
1849 state = rb_entry(node, struct extent_state, rb_node); 1849 state = rb_entry(node, struct extent_state, rb_node);
1850 if (state->start != start) { 1850 if (state->start != start) {
1851 ret = -ENOENT; 1851 ret = -ENOENT;
1852 goto out; 1852 goto out;
1853 } 1853 }
1854 state->private = private; 1854 state->private = private;
1855 out: 1855 out:
1856 spin_unlock(&tree->lock); 1856 spin_unlock(&tree->lock);
1857 return ret; 1857 return ret;
1858 } 1858 }
1859 1859
1860 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private) 1860 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1861 { 1861 {
1862 struct rb_node *node; 1862 struct rb_node *node;
1863 struct extent_state *state; 1863 struct extent_state *state;
1864 int ret = 0; 1864 int ret = 0;
1865 1865
1866 spin_lock(&tree->lock); 1866 spin_lock(&tree->lock);
1867 /* 1867 /*
1868 * this search will find all the extents that end after 1868 * this search will find all the extents that end after
1869 * our range starts. 1869 * our range starts.
1870 */ 1870 */
1871 node = tree_search(tree, start); 1871 node = tree_search(tree, start);
1872 if (!node) { 1872 if (!node) {
1873 ret = -ENOENT; 1873 ret = -ENOENT;
1874 goto out; 1874 goto out;
1875 } 1875 }
1876 state = rb_entry(node, struct extent_state, rb_node); 1876 state = rb_entry(node, struct extent_state, rb_node);
1877 if (state->start != start) { 1877 if (state->start != start) {
1878 ret = -ENOENT; 1878 ret = -ENOENT;
1879 goto out; 1879 goto out;
1880 } 1880 }
1881 *private = state->private; 1881 *private = state->private;
1882 out: 1882 out:
1883 spin_unlock(&tree->lock); 1883 spin_unlock(&tree->lock);
1884 return ret; 1884 return ret;
1885 } 1885 }
1886 1886
1887 /* 1887 /*
1888 * searches a range in the state tree for a given mask. 1888 * searches a range in the state tree for a given mask.
1889 * If 'filled' == 1, this returns 1 only if every extent in the tree 1889 * If 'filled' == 1, this returns 1 only if every extent in the tree
1890 * has the bits set. Otherwise, 1 is returned if any bit in the 1890 * has the bits set. Otherwise, 1 is returned if any bit in the
1891 * range is found set. 1891 * range is found set.
1892 */ 1892 */
1893 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, 1893 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1894 unsigned long bits, int filled, struct extent_state *cached) 1894 unsigned long bits, int filled, struct extent_state *cached)
1895 { 1895 {
1896 struct extent_state *state = NULL; 1896 struct extent_state *state = NULL;
1897 struct rb_node *node; 1897 struct rb_node *node;
1898 int bitset = 0; 1898 int bitset = 0;
1899 1899
1900 spin_lock(&tree->lock); 1900 spin_lock(&tree->lock);
1901 if (cached && cached->tree && cached->start <= start && 1901 if (cached && cached->tree && cached->start <= start &&
1902 cached->end > start) 1902 cached->end > start)
1903 node = &cached->rb_node; 1903 node = &cached->rb_node;
1904 else 1904 else
1905 node = tree_search(tree, start); 1905 node = tree_search(tree, start);
1906 while (node && start <= end) { 1906 while (node && start <= end) {
1907 state = rb_entry(node, struct extent_state, rb_node); 1907 state = rb_entry(node, struct extent_state, rb_node);
1908 1908
1909 if (filled && state->start > start) { 1909 if (filled && state->start > start) {
1910 bitset = 0; 1910 bitset = 0;
1911 break; 1911 break;
1912 } 1912 }
1913 1913
1914 if (state->start > end) 1914 if (state->start > end)
1915 break; 1915 break;
1916 1916
1917 if (state->state & bits) { 1917 if (state->state & bits) {
1918 bitset = 1; 1918 bitset = 1;
1919 if (!filled) 1919 if (!filled)
1920 break; 1920 break;
1921 } else if (filled) { 1921 } else if (filled) {
1922 bitset = 0; 1922 bitset = 0;
1923 break; 1923 break;
1924 } 1924 }
1925 1925
1926 if (state->end == (u64)-1) 1926 if (state->end == (u64)-1)
1927 break; 1927 break;
1928 1928
1929 start = state->end + 1; 1929 start = state->end + 1;
1930 if (start > end) 1930 if (start > end)
1931 break; 1931 break;
1932 node = rb_next(node); 1932 node = rb_next(node);
1933 if (!node) { 1933 if (!node) {
1934 if (filled) 1934 if (filled)
1935 bitset = 0; 1935 bitset = 0;
1936 break; 1936 break;
1937 } 1937 }
1938 } 1938 }
1939 spin_unlock(&tree->lock); 1939 spin_unlock(&tree->lock);
1940 return bitset; 1940 return bitset;
1941 } 1941 }
1942 1942
1943 /* 1943 /*
1944 * helper function to set a given page up to date if all the 1944 * helper function to set a given page up to date if all the
1945 * extents in the tree for that page are up to date 1945 * extents in the tree for that page are up to date
1946 */ 1946 */
1947 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page) 1947 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1948 { 1948 {
1949 u64 start = page_offset(page); 1949 u64 start = page_offset(page);
1950 u64 end = start + PAGE_CACHE_SIZE - 1; 1950 u64 end = start + PAGE_CACHE_SIZE - 1;
1951 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL)) 1951 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1952 SetPageUptodate(page); 1952 SetPageUptodate(page);
1953 } 1953 }
1954 1954
1955 /* 1955 /*
1956 * When IO fails, either with EIO or csum verification fails, we 1956 * When IO fails, either with EIO or csum verification fails, we
1957 * try other mirrors that might have a good copy of the data. This 1957 * try other mirrors that might have a good copy of the data. This
1958 * io_failure_record is used to record state as we go through all the 1958 * io_failure_record is used to record state as we go through all the
1959 * mirrors. If another mirror has good data, the page is set up to date 1959 * mirrors. If another mirror has good data, the page is set up to date
1960 * and things continue. If a good mirror can't be found, the original 1960 * and things continue. If a good mirror can't be found, the original
1961 * bio end_io callback is called to indicate things have failed. 1961 * bio end_io callback is called to indicate things have failed.
1962 */ 1962 */
1963 struct io_failure_record { 1963 struct io_failure_record {
1964 struct page *page; 1964 struct page *page;
1965 u64 start; 1965 u64 start;
1966 u64 len; 1966 u64 len;
1967 u64 logical; 1967 u64 logical;
1968 unsigned long bio_flags; 1968 unsigned long bio_flags;
1969 int this_mirror; 1969 int this_mirror;
1970 int failed_mirror; 1970 int failed_mirror;
1971 int in_validation; 1971 int in_validation;
1972 }; 1972 };
1973 1973
1974 static int free_io_failure(struct inode *inode, struct io_failure_record *rec, 1974 static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1975 int did_repair) 1975 int did_repair)
1976 { 1976 {
1977 int ret; 1977 int ret;
1978 int err = 0; 1978 int err = 0;
1979 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 1979 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1980 1980
1981 set_state_private(failure_tree, rec->start, 0); 1981 set_state_private(failure_tree, rec->start, 0);
1982 ret = clear_extent_bits(failure_tree, rec->start, 1982 ret = clear_extent_bits(failure_tree, rec->start,
1983 rec->start + rec->len - 1, 1983 rec->start + rec->len - 1,
1984 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS); 1984 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1985 if (ret) 1985 if (ret)
1986 err = ret; 1986 err = ret;
1987 1987
1988 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start, 1988 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1989 rec->start + rec->len - 1, 1989 rec->start + rec->len - 1,
1990 EXTENT_DAMAGED, GFP_NOFS); 1990 EXTENT_DAMAGED, GFP_NOFS);
1991 if (ret && !err) 1991 if (ret && !err)
1992 err = ret; 1992 err = ret;
1993 1993
1994 kfree(rec); 1994 kfree(rec);
1995 return err; 1995 return err;
1996 } 1996 }
1997 1997
1998 /* 1998 /*
1999 * this bypasses the standard btrfs submit functions deliberately, as 1999 * this bypasses the standard btrfs submit functions deliberately, as
2000 * the standard behavior is to write all copies in a raid setup. here we only 2000 * the standard behavior is to write all copies in a raid setup. here we only
2001 * want to write the one bad copy. so we do the mapping for ourselves and issue 2001 * want to write the one bad copy. so we do the mapping for ourselves and issue
2002 * submit_bio directly. 2002 * submit_bio directly.
2003 * to avoid any synchronization issues, wait for the data after writing, which 2003 * to avoid any synchronization issues, wait for the data after writing, which
2004 * actually prevents the read that triggered the error from finishing. 2004 * actually prevents the read that triggered the error from finishing.
2005 * currently, there can be no more than two copies of every data bit. thus, 2005 * currently, there can be no more than two copies of every data bit. thus,
2006 * exactly one rewrite is required. 2006 * exactly one rewrite is required.
2007 */ 2007 */
2008 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start, 2008 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
2009 u64 length, u64 logical, struct page *page, 2009 u64 length, u64 logical, struct page *page,
2010 int mirror_num) 2010 int mirror_num)
2011 { 2011 {
2012 struct bio *bio; 2012 struct bio *bio;
2013 struct btrfs_device *dev; 2013 struct btrfs_device *dev;
2014 u64 map_length = 0; 2014 u64 map_length = 0;
2015 u64 sector; 2015 u64 sector;
2016 struct btrfs_bio *bbio = NULL; 2016 struct btrfs_bio *bbio = NULL;
2017 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree; 2017 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
2018 int ret; 2018 int ret;
2019 2019
2020 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY)); 2020 ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
2021 BUG_ON(!mirror_num); 2021 BUG_ON(!mirror_num);
2022 2022
2023 /* we can't repair anything in raid56 yet */ 2023 /* we can't repair anything in raid56 yet */
2024 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num)) 2024 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2025 return 0; 2025 return 0;
2026 2026
2027 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 2027 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2028 if (!bio) 2028 if (!bio)
2029 return -EIO; 2029 return -EIO;
2030 bio->bi_iter.bi_size = 0; 2030 bio->bi_iter.bi_size = 0;
2031 map_length = length; 2031 map_length = length;
2032 2032
2033 ret = btrfs_map_block(fs_info, WRITE, logical, 2033 ret = btrfs_map_block(fs_info, WRITE, logical,
2034 &map_length, &bbio, mirror_num); 2034 &map_length, &bbio, mirror_num);
2035 if (ret) { 2035 if (ret) {
2036 bio_put(bio); 2036 bio_put(bio);
2037 return -EIO; 2037 return -EIO;
2038 } 2038 }
2039 BUG_ON(mirror_num != bbio->mirror_num); 2039 BUG_ON(mirror_num != bbio->mirror_num);
2040 sector = bbio->stripes[mirror_num-1].physical >> 9; 2040 sector = bbio->stripes[mirror_num-1].physical >> 9;
2041 bio->bi_iter.bi_sector = sector; 2041 bio->bi_iter.bi_sector = sector;
2042 dev = bbio->stripes[mirror_num-1].dev; 2042 dev = bbio->stripes[mirror_num-1].dev;
2043 kfree(bbio); 2043 kfree(bbio);
2044 if (!dev || !dev->bdev || !dev->writeable) { 2044 if (!dev || !dev->bdev || !dev->writeable) {
2045 bio_put(bio); 2045 bio_put(bio);
2046 return -EIO; 2046 return -EIO;
2047 } 2047 }
2048 bio->bi_bdev = dev->bdev; 2048 bio->bi_bdev = dev->bdev;
2049 bio_add_page(bio, page, length, start - page_offset(page)); 2049 bio_add_page(bio, page, length, start - page_offset(page));
2050 2050
2051 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) { 2051 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
2052 /* try to remap that extent elsewhere? */ 2052 /* try to remap that extent elsewhere? */
2053 bio_put(bio); 2053 bio_put(bio);
2054 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); 2054 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2055 return -EIO; 2055 return -EIO;
2056 } 2056 }
2057 2057
2058 printk_ratelimited_in_rcu(KERN_INFO 2058 printk_ratelimited_in_rcu(KERN_INFO
2059 "BTRFS: read error corrected: ino %lu off %llu " 2059 "BTRFS: read error corrected: ino %lu off %llu "
2060 "(dev %s sector %llu)\n", page->mapping->host->i_ino, 2060 "(dev %s sector %llu)\n", page->mapping->host->i_ino,
2061 start, rcu_str_deref(dev->name), sector); 2061 start, rcu_str_deref(dev->name), sector);
2062 2062
2063 bio_put(bio); 2063 bio_put(bio);
2064 return 0; 2064 return 0;
2065 } 2065 }
2066 2066
2067 int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb, 2067 int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2068 int mirror_num) 2068 int mirror_num)
2069 { 2069 {
2070 u64 start = eb->start; 2070 u64 start = eb->start;
2071 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len); 2071 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
2072 int ret = 0; 2072 int ret = 0;
2073 2073
2074 if (root->fs_info->sb->s_flags & MS_RDONLY) 2074 if (root->fs_info->sb->s_flags & MS_RDONLY)
2075 return -EROFS; 2075 return -EROFS;
2076 2076
2077 for (i = 0; i < num_pages; i++) { 2077 for (i = 0; i < num_pages; i++) {
2078 struct page *p = extent_buffer_page(eb, i); 2078 struct page *p = extent_buffer_page(eb, i);
2079 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE, 2079 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
2080 start, p, mirror_num); 2080 start, p, mirror_num);
2081 if (ret) 2081 if (ret)
2082 break; 2082 break;
2083 start += PAGE_CACHE_SIZE; 2083 start += PAGE_CACHE_SIZE;
2084 } 2084 }
2085 2085
2086 return ret; 2086 return ret;
2087 } 2087 }
2088 2088
2089 /* 2089 /*
2090 * each time an IO finishes, we do a fast check in the IO failure tree 2090 * each time an IO finishes, we do a fast check in the IO failure tree
2091 * to see if we need to process or clean up an io_failure_record 2091 * to see if we need to process or clean up an io_failure_record
2092 */ 2092 */
2093 static int clean_io_failure(u64 start, struct page *page) 2093 static int clean_io_failure(u64 start, struct page *page)
2094 { 2094 {
2095 u64 private; 2095 u64 private;
2096 u64 private_failure; 2096 u64 private_failure;
2097 struct io_failure_record *failrec; 2097 struct io_failure_record *failrec;
2098 struct inode *inode = page->mapping->host; 2098 struct inode *inode = page->mapping->host;
2099 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 2099 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2100 struct extent_state *state; 2100 struct extent_state *state;
2101 int num_copies; 2101 int num_copies;
2102 int did_repair = 0; 2102 int did_repair = 0;
2103 int ret; 2103 int ret;
2104 2104
2105 private = 0; 2105 private = 0;
2106 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private, 2106 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2107 (u64)-1, 1, EXTENT_DIRTY, 0); 2107 (u64)-1, 1, EXTENT_DIRTY, 0);
2108 if (!ret) 2108 if (!ret)
2109 return 0; 2109 return 0;
2110 2110
2111 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start, 2111 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2112 &private_failure); 2112 &private_failure);
2113 if (ret) 2113 if (ret)
2114 return 0; 2114 return 0;
2115 2115
2116 failrec = (struct io_failure_record *)(unsigned long) private_failure; 2116 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2117 BUG_ON(!failrec->this_mirror); 2117 BUG_ON(!failrec->this_mirror);
2118 2118
2119 if (failrec->in_validation) { 2119 if (failrec->in_validation) {
2120 /* there was no real error, just free the record */ 2120 /* there was no real error, just free the record */
2121 pr_debug("clean_io_failure: freeing dummy error at %llu\n", 2121 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2122 failrec->start); 2122 failrec->start);
2123 did_repair = 1; 2123 did_repair = 1;
2124 goto out; 2124 goto out;
2125 } 2125 }
2126 if (fs_info->sb->s_flags & MS_RDONLY) 2126 if (fs_info->sb->s_flags & MS_RDONLY)
2127 goto out; 2127 goto out;
2128 2128
2129 spin_lock(&BTRFS_I(inode)->io_tree.lock); 2129 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2130 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree, 2130 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2131 failrec->start, 2131 failrec->start,
2132 EXTENT_LOCKED); 2132 EXTENT_LOCKED);
2133 spin_unlock(&BTRFS_I(inode)->io_tree.lock); 2133 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2134 2134
2135 if (state && state->start <= failrec->start && 2135 if (state && state->start <= failrec->start &&
2136 state->end >= failrec->start + failrec->len - 1) { 2136 state->end >= failrec->start + failrec->len - 1) {
2137 num_copies = btrfs_num_copies(fs_info, failrec->logical, 2137 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2138 failrec->len); 2138 failrec->len);
2139 if (num_copies > 1) { 2139 if (num_copies > 1) {
2140 ret = repair_io_failure(fs_info, start, failrec->len, 2140 ret = repair_io_failure(fs_info, start, failrec->len,
2141 failrec->logical, page, 2141 failrec->logical, page,
2142 failrec->failed_mirror); 2142 failrec->failed_mirror);
2143 did_repair = !ret; 2143 did_repair = !ret;
2144 } 2144 }
2145 ret = 0; 2145 ret = 0;
2146 } 2146 }
2147 2147
2148 out: 2148 out:
2149 if (!ret) 2149 if (!ret)
2150 ret = free_io_failure(inode, failrec, did_repair); 2150 ret = free_io_failure(inode, failrec, did_repair);
2151 2151
2152 return ret; 2152 return ret;
2153 } 2153 }
2154 2154
2155 /* 2155 /*
2156 * this is a generic handler for readpage errors (default 2156 * this is a generic handler for readpage errors (default
2157 * readpage_io_failed_hook). if other copies exist, read those and write back 2157 * readpage_io_failed_hook). if other copies exist, read those and write back
2158 * good data to the failed position. does not investigate in remapping the 2158 * good data to the failed position. does not investigate in remapping the
2159 * failed extent elsewhere, hoping the device will be smart enough to do this as 2159 * failed extent elsewhere, hoping the device will be smart enough to do this as
2160 * needed 2160 * needed
2161 */ 2161 */
2162 2162
2163 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset, 2163 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2164 struct page *page, u64 start, u64 end, 2164 struct page *page, u64 start, u64 end,
2165 int failed_mirror) 2165 int failed_mirror)
2166 { 2166 {
2167 struct io_failure_record *failrec = NULL; 2167 struct io_failure_record *failrec = NULL;
2168 u64 private; 2168 u64 private;
2169 struct extent_map *em; 2169 struct extent_map *em;
2170 struct inode *inode = page->mapping->host; 2170 struct inode *inode = page->mapping->host;
2171 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; 2171 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2172 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; 2172 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2173 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; 2173 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2174 struct bio *bio; 2174 struct bio *bio;
2175 struct btrfs_io_bio *btrfs_failed_bio; 2175 struct btrfs_io_bio *btrfs_failed_bio;
2176 struct btrfs_io_bio *btrfs_bio; 2176 struct btrfs_io_bio *btrfs_bio;
2177 int num_copies; 2177 int num_copies;
2178 int ret; 2178 int ret;
2179 int read_mode; 2179 int read_mode;
2180 u64 logical; 2180 u64 logical;
2181 2181
2182 BUG_ON(failed_bio->bi_rw & REQ_WRITE); 2182 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2183 2183
2184 ret = get_state_private(failure_tree, start, &private); 2184 ret = get_state_private(failure_tree, start, &private);
2185 if (ret) { 2185 if (ret) {
2186 failrec = kzalloc(sizeof(*failrec), GFP_NOFS); 2186 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2187 if (!failrec) 2187 if (!failrec)
2188 return -ENOMEM; 2188 return -ENOMEM;
2189 failrec->start = start; 2189 failrec->start = start;
2190 failrec->len = end - start + 1; 2190 failrec->len = end - start + 1;
2191 failrec->this_mirror = 0; 2191 failrec->this_mirror = 0;
2192 failrec->bio_flags = 0; 2192 failrec->bio_flags = 0;
2193 failrec->in_validation = 0; 2193 failrec->in_validation = 0;
2194 2194
2195 read_lock(&em_tree->lock); 2195 read_lock(&em_tree->lock);
2196 em = lookup_extent_mapping(em_tree, start, failrec->len); 2196 em = lookup_extent_mapping(em_tree, start, failrec->len);
2197 if (!em) { 2197 if (!em) {
2198 read_unlock(&em_tree->lock); 2198 read_unlock(&em_tree->lock);
2199 kfree(failrec); 2199 kfree(failrec);
2200 return -EIO; 2200 return -EIO;
2201 } 2201 }
2202 2202
2203 if (em->start > start || em->start + em->len <= start) { 2203 if (em->start > start || em->start + em->len <= start) {
2204 free_extent_map(em); 2204 free_extent_map(em);
2205 em = NULL; 2205 em = NULL;
2206 } 2206 }
2207 read_unlock(&em_tree->lock); 2207 read_unlock(&em_tree->lock);
2208 2208
2209 if (!em) { 2209 if (!em) {
2210 kfree(failrec); 2210 kfree(failrec);
2211 return -EIO; 2211 return -EIO;
2212 } 2212 }
2213 logical = start - em->start; 2213 logical = start - em->start;
2214 logical = em->block_start + logical; 2214 logical = em->block_start + logical;
2215 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 2215 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2216 logical = em->block_start; 2216 logical = em->block_start;
2217 failrec->bio_flags = EXTENT_BIO_COMPRESSED; 2217 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2218 extent_set_compress_type(&failrec->bio_flags, 2218 extent_set_compress_type(&failrec->bio_flags,
2219 em->compress_type); 2219 em->compress_type);
2220 } 2220 }
2221 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, " 2221 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2222 "len=%llu\n", logical, start, failrec->len); 2222 "len=%llu\n", logical, start, failrec->len);
2223 failrec->logical = logical; 2223 failrec->logical = logical;
2224 free_extent_map(em); 2224 free_extent_map(em);
2225 2225
2226 /* set the bits in the private failure tree */ 2226 /* set the bits in the private failure tree */
2227 ret = set_extent_bits(failure_tree, start, end, 2227 ret = set_extent_bits(failure_tree, start, end,
2228 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS); 2228 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2229 if (ret >= 0) 2229 if (ret >= 0)
2230 ret = set_state_private(failure_tree, start, 2230 ret = set_state_private(failure_tree, start,
2231 (u64)(unsigned long)failrec); 2231 (u64)(unsigned long)failrec);
2232 /* set the bits in the inode's tree */ 2232 /* set the bits in the inode's tree */
2233 if (ret >= 0) 2233 if (ret >= 0)
2234 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED, 2234 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2235 GFP_NOFS); 2235 GFP_NOFS);
2236 if (ret < 0) { 2236 if (ret < 0) {
2237 kfree(failrec); 2237 kfree(failrec);
2238 return ret; 2238 return ret;
2239 } 2239 }
2240 } else { 2240 } else {
2241 failrec = (struct io_failure_record *)(unsigned long)private; 2241 failrec = (struct io_failure_record *)(unsigned long)private;
2242 pr_debug("bio_readpage_error: (found) logical=%llu, " 2242 pr_debug("bio_readpage_error: (found) logical=%llu, "
2243 "start=%llu, len=%llu, validation=%d\n", 2243 "start=%llu, len=%llu, validation=%d\n",
2244 failrec->logical, failrec->start, failrec->len, 2244 failrec->logical, failrec->start, failrec->len,
2245 failrec->in_validation); 2245 failrec->in_validation);
2246 /* 2246 /*
2247 * when data can be on disk more than twice, add to failrec here 2247 * when data can be on disk more than twice, add to failrec here
2248 * (e.g. with a list for failed_mirror) to make 2248 * (e.g. with a list for failed_mirror) to make
2249 * clean_io_failure() clean all those errors at once. 2249 * clean_io_failure() clean all those errors at once.
2250 */ 2250 */
2251 } 2251 }
2252 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info, 2252 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2253 failrec->logical, failrec->len); 2253 failrec->logical, failrec->len);
2254 if (num_copies == 1) { 2254 if (num_copies == 1) {
2255 /* 2255 /*
2256 * we only have a single copy of the data, so don't bother with 2256 * we only have a single copy of the data, so don't bother with
2257 * all the retry and error correction code that follows. no 2257 * all the retry and error correction code that follows. no
2258 * matter what the error is, it is very likely to persist. 2258 * matter what the error is, it is very likely to persist.
2259 */ 2259 */
2260 pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n", 2260 pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
2261 num_copies, failrec->this_mirror, failed_mirror); 2261 num_copies, failrec->this_mirror, failed_mirror);
2262 free_io_failure(inode, failrec, 0); 2262 free_io_failure(inode, failrec, 0);
2263 return -EIO; 2263 return -EIO;
2264 } 2264 }
2265 2265
2266 /* 2266 /*
2267 * there are two premises: 2267 * there are two premises:
2268 * a) deliver good data to the caller 2268 * a) deliver good data to the caller
2269 * b) correct the bad sectors on disk 2269 * b) correct the bad sectors on disk
2270 */ 2270 */
2271 if (failed_bio->bi_vcnt > 1) { 2271 if (failed_bio->bi_vcnt > 1) {
2272 /* 2272 /*
2273 * to fulfill b), we need to know the exact failing sectors, as 2273 * to fulfill b), we need to know the exact failing sectors, as
2274 * we don't want to rewrite any more than the failed ones. thus, 2274 * we don't want to rewrite any more than the failed ones. thus,
2275 * we need separate read requests for the failed bio 2275 * we need separate read requests for the failed bio
2276 * 2276 *
2277 * if the following BUG_ON triggers, our validation request got 2277 * if the following BUG_ON triggers, our validation request got
2278 * merged. we need separate requests for our algorithm to work. 2278 * merged. we need separate requests for our algorithm to work.
2279 */ 2279 */
2280 BUG_ON(failrec->in_validation); 2280 BUG_ON(failrec->in_validation);
2281 failrec->in_validation = 1; 2281 failrec->in_validation = 1;
2282 failrec->this_mirror = failed_mirror; 2282 failrec->this_mirror = failed_mirror;
2283 read_mode = READ_SYNC | REQ_FAILFAST_DEV; 2283 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2284 } else { 2284 } else {
2285 /* 2285 /*
2286 * we're ready to fulfill a) and b) alongside. get a good copy 2286 * we're ready to fulfill a) and b) alongside. get a good copy
2287 * of the failed sector and if we succeed, we have setup 2287 * of the failed sector and if we succeed, we have setup
2288 * everything for repair_io_failure to do the rest for us. 2288 * everything for repair_io_failure to do the rest for us.
2289 */ 2289 */
2290 if (failrec->in_validation) { 2290 if (failrec->in_validation) {
2291 BUG_ON(failrec->this_mirror != failed_mirror); 2291 BUG_ON(failrec->this_mirror != failed_mirror);
2292 failrec->in_validation = 0; 2292 failrec->in_validation = 0;
2293 failrec->this_mirror = 0; 2293 failrec->this_mirror = 0;
2294 } 2294 }
2295 failrec->failed_mirror = failed_mirror; 2295 failrec->failed_mirror = failed_mirror;
2296 failrec->this_mirror++; 2296 failrec->this_mirror++;
2297 if (failrec->this_mirror == failed_mirror) 2297 if (failrec->this_mirror == failed_mirror)
2298 failrec->this_mirror++; 2298 failrec->this_mirror++;
2299 read_mode = READ_SYNC; 2299 read_mode = READ_SYNC;
2300 } 2300 }
2301 2301
2302 if (failrec->this_mirror > num_copies) { 2302 if (failrec->this_mirror > num_copies) {
2303 pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n", 2303 pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
2304 num_copies, failrec->this_mirror, failed_mirror); 2304 num_copies, failrec->this_mirror, failed_mirror);
2305 free_io_failure(inode, failrec, 0); 2305 free_io_failure(inode, failrec, 0);
2306 return -EIO; 2306 return -EIO;
2307 } 2307 }
2308 2308
2309 bio = btrfs_io_bio_alloc(GFP_NOFS, 1); 2309 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2310 if (!bio) { 2310 if (!bio) {
2311 free_io_failure(inode, failrec, 0); 2311 free_io_failure(inode, failrec, 0);
2312 return -EIO; 2312 return -EIO;
2313 } 2313 }
2314 bio->bi_end_io = failed_bio->bi_end_io; 2314 bio->bi_end_io = failed_bio->bi_end_io;
2315 bio->bi_iter.bi_sector = failrec->logical >> 9; 2315 bio->bi_iter.bi_sector = failrec->logical >> 9;
2316 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev; 2316 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2317 bio->bi_iter.bi_size = 0; 2317 bio->bi_iter.bi_size = 0;
2318 2318
2319 btrfs_failed_bio = btrfs_io_bio(failed_bio); 2319 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2320 if (btrfs_failed_bio->csum) { 2320 if (btrfs_failed_bio->csum) {
2321 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; 2321 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2322 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); 2322 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2323 2323
2324 btrfs_bio = btrfs_io_bio(bio); 2324 btrfs_bio = btrfs_io_bio(bio);
2325 btrfs_bio->csum = btrfs_bio->csum_inline; 2325 btrfs_bio->csum = btrfs_bio->csum_inline;
2326 phy_offset >>= inode->i_sb->s_blocksize_bits; 2326 phy_offset >>= inode->i_sb->s_blocksize_bits;
2327 phy_offset *= csum_size; 2327 phy_offset *= csum_size;
2328 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset, 2328 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset,
2329 csum_size); 2329 csum_size);
2330 } 2330 }
2331 2331
2332 bio_add_page(bio, page, failrec->len, start - page_offset(page)); 2332 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2333 2333
2334 pr_debug("bio_readpage_error: submitting new read[%#x] to " 2334 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2335 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode, 2335 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2336 failrec->this_mirror, num_copies, failrec->in_validation); 2336 failrec->this_mirror, num_copies, failrec->in_validation);
2337 2337
2338 ret = tree->ops->submit_bio_hook(inode, read_mode, bio, 2338 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2339 failrec->this_mirror, 2339 failrec->this_mirror,
2340 failrec->bio_flags, 0); 2340 failrec->bio_flags, 0);
2341 return ret; 2341 return ret;
2342 } 2342 }
2343 2343
2344 /* lots and lots of room for performance fixes in the end_bio funcs */ 2344 /* lots and lots of room for performance fixes in the end_bio funcs */
2345 2345
2346 int end_extent_writepage(struct page *page, int err, u64 start, u64 end) 2346 int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2347 { 2347 {
2348 int uptodate = (err == 0); 2348 int uptodate = (err == 0);
2349 struct extent_io_tree *tree; 2349 struct extent_io_tree *tree;
2350 int ret = 0; 2350 int ret = 0;
2351 2351
2352 tree = &BTRFS_I(page->mapping->host)->io_tree; 2352 tree = &BTRFS_I(page->mapping->host)->io_tree;
2353 2353
2354 if (tree->ops && tree->ops->writepage_end_io_hook) { 2354 if (tree->ops && tree->ops->writepage_end_io_hook) {
2355 ret = tree->ops->writepage_end_io_hook(page, start, 2355 ret = tree->ops->writepage_end_io_hook(page, start,
2356 end, NULL, uptodate); 2356 end, NULL, uptodate);
2357 if (ret) 2357 if (ret)
2358 uptodate = 0; 2358 uptodate = 0;
2359 } 2359 }
2360 2360
2361 if (!uptodate) { 2361 if (!uptodate) {
2362 ClearPageUptodate(page); 2362 ClearPageUptodate(page);
2363 SetPageError(page); 2363 SetPageError(page);
2364 ret = ret < 0 ? ret : -EIO; 2364 ret = ret < 0 ? ret : -EIO;
2365 mapping_set_error(page->mapping, ret); 2365 mapping_set_error(page->mapping, ret);
2366 } 2366 }
2367 return 0; 2367 return 0;
2368 } 2368 }
2369 2369
2370 /* 2370 /*
2371 * after a writepage IO is done, we need to: 2371 * after a writepage IO is done, we need to:
2372 * clear the uptodate bits on error 2372 * clear the uptodate bits on error
2373 * clear the writeback bits in the extent tree for this IO 2373 * clear the writeback bits in the extent tree for this IO
2374 * end_page_writeback if the page has no more pending IO 2374 * end_page_writeback if the page has no more pending IO
2375 * 2375 *
2376 * Scheduling is not allowed, so the extent state tree is expected 2376 * Scheduling is not allowed, so the extent state tree is expected
2377 * to have one and only one object corresponding to this IO. 2377 * to have one and only one object corresponding to this IO.
2378 */ 2378 */
2379 static void end_bio_extent_writepage(struct bio *bio, int err) 2379 static void end_bio_extent_writepage(struct bio *bio, int err)
2380 { 2380 {
2381 struct bio_vec *bvec; 2381 struct bio_vec *bvec;
2382 u64 start; 2382 u64 start;
2383 u64 end; 2383 u64 end;
2384 int i; 2384 int i;
2385 2385
2386 bio_for_each_segment_all(bvec, bio, i) { 2386 bio_for_each_segment_all(bvec, bio, i) {
2387 struct page *page = bvec->bv_page; 2387 struct page *page = bvec->bv_page;
2388 2388
2389 /* We always issue full-page reads, but if some block 2389 /* We always issue full-page reads, but if some block
2390 * in a page fails to read, blk_update_request() will 2390 * in a page fails to read, blk_update_request() will
2391 * advance bv_offset and adjust bv_len to compensate. 2391 * advance bv_offset and adjust bv_len to compensate.
2392 * Print a warning for nonzero offsets, and an error 2392 * Print a warning for nonzero offsets, and an error
2393 * if they don't add up to a full page. */ 2393 * if they don't add up to a full page. */
2394 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) { 2394 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2395 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE) 2395 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2396 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info, 2396 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2397 "partial page write in btrfs with offset %u and length %u", 2397 "partial page write in btrfs with offset %u and length %u",
2398 bvec->bv_offset, bvec->bv_len); 2398 bvec->bv_offset, bvec->bv_len);
2399 else 2399 else
2400 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info, 2400 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2401 "incomplete page write in btrfs with offset %u and " 2401 "incomplete page write in btrfs with offset %u and "
2402 "length %u", 2402 "length %u",
2403 bvec->bv_offset, bvec->bv_len); 2403 bvec->bv_offset, bvec->bv_len);
2404 } 2404 }
2405 2405
2406 start = page_offset(page); 2406 start = page_offset(page);
2407 end = start + bvec->bv_offset + bvec->bv_len - 1; 2407 end = start + bvec->bv_offset + bvec->bv_len - 1;
2408 2408
2409 if (end_extent_writepage(page, err, start, end)) 2409 if (end_extent_writepage(page, err, start, end))
2410 continue; 2410 continue;
2411 2411
2412 end_page_writeback(page); 2412 end_page_writeback(page);
2413 } 2413 }
2414 2414
2415 bio_put(bio); 2415 bio_put(bio);
2416 } 2416 }
2417 2417
2418 static void 2418 static void
2419 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len, 2419 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2420 int uptodate) 2420 int uptodate)
2421 { 2421 {
2422 struct extent_state *cached = NULL; 2422 struct extent_state *cached = NULL;
2423 u64 end = start + len - 1; 2423 u64 end = start + len - 1;
2424 2424
2425 if (uptodate && tree->track_uptodate) 2425 if (uptodate && tree->track_uptodate)
2426 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC); 2426 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2427 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC); 2427 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2428 } 2428 }
2429 2429
2430 /* 2430 /*
2431 * after a readpage IO is done, we need to: 2431 * after a readpage IO is done, we need to:
2432 * clear the uptodate bits on error 2432 * clear the uptodate bits on error
2433 * set the uptodate bits if things worked 2433 * set the uptodate bits if things worked
2434 * set the page up to date if all extents in the tree are uptodate 2434 * set the page up to date if all extents in the tree are uptodate
2435 * clear the lock bit in the extent tree 2435 * clear the lock bit in the extent tree
2436 * unlock the page if there are no other extents locked for it 2436 * unlock the page if there are no other extents locked for it
2437 * 2437 *
2438 * Scheduling is not allowed, so the extent state tree is expected 2438 * Scheduling is not allowed, so the extent state tree is expected
2439 * to have one and only one object corresponding to this IO. 2439 * to have one and only one object corresponding to this IO.
2440 */ 2440 */
2441 static void end_bio_extent_readpage(struct bio *bio, int err) 2441 static void end_bio_extent_readpage(struct bio *bio, int err)
2442 { 2442 {
2443 struct bio_vec *bvec; 2443 struct bio_vec *bvec;
2444 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 2444 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
2445 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); 2445 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2446 struct extent_io_tree *tree; 2446 struct extent_io_tree *tree;
2447 u64 offset = 0; 2447 u64 offset = 0;
2448 u64 start; 2448 u64 start;
2449 u64 end; 2449 u64 end;
2450 u64 len; 2450 u64 len;
2451 u64 extent_start = 0; 2451 u64 extent_start = 0;
2452 u64 extent_len = 0; 2452 u64 extent_len = 0;
2453 int mirror; 2453 int mirror;
2454 int ret; 2454 int ret;
2455 int i; 2455 int i;
2456 2456
2457 if (err) 2457 if (err)
2458 uptodate = 0; 2458 uptodate = 0;
2459 2459
2460 bio_for_each_segment_all(bvec, bio, i) { 2460 bio_for_each_segment_all(bvec, bio, i) {
2461 struct page *page = bvec->bv_page; 2461 struct page *page = bvec->bv_page;
2462 struct inode *inode = page->mapping->host; 2462 struct inode *inode = page->mapping->host;
2463 2463
2464 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, " 2464 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2465 "mirror=%lu\n", (u64)bio->bi_iter.bi_sector, err, 2465 "mirror=%lu\n", (u64)bio->bi_iter.bi_sector, err,
2466 io_bio->mirror_num); 2466 io_bio->mirror_num);
2467 tree = &BTRFS_I(inode)->io_tree; 2467 tree = &BTRFS_I(inode)->io_tree;
2468 2468
2469 /* We always issue full-page reads, but if some block 2469 /* We always issue full-page reads, but if some block
2470 * in a page fails to read, blk_update_request() will 2470 * in a page fails to read, blk_update_request() will
2471 * advance bv_offset and adjust bv_len to compensate. 2471 * advance bv_offset and adjust bv_len to compensate.
2472 * Print a warning for nonzero offsets, and an error 2472 * Print a warning for nonzero offsets, and an error
2473 * if they don't add up to a full page. */ 2473 * if they don't add up to a full page. */
2474 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) { 2474 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE) {
2475 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE) 2475 if (bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE)
2476 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info, 2476 btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2477 "partial page read in btrfs with offset %u and length %u", 2477 "partial page read in btrfs with offset %u and length %u",
2478 bvec->bv_offset, bvec->bv_len); 2478 bvec->bv_offset, bvec->bv_len);
2479 else 2479 else
2480 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info, 2480 btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2481 "incomplete page read in btrfs with offset %u and " 2481 "incomplete page read in btrfs with offset %u and "
2482 "length %u", 2482 "length %u",
2483 bvec->bv_offset, bvec->bv_len); 2483 bvec->bv_offset, bvec->bv_len);
2484 } 2484 }
2485 2485
2486 start = page_offset(page); 2486 start = page_offset(page);
2487 end = start + bvec->bv_offset + bvec->bv_len - 1; 2487 end = start + bvec->bv_offset + bvec->bv_len - 1;
2488 len = bvec->bv_len; 2488 len = bvec->bv_len;
2489 2489
2490 mirror = io_bio->mirror_num; 2490 mirror = io_bio->mirror_num;
2491 if (likely(uptodate && tree->ops && 2491 if (likely(uptodate && tree->ops &&
2492 tree->ops->readpage_end_io_hook)) { 2492 tree->ops->readpage_end_io_hook)) {
2493 ret = tree->ops->readpage_end_io_hook(io_bio, offset, 2493 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2494 page, start, end, 2494 page, start, end,
2495 mirror); 2495 mirror);
2496 if (ret) 2496 if (ret)
2497 uptodate = 0; 2497 uptodate = 0;
2498 else 2498 else
2499 clean_io_failure(start, page); 2499 clean_io_failure(start, page);
2500 } 2500 }
2501 2501
2502 if (likely(uptodate)) 2502 if (likely(uptodate))
2503 goto readpage_ok; 2503 goto readpage_ok;
2504 2504
2505 if (tree->ops && tree->ops->readpage_io_failed_hook) { 2505 if (tree->ops && tree->ops->readpage_io_failed_hook) {
2506 ret = tree->ops->readpage_io_failed_hook(page, mirror); 2506 ret = tree->ops->readpage_io_failed_hook(page, mirror);
2507 if (!ret && !err && 2507 if (!ret && !err &&
2508 test_bit(BIO_UPTODATE, &bio->bi_flags)) 2508 test_bit(BIO_UPTODATE, &bio->bi_flags))
2509 uptodate = 1; 2509 uptodate = 1;
2510 } else { 2510 } else {
2511 /* 2511 /*
2512 * The generic bio_readpage_error handles errors the 2512 * The generic bio_readpage_error handles errors the
2513 * following way: If possible, new read requests are 2513 * following way: If possible, new read requests are
2514 * created and submitted and will end up in 2514 * created and submitted and will end up in
2515 * end_bio_extent_readpage as well (if we're lucky, not 2515 * end_bio_extent_readpage as well (if we're lucky, not
2516 * in the !uptodate case). In that case it returns 0 and 2516 * in the !uptodate case). In that case it returns 0 and
2517 * we just go on with the next page in our bio. If it 2517 * we just go on with the next page in our bio. If it
2518 * can't handle the error it will return -EIO and we 2518 * can't handle the error it will return -EIO and we
2519 * remain responsible for that page. 2519 * remain responsible for that page.
2520 */ 2520 */
2521 ret = bio_readpage_error(bio, offset, page, start, end, 2521 ret = bio_readpage_error(bio, offset, page, start, end,
2522 mirror); 2522 mirror);
2523 if (ret == 0) { 2523 if (ret == 0) {
2524 uptodate = 2524 uptodate =
2525 test_bit(BIO_UPTODATE, &bio->bi_flags); 2525 test_bit(BIO_UPTODATE, &bio->bi_flags);
2526 if (err) 2526 if (err)
2527 uptodate = 0; 2527 uptodate = 0;
2528 offset += len;
2528 continue; 2529 continue;
2529 } 2530 }
2530 } 2531 }
2531 readpage_ok: 2532 readpage_ok:
2532 if (likely(uptodate)) { 2533 if (likely(uptodate)) {
2533 loff_t i_size = i_size_read(inode); 2534 loff_t i_size = i_size_read(inode);
2534 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT; 2535 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2535 unsigned offset; 2536 unsigned offset;
2536 2537
2537 /* Zero out the end if this page straddles i_size */ 2538 /* Zero out the end if this page straddles i_size */
2538 offset = i_size & (PAGE_CACHE_SIZE-1); 2539 offset = i_size & (PAGE_CACHE_SIZE-1);
2539 if (page->index == end_index && offset) 2540 if (page->index == end_index && offset)
2540 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 2541 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2541 SetPageUptodate(page); 2542 SetPageUptodate(page);
2542 } else { 2543 } else {
2543 ClearPageUptodate(page); 2544 ClearPageUptodate(page);
2544 SetPageError(page); 2545 SetPageError(page);
2545 } 2546 }
2546 unlock_page(page); 2547 unlock_page(page);
2547 offset += len; 2548 offset += len;
2548 2549
2549 if (unlikely(!uptodate)) { 2550 if (unlikely(!uptodate)) {
2550 if (extent_len) { 2551 if (extent_len) {
2551 endio_readpage_release_extent(tree, 2552 endio_readpage_release_extent(tree,
2552 extent_start, 2553 extent_start,
2553 extent_len, 1); 2554 extent_len, 1);
2554 extent_start = 0; 2555 extent_start = 0;
2555 extent_len = 0; 2556 extent_len = 0;
2556 } 2557 }
2557 endio_readpage_release_extent(tree, start, 2558 endio_readpage_release_extent(tree, start,
2558 end - start + 1, 0); 2559 end - start + 1, 0);
2559 } else if (!extent_len) { 2560 } else if (!extent_len) {
2560 extent_start = start; 2561 extent_start = start;
2561 extent_len = end + 1 - start; 2562 extent_len = end + 1 - start;
2562 } else if (extent_start + extent_len == start) { 2563 } else if (extent_start + extent_len == start) {
2563 extent_len += end + 1 - start; 2564 extent_len += end + 1 - start;
2564 } else { 2565 } else {
2565 endio_readpage_release_extent(tree, extent_start, 2566 endio_readpage_release_extent(tree, extent_start,
2566 extent_len, uptodate); 2567 extent_len, uptodate);
2567 extent_start = start; 2568 extent_start = start;
2568 extent_len = end + 1 - start; 2569 extent_len = end + 1 - start;
2569 } 2570 }
2570 } 2571 }
2571 2572
2572 if (extent_len) 2573 if (extent_len)
2573 endio_readpage_release_extent(tree, extent_start, extent_len, 2574 endio_readpage_release_extent(tree, extent_start, extent_len,
2574 uptodate); 2575 uptodate);
2575 if (io_bio->end_io) 2576 if (io_bio->end_io)
2576 io_bio->end_io(io_bio, err); 2577 io_bio->end_io(io_bio, err);
2577 bio_put(bio); 2578 bio_put(bio);
2578 } 2579 }
2579 2580
2580 /* 2581 /*
2581 * this allocates from the btrfs_bioset. We're returning a bio right now 2582 * this allocates from the btrfs_bioset. We're returning a bio right now
2582 * but you can call btrfs_io_bio for the appropriate container_of magic 2583 * but you can call btrfs_io_bio for the appropriate container_of magic
2583 */ 2584 */
2584 struct bio * 2585 struct bio *
2585 btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs, 2586 btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2586 gfp_t gfp_flags) 2587 gfp_t gfp_flags)
2587 { 2588 {
2588 struct btrfs_io_bio *btrfs_bio; 2589 struct btrfs_io_bio *btrfs_bio;
2589 struct bio *bio; 2590 struct bio *bio;
2590 2591
2591 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset); 2592 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
2592 2593
2593 if (bio == NULL && (current->flags & PF_MEMALLOC)) { 2594 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2594 while (!bio && (nr_vecs /= 2)) { 2595 while (!bio && (nr_vecs /= 2)) {
2595 bio = bio_alloc_bioset(gfp_flags, 2596 bio = bio_alloc_bioset(gfp_flags,
2596 nr_vecs, btrfs_bioset); 2597 nr_vecs, btrfs_bioset);
2597 } 2598 }
2598 } 2599 }
2599 2600
2600 if (bio) { 2601 if (bio) {
2601 bio->bi_bdev = bdev; 2602 bio->bi_bdev = bdev;
2602 bio->bi_iter.bi_sector = first_sector; 2603 bio->bi_iter.bi_sector = first_sector;
2603 btrfs_bio = btrfs_io_bio(bio); 2604 btrfs_bio = btrfs_io_bio(bio);
2604 btrfs_bio->csum = NULL; 2605 btrfs_bio->csum = NULL;
2605 btrfs_bio->csum_allocated = NULL; 2606 btrfs_bio->csum_allocated = NULL;
2606 btrfs_bio->end_io = NULL; 2607 btrfs_bio->end_io = NULL;
2607 } 2608 }
2608 return bio; 2609 return bio;
2609 } 2610 }
2610 2611
2611 struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask) 2612 struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2612 { 2613 {
2613 return bio_clone_bioset(bio, gfp_mask, btrfs_bioset); 2614 return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2614 } 2615 }
2615 2616
2616 2617
2617 /* this also allocates from the btrfs_bioset */ 2618 /* this also allocates from the btrfs_bioset */
2618 struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs) 2619 struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2619 { 2620 {
2620 struct btrfs_io_bio *btrfs_bio; 2621 struct btrfs_io_bio *btrfs_bio;
2621 struct bio *bio; 2622 struct bio *bio;
2622 2623
2623 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset); 2624 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2624 if (bio) { 2625 if (bio) {
2625 btrfs_bio = btrfs_io_bio(bio); 2626 btrfs_bio = btrfs_io_bio(bio);
2626 btrfs_bio->csum = NULL; 2627 btrfs_bio->csum = NULL;
2627 btrfs_bio->csum_allocated = NULL; 2628 btrfs_bio->csum_allocated = NULL;
2628 btrfs_bio->end_io = NULL; 2629 btrfs_bio->end_io = NULL;
2629 } 2630 }
2630 return bio; 2631 return bio;
2631 } 2632 }
2632 2633
2633 2634
2634 static int __must_check submit_one_bio(int rw, struct bio *bio, 2635 static int __must_check submit_one_bio(int rw, struct bio *bio,
2635 int mirror_num, unsigned long bio_flags) 2636 int mirror_num, unsigned long bio_flags)
2636 { 2637 {
2637 int ret = 0; 2638 int ret = 0;
2638 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 2639 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2639 struct page *page = bvec->bv_page; 2640 struct page *page = bvec->bv_page;
2640 struct extent_io_tree *tree = bio->bi_private; 2641 struct extent_io_tree *tree = bio->bi_private;
2641 u64 start; 2642 u64 start;
2642 2643
2643 start = page_offset(page) + bvec->bv_offset; 2644 start = page_offset(page) + bvec->bv_offset;
2644 2645
2645 bio->bi_private = NULL; 2646 bio->bi_private = NULL;
2646 2647
2647 bio_get(bio); 2648 bio_get(bio);
2648 2649
2649 if (tree->ops && tree->ops->submit_bio_hook) 2650 if (tree->ops && tree->ops->submit_bio_hook)
2650 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio, 2651 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
2651 mirror_num, bio_flags, start); 2652 mirror_num, bio_flags, start);
2652 else 2653 else
2653 btrfsic_submit_bio(rw, bio); 2654 btrfsic_submit_bio(rw, bio);
2654 2655
2655 if (bio_flagged(bio, BIO_EOPNOTSUPP)) 2656 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2656 ret = -EOPNOTSUPP; 2657 ret = -EOPNOTSUPP;
2657 bio_put(bio); 2658 bio_put(bio);
2658 return ret; 2659 return ret;
2659 } 2660 }
2660 2661
2661 static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page, 2662 static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
2662 unsigned long offset, size_t size, struct bio *bio, 2663 unsigned long offset, size_t size, struct bio *bio,
2663 unsigned long bio_flags) 2664 unsigned long bio_flags)
2664 { 2665 {
2665 int ret = 0; 2666 int ret = 0;
2666 if (tree->ops && tree->ops->merge_bio_hook) 2667 if (tree->ops && tree->ops->merge_bio_hook)
2667 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio, 2668 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
2668 bio_flags); 2669 bio_flags);
2669 BUG_ON(ret < 0); 2670 BUG_ON(ret < 0);
2670 return ret; 2671 return ret;
2671 2672
2672 } 2673 }
2673 2674
2674 static int submit_extent_page(int rw, struct extent_io_tree *tree, 2675 static int submit_extent_page(int rw, struct extent_io_tree *tree,
2675 struct page *page, sector_t sector, 2676 struct page *page, sector_t sector,
2676 size_t size, unsigned long offset, 2677 size_t size, unsigned long offset,
2677 struct block_device *bdev, 2678 struct block_device *bdev,
2678 struct bio **bio_ret, 2679 struct bio **bio_ret,
2679 unsigned long max_pages, 2680 unsigned long max_pages,
2680 bio_end_io_t end_io_func, 2681 bio_end_io_t end_io_func,
2681 int mirror_num, 2682 int mirror_num,
2682 unsigned long prev_bio_flags, 2683 unsigned long prev_bio_flags,
2683 unsigned long bio_flags) 2684 unsigned long bio_flags)
2684 { 2685 {
2685 int ret = 0; 2686 int ret = 0;
2686 struct bio *bio; 2687 struct bio *bio;
2687 int nr; 2688 int nr;
2688 int contig = 0; 2689 int contig = 0;
2689 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED; 2690 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2690 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED; 2691 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2691 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE); 2692 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
2692 2693
2693 if (bio_ret && *bio_ret) { 2694 if (bio_ret && *bio_ret) {
2694 bio = *bio_ret; 2695 bio = *bio_ret;
2695 if (old_compressed) 2696 if (old_compressed)
2696 contig = bio->bi_iter.bi_sector == sector; 2697 contig = bio->bi_iter.bi_sector == sector;
2697 else 2698 else
2698 contig = bio_end_sector(bio) == sector; 2699 contig = bio_end_sector(bio) == sector;
2699 2700
2700 if (prev_bio_flags != bio_flags || !contig || 2701 if (prev_bio_flags != bio_flags || !contig ||
2701 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) || 2702 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
2702 bio_add_page(bio, page, page_size, offset) < page_size) { 2703 bio_add_page(bio, page, page_size, offset) < page_size) {
2703 ret = submit_one_bio(rw, bio, mirror_num, 2704 ret = submit_one_bio(rw, bio, mirror_num,
2704 prev_bio_flags); 2705 prev_bio_flags);
2705 if (ret < 0) 2706 if (ret < 0)
2706 return ret; 2707 return ret;
2707 bio = NULL; 2708 bio = NULL;
2708 } else { 2709 } else {
2709 return 0; 2710 return 0;
2710 } 2711 }
2711 } 2712 }
2712 if (this_compressed) 2713 if (this_compressed)
2713 nr = BIO_MAX_PAGES; 2714 nr = BIO_MAX_PAGES;
2714 else 2715 else
2715 nr = bio_get_nr_vecs(bdev); 2716 nr = bio_get_nr_vecs(bdev);
2716 2717
2717 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH); 2718 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
2718 if (!bio) 2719 if (!bio)
2719 return -ENOMEM; 2720 return -ENOMEM;
2720 2721
2721 bio_add_page(bio, page, page_size, offset); 2722 bio_add_page(bio, page, page_size, offset);
2722 bio->bi_end_io = end_io_func; 2723 bio->bi_end_io = end_io_func;
2723 bio->bi_private = tree; 2724 bio->bi_private = tree;
2724 2725
2725 if (bio_ret) 2726 if (bio_ret)
2726 *bio_ret = bio; 2727 *bio_ret = bio;
2727 else 2728 else
2728 ret = submit_one_bio(rw, bio, mirror_num, bio_flags); 2729 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
2729 2730
2730 return ret; 2731 return ret;
2731 } 2732 }
2732 2733
2733 static void attach_extent_buffer_page(struct extent_buffer *eb, 2734 static void attach_extent_buffer_page(struct extent_buffer *eb,
2734 struct page *page) 2735 struct page *page)
2735 { 2736 {
2736 if (!PagePrivate(page)) { 2737 if (!PagePrivate(page)) {
2737 SetPagePrivate(page); 2738 SetPagePrivate(page);
2738 page_cache_get(page); 2739 page_cache_get(page);
2739 set_page_private(page, (unsigned long)eb); 2740 set_page_private(page, (unsigned long)eb);
2740 } else { 2741 } else {
2741 WARN_ON(page->private != (unsigned long)eb); 2742 WARN_ON(page->private != (unsigned long)eb);
2742 } 2743 }
2743 } 2744 }
2744 2745
2745 void set_page_extent_mapped(struct page *page) 2746 void set_page_extent_mapped(struct page *page)
2746 { 2747 {
2747 if (!PagePrivate(page)) { 2748 if (!PagePrivate(page)) {
2748 SetPagePrivate(page); 2749 SetPagePrivate(page);
2749 page_cache_get(page); 2750 page_cache_get(page);
2750 set_page_private(page, EXTENT_PAGE_PRIVATE); 2751 set_page_private(page, EXTENT_PAGE_PRIVATE);
2751 } 2752 }
2752 } 2753 }
2753 2754
2754 static struct extent_map * 2755 static struct extent_map *
2755 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, 2756 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2756 u64 start, u64 len, get_extent_t *get_extent, 2757 u64 start, u64 len, get_extent_t *get_extent,
2757 struct extent_map **em_cached) 2758 struct extent_map **em_cached)
2758 { 2759 {
2759 struct extent_map *em; 2760 struct extent_map *em;
2760 2761
2761 if (em_cached && *em_cached) { 2762 if (em_cached && *em_cached) {
2762 em = *em_cached; 2763 em = *em_cached;
2763 if (em->in_tree && start >= em->start && 2764 if (em->in_tree && start >= em->start &&
2764 start < extent_map_end(em)) { 2765 start < extent_map_end(em)) {
2765 atomic_inc(&em->refs); 2766 atomic_inc(&em->refs);
2766 return em; 2767 return em;
2767 } 2768 }
2768 2769
2769 free_extent_map(em); 2770 free_extent_map(em);
2770 *em_cached = NULL; 2771 *em_cached = NULL;
2771 } 2772 }
2772 2773
2773 em = get_extent(inode, page, pg_offset, start, len, 0); 2774 em = get_extent(inode, page, pg_offset, start, len, 0);
2774 if (em_cached && !IS_ERR_OR_NULL(em)) { 2775 if (em_cached && !IS_ERR_OR_NULL(em)) {
2775 BUG_ON(*em_cached); 2776 BUG_ON(*em_cached);
2776 atomic_inc(&em->refs); 2777 atomic_inc(&em->refs);
2777 *em_cached = em; 2778 *em_cached = em;
2778 } 2779 }
2779 return em; 2780 return em;
2780 } 2781 }
2781 /* 2782 /*
2782 * basic readpage implementation. Locked extent state structs are inserted 2783 * basic readpage implementation. Locked extent state structs are inserted
2783 * into the tree that are removed when the IO is done (by the end_io 2784 * into the tree that are removed when the IO is done (by the end_io
2784 * handlers) 2785 * handlers)
2785 * XXX JDM: This needs looking at to ensure proper page locking 2786 * XXX JDM: This needs looking at to ensure proper page locking
2786 */ 2787 */
2787 static int __do_readpage(struct extent_io_tree *tree, 2788 static int __do_readpage(struct extent_io_tree *tree,
2788 struct page *page, 2789 struct page *page,
2789 get_extent_t *get_extent, 2790 get_extent_t *get_extent,
2790 struct extent_map **em_cached, 2791 struct extent_map **em_cached,
2791 struct bio **bio, int mirror_num, 2792 struct bio **bio, int mirror_num,
2792 unsigned long *bio_flags, int rw) 2793 unsigned long *bio_flags, int rw)
2793 { 2794 {
2794 struct inode *inode = page->mapping->host; 2795 struct inode *inode = page->mapping->host;
2795 u64 start = page_offset(page); 2796 u64 start = page_offset(page);
2796 u64 page_end = start + PAGE_CACHE_SIZE - 1; 2797 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2797 u64 end; 2798 u64 end;
2798 u64 cur = start; 2799 u64 cur = start;
2799 u64 extent_offset; 2800 u64 extent_offset;
2800 u64 last_byte = i_size_read(inode); 2801 u64 last_byte = i_size_read(inode);
2801 u64 block_start; 2802 u64 block_start;
2802 u64 cur_end; 2803 u64 cur_end;
2803 sector_t sector; 2804 sector_t sector;
2804 struct extent_map *em; 2805 struct extent_map *em;
2805 struct block_device *bdev; 2806 struct block_device *bdev;
2806 int ret; 2807 int ret;
2807 int nr = 0; 2808 int nr = 0;
2808 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED; 2809 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2809 size_t pg_offset = 0; 2810 size_t pg_offset = 0;
2810 size_t iosize; 2811 size_t iosize;
2811 size_t disk_io_size; 2812 size_t disk_io_size;
2812 size_t blocksize = inode->i_sb->s_blocksize; 2813 size_t blocksize = inode->i_sb->s_blocksize;
2813 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED; 2814 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2814 2815
2815 set_page_extent_mapped(page); 2816 set_page_extent_mapped(page);
2816 2817
2817 end = page_end; 2818 end = page_end;
2818 if (!PageUptodate(page)) { 2819 if (!PageUptodate(page)) {
2819 if (cleancache_get_page(page) == 0) { 2820 if (cleancache_get_page(page) == 0) {
2820 BUG_ON(blocksize != PAGE_SIZE); 2821 BUG_ON(blocksize != PAGE_SIZE);
2821 unlock_extent(tree, start, end); 2822 unlock_extent(tree, start, end);
2822 goto out; 2823 goto out;
2823 } 2824 }
2824 } 2825 }
2825 2826
2826 if (page->index == last_byte >> PAGE_CACHE_SHIFT) { 2827 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2827 char *userpage; 2828 char *userpage;
2828 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1); 2829 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2829 2830
2830 if (zero_offset) { 2831 if (zero_offset) {
2831 iosize = PAGE_CACHE_SIZE - zero_offset; 2832 iosize = PAGE_CACHE_SIZE - zero_offset;
2832 userpage = kmap_atomic(page); 2833 userpage = kmap_atomic(page);
2833 memset(userpage + zero_offset, 0, iosize); 2834 memset(userpage + zero_offset, 0, iosize);
2834 flush_dcache_page(page); 2835 flush_dcache_page(page);
2835 kunmap_atomic(userpage); 2836 kunmap_atomic(userpage);
2836 } 2837 }
2837 } 2838 }
2838 while (cur <= end) { 2839 while (cur <= end) {
2839 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1; 2840 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2840 2841
2841 if (cur >= last_byte) { 2842 if (cur >= last_byte) {
2842 char *userpage; 2843 char *userpage;
2843 struct extent_state *cached = NULL; 2844 struct extent_state *cached = NULL;
2844 2845
2845 iosize = PAGE_CACHE_SIZE - pg_offset; 2846 iosize = PAGE_CACHE_SIZE - pg_offset;
2846 userpage = kmap_atomic(page); 2847 userpage = kmap_atomic(page);
2847 memset(userpage + pg_offset, 0, iosize); 2848 memset(userpage + pg_offset, 0, iosize);
2848 flush_dcache_page(page); 2849 flush_dcache_page(page);
2849 kunmap_atomic(userpage); 2850 kunmap_atomic(userpage);
2850 set_extent_uptodate(tree, cur, cur + iosize - 1, 2851 set_extent_uptodate(tree, cur, cur + iosize - 1,
2851 &cached, GFP_NOFS); 2852 &cached, GFP_NOFS);
2852 if (!parent_locked) 2853 if (!parent_locked)
2853 unlock_extent_cached(tree, cur, 2854 unlock_extent_cached(tree, cur,
2854 cur + iosize - 1, 2855 cur + iosize - 1,
2855 &cached, GFP_NOFS); 2856 &cached, GFP_NOFS);
2856 break; 2857 break;
2857 } 2858 }
2858 em = __get_extent_map(inode, page, pg_offset, cur, 2859 em = __get_extent_map(inode, page, pg_offset, cur,
2859 end - cur + 1, get_extent, em_cached); 2860 end - cur + 1, get_extent, em_cached);
2860 if (IS_ERR_OR_NULL(em)) { 2861 if (IS_ERR_OR_NULL(em)) {
2861 SetPageError(page); 2862 SetPageError(page);
2862 if (!parent_locked) 2863 if (!parent_locked)
2863 unlock_extent(tree, cur, end); 2864 unlock_extent(tree, cur, end);
2864 break; 2865 break;
2865 } 2866 }
2866 extent_offset = cur - em->start; 2867 extent_offset = cur - em->start;
2867 BUG_ON(extent_map_end(em) <= cur); 2868 BUG_ON(extent_map_end(em) <= cur);
2868 BUG_ON(end < cur); 2869 BUG_ON(end < cur);
2869 2870
2870 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) { 2871 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2871 this_bio_flag |= EXTENT_BIO_COMPRESSED; 2872 this_bio_flag |= EXTENT_BIO_COMPRESSED;
2872 extent_set_compress_type(&this_bio_flag, 2873 extent_set_compress_type(&this_bio_flag,
2873 em->compress_type); 2874 em->compress_type);
2874 } 2875 }
2875 2876
2876 iosize = min(extent_map_end(em) - cur, end - cur + 1); 2877 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2877 cur_end = min(extent_map_end(em) - 1, end); 2878 cur_end = min(extent_map_end(em) - 1, end);
2878 iosize = ALIGN(iosize, blocksize); 2879 iosize = ALIGN(iosize, blocksize);
2879 if (this_bio_flag & EXTENT_BIO_COMPRESSED) { 2880 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2880 disk_io_size = em->block_len; 2881 disk_io_size = em->block_len;
2881 sector = em->block_start >> 9; 2882 sector = em->block_start >> 9;
2882 } else { 2883 } else {
2883 sector = (em->block_start + extent_offset) >> 9; 2884 sector = (em->block_start + extent_offset) >> 9;
2884 disk_io_size = iosize; 2885 disk_io_size = iosize;
2885 } 2886 }
2886 bdev = em->bdev; 2887 bdev = em->bdev;
2887 block_start = em->block_start; 2888 block_start = em->block_start;
2888 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) 2889 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2889 block_start = EXTENT_MAP_HOLE; 2890 block_start = EXTENT_MAP_HOLE;
2890 free_extent_map(em); 2891 free_extent_map(em);
2891 em = NULL; 2892 em = NULL;
2892 2893
2893 /* we've found a hole, just zero and go on */ 2894 /* we've found a hole, just zero and go on */
2894 if (block_start == EXTENT_MAP_HOLE) { 2895 if (block_start == EXTENT_MAP_HOLE) {
2895 char *userpage; 2896 char *userpage;
2896 struct extent_state *cached = NULL; 2897 struct extent_state *cached = NULL;
2897 2898
2898 userpage = kmap_atomic(page); 2899 userpage = kmap_atomic(page);
2899 memset(userpage + pg_offset, 0, iosize); 2900 memset(userpage + pg_offset, 0, iosize);
2900 flush_dcache_page(page); 2901 flush_dcache_page(page);
2901 kunmap_atomic(userpage); 2902 kunmap_atomic(userpage);
2902 2903
2903 set_extent_uptodate(tree, cur, cur + iosize - 1, 2904 set_extent_uptodate(tree, cur, cur + iosize - 1,
2904 &cached, GFP_NOFS); 2905 &cached, GFP_NOFS);
2905 unlock_extent_cached(tree, cur, cur + iosize - 1, 2906 unlock_extent_cached(tree, cur, cur + iosize - 1,
2906 &cached, GFP_NOFS); 2907 &cached, GFP_NOFS);
2907 cur = cur + iosize; 2908 cur = cur + iosize;
2908 pg_offset += iosize; 2909 pg_offset += iosize;
2909 continue; 2910 continue;
2910 } 2911 }
2911 /* the get_extent function already copied into the page */ 2912 /* the get_extent function already copied into the page */
2912 if (test_range_bit(tree, cur, cur_end, 2913 if (test_range_bit(tree, cur, cur_end,
2913 EXTENT_UPTODATE, 1, NULL)) { 2914 EXTENT_UPTODATE, 1, NULL)) {
2914 check_page_uptodate(tree, page); 2915 check_page_uptodate(tree, page);
2915 if (!parent_locked) 2916 if (!parent_locked)
2916 unlock_extent(tree, cur, cur + iosize - 1); 2917 unlock_extent(tree, cur, cur + iosize - 1);
2917 cur = cur + iosize; 2918 cur = cur + iosize;
2918 pg_offset += iosize; 2919 pg_offset += iosize;
2919 continue; 2920 continue;
2920 } 2921 }
2921 /* we have an inline extent but it didn't get marked up 2922 /* we have an inline extent but it didn't get marked up
2922 * to date. Error out 2923 * to date. Error out
2923 */ 2924 */
2924 if (block_start == EXTENT_MAP_INLINE) { 2925 if (block_start == EXTENT_MAP_INLINE) {
2925 SetPageError(page); 2926 SetPageError(page);
2926 if (!parent_locked) 2927 if (!parent_locked)
2927 unlock_extent(tree, cur, cur + iosize - 1); 2928 unlock_extent(tree, cur, cur + iosize - 1);
2928 cur = cur + iosize; 2929 cur = cur + iosize;
2929 pg_offset += iosize; 2930 pg_offset += iosize;
2930 continue; 2931 continue;
2931 } 2932 }
2932 2933
2933 pnr -= page->index; 2934 pnr -= page->index;
2934 ret = submit_extent_page(rw, tree, page, 2935 ret = submit_extent_page(rw, tree, page,
2935 sector, disk_io_size, pg_offset, 2936 sector, disk_io_size, pg_offset,
2936 bdev, bio, pnr, 2937 bdev, bio, pnr,
2937 end_bio_extent_readpage, mirror_num, 2938 end_bio_extent_readpage, mirror_num,
2938 *bio_flags, 2939 *bio_flags,
2939 this_bio_flag); 2940 this_bio_flag);
2940 if (!ret) { 2941 if (!ret) {
2941 nr++; 2942 nr++;
2942 *bio_flags = this_bio_flag; 2943 *bio_flags = this_bio_flag;
2943 } else { 2944 } else {
2944 SetPageError(page); 2945 SetPageError(page);
2945 if (!parent_locked) 2946 if (!parent_locked)
2946 unlock_extent(tree, cur, cur + iosize - 1); 2947 unlock_extent(tree, cur, cur + iosize - 1);
2947 } 2948 }
2948 cur = cur + iosize; 2949 cur = cur + iosize;
2949 pg_offset += iosize; 2950 pg_offset += iosize;
2950 } 2951 }
2951 out: 2952 out:
2952 if (!nr) { 2953 if (!nr) {
2953 if (!PageError(page)) 2954 if (!PageError(page))
2954 SetPageUptodate(page); 2955 SetPageUptodate(page);
2955 unlock_page(page); 2956 unlock_page(page);
2956 } 2957 }
2957 return 0; 2958 return 0;
2958 } 2959 }
2959 2960
2960 static inline void __do_contiguous_readpages(struct extent_io_tree *tree, 2961 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
2961 struct page *pages[], int nr_pages, 2962 struct page *pages[], int nr_pages,
2962 u64 start, u64 end, 2963 u64 start, u64 end,
2963 get_extent_t *get_extent, 2964 get_extent_t *get_extent,
2964 struct extent_map **em_cached, 2965 struct extent_map **em_cached,
2965 struct bio **bio, int mirror_num, 2966 struct bio **bio, int mirror_num,
2966 unsigned long *bio_flags, int rw) 2967 unsigned long *bio_flags, int rw)
2967 { 2968 {
2968 struct inode *inode; 2969 struct inode *inode;
2969 struct btrfs_ordered_extent *ordered; 2970 struct btrfs_ordered_extent *ordered;
2970 int index; 2971 int index;
2971 2972
2972 inode = pages[0]->mapping->host; 2973 inode = pages[0]->mapping->host;
2973 while (1) { 2974 while (1) {
2974 lock_extent(tree, start, end); 2975 lock_extent(tree, start, end);
2975 ordered = btrfs_lookup_ordered_range(inode, start, 2976 ordered = btrfs_lookup_ordered_range(inode, start,
2976 end - start + 1); 2977 end - start + 1);
2977 if (!ordered) 2978 if (!ordered)
2978 break; 2979 break;
2979 unlock_extent(tree, start, end); 2980 unlock_extent(tree, start, end);
2980 btrfs_start_ordered_extent(inode, ordered, 1); 2981 btrfs_start_ordered_extent(inode, ordered, 1);
2981 btrfs_put_ordered_extent(ordered); 2982 btrfs_put_ordered_extent(ordered);
2982 } 2983 }
2983 2984
2984 for (index = 0; index < nr_pages; index++) { 2985 for (index = 0; index < nr_pages; index++) {
2985 __do_readpage(tree, pages[index], get_extent, em_cached, bio, 2986 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
2986 mirror_num, bio_flags, rw); 2987 mirror_num, bio_flags, rw);
2987 page_cache_release(pages[index]); 2988 page_cache_release(pages[index]);
2988 } 2989 }
2989 } 2990 }
2990 2991
2991 static void __extent_readpages(struct extent_io_tree *tree, 2992 static void __extent_readpages(struct extent_io_tree *tree,
2992 struct page *pages[], 2993 struct page *pages[],
2993 int nr_pages, get_extent_t *get_extent, 2994 int nr_pages, get_extent_t *get_extent,
2994 struct extent_map **em_cached, 2995 struct extent_map **em_cached,
2995 struct bio **bio, int mirror_num, 2996 struct bio **bio, int mirror_num,
2996 unsigned long *bio_flags, int rw) 2997 unsigned long *bio_flags, int rw)
2997 { 2998 {
2998 u64 start = 0; 2999 u64 start = 0;
2999 u64 end = 0; 3000 u64 end = 0;
3000 u64 page_start; 3001 u64 page_start;
3001 int index; 3002 int index;
3002 int first_index = 0; 3003 int first_index = 0;
3003 3004
3004 for (index = 0; index < nr_pages; index++) { 3005 for (index = 0; index < nr_pages; index++) {
3005 page_start = page_offset(pages[index]); 3006 page_start = page_offset(pages[index]);
3006 if (!end) { 3007 if (!end) {
3007 start = page_start; 3008 start = page_start;
3008 end = start + PAGE_CACHE_SIZE - 1; 3009 end = start + PAGE_CACHE_SIZE - 1;
3009 first_index = index; 3010 first_index = index;
3010 } else if (end + 1 == page_start) { 3011 } else if (end + 1 == page_start) {
3011 end += PAGE_CACHE_SIZE; 3012 end += PAGE_CACHE_SIZE;
3012 } else { 3013 } else {
3013 __do_contiguous_readpages(tree, &pages[first_index], 3014 __do_contiguous_readpages(tree, &pages[first_index],
3014 index - first_index, start, 3015 index - first_index, start,
3015 end, get_extent, em_cached, 3016 end, get_extent, em_cached,
3016 bio, mirror_num, bio_flags, 3017 bio, mirror_num, bio_flags,
3017 rw); 3018 rw);
3018 start = page_start; 3019 start = page_start;
3019 end = start + PAGE_CACHE_SIZE - 1; 3020 end = start + PAGE_CACHE_SIZE - 1;
3020 first_index = index; 3021 first_index = index;
3021 } 3022 }
3022 } 3023 }
3023 3024
3024 if (end) 3025 if (end)
3025 __do_contiguous_readpages(tree, &pages[first_index], 3026 __do_contiguous_readpages(tree, &pages[first_index],
3026 index - first_index, start, 3027 index - first_index, start,
3027 end, get_extent, em_cached, bio, 3028 end, get_extent, em_cached, bio,
3028 mirror_num, bio_flags, rw); 3029 mirror_num, bio_flags, rw);
3029 } 3030 }
3030 3031
3031 static int __extent_read_full_page(struct extent_io_tree *tree, 3032 static int __extent_read_full_page(struct extent_io_tree *tree,
3032 struct page *page, 3033 struct page *page,
3033 get_extent_t *get_extent, 3034 get_extent_t *get_extent,
3034 struct bio **bio, int mirror_num, 3035 struct bio **bio, int mirror_num,
3035 unsigned long *bio_flags, int rw) 3036 unsigned long *bio_flags, int rw)
3036 { 3037 {
3037 struct inode *inode = page->mapping->host; 3038 struct inode *inode = page->mapping->host;
3038 struct btrfs_ordered_extent *ordered; 3039 struct btrfs_ordered_extent *ordered;
3039 u64 start = page_offset(page); 3040 u64 start = page_offset(page);
3040 u64 end = start + PAGE_CACHE_SIZE - 1; 3041 u64 end = start + PAGE_CACHE_SIZE - 1;
3041 int ret; 3042 int ret;
3042 3043
3043 while (1) { 3044 while (1) {
3044 lock_extent(tree, start, end); 3045 lock_extent(tree, start, end);
3045 ordered = btrfs_lookup_ordered_extent(inode, start); 3046 ordered = btrfs_lookup_ordered_extent(inode, start);
3046 if (!ordered) 3047 if (!ordered)
3047 break; 3048 break;
3048 unlock_extent(tree, start, end); 3049 unlock_extent(tree, start, end);
3049 btrfs_start_ordered_extent(inode, ordered, 1); 3050 btrfs_start_ordered_extent(inode, ordered, 1);
3050 btrfs_put_ordered_extent(ordered); 3051 btrfs_put_ordered_extent(ordered);
3051 } 3052 }
3052 3053
3053 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num, 3054 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3054 bio_flags, rw); 3055 bio_flags, rw);
3055 return ret; 3056 return ret;
3056 } 3057 }
3057 3058
3058 int extent_read_full_page(struct extent_io_tree *tree, struct page *page, 3059 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
3059 get_extent_t *get_extent, int mirror_num) 3060 get_extent_t *get_extent, int mirror_num)
3060 { 3061 {
3061 struct bio *bio = NULL; 3062 struct bio *bio = NULL;
3062 unsigned long bio_flags = 0; 3063 unsigned long bio_flags = 0;
3063 int ret; 3064 int ret;
3064 3065
3065 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num, 3066 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
3066 &bio_flags, READ); 3067 &bio_flags, READ);
3067 if (bio) 3068 if (bio)
3068 ret = submit_one_bio(READ, bio, mirror_num, bio_flags); 3069 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3069 return ret; 3070 return ret;
3070 } 3071 }
3071 3072
3072 int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page, 3073 int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3073 get_extent_t *get_extent, int mirror_num) 3074 get_extent_t *get_extent, int mirror_num)
3074 { 3075 {
3075 struct bio *bio = NULL; 3076 struct bio *bio = NULL;
3076 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED; 3077 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3077 int ret; 3078 int ret;
3078 3079
3079 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num, 3080 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
3080 &bio_flags, READ); 3081 &bio_flags, READ);
3081 if (bio) 3082 if (bio)
3082 ret = submit_one_bio(READ, bio, mirror_num, bio_flags); 3083 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3083 return ret; 3084 return ret;
3084 } 3085 }
3085 3086
3086 static noinline void update_nr_written(struct page *page, 3087 static noinline void update_nr_written(struct page *page,
3087 struct writeback_control *wbc, 3088 struct writeback_control *wbc,
3088 unsigned long nr_written) 3089 unsigned long nr_written)
3089 { 3090 {
3090 wbc->nr_to_write -= nr_written; 3091 wbc->nr_to_write -= nr_written;
3091 if (wbc->range_cyclic || (wbc->nr_to_write > 0 && 3092 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3092 wbc->range_start == 0 && wbc->range_end == LLONG_MAX)) 3093 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3093 page->mapping->writeback_index = page->index + nr_written; 3094 page->mapping->writeback_index = page->index + nr_written;
3094 } 3095 }
3095 3096
3096 /* 3097 /*
3097 * the writepage semantics are similar to regular writepage. extent 3098 * the writepage semantics are similar to regular writepage. extent
3098 * records are inserted to lock ranges in the tree, and as dirty areas 3099 * records are inserted to lock ranges in the tree, and as dirty areas
3099 * are found, they are marked writeback. Then the lock bits are removed 3100 * are found, they are marked writeback. Then the lock bits are removed
3100 * and the end_io handler clears the writeback ranges 3101 * and the end_io handler clears the writeback ranges
3101 */ 3102 */
3102 static int __extent_writepage(struct page *page, struct writeback_control *wbc, 3103 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3103 void *data) 3104 void *data)
3104 { 3105 {
3105 struct inode *inode = page->mapping->host; 3106 struct inode *inode = page->mapping->host;
3106 struct extent_page_data *epd = data; 3107 struct extent_page_data *epd = data;
3107 struct extent_io_tree *tree = epd->tree; 3108 struct extent_io_tree *tree = epd->tree;
3108 u64 start = page_offset(page); 3109 u64 start = page_offset(page);
3109 u64 delalloc_start; 3110 u64 delalloc_start;
3110 u64 page_end = start + PAGE_CACHE_SIZE - 1; 3111 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3111 u64 end; 3112 u64 end;
3112 u64 cur = start; 3113 u64 cur = start;
3113 u64 extent_offset; 3114 u64 extent_offset;
3114 u64 last_byte = i_size_read(inode); 3115 u64 last_byte = i_size_read(inode);
3115 u64 block_start; 3116 u64 block_start;
3116 u64 iosize; 3117 u64 iosize;
3117 sector_t sector; 3118 sector_t sector;
3118 struct extent_state *cached_state = NULL; 3119 struct extent_state *cached_state = NULL;
3119 struct extent_map *em; 3120 struct extent_map *em;
3120 struct block_device *bdev; 3121 struct block_device *bdev;
3121 int ret; 3122 int ret;
3122 int nr = 0; 3123 int nr = 0;
3123 size_t pg_offset = 0; 3124 size_t pg_offset = 0;
3124 size_t blocksize; 3125 size_t blocksize;
3125 loff_t i_size = i_size_read(inode); 3126 loff_t i_size = i_size_read(inode);
3126 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT; 3127 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3127 u64 nr_delalloc; 3128 u64 nr_delalloc;
3128 u64 delalloc_end; 3129 u64 delalloc_end;
3129 int page_started; 3130 int page_started;
3130 int compressed; 3131 int compressed;
3131 int write_flags; 3132 int write_flags;
3132 unsigned long nr_written = 0; 3133 unsigned long nr_written = 0;
3133 bool fill_delalloc = true; 3134 bool fill_delalloc = true;
3134 3135
3135 if (wbc->sync_mode == WB_SYNC_ALL) 3136 if (wbc->sync_mode == WB_SYNC_ALL)
3136 write_flags = WRITE_SYNC; 3137 write_flags = WRITE_SYNC;
3137 else 3138 else
3138 write_flags = WRITE; 3139 write_flags = WRITE;
3139 3140
3140 trace___extent_writepage(page, inode, wbc); 3141 trace___extent_writepage(page, inode, wbc);
3141 3142
3142 WARN_ON(!PageLocked(page)); 3143 WARN_ON(!PageLocked(page));
3143 3144
3144 ClearPageError(page); 3145 ClearPageError(page);
3145 3146
3146 pg_offset = i_size & (PAGE_CACHE_SIZE - 1); 3147 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3147 if (page->index > end_index || 3148 if (page->index > end_index ||
3148 (page->index == end_index && !pg_offset)) { 3149 (page->index == end_index && !pg_offset)) {
3149 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE); 3150 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3150 unlock_page(page); 3151 unlock_page(page);
3151 return 0; 3152 return 0;
3152 } 3153 }
3153 3154
3154 if (page->index == end_index) { 3155 if (page->index == end_index) {
3155 char *userpage; 3156 char *userpage;
3156 3157
3157 userpage = kmap_atomic(page); 3158 userpage = kmap_atomic(page);
3158 memset(userpage + pg_offset, 0, 3159 memset(userpage + pg_offset, 0,
3159 PAGE_CACHE_SIZE - pg_offset); 3160 PAGE_CACHE_SIZE - pg_offset);
3160 kunmap_atomic(userpage); 3161 kunmap_atomic(userpage);
3161 flush_dcache_page(page); 3162 flush_dcache_page(page);
3162 } 3163 }
3163 pg_offset = 0; 3164 pg_offset = 0;
3164 3165
3165 set_page_extent_mapped(page); 3166 set_page_extent_mapped(page);
3166 3167
3167 if (!tree->ops || !tree->ops->fill_delalloc) 3168 if (!tree->ops || !tree->ops->fill_delalloc)
3168 fill_delalloc = false; 3169 fill_delalloc = false;
3169 3170
3170 delalloc_start = start; 3171 delalloc_start = start;
3171 delalloc_end = 0; 3172 delalloc_end = 0;
3172 page_started = 0; 3173 page_started = 0;
3173 if (!epd->extent_locked && fill_delalloc) { 3174 if (!epd->extent_locked && fill_delalloc) {
3174 u64 delalloc_to_write = 0; 3175 u64 delalloc_to_write = 0;
3175 /* 3176 /*
3176 * make sure the wbc mapping index is at least updated 3177 * make sure the wbc mapping index is at least updated
3177 * to this page. 3178 * to this page.
3178 */ 3179 */
3179 update_nr_written(page, wbc, 0); 3180 update_nr_written(page, wbc, 0);
3180 3181
3181 while (delalloc_end < page_end) { 3182 while (delalloc_end < page_end) {
3182 nr_delalloc = find_lock_delalloc_range(inode, tree, 3183 nr_delalloc = find_lock_delalloc_range(inode, tree,
3183 page, 3184 page,
3184 &delalloc_start, 3185 &delalloc_start,
3185 &delalloc_end, 3186 &delalloc_end,
3186 128 * 1024 * 1024); 3187 128 * 1024 * 1024);
3187 if (nr_delalloc == 0) { 3188 if (nr_delalloc == 0) {
3188 delalloc_start = delalloc_end + 1; 3189 delalloc_start = delalloc_end + 1;
3189 continue; 3190 continue;
3190 } 3191 }
3191 ret = tree->ops->fill_delalloc(inode, page, 3192 ret = tree->ops->fill_delalloc(inode, page,
3192 delalloc_start, 3193 delalloc_start,
3193 delalloc_end, 3194 delalloc_end,
3194 &page_started, 3195 &page_started,
3195 &nr_written); 3196 &nr_written);
3196 /* File system has been set read-only */ 3197 /* File system has been set read-only */
3197 if (ret) { 3198 if (ret) {
3198 SetPageError(page); 3199 SetPageError(page);
3199 goto done; 3200 goto done;
3200 } 3201 }
3201 /* 3202 /*
3202 * delalloc_end is already one less than the total 3203 * delalloc_end is already one less than the total
3203 * length, so we don't subtract one from 3204 * length, so we don't subtract one from
3204 * PAGE_CACHE_SIZE 3205 * PAGE_CACHE_SIZE
3205 */ 3206 */
3206 delalloc_to_write += (delalloc_end - delalloc_start + 3207 delalloc_to_write += (delalloc_end - delalloc_start +
3207 PAGE_CACHE_SIZE) >> 3208 PAGE_CACHE_SIZE) >>
3208 PAGE_CACHE_SHIFT; 3209 PAGE_CACHE_SHIFT;
3209 delalloc_start = delalloc_end + 1; 3210 delalloc_start = delalloc_end + 1;
3210 } 3211 }
3211 if (wbc->nr_to_write < delalloc_to_write) { 3212 if (wbc->nr_to_write < delalloc_to_write) {
3212 int thresh = 8192; 3213 int thresh = 8192;
3213 3214
3214 if (delalloc_to_write < thresh * 2) 3215 if (delalloc_to_write < thresh * 2)
3215 thresh = delalloc_to_write; 3216 thresh = delalloc_to_write;
3216 wbc->nr_to_write = min_t(u64, delalloc_to_write, 3217 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3217 thresh); 3218 thresh);
3218 } 3219 }
3219 3220
3220 /* did the fill delalloc function already unlock and start 3221 /* did the fill delalloc function already unlock and start
3221 * the IO? 3222 * the IO?
3222 */ 3223 */
3223 if (page_started) { 3224 if (page_started) {
3224 ret = 0; 3225 ret = 0;
3225 /* 3226 /*
3226 * we've unlocked the page, so we can't update 3227 * we've unlocked the page, so we can't update
3227 * the mapping's writeback index, just update 3228 * the mapping's writeback index, just update
3228 * nr_to_write. 3229 * nr_to_write.
3229 */ 3230 */
3230 wbc->nr_to_write -= nr_written; 3231 wbc->nr_to_write -= nr_written;
3231 goto done_unlocked; 3232 goto done_unlocked;
3232 } 3233 }
3233 } 3234 }
3234 if (tree->ops && tree->ops->writepage_start_hook) { 3235 if (tree->ops && tree->ops->writepage_start_hook) {
3235 ret = tree->ops->writepage_start_hook(page, start, 3236 ret = tree->ops->writepage_start_hook(page, start,
3236 page_end); 3237 page_end);
3237 if (ret) { 3238 if (ret) {
3238 /* Fixup worker will requeue */ 3239 /* Fixup worker will requeue */
3239 if (ret == -EBUSY) 3240 if (ret == -EBUSY)
3240 wbc->pages_skipped++; 3241 wbc->pages_skipped++;
3241 else 3242 else
3242 redirty_page_for_writepage(wbc, page); 3243 redirty_page_for_writepage(wbc, page);
3243 update_nr_written(page, wbc, nr_written); 3244 update_nr_written(page, wbc, nr_written);
3244 unlock_page(page); 3245 unlock_page(page);
3245 ret = 0; 3246 ret = 0;
3246 goto done_unlocked; 3247 goto done_unlocked;
3247 } 3248 }
3248 } 3249 }
3249 3250
3250 /* 3251 /*
3251 * we don't want to touch the inode after unlocking the page, 3252 * we don't want to touch the inode after unlocking the page,
3252 * so we update the mapping writeback index now 3253 * so we update the mapping writeback index now
3253 */ 3254 */
3254 update_nr_written(page, wbc, nr_written + 1); 3255 update_nr_written(page, wbc, nr_written + 1);
3255 3256
3256 end = page_end; 3257 end = page_end;
3257 if (last_byte <= start) { 3258 if (last_byte <= start) {
3258 if (tree->ops && tree->ops->writepage_end_io_hook) 3259 if (tree->ops && tree->ops->writepage_end_io_hook)
3259 tree->ops->writepage_end_io_hook(page, start, 3260 tree->ops->writepage_end_io_hook(page, start,
3260 page_end, NULL, 1); 3261 page_end, NULL, 1);
3261 goto done; 3262 goto done;
3262 } 3263 }
3263 3264
3264 blocksize = inode->i_sb->s_blocksize; 3265 blocksize = inode->i_sb->s_blocksize;
3265 3266
3266 while (cur <= end) { 3267 while (cur <= end) {
3267 if (cur >= last_byte) { 3268 if (cur >= last_byte) {
3268 if (tree->ops && tree->ops->writepage_end_io_hook) 3269 if (tree->ops && tree->ops->writepage_end_io_hook)
3269 tree->ops->writepage_end_io_hook(page, cur, 3270 tree->ops->writepage_end_io_hook(page, cur,
3270 page_end, NULL, 1); 3271 page_end, NULL, 1);
3271 break; 3272 break;
3272 } 3273 }
3273 em = epd->get_extent(inode, page, pg_offset, cur, 3274 em = epd->get_extent(inode, page, pg_offset, cur,
3274 end - cur + 1, 1); 3275 end - cur + 1, 1);
3275 if (IS_ERR_OR_NULL(em)) { 3276 if (IS_ERR_OR_NULL(em)) {
3276 SetPageError(page); 3277 SetPageError(page);
3277 break; 3278 break;
3278 } 3279 }
3279 3280
3280 extent_offset = cur - em->start; 3281 extent_offset = cur - em->start;
3281 BUG_ON(extent_map_end(em) <= cur); 3282 BUG_ON(extent_map_end(em) <= cur);
3282 BUG_ON(end < cur); 3283 BUG_ON(end < cur);
3283 iosize = min(extent_map_end(em) - cur, end - cur + 1); 3284 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3284 iosize = ALIGN(iosize, blocksize); 3285 iosize = ALIGN(iosize, blocksize);
3285 sector = (em->block_start + extent_offset) >> 9; 3286 sector = (em->block_start + extent_offset) >> 9;
3286 bdev = em->bdev; 3287 bdev = em->bdev;
3287 block_start = em->block_start; 3288 block_start = em->block_start;
3288 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); 3289 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
3289 free_extent_map(em); 3290 free_extent_map(em);
3290 em = NULL; 3291 em = NULL;
3291 3292
3292 /* 3293 /*
3293 * compressed and inline extents are written through other 3294 * compressed and inline extents are written through other
3294 * paths in the FS 3295 * paths in the FS
3295 */ 3296 */
3296 if (compressed || block_start == EXTENT_MAP_HOLE || 3297 if (compressed || block_start == EXTENT_MAP_HOLE ||
3297 block_start == EXTENT_MAP_INLINE) { 3298 block_start == EXTENT_MAP_INLINE) {
3298 /* 3299 /*
3299 * end_io notification does not happen here for 3300 * end_io notification does not happen here for
3300 * compressed extents 3301 * compressed extents
3301 */ 3302 */
3302 if (!compressed && tree->ops && 3303 if (!compressed && tree->ops &&
3303 tree->ops->writepage_end_io_hook) 3304 tree->ops->writepage_end_io_hook)
3304 tree->ops->writepage_end_io_hook(page, cur, 3305 tree->ops->writepage_end_io_hook(page, cur,
3305 cur + iosize - 1, 3306 cur + iosize - 1,
3306 NULL, 1); 3307 NULL, 1);
3307 else if (compressed) { 3308 else if (compressed) {
3308 /* we don't want to end_page_writeback on 3309 /* we don't want to end_page_writeback on
3309 * a compressed extent. this happens 3310 * a compressed extent. this happens
3310 * elsewhere 3311 * elsewhere
3311 */ 3312 */
3312 nr++; 3313 nr++;
3313 } 3314 }
3314 3315
3315 cur += iosize; 3316 cur += iosize;
3316 pg_offset += iosize; 3317 pg_offset += iosize;
3317 continue; 3318 continue;
3318 } 3319 }
3319 /* leave this out until we have a page_mkwrite call */ 3320 /* leave this out until we have a page_mkwrite call */
3320 if (0 && !test_range_bit(tree, cur, cur + iosize - 1, 3321 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
3321 EXTENT_DIRTY, 0, NULL)) { 3322 EXTENT_DIRTY, 0, NULL)) {
3322 cur = cur + iosize; 3323 cur = cur + iosize;
3323 pg_offset += iosize; 3324 pg_offset += iosize;
3324 continue; 3325 continue;
3325 } 3326 }
3326 3327
3327 if (tree->ops && tree->ops->writepage_io_hook) { 3328 if (tree->ops && tree->ops->writepage_io_hook) {
3328 ret = tree->ops->writepage_io_hook(page, cur, 3329 ret = tree->ops->writepage_io_hook(page, cur,
3329 cur + iosize - 1); 3330 cur + iosize - 1);
3330 } else { 3331 } else {
3331 ret = 0; 3332 ret = 0;
3332 } 3333 }
3333 if (ret) { 3334 if (ret) {
3334 SetPageError(page); 3335 SetPageError(page);
3335 } else { 3336 } else {
3336 unsigned long max_nr = end_index + 1; 3337 unsigned long max_nr = end_index + 1;
3337 3338
3338 set_range_writeback(tree, cur, cur + iosize - 1); 3339 set_range_writeback(tree, cur, cur + iosize - 1);
3339 if (!PageWriteback(page)) { 3340 if (!PageWriteback(page)) {
3340 btrfs_err(BTRFS_I(inode)->root->fs_info, 3341 btrfs_err(BTRFS_I(inode)->root->fs_info,
3341 "page %lu not writeback, cur %llu end %llu", 3342 "page %lu not writeback, cur %llu end %llu",
3342 page->index, cur, end); 3343 page->index, cur, end);
3343 } 3344 }
3344 3345
3345 ret = submit_extent_page(write_flags, tree, page, 3346 ret = submit_extent_page(write_flags, tree, page,
3346 sector, iosize, pg_offset, 3347 sector, iosize, pg_offset,
3347 bdev, &epd->bio, max_nr, 3348 bdev, &epd->bio, max_nr,
3348 end_bio_extent_writepage, 3349 end_bio_extent_writepage,
3349 0, 0, 0); 3350 0, 0, 0);
3350 if (ret) 3351 if (ret)
3351 SetPageError(page); 3352 SetPageError(page);
3352 } 3353 }
3353 cur = cur + iosize; 3354 cur = cur + iosize;
3354 pg_offset += iosize; 3355 pg_offset += iosize;
3355 nr++; 3356 nr++;
3356 } 3357 }
3357 done: 3358 done:
3358 if (nr == 0) { 3359 if (nr == 0) {
3359 /* make sure the mapping tag for page dirty gets cleared */ 3360 /* make sure the mapping tag for page dirty gets cleared */
3360 set_page_writeback(page); 3361 set_page_writeback(page);
3361 end_page_writeback(page); 3362 end_page_writeback(page);
3362 } 3363 }
3363 unlock_page(page); 3364 unlock_page(page);
3364 3365
3365 done_unlocked: 3366 done_unlocked:
3366 3367
3367 /* drop our reference on any cached states */ 3368 /* drop our reference on any cached states */
3368 free_extent_state(cached_state); 3369 free_extent_state(cached_state);
3369 return 0; 3370 return 0;
3370 } 3371 }
3371 3372
3372 static int eb_wait(void *word) 3373 static int eb_wait(void *word)
3373 { 3374 {
3374 io_schedule(); 3375 io_schedule();
3375 return 0; 3376 return 0;
3376 } 3377 }
3377 3378
3378 void wait_on_extent_buffer_writeback(struct extent_buffer *eb) 3379 void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3379 { 3380 {
3380 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait, 3381 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3381 TASK_UNINTERRUPTIBLE); 3382 TASK_UNINTERRUPTIBLE);
3382 } 3383 }
3383 3384
3384 static int lock_extent_buffer_for_io(struct extent_buffer *eb, 3385 static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3385 struct btrfs_fs_info *fs_info, 3386 struct btrfs_fs_info *fs_info,
3386 struct extent_page_data *epd) 3387 struct extent_page_data *epd)
3387 { 3388 {
3388 unsigned long i, num_pages; 3389 unsigned long i, num_pages;
3389 int flush = 0; 3390 int flush = 0;
3390 int ret = 0; 3391 int ret = 0;
3391 3392
3392 if (!btrfs_try_tree_write_lock(eb)) { 3393 if (!btrfs_try_tree_write_lock(eb)) {
3393 flush = 1; 3394 flush = 1;
3394 flush_write_bio(epd); 3395 flush_write_bio(epd);
3395 btrfs_tree_lock(eb); 3396 btrfs_tree_lock(eb);
3396 } 3397 }
3397 3398
3398 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) { 3399 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3399 btrfs_tree_unlock(eb); 3400 btrfs_tree_unlock(eb);
3400 if (!epd->sync_io) 3401 if (!epd->sync_io)
3401 return 0; 3402 return 0;
3402 if (!flush) { 3403 if (!flush) {
3403 flush_write_bio(epd); 3404 flush_write_bio(epd);
3404 flush = 1; 3405 flush = 1;
3405 } 3406 }
3406 while (1) { 3407 while (1) {
3407 wait_on_extent_buffer_writeback(eb); 3408 wait_on_extent_buffer_writeback(eb);
3408 btrfs_tree_lock(eb); 3409 btrfs_tree_lock(eb);
3409 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) 3410 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3410 break; 3411 break;
3411 btrfs_tree_unlock(eb); 3412 btrfs_tree_unlock(eb);
3412 } 3413 }
3413 } 3414 }
3414 3415
3415 /* 3416 /*
3416 * We need to do this to prevent races in people who check if the eb is 3417 * We need to do this to prevent races in people who check if the eb is
3417 * under IO since we can end up having no IO bits set for a short period 3418 * under IO since we can end up having no IO bits set for a short period
3418 * of time. 3419 * of time.
3419 */ 3420 */
3420 spin_lock(&eb->refs_lock); 3421 spin_lock(&eb->refs_lock);
3421 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { 3422 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3422 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); 3423 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3423 spin_unlock(&eb->refs_lock); 3424 spin_unlock(&eb->refs_lock);
3424 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); 3425 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3425 __percpu_counter_add(&fs_info->dirty_metadata_bytes, 3426 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3426 -eb->len, 3427 -eb->len,
3427 fs_info->dirty_metadata_batch); 3428 fs_info->dirty_metadata_batch);
3428 ret = 1; 3429 ret = 1;
3429 } else { 3430 } else {
3430 spin_unlock(&eb->refs_lock); 3431 spin_unlock(&eb->refs_lock);
3431 } 3432 }
3432 3433
3433 btrfs_tree_unlock(eb); 3434 btrfs_tree_unlock(eb);
3434 3435
3435 if (!ret) 3436 if (!ret)
3436 return ret; 3437 return ret;
3437 3438
3438 num_pages = num_extent_pages(eb->start, eb->len); 3439 num_pages = num_extent_pages(eb->start, eb->len);
3439 for (i = 0; i < num_pages; i++) { 3440 for (i = 0; i < num_pages; i++) {
3440 struct page *p = extent_buffer_page(eb, i); 3441 struct page *p = extent_buffer_page(eb, i);
3441 3442
3442 if (!trylock_page(p)) { 3443 if (!trylock_page(p)) {
3443 if (!flush) { 3444 if (!flush) {
3444 flush_write_bio(epd); 3445 flush_write_bio(epd);
3445 flush = 1; 3446 flush = 1;
3446 } 3447 }
3447 lock_page(p); 3448 lock_page(p);
3448 } 3449 }
3449 } 3450 }
3450 3451
3451 return ret; 3452 return ret;
3452 } 3453 }
3453 3454
3454 static void end_extent_buffer_writeback(struct extent_buffer *eb) 3455 static void end_extent_buffer_writeback(struct extent_buffer *eb)
3455 { 3456 {
3456 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); 3457 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3457 smp_mb__after_clear_bit(); 3458 smp_mb__after_clear_bit();
3458 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK); 3459 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3459 } 3460 }
3460 3461
3461 static void end_bio_extent_buffer_writepage(struct bio *bio, int err) 3462 static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3462 { 3463 {
3463 struct bio_vec *bvec; 3464 struct bio_vec *bvec;
3464 struct extent_buffer *eb; 3465 struct extent_buffer *eb;
3465 int i, done; 3466 int i, done;
3466 3467
3467 bio_for_each_segment_all(bvec, bio, i) { 3468 bio_for_each_segment_all(bvec, bio, i) {
3468 struct page *page = bvec->bv_page; 3469 struct page *page = bvec->bv_page;
3469 3470
3470 eb = (struct extent_buffer *)page->private; 3471 eb = (struct extent_buffer *)page->private;
3471 BUG_ON(!eb); 3472 BUG_ON(!eb);
3472 done = atomic_dec_and_test(&eb->io_pages); 3473 done = atomic_dec_and_test(&eb->io_pages);
3473 3474
3474 if (err || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) { 3475 if (err || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3475 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags); 3476 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3476 ClearPageUptodate(page); 3477 ClearPageUptodate(page);
3477 SetPageError(page); 3478 SetPageError(page);
3478 } 3479 }
3479 3480
3480 end_page_writeback(page); 3481 end_page_writeback(page);
3481 3482
3482 if (!done) 3483 if (!done)
3483 continue; 3484 continue;
3484 3485
3485 end_extent_buffer_writeback(eb); 3486 end_extent_buffer_writeback(eb);
3486 } 3487 }
3487 3488
3488 bio_put(bio); 3489 bio_put(bio);
3489 } 3490 }
3490 3491
3491 static int write_one_eb(struct extent_buffer *eb, 3492 static int write_one_eb(struct extent_buffer *eb,
3492 struct btrfs_fs_info *fs_info, 3493 struct btrfs_fs_info *fs_info,
3493 struct writeback_control *wbc, 3494 struct writeback_control *wbc,
3494 struct extent_page_data *epd) 3495 struct extent_page_data *epd)
3495 { 3496 {
3496 struct block_device *bdev = fs_info->fs_devices->latest_bdev; 3497 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3497 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree; 3498 struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
3498 u64 offset = eb->start; 3499 u64 offset = eb->start;
3499 unsigned long i, num_pages; 3500 unsigned long i, num_pages;
3500 unsigned long bio_flags = 0; 3501 unsigned long bio_flags = 0;
3501 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META; 3502 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
3502 int ret = 0; 3503 int ret = 0;
3503 3504
3504 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags); 3505 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3505 num_pages = num_extent_pages(eb->start, eb->len); 3506 num_pages = num_extent_pages(eb->start, eb->len);
3506 atomic_set(&eb->io_pages, num_pages); 3507 atomic_set(&eb->io_pages, num_pages);
3507 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID) 3508 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3508 bio_flags = EXTENT_BIO_TREE_LOG; 3509 bio_flags = EXTENT_BIO_TREE_LOG;
3509 3510
3510 for (i = 0; i < num_pages; i++) { 3511 for (i = 0; i < num_pages; i++) {
3511 struct page *p = extent_buffer_page(eb, i); 3512 struct page *p = extent_buffer_page(eb, i);
3512 3513
3513 clear_page_dirty_for_io(p); 3514 clear_page_dirty_for_io(p);
3514 set_page_writeback(p); 3515 set_page_writeback(p);
3515 ret = submit_extent_page(rw, tree, p, offset >> 9, 3516 ret = submit_extent_page(rw, tree, p, offset >> 9,
3516 PAGE_CACHE_SIZE, 0, bdev, &epd->bio, 3517 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3517 -1, end_bio_extent_buffer_writepage, 3518 -1, end_bio_extent_buffer_writepage,
3518 0, epd->bio_flags, bio_flags); 3519 0, epd->bio_flags, bio_flags);
3519 epd->bio_flags = bio_flags; 3520 epd->bio_flags = bio_flags;
3520 if (ret) { 3521 if (ret) {
3521 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags); 3522 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3522 SetPageError(p); 3523 SetPageError(p);
3523 if (atomic_sub_and_test(num_pages - i, &eb->io_pages)) 3524 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3524 end_extent_buffer_writeback(eb); 3525 end_extent_buffer_writeback(eb);
3525 ret = -EIO; 3526 ret = -EIO;
3526 break; 3527 break;
3527 } 3528 }
3528 offset += PAGE_CACHE_SIZE; 3529 offset += PAGE_CACHE_SIZE;
3529 update_nr_written(p, wbc, 1); 3530 update_nr_written(p, wbc, 1);
3530 unlock_page(p); 3531 unlock_page(p);
3531 } 3532 }
3532 3533
3533 if (unlikely(ret)) { 3534 if (unlikely(ret)) {
3534 for (; i < num_pages; i++) { 3535 for (; i < num_pages; i++) {
3535 struct page *p = extent_buffer_page(eb, i); 3536 struct page *p = extent_buffer_page(eb, i);
3536 unlock_page(p); 3537 unlock_page(p);
3537 } 3538 }
3538 } 3539 }
3539 3540
3540 return ret; 3541 return ret;
3541 } 3542 }
3542 3543
3543 int btree_write_cache_pages(struct address_space *mapping, 3544 int btree_write_cache_pages(struct address_space *mapping,
3544 struct writeback_control *wbc) 3545 struct writeback_control *wbc)
3545 { 3546 {
3546 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree; 3547 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3547 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info; 3548 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3548 struct extent_buffer *eb, *prev_eb = NULL; 3549 struct extent_buffer *eb, *prev_eb = NULL;
3549 struct extent_page_data epd = { 3550 struct extent_page_data epd = {
3550 .bio = NULL, 3551 .bio = NULL,
3551 .tree = tree, 3552 .tree = tree,
3552 .extent_locked = 0, 3553 .extent_locked = 0,
3553 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 3554 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3554 .bio_flags = 0, 3555 .bio_flags = 0,
3555 }; 3556 };
3556 int ret = 0; 3557 int ret = 0;
3557 int done = 0; 3558 int done = 0;
3558 int nr_to_write_done = 0; 3559 int nr_to_write_done = 0;
3559 struct pagevec pvec; 3560 struct pagevec pvec;
3560 int nr_pages; 3561 int nr_pages;
3561 pgoff_t index; 3562 pgoff_t index;
3562 pgoff_t end; /* Inclusive */ 3563 pgoff_t end; /* Inclusive */
3563 int scanned = 0; 3564 int scanned = 0;
3564 int tag; 3565 int tag;
3565 3566
3566 pagevec_init(&pvec, 0); 3567 pagevec_init(&pvec, 0);
3567 if (wbc->range_cyclic) { 3568 if (wbc->range_cyclic) {
3568 index = mapping->writeback_index; /* Start from prev offset */ 3569 index = mapping->writeback_index; /* Start from prev offset */
3569 end = -1; 3570 end = -1;
3570 } else { 3571 } else {
3571 index = wbc->range_start >> PAGE_CACHE_SHIFT; 3572 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3572 end = wbc->range_end >> PAGE_CACHE_SHIFT; 3573 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3573 scanned = 1; 3574 scanned = 1;
3574 } 3575 }
3575 if (wbc->sync_mode == WB_SYNC_ALL) 3576 if (wbc->sync_mode == WB_SYNC_ALL)
3576 tag = PAGECACHE_TAG_TOWRITE; 3577 tag = PAGECACHE_TAG_TOWRITE;
3577 else 3578 else
3578 tag = PAGECACHE_TAG_DIRTY; 3579 tag = PAGECACHE_TAG_DIRTY;
3579 retry: 3580 retry:
3580 if (wbc->sync_mode == WB_SYNC_ALL) 3581 if (wbc->sync_mode == WB_SYNC_ALL)
3581 tag_pages_for_writeback(mapping, index, end); 3582 tag_pages_for_writeback(mapping, index, end);
3582 while (!done && !nr_to_write_done && (index <= end) && 3583 while (!done && !nr_to_write_done && (index <= end) &&
3583 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 3584 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3584 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) { 3585 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3585 unsigned i; 3586 unsigned i;
3586 3587
3587 scanned = 1; 3588 scanned = 1;
3588 for (i = 0; i < nr_pages; i++) { 3589 for (i = 0; i < nr_pages; i++) {
3589 struct page *page = pvec.pages[i]; 3590 struct page *page = pvec.pages[i];
3590 3591
3591 if (!PagePrivate(page)) 3592 if (!PagePrivate(page))
3592 continue; 3593 continue;
3593 3594
3594 if (!wbc->range_cyclic && page->index > end) { 3595 if (!wbc->range_cyclic && page->index > end) {
3595 done = 1; 3596 done = 1;
3596 break; 3597 break;
3597 } 3598 }
3598 3599
3599 spin_lock(&mapping->private_lock); 3600 spin_lock(&mapping->private_lock);
3600 if (!PagePrivate(page)) { 3601 if (!PagePrivate(page)) {
3601 spin_unlock(&mapping->private_lock); 3602 spin_unlock(&mapping->private_lock);
3602 continue; 3603 continue;
3603 } 3604 }
3604 3605
3605 eb = (struct extent_buffer *)page->private; 3606 eb = (struct extent_buffer *)page->private;
3606 3607
3607 /* 3608 /*
3608 * Shouldn't happen and normally this would be a BUG_ON 3609 * Shouldn't happen and normally this would be a BUG_ON
3609 * but no sense in crashing the users box for something 3610 * but no sense in crashing the users box for something
3610 * we can survive anyway. 3611 * we can survive anyway.
3611 */ 3612 */
3612 if (WARN_ON(!eb)) { 3613 if (WARN_ON(!eb)) {
3613 spin_unlock(&mapping->private_lock); 3614 spin_unlock(&mapping->private_lock);
3614 continue; 3615 continue;
3615 } 3616 }
3616 3617
3617 if (eb == prev_eb) { 3618 if (eb == prev_eb) {
3618 spin_unlock(&mapping->private_lock); 3619 spin_unlock(&mapping->private_lock);
3619 continue; 3620 continue;
3620 } 3621 }
3621 3622
3622 ret = atomic_inc_not_zero(&eb->refs); 3623 ret = atomic_inc_not_zero(&eb->refs);
3623 spin_unlock(&mapping->private_lock); 3624 spin_unlock(&mapping->private_lock);
3624 if (!ret) 3625 if (!ret)
3625 continue; 3626 continue;
3626 3627
3627 prev_eb = eb; 3628 prev_eb = eb;
3628 ret = lock_extent_buffer_for_io(eb, fs_info, &epd); 3629 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3629 if (!ret) { 3630 if (!ret) {
3630 free_extent_buffer(eb); 3631 free_extent_buffer(eb);
3631 continue; 3632 continue;
3632 } 3633 }
3633 3634
3634 ret = write_one_eb(eb, fs_info, wbc, &epd); 3635 ret = write_one_eb(eb, fs_info, wbc, &epd);
3635 if (ret) { 3636 if (ret) {
3636 done = 1; 3637 done = 1;
3637 free_extent_buffer(eb); 3638 free_extent_buffer(eb);
3638 break; 3639 break;
3639 } 3640 }
3640 free_extent_buffer(eb); 3641 free_extent_buffer(eb);
3641 3642
3642 /* 3643 /*
3643 * the filesystem may choose to bump up nr_to_write. 3644 * the filesystem may choose to bump up nr_to_write.
3644 * We have to make sure to honor the new nr_to_write 3645 * We have to make sure to honor the new nr_to_write
3645 * at any time 3646 * at any time
3646 */ 3647 */
3647 nr_to_write_done = wbc->nr_to_write <= 0; 3648 nr_to_write_done = wbc->nr_to_write <= 0;
3648 } 3649 }
3649 pagevec_release(&pvec); 3650 pagevec_release(&pvec);
3650 cond_resched(); 3651 cond_resched();
3651 } 3652 }
3652 if (!scanned && !done) { 3653 if (!scanned && !done) {
3653 /* 3654 /*
3654 * We hit the last page and there is more work to be done: wrap 3655 * We hit the last page and there is more work to be done: wrap
3655 * back to the start of the file 3656 * back to the start of the file
3656 */ 3657 */
3657 scanned = 1; 3658 scanned = 1;
3658 index = 0; 3659 index = 0;
3659 goto retry; 3660 goto retry;
3660 } 3661 }
3661 flush_write_bio(&epd); 3662 flush_write_bio(&epd);
3662 return ret; 3663 return ret;
3663 } 3664 }
3664 3665
3665 /** 3666 /**
3666 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 3667 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3667 * @mapping: address space structure to write 3668 * @mapping: address space structure to write
3668 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 3669 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3669 * @writepage: function called for each page 3670 * @writepage: function called for each page
3670 * @data: data passed to writepage function 3671 * @data: data passed to writepage function
3671 * 3672 *
3672 * If a page is already under I/O, write_cache_pages() skips it, even 3673 * If a page is already under I/O, write_cache_pages() skips it, even
3673 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 3674 * if it's dirty. This is desirable behaviour for memory-cleaning writeback,
3674 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 3675 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync()
3675 * and msync() need to guarantee that all the data which was dirty at the time 3676 * and msync() need to guarantee that all the data which was dirty at the time
3676 * the call was made get new I/O started against them. If wbc->sync_mode is 3677 * the call was made get new I/O started against them. If wbc->sync_mode is
3677 * WB_SYNC_ALL then we were called for data integrity and we must wait for 3678 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3678 * existing IO to complete. 3679 * existing IO to complete.
3679 */ 3680 */
3680 static int extent_write_cache_pages(struct extent_io_tree *tree, 3681 static int extent_write_cache_pages(struct extent_io_tree *tree,
3681 struct address_space *mapping, 3682 struct address_space *mapping,
3682 struct writeback_control *wbc, 3683 struct writeback_control *wbc,
3683 writepage_t writepage, void *data, 3684 writepage_t writepage, void *data,
3684 void (*flush_fn)(void *)) 3685 void (*flush_fn)(void *))
3685 { 3686 {
3686 struct inode *inode = mapping->host; 3687 struct inode *inode = mapping->host;
3687 int ret = 0; 3688 int ret = 0;
3688 int done = 0; 3689 int done = 0;
3689 int nr_to_write_done = 0; 3690 int nr_to_write_done = 0;
3690 struct pagevec pvec; 3691 struct pagevec pvec;
3691 int nr_pages; 3692 int nr_pages;
3692 pgoff_t index; 3693 pgoff_t index;
3693 pgoff_t end; /* Inclusive */ 3694 pgoff_t end; /* Inclusive */
3694 int scanned = 0; 3695 int scanned = 0;
3695 int tag; 3696 int tag;
3696 3697
3697 /* 3698 /*
3698 * We have to hold onto the inode so that ordered extents can do their 3699 * We have to hold onto the inode so that ordered extents can do their
3699 * work when the IO finishes. The alternative to this is failing to add 3700 * work when the IO finishes. The alternative to this is failing to add
3700 * an ordered extent if the igrab() fails there and that is a huge pain 3701 * an ordered extent if the igrab() fails there and that is a huge pain
3701 * to deal with, so instead just hold onto the inode throughout the 3702 * to deal with, so instead just hold onto the inode throughout the
3702 * writepages operation. If it fails here we are freeing up the inode 3703 * writepages operation. If it fails here we are freeing up the inode
3703 * anyway and we'd rather not waste our time writing out stuff that is 3704 * anyway and we'd rather not waste our time writing out stuff that is
3704 * going to be truncated anyway. 3705 * going to be truncated anyway.
3705 */ 3706 */
3706 if (!igrab(inode)) 3707 if (!igrab(inode))
3707 return 0; 3708 return 0;
3708 3709
3709 pagevec_init(&pvec, 0); 3710 pagevec_init(&pvec, 0);
3710 if (wbc->range_cyclic) { 3711 if (wbc->range_cyclic) {
3711 index = mapping->writeback_index; /* Start from prev offset */ 3712 index = mapping->writeback_index; /* Start from prev offset */
3712 end = -1; 3713 end = -1;
3713 } else { 3714 } else {
3714 index = wbc->range_start >> PAGE_CACHE_SHIFT; 3715 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3715 end = wbc->range_end >> PAGE_CACHE_SHIFT; 3716 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3716 scanned = 1; 3717 scanned = 1;
3717 } 3718 }
3718 if (wbc->sync_mode == WB_SYNC_ALL) 3719 if (wbc->sync_mode == WB_SYNC_ALL)
3719 tag = PAGECACHE_TAG_TOWRITE; 3720 tag = PAGECACHE_TAG_TOWRITE;
3720 else 3721 else
3721 tag = PAGECACHE_TAG_DIRTY; 3722 tag = PAGECACHE_TAG_DIRTY;
3722 retry: 3723 retry:
3723 if (wbc->sync_mode == WB_SYNC_ALL) 3724 if (wbc->sync_mode == WB_SYNC_ALL)
3724 tag_pages_for_writeback(mapping, index, end); 3725 tag_pages_for_writeback(mapping, index, end);
3725 while (!done && !nr_to_write_done && (index <= end) && 3726 while (!done && !nr_to_write_done && (index <= end) &&
3726 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 3727 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3727 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) { 3728 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3728 unsigned i; 3729 unsigned i;
3729 3730
3730 scanned = 1; 3731 scanned = 1;
3731 for (i = 0; i < nr_pages; i++) { 3732 for (i = 0; i < nr_pages; i++) {
3732 struct page *page = pvec.pages[i]; 3733 struct page *page = pvec.pages[i];
3733 3734
3734 /* 3735 /*
3735 * At this point we hold neither mapping->tree_lock nor 3736 * At this point we hold neither mapping->tree_lock nor
3736 * lock on the page itself: the page may be truncated or 3737 * lock on the page itself: the page may be truncated or
3737 * invalidated (changing page->mapping to NULL), or even 3738 * invalidated (changing page->mapping to NULL), or even
3738 * swizzled back from swapper_space to tmpfs file 3739 * swizzled back from swapper_space to tmpfs file
3739 * mapping 3740 * mapping
3740 */ 3741 */
3741 if (!trylock_page(page)) { 3742 if (!trylock_page(page)) {
3742 flush_fn(data); 3743 flush_fn(data);
3743 lock_page(page); 3744 lock_page(page);
3744 } 3745 }
3745 3746
3746 if (unlikely(page->mapping != mapping)) { 3747 if (unlikely(page->mapping != mapping)) {
3747 unlock_page(page); 3748 unlock_page(page);
3748 continue; 3749 continue;
3749 } 3750 }
3750 3751
3751 if (!wbc->range_cyclic && page->index > end) { 3752 if (!wbc->range_cyclic && page->index > end) {
3752 done = 1; 3753 done = 1;
3753 unlock_page(page); 3754 unlock_page(page);
3754 continue; 3755 continue;
3755 } 3756 }
3756 3757
3757 if (wbc->sync_mode != WB_SYNC_NONE) { 3758 if (wbc->sync_mode != WB_SYNC_NONE) {
3758 if (PageWriteback(page)) 3759 if (PageWriteback(page))
3759 flush_fn(data); 3760 flush_fn(data);
3760 wait_on_page_writeback(page); 3761 wait_on_page_writeback(page);
3761 } 3762 }
3762 3763
3763 if (PageWriteback(page) || 3764 if (PageWriteback(page) ||
3764 !clear_page_dirty_for_io(page)) { 3765 !clear_page_dirty_for_io(page)) {
3765 unlock_page(page); 3766 unlock_page(page);
3766 continue; 3767 continue;
3767 } 3768 }
3768 3769
3769 ret = (*writepage)(page, wbc, data); 3770 ret = (*writepage)(page, wbc, data);
3770 3771
3771 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) { 3772 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3772 unlock_page(page); 3773 unlock_page(page);
3773 ret = 0; 3774 ret = 0;
3774 } 3775 }
3775 if (ret) 3776 if (ret)
3776 done = 1; 3777 done = 1;
3777 3778
3778 /* 3779 /*
3779 * the filesystem may choose to bump up nr_to_write. 3780 * the filesystem may choose to bump up nr_to_write.
3780 * We have to make sure to honor the new nr_to_write 3781 * We have to make sure to honor the new nr_to_write
3781 * at any time 3782 * at any time
3782 */ 3783 */
3783 nr_to_write_done = wbc->nr_to_write <= 0; 3784 nr_to_write_done = wbc->nr_to_write <= 0;
3784 } 3785 }
3785 pagevec_release(&pvec); 3786 pagevec_release(&pvec);
3786 cond_resched(); 3787 cond_resched();
3787 } 3788 }
3788 if (!scanned && !done) { 3789 if (!scanned && !done) {
3789 /* 3790 /*
3790 * We hit the last page and there is more work to be done: wrap 3791 * We hit the last page and there is more work to be done: wrap
3791 * back to the start of the file 3792 * back to the start of the file
3792 */ 3793 */
3793 scanned = 1; 3794 scanned = 1;
3794 index = 0; 3795 index = 0;
3795 goto retry; 3796 goto retry;
3796 } 3797 }
3797 btrfs_add_delayed_iput(inode); 3798 btrfs_add_delayed_iput(inode);
3798 return ret; 3799 return ret;
3799 } 3800 }
3800 3801
3801 static void flush_epd_write_bio(struct extent_page_data *epd) 3802 static void flush_epd_write_bio(struct extent_page_data *epd)
3802 { 3803 {
3803 if (epd->bio) { 3804 if (epd->bio) {
3804 int rw = WRITE; 3805 int rw = WRITE;
3805 int ret; 3806 int ret;
3806 3807
3807 if (epd->sync_io) 3808 if (epd->sync_io)
3808 rw = WRITE_SYNC; 3809 rw = WRITE_SYNC;
3809 3810
3810 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags); 3811 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
3811 BUG_ON(ret < 0); /* -ENOMEM */ 3812 BUG_ON(ret < 0); /* -ENOMEM */
3812 epd->bio = NULL; 3813 epd->bio = NULL;
3813 } 3814 }
3814 } 3815 }
3815 3816
3816 static noinline void flush_write_bio(void *data) 3817 static noinline void flush_write_bio(void *data)
3817 { 3818 {
3818 struct extent_page_data *epd = data; 3819 struct extent_page_data *epd = data;
3819 flush_epd_write_bio(epd); 3820 flush_epd_write_bio(epd);
3820 } 3821 }
3821 3822
3822 int extent_write_full_page(struct extent_io_tree *tree, struct page *page, 3823 int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3823 get_extent_t *get_extent, 3824 get_extent_t *get_extent,
3824 struct writeback_control *wbc) 3825 struct writeback_control *wbc)
3825 { 3826 {
3826 int ret; 3827 int ret;
3827 struct extent_page_data epd = { 3828 struct extent_page_data epd = {
3828 .bio = NULL, 3829 .bio = NULL,
3829 .tree = tree, 3830 .tree = tree,
3830 .get_extent = get_extent, 3831 .get_extent = get_extent,
3831 .extent_locked = 0, 3832 .extent_locked = 0,
3832 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 3833 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3833 .bio_flags = 0, 3834 .bio_flags = 0,
3834 }; 3835 };
3835 3836
3836 ret = __extent_writepage(page, wbc, &epd); 3837 ret = __extent_writepage(page, wbc, &epd);
3837 3838
3838 flush_epd_write_bio(&epd); 3839 flush_epd_write_bio(&epd);
3839 return ret; 3840 return ret;
3840 } 3841 }
3841 3842
3842 int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode, 3843 int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3843 u64 start, u64 end, get_extent_t *get_extent, 3844 u64 start, u64 end, get_extent_t *get_extent,
3844 int mode) 3845 int mode)
3845 { 3846 {
3846 int ret = 0; 3847 int ret = 0;
3847 struct address_space *mapping = inode->i_mapping; 3848 struct address_space *mapping = inode->i_mapping;
3848 struct page *page; 3849 struct page *page;
3849 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >> 3850 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3850 PAGE_CACHE_SHIFT; 3851 PAGE_CACHE_SHIFT;
3851 3852
3852 struct extent_page_data epd = { 3853 struct extent_page_data epd = {
3853 .bio = NULL, 3854 .bio = NULL,
3854 .tree = tree, 3855 .tree = tree,
3855 .get_extent = get_extent, 3856 .get_extent = get_extent,
3856 .extent_locked = 1, 3857 .extent_locked = 1,
3857 .sync_io = mode == WB_SYNC_ALL, 3858 .sync_io = mode == WB_SYNC_ALL,
3858 .bio_flags = 0, 3859 .bio_flags = 0,
3859 }; 3860 };
3860 struct writeback_control wbc_writepages = { 3861 struct writeback_control wbc_writepages = {
3861 .sync_mode = mode, 3862 .sync_mode = mode,
3862 .nr_to_write = nr_pages * 2, 3863 .nr_to_write = nr_pages * 2,
3863 .range_start = start, 3864 .range_start = start,
3864 .range_end = end + 1, 3865 .range_end = end + 1,
3865 }; 3866 };
3866 3867
3867 while (start <= end) { 3868 while (start <= end) {
3868 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT); 3869 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3869 if (clear_page_dirty_for_io(page)) 3870 if (clear_page_dirty_for_io(page))
3870 ret = __extent_writepage(page, &wbc_writepages, &epd); 3871 ret = __extent_writepage(page, &wbc_writepages, &epd);
3871 else { 3872 else {
3872 if (tree->ops && tree->ops->writepage_end_io_hook) 3873 if (tree->ops && tree->ops->writepage_end_io_hook)
3873 tree->ops->writepage_end_io_hook(page, start, 3874 tree->ops->writepage_end_io_hook(page, start,
3874 start + PAGE_CACHE_SIZE - 1, 3875 start + PAGE_CACHE_SIZE - 1,
3875 NULL, 1); 3876 NULL, 1);
3876 unlock_page(page); 3877 unlock_page(page);
3877 } 3878 }
3878 page_cache_release(page); 3879 page_cache_release(page);
3879 start += PAGE_CACHE_SIZE; 3880 start += PAGE_CACHE_SIZE;
3880 } 3881 }
3881 3882
3882 flush_epd_write_bio(&epd); 3883 flush_epd_write_bio(&epd);
3883 return ret; 3884 return ret;
3884 } 3885 }
3885 3886
3886 int extent_writepages(struct extent_io_tree *tree, 3887 int extent_writepages(struct extent_io_tree *tree,
3887 struct address_space *mapping, 3888 struct address_space *mapping,
3888 get_extent_t *get_extent, 3889 get_extent_t *get_extent,
3889 struct writeback_control *wbc) 3890 struct writeback_control *wbc)
3890 { 3891 {
3891 int ret = 0; 3892 int ret = 0;
3892 struct extent_page_data epd = { 3893 struct extent_page_data epd = {
3893 .bio = NULL, 3894 .bio = NULL,
3894 .tree = tree, 3895 .tree = tree,
3895 .get_extent = get_extent, 3896 .get_extent = get_extent,
3896 .extent_locked = 0, 3897 .extent_locked = 0,
3897 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 3898 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3898 .bio_flags = 0, 3899 .bio_flags = 0,
3899 }; 3900 };
3900 3901
3901 ret = extent_write_cache_pages(tree, mapping, wbc, 3902 ret = extent_write_cache_pages(tree, mapping, wbc,
3902 __extent_writepage, &epd, 3903 __extent_writepage, &epd,
3903 flush_write_bio); 3904 flush_write_bio);
3904 flush_epd_write_bio(&epd); 3905 flush_epd_write_bio(&epd);
3905 return ret; 3906 return ret;
3906 } 3907 }
3907 3908
3908 int extent_readpages(struct extent_io_tree *tree, 3909 int extent_readpages(struct extent_io_tree *tree,
3909 struct address_space *mapping, 3910 struct address_space *mapping,
3910 struct list_head *pages, unsigned nr_pages, 3911 struct list_head *pages, unsigned nr_pages,
3911 get_extent_t get_extent) 3912 get_extent_t get_extent)
3912 { 3913 {
3913 struct bio *bio = NULL; 3914 struct bio *bio = NULL;
3914 unsigned page_idx; 3915 unsigned page_idx;
3915 unsigned long bio_flags = 0; 3916 unsigned long bio_flags = 0;
3916 struct page *pagepool[16]; 3917 struct page *pagepool[16];
3917 struct page *page; 3918 struct page *page;
3918 struct extent_map *em_cached = NULL; 3919 struct extent_map *em_cached = NULL;
3919 int nr = 0; 3920 int nr = 0;
3920 3921
3921 for (page_idx = 0; page_idx < nr_pages; page_idx++) { 3922 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3922 page = list_entry(pages->prev, struct page, lru); 3923 page = list_entry(pages->prev, struct page, lru);
3923 3924
3924 prefetchw(&page->flags); 3925 prefetchw(&page->flags);
3925 list_del(&page->lru); 3926 list_del(&page->lru);
3926 if (add_to_page_cache_lru(page, mapping, 3927 if (add_to_page_cache_lru(page, mapping,
3927 page->index, GFP_NOFS)) { 3928 page->index, GFP_NOFS)) {
3928 page_cache_release(page); 3929 page_cache_release(page);
3929 continue; 3930 continue;
3930 } 3931 }
3931 3932
3932 pagepool[nr++] = page; 3933 pagepool[nr++] = page;
3933 if (nr < ARRAY_SIZE(pagepool)) 3934 if (nr < ARRAY_SIZE(pagepool))
3934 continue; 3935 continue;
3935 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached, 3936 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3936 &bio, 0, &bio_flags, READ); 3937 &bio, 0, &bio_flags, READ);
3937 nr = 0; 3938 nr = 0;
3938 } 3939 }
3939 if (nr) 3940 if (nr)
3940 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached, 3941 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3941 &bio, 0, &bio_flags, READ); 3942 &bio, 0, &bio_flags, READ);
3942 3943
3943 if (em_cached) 3944 if (em_cached)
3944 free_extent_map(em_cached); 3945 free_extent_map(em_cached);
3945 3946
3946 BUG_ON(!list_empty(pages)); 3947 BUG_ON(!list_empty(pages));
3947 if (bio) 3948 if (bio)
3948 return submit_one_bio(READ, bio, 0, bio_flags); 3949 return submit_one_bio(READ, bio, 0, bio_flags);
3949 return 0; 3950 return 0;
3950 } 3951 }
3951 3952
3952 /* 3953 /*
3953 * basic invalidatepage code, this waits on any locked or writeback 3954 * basic invalidatepage code, this waits on any locked or writeback
3954 * ranges corresponding to the page, and then deletes any extent state 3955 * ranges corresponding to the page, and then deletes any extent state
3955 * records from the tree 3956 * records from the tree
3956 */ 3957 */
3957 int extent_invalidatepage(struct extent_io_tree *tree, 3958 int extent_invalidatepage(struct extent_io_tree *tree,
3958 struct page *page, unsigned long offset) 3959 struct page *page, unsigned long offset)
3959 { 3960 {
3960 struct extent_state *cached_state = NULL; 3961 struct extent_state *cached_state = NULL;
3961 u64 start = page_offset(page); 3962 u64 start = page_offset(page);
3962 u64 end = start + PAGE_CACHE_SIZE - 1; 3963 u64 end = start + PAGE_CACHE_SIZE - 1;
3963 size_t blocksize = page->mapping->host->i_sb->s_blocksize; 3964 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3964 3965
3965 start += ALIGN(offset, blocksize); 3966 start += ALIGN(offset, blocksize);
3966 if (start > end) 3967 if (start > end)
3967 return 0; 3968 return 0;
3968 3969
3969 lock_extent_bits(tree, start, end, 0, &cached_state); 3970 lock_extent_bits(tree, start, end, 0, &cached_state);
3970 wait_on_page_writeback(page); 3971 wait_on_page_writeback(page);
3971 clear_extent_bit(tree, start, end, 3972 clear_extent_bit(tree, start, end,
3972 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC | 3973 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3973 EXTENT_DO_ACCOUNTING, 3974 EXTENT_DO_ACCOUNTING,
3974 1, 1, &cached_state, GFP_NOFS); 3975 1, 1, &cached_state, GFP_NOFS);
3975 return 0; 3976 return 0;
3976 } 3977 }
3977 3978
3978 /* 3979 /*
3979 * a helper for releasepage, this tests for areas of the page that 3980 * a helper for releasepage, this tests for areas of the page that
3980 * are locked or under IO and drops the related state bits if it is safe 3981 * are locked or under IO and drops the related state bits if it is safe
3981 * to drop the page. 3982 * to drop the page.
3982 */ 3983 */
3983 static int try_release_extent_state(struct extent_map_tree *map, 3984 static int try_release_extent_state(struct extent_map_tree *map,
3984 struct extent_io_tree *tree, 3985 struct extent_io_tree *tree,
3985 struct page *page, gfp_t mask) 3986 struct page *page, gfp_t mask)
3986 { 3987 {
3987 u64 start = page_offset(page); 3988 u64 start = page_offset(page);
3988 u64 end = start + PAGE_CACHE_SIZE - 1; 3989 u64 end = start + PAGE_CACHE_SIZE - 1;
3989 int ret = 1; 3990 int ret = 1;
3990 3991
3991 if (test_range_bit(tree, start, end, 3992 if (test_range_bit(tree, start, end,
3992 EXTENT_IOBITS, 0, NULL)) 3993 EXTENT_IOBITS, 0, NULL))
3993 ret = 0; 3994 ret = 0;
3994 else { 3995 else {
3995 if ((mask & GFP_NOFS) == GFP_NOFS) 3996 if ((mask & GFP_NOFS) == GFP_NOFS)
3996 mask = GFP_NOFS; 3997 mask = GFP_NOFS;
3997 /* 3998 /*
3998 * at this point we can safely clear everything except the 3999 * at this point we can safely clear everything except the
3999 * locked bit and the nodatasum bit 4000 * locked bit and the nodatasum bit
4000 */ 4001 */
4001 ret = clear_extent_bit(tree, start, end, 4002 ret = clear_extent_bit(tree, start, end,
4002 ~(EXTENT_LOCKED | EXTENT_NODATASUM), 4003 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4003 0, 0, NULL, mask); 4004 0, 0, NULL, mask);
4004 4005
4005 /* if clear_extent_bit failed for enomem reasons, 4006 /* if clear_extent_bit failed for enomem reasons,
4006 * we can't allow the release to continue. 4007 * we can't allow the release to continue.
4007 */ 4008 */
4008 if (ret < 0) 4009 if (ret < 0)
4009 ret = 0; 4010 ret = 0;
4010 else 4011 else
4011 ret = 1; 4012 ret = 1;
4012 } 4013 }
4013 return ret; 4014 return ret;
4014 } 4015 }
4015 4016
4016 /* 4017 /*
4017 * a helper for releasepage. As long as there are no locked extents 4018 * a helper for releasepage. As long as there are no locked extents
4018 * in the range corresponding to the page, both state records and extent 4019 * in the range corresponding to the page, both state records and extent
4019 * map records are removed 4020 * map records are removed
4020 */ 4021 */
4021 int try_release_extent_mapping(struct extent_map_tree *map, 4022 int try_release_extent_mapping(struct extent_map_tree *map,
4022 struct extent_io_tree *tree, struct page *page, 4023 struct extent_io_tree *tree, struct page *page,
4023 gfp_t mask) 4024 gfp_t mask)
4024 { 4025 {
4025 struct extent_map *em; 4026 struct extent_map *em;
4026 u64 start = page_offset(page); 4027 u64 start = page_offset(page);
4027 u64 end = start + PAGE_CACHE_SIZE - 1; 4028 u64 end = start + PAGE_CACHE_SIZE - 1;
4028 4029
4029 if ((mask & __GFP_WAIT) && 4030 if ((mask & __GFP_WAIT) &&
4030 page->mapping->host->i_size > 16 * 1024 * 1024) { 4031 page->mapping->host->i_size > 16 * 1024 * 1024) {
4031 u64 len; 4032 u64 len;
4032 while (start <= end) { 4033 while (start <= end) {
4033 len = end - start + 1; 4034 len = end - start + 1;
4034 write_lock(&map->lock); 4035 write_lock(&map->lock);
4035 em = lookup_extent_mapping(map, start, len); 4036 em = lookup_extent_mapping(map, start, len);
4036 if (!em) { 4037 if (!em) {
4037 write_unlock(&map->lock); 4038 write_unlock(&map->lock);
4038 break; 4039 break;
4039 } 4040 }
4040 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) || 4041 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4041 em->start != start) { 4042 em->start != start) {
4042 write_unlock(&map->lock); 4043 write_unlock(&map->lock);
4043 free_extent_map(em); 4044 free_extent_map(em);
4044 break; 4045 break;
4045 } 4046 }
4046 if (!test_range_bit(tree, em->start, 4047 if (!test_range_bit(tree, em->start,
4047 extent_map_end(em) - 1, 4048 extent_map_end(em) - 1,
4048 EXTENT_LOCKED | EXTENT_WRITEBACK, 4049 EXTENT_LOCKED | EXTENT_WRITEBACK,
4049 0, NULL)) { 4050 0, NULL)) {
4050 remove_extent_mapping(map, em); 4051 remove_extent_mapping(map, em);
4051 /* once for the rb tree */ 4052 /* once for the rb tree */
4052 free_extent_map(em); 4053 free_extent_map(em);
4053 } 4054 }
4054 start = extent_map_end(em); 4055 start = extent_map_end(em);
4055 write_unlock(&map->lock); 4056 write_unlock(&map->lock);
4056 4057
4057 /* once for us */ 4058 /* once for us */
4058 free_extent_map(em); 4059 free_extent_map(em);
4059 } 4060 }
4060 } 4061 }
4061 return try_release_extent_state(map, tree, page, mask); 4062 return try_release_extent_state(map, tree, page, mask);
4062 } 4063 }
4063 4064
4064 /* 4065 /*
4065 * helper function for fiemap, which doesn't want to see any holes. 4066 * helper function for fiemap, which doesn't want to see any holes.
4066 * This maps until we find something past 'last' 4067 * This maps until we find something past 'last'
4067 */ 4068 */
4068 static struct extent_map *get_extent_skip_holes(struct inode *inode, 4069 static struct extent_map *get_extent_skip_holes(struct inode *inode,
4069 u64 offset, 4070 u64 offset,
4070 u64 last, 4071 u64 last,
4071 get_extent_t *get_extent) 4072 get_extent_t *get_extent)
4072 { 4073 {
4073 u64 sectorsize = BTRFS_I(inode)->root->sectorsize; 4074 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4074 struct extent_map *em; 4075 struct extent_map *em;
4075 u64 len; 4076 u64 len;
4076 4077
4077 if (offset >= last) 4078 if (offset >= last)
4078 return NULL; 4079 return NULL;
4079 4080
4080 while (1) { 4081 while (1) {
4081 len = last - offset; 4082 len = last - offset;
4082 if (len == 0) 4083 if (len == 0)
4083 break; 4084 break;
4084 len = ALIGN(len, sectorsize); 4085 len = ALIGN(len, sectorsize);
4085 em = get_extent(inode, NULL, 0, offset, len, 0); 4086 em = get_extent(inode, NULL, 0, offset, len, 0);
4086 if (IS_ERR_OR_NULL(em)) 4087 if (IS_ERR_OR_NULL(em))
4087 return em; 4088 return em;
4088 4089
4089 /* if this isn't a hole return it */ 4090 /* if this isn't a hole return it */
4090 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) && 4091 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4091 em->block_start != EXTENT_MAP_HOLE) { 4092 em->block_start != EXTENT_MAP_HOLE) {
4092 return em; 4093 return em;
4093 } 4094 }
4094 4095
4095 /* this is a hole, advance to the next extent */ 4096 /* this is a hole, advance to the next extent */
4096 offset = extent_map_end(em); 4097 offset = extent_map_end(em);
4097 free_extent_map(em); 4098 free_extent_map(em);
4098 if (offset >= last) 4099 if (offset >= last)
4099 break; 4100 break;
4100 } 4101 }
4101 return NULL; 4102 return NULL;
4102 } 4103 }
4103 4104
4104 static noinline int count_ext_ref(u64 inum, u64 offset, u64 root_id, void *ctx) 4105 static noinline int count_ext_ref(u64 inum, u64 offset, u64 root_id, void *ctx)
4105 { 4106 {
4106 unsigned long cnt = *((unsigned long *)ctx); 4107 unsigned long cnt = *((unsigned long *)ctx);
4107 4108
4108 cnt++; 4109 cnt++;
4109 *((unsigned long *)ctx) = cnt; 4110 *((unsigned long *)ctx) = cnt;
4110 4111
4111 /* Now we're sure that the extent is shared. */ 4112 /* Now we're sure that the extent is shared. */
4112 if (cnt > 1) 4113 if (cnt > 1)
4113 return 1; 4114 return 1;
4114 return 0; 4115 return 0;
4115 } 4116 }
4116 4117
4117 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 4118 int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4118 __u64 start, __u64 len, get_extent_t *get_extent) 4119 __u64 start, __u64 len, get_extent_t *get_extent)
4119 { 4120 {
4120 int ret = 0; 4121 int ret = 0;
4121 u64 off = start; 4122 u64 off = start;
4122 u64 max = start + len; 4123 u64 max = start + len;
4123 u32 flags = 0; 4124 u32 flags = 0;
4124 u32 found_type; 4125 u32 found_type;
4125 u64 last; 4126 u64 last;
4126 u64 last_for_get_extent = 0; 4127 u64 last_for_get_extent = 0;
4127 u64 disko = 0; 4128 u64 disko = 0;
4128 u64 isize = i_size_read(inode); 4129 u64 isize = i_size_read(inode);
4129 struct btrfs_key found_key; 4130 struct btrfs_key found_key;
4130 struct extent_map *em = NULL; 4131 struct extent_map *em = NULL;
4131 struct extent_state *cached_state = NULL; 4132 struct extent_state *cached_state = NULL;
4132 struct btrfs_path *path; 4133 struct btrfs_path *path;
4133 int end = 0; 4134 int end = 0;
4134 u64 em_start = 0; 4135 u64 em_start = 0;
4135 u64 em_len = 0; 4136 u64 em_len = 0;
4136 u64 em_end = 0; 4137 u64 em_end = 0;
4137 4138
4138 if (len == 0) 4139 if (len == 0)
4139 return -EINVAL; 4140 return -EINVAL;
4140 4141
4141 path = btrfs_alloc_path(); 4142 path = btrfs_alloc_path();
4142 if (!path) 4143 if (!path)
4143 return -ENOMEM; 4144 return -ENOMEM;
4144 path->leave_spinning = 1; 4145 path->leave_spinning = 1;
4145 4146
4146 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize); 4147 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
4147 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize); 4148 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
4148 4149
4149 /* 4150 /*
4150 * lookup the last file extent. We're not using i_size here 4151 * lookup the last file extent. We're not using i_size here
4151 * because there might be preallocation past i_size 4152 * because there might be preallocation past i_size
4152 */ 4153 */
4153 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root, 4154 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
4154 path, btrfs_ino(inode), -1, 0); 4155 path, btrfs_ino(inode), -1, 0);
4155 if (ret < 0) { 4156 if (ret < 0) {
4156 btrfs_free_path(path); 4157 btrfs_free_path(path);
4157 return ret; 4158 return ret;
4158 } 4159 }
4159 WARN_ON(!ret); 4160 WARN_ON(!ret);
4160 path->slots[0]--; 4161 path->slots[0]--;
4161 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); 4162 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4162 found_type = btrfs_key_type(&found_key); 4163 found_type = btrfs_key_type(&found_key);
4163 4164
4164 /* No extents, but there might be delalloc bits */ 4165 /* No extents, but there might be delalloc bits */
4165 if (found_key.objectid != btrfs_ino(inode) || 4166 if (found_key.objectid != btrfs_ino(inode) ||
4166 found_type != BTRFS_EXTENT_DATA_KEY) { 4167 found_type != BTRFS_EXTENT_DATA_KEY) {
4167 /* have to trust i_size as the end */ 4168 /* have to trust i_size as the end */
4168 last = (u64)-1; 4169 last = (u64)-1;
4169 last_for_get_extent = isize; 4170 last_for_get_extent = isize;
4170 } else { 4171 } else {
4171 /* 4172 /*
4172 * remember the start of the last extent. There are a 4173 * remember the start of the last extent. There are a
4173 * bunch of different factors that go into the length of the 4174 * bunch of different factors that go into the length of the
4174 * extent, so its much less complex to remember where it started 4175 * extent, so its much less complex to remember where it started
4175 */ 4176 */
4176 last = found_key.offset; 4177 last = found_key.offset;
4177 last_for_get_extent = last + 1; 4178 last_for_get_extent = last + 1;
4178 } 4179 }
4179 btrfs_release_path(path); 4180 btrfs_release_path(path);
4180 4181
4181 /* 4182 /*
4182 * we might have some extents allocated but more delalloc past those 4183 * we might have some extents allocated but more delalloc past those
4183 * extents. so, we trust isize unless the start of the last extent is 4184 * extents. so, we trust isize unless the start of the last extent is
4184 * beyond isize 4185 * beyond isize
4185 */ 4186 */
4186 if (last < isize) { 4187 if (last < isize) {
4187 last = (u64)-1; 4188 last = (u64)-1;
4188 last_for_get_extent = isize; 4189 last_for_get_extent = isize;
4189 } 4190 }
4190 4191
4191 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0, 4192 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
4192 &cached_state); 4193 &cached_state);
4193 4194
4194 em = get_extent_skip_holes(inode, start, last_for_get_extent, 4195 em = get_extent_skip_holes(inode, start, last_for_get_extent,
4195 get_extent); 4196 get_extent);
4196 if (!em) 4197 if (!em)
4197 goto out; 4198 goto out;
4198 if (IS_ERR(em)) { 4199 if (IS_ERR(em)) {
4199 ret = PTR_ERR(em); 4200 ret = PTR_ERR(em);
4200 goto out; 4201 goto out;
4201 } 4202 }
4202 4203
4203 while (!end) { 4204 while (!end) {
4204 u64 offset_in_extent = 0; 4205 u64 offset_in_extent = 0;
4205 4206
4206 /* break if the extent we found is outside the range */ 4207 /* break if the extent we found is outside the range */
4207 if (em->start >= max || extent_map_end(em) < off) 4208 if (em->start >= max || extent_map_end(em) < off)
4208 break; 4209 break;
4209 4210
4210 /* 4211 /*
4211 * get_extent may return an extent that starts before our 4212 * get_extent may return an extent that starts before our
4212 * requested range. We have to make sure the ranges 4213 * requested range. We have to make sure the ranges
4213 * we return to fiemap always move forward and don't 4214 * we return to fiemap always move forward and don't
4214 * overlap, so adjust the offsets here 4215 * overlap, so adjust the offsets here
4215 */ 4216 */
4216 em_start = max(em->start, off); 4217 em_start = max(em->start, off);
4217 4218
4218 /* 4219 /*
4219 * record the offset from the start of the extent 4220 * record the offset from the start of the extent
4220 * for adjusting the disk offset below. Only do this if the 4221 * for adjusting the disk offset below. Only do this if the
4221 * extent isn't compressed since our in ram offset may be past 4222 * extent isn't compressed since our in ram offset may be past
4222 * what we have actually allocated on disk. 4223 * what we have actually allocated on disk.
4223 */ 4224 */
4224 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 4225 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4225 offset_in_extent = em_start - em->start; 4226 offset_in_extent = em_start - em->start;
4226 em_end = extent_map_end(em); 4227 em_end = extent_map_end(em);
4227 em_len = em_end - em_start; 4228 em_len = em_end - em_start;
4228 disko = 0; 4229 disko = 0;
4229 flags = 0; 4230 flags = 0;
4230 4231
4231 /* 4232 /*
4232 * bump off for our next call to get_extent 4233 * bump off for our next call to get_extent
4233 */ 4234 */
4234 off = extent_map_end(em); 4235 off = extent_map_end(em);
4235 if (off >= max) 4236 if (off >= max)
4236 end = 1; 4237 end = 1;
4237 4238
4238 if (em->block_start == EXTENT_MAP_LAST_BYTE) { 4239 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
4239 end = 1; 4240 end = 1;
4240 flags |= FIEMAP_EXTENT_LAST; 4241 flags |= FIEMAP_EXTENT_LAST;
4241 } else if (em->block_start == EXTENT_MAP_INLINE) { 4242 } else if (em->block_start == EXTENT_MAP_INLINE) {
4242 flags |= (FIEMAP_EXTENT_DATA_INLINE | 4243 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4243 FIEMAP_EXTENT_NOT_ALIGNED); 4244 FIEMAP_EXTENT_NOT_ALIGNED);
4244 } else if (em->block_start == EXTENT_MAP_DELALLOC) { 4245 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
4245 flags |= (FIEMAP_EXTENT_DELALLOC | 4246 flags |= (FIEMAP_EXTENT_DELALLOC |
4246 FIEMAP_EXTENT_UNKNOWN); 4247 FIEMAP_EXTENT_UNKNOWN);
4247 } else { 4248 } else {
4248 unsigned long ref_cnt = 0; 4249 unsigned long ref_cnt = 0;
4249 4250
4250 disko = em->block_start + offset_in_extent; 4251 disko = em->block_start + offset_in_extent;
4251 4252
4252 /* 4253 /*
4253 * As btrfs supports shared space, this information 4254 * As btrfs supports shared space, this information
4254 * can be exported to userspace tools via 4255 * can be exported to userspace tools via
4255 * flag FIEMAP_EXTENT_SHARED. 4256 * flag FIEMAP_EXTENT_SHARED.
4256 */ 4257 */
4257 ret = iterate_inodes_from_logical( 4258 ret = iterate_inodes_from_logical(
4258 em->block_start, 4259 em->block_start,
4259 BTRFS_I(inode)->root->fs_info, 4260 BTRFS_I(inode)->root->fs_info,
4260 path, count_ext_ref, &ref_cnt); 4261 path, count_ext_ref, &ref_cnt);
4261 if (ret < 0 && ret != -ENOENT) 4262 if (ret < 0 && ret != -ENOENT)
4262 goto out_free; 4263 goto out_free;
4263 4264
4264 if (ref_cnt > 1) 4265 if (ref_cnt > 1)
4265 flags |= FIEMAP_EXTENT_SHARED; 4266 flags |= FIEMAP_EXTENT_SHARED;
4266 } 4267 }
4267 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 4268 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4268 flags |= FIEMAP_EXTENT_ENCODED; 4269 flags |= FIEMAP_EXTENT_ENCODED;
4269 4270
4270 free_extent_map(em); 4271 free_extent_map(em);
4271 em = NULL; 4272 em = NULL;
4272 if ((em_start >= last) || em_len == (u64)-1 || 4273 if ((em_start >= last) || em_len == (u64)-1 ||
4273 (last == (u64)-1 && isize <= em_end)) { 4274 (last == (u64)-1 && isize <= em_end)) {
4274 flags |= FIEMAP_EXTENT_LAST; 4275 flags |= FIEMAP_EXTENT_LAST;
4275 end = 1; 4276 end = 1;
4276 } 4277 }
4277 4278
4278 /* now scan forward to see if this is really the last extent. */ 4279 /* now scan forward to see if this is really the last extent. */
4279 em = get_extent_skip_holes(inode, off, last_for_get_extent, 4280 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4280 get_extent); 4281 get_extent);
4281 if (IS_ERR(em)) { 4282 if (IS_ERR(em)) {
4282 ret = PTR_ERR(em); 4283 ret = PTR_ERR(em);
4283 goto out; 4284 goto out;
4284 } 4285 }
4285 if (!em) { 4286 if (!em) {
4286 flags |= FIEMAP_EXTENT_LAST; 4287 flags |= FIEMAP_EXTENT_LAST;
4287 end = 1; 4288 end = 1;
4288 } 4289 }
4289 ret = fiemap_fill_next_extent(fieinfo, em_start, disko, 4290 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4290 em_len, flags); 4291 em_len, flags);
4291 if (ret) 4292 if (ret)
4292 goto out_free; 4293 goto out_free;
4293 } 4294 }
4294 out_free: 4295 out_free:
4295 free_extent_map(em); 4296 free_extent_map(em);
4296 out: 4297 out:
4297 btrfs_free_path(path); 4298 btrfs_free_path(path);
4298 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1, 4299 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
4299 &cached_state, GFP_NOFS); 4300 &cached_state, GFP_NOFS);
4300 return ret; 4301 return ret;
4301 } 4302 }
4302 4303
4303 static void __free_extent_buffer(struct extent_buffer *eb) 4304 static void __free_extent_buffer(struct extent_buffer *eb)
4304 { 4305 {
4305 btrfs_leak_debug_del(&eb->leak_list); 4306 btrfs_leak_debug_del(&eb->leak_list);
4306 kmem_cache_free(extent_buffer_cache, eb); 4307 kmem_cache_free(extent_buffer_cache, eb);
4307 } 4308 }
4308 4309
4309 static int extent_buffer_under_io(struct extent_buffer *eb) 4310 static int extent_buffer_under_io(struct extent_buffer *eb)
4310 { 4311 {
4311 return (atomic_read(&eb->io_pages) || 4312 return (atomic_read(&eb->io_pages) ||
4312 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || 4313 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4313 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); 4314 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4314 } 4315 }
4315 4316
4316 /* 4317 /*
4317 * Helper for releasing extent buffer page. 4318 * Helper for releasing extent buffer page.
4318 */ 4319 */
4319 static void btrfs_release_extent_buffer_page(struct extent_buffer *eb, 4320 static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4320 unsigned long start_idx) 4321 unsigned long start_idx)
4321 { 4322 {
4322 unsigned long index; 4323 unsigned long index;
4323 unsigned long num_pages; 4324 unsigned long num_pages;
4324 struct page *page; 4325 struct page *page;
4325 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags); 4326 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4326 4327
4327 BUG_ON(extent_buffer_under_io(eb)); 4328 BUG_ON(extent_buffer_under_io(eb));
4328 4329
4329 num_pages = num_extent_pages(eb->start, eb->len); 4330 num_pages = num_extent_pages(eb->start, eb->len);
4330 index = start_idx + num_pages; 4331 index = start_idx + num_pages;
4331 if (start_idx >= index) 4332 if (start_idx >= index)
4332 return; 4333 return;
4333 4334
4334 do { 4335 do {
4335 index--; 4336 index--;
4336 page = extent_buffer_page(eb, index); 4337 page = extent_buffer_page(eb, index);
4337 if (page && mapped) { 4338 if (page && mapped) {
4338 spin_lock(&page->mapping->private_lock); 4339 spin_lock(&page->mapping->private_lock);
4339 /* 4340 /*
4340 * We do this since we'll remove the pages after we've 4341 * We do this since we'll remove the pages after we've
4341 * removed the eb from the radix tree, so we could race 4342 * removed the eb from the radix tree, so we could race
4342 * and have this page now attached to the new eb. So 4343 * and have this page now attached to the new eb. So
4343 * only clear page_private if it's still connected to 4344 * only clear page_private if it's still connected to
4344 * this eb. 4345 * this eb.
4345 */ 4346 */
4346 if (PagePrivate(page) && 4347 if (PagePrivate(page) &&
4347 page->private == (unsigned long)eb) { 4348 page->private == (unsigned long)eb) {
4348 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); 4349 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4349 BUG_ON(PageDirty(page)); 4350 BUG_ON(PageDirty(page));
4350 BUG_ON(PageWriteback(page)); 4351 BUG_ON(PageWriteback(page));
4351 /* 4352 /*
4352 * We need to make sure we haven't be attached 4353 * We need to make sure we haven't be attached
4353 * to a new eb. 4354 * to a new eb.
4354 */ 4355 */
4355 ClearPagePrivate(page); 4356 ClearPagePrivate(page);
4356 set_page_private(page, 0); 4357 set_page_private(page, 0);
4357 /* One for the page private */ 4358 /* One for the page private */
4358 page_cache_release(page); 4359 page_cache_release(page);
4359 } 4360 }
4360 spin_unlock(&page->mapping->private_lock); 4361 spin_unlock(&page->mapping->private_lock);
4361 4362
4362 } 4363 }
4363 if (page) { 4364 if (page) {
4364 /* One for when we alloced the page */ 4365 /* One for when we alloced the page */
4365 page_cache_release(page); 4366 page_cache_release(page);
4366 } 4367 }
4367 } while (index != start_idx); 4368 } while (index != start_idx);
4368 } 4369 }
4369 4370
4370 /* 4371 /*
4371 * Helper for releasing the extent buffer. 4372 * Helper for releasing the extent buffer.
4372 */ 4373 */
4373 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) 4374 static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4374 { 4375 {
4375 btrfs_release_extent_buffer_page(eb, 0); 4376 btrfs_release_extent_buffer_page(eb, 0);
4376 __free_extent_buffer(eb); 4377 __free_extent_buffer(eb);
4377 } 4378 }
4378 4379
4379 static struct extent_buffer * 4380 static struct extent_buffer *
4380 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, 4381 __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
4381 unsigned long len, gfp_t mask) 4382 unsigned long len, gfp_t mask)
4382 { 4383 {
4383 struct extent_buffer *eb = NULL; 4384 struct extent_buffer *eb = NULL;
4384 4385
4385 eb = kmem_cache_zalloc(extent_buffer_cache, mask); 4386 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
4386 if (eb == NULL) 4387 if (eb == NULL)
4387 return NULL; 4388 return NULL;
4388 eb->start = start; 4389 eb->start = start;
4389 eb->len = len; 4390 eb->len = len;
4390 eb->fs_info = fs_info; 4391 eb->fs_info = fs_info;
4391 eb->bflags = 0; 4392 eb->bflags = 0;
4392 rwlock_init(&eb->lock); 4393 rwlock_init(&eb->lock);
4393 atomic_set(&eb->write_locks, 0); 4394 atomic_set(&eb->write_locks, 0);
4394 atomic_set(&eb->read_locks, 0); 4395 atomic_set(&eb->read_locks, 0);
4395 atomic_set(&eb->blocking_readers, 0); 4396 atomic_set(&eb->blocking_readers, 0);
4396 atomic_set(&eb->blocking_writers, 0); 4397 atomic_set(&eb->blocking_writers, 0);
4397 atomic_set(&eb->spinning_readers, 0); 4398 atomic_set(&eb->spinning_readers, 0);
4398 atomic_set(&eb->spinning_writers, 0); 4399 atomic_set(&eb->spinning_writers, 0);
4399 eb->lock_nested = 0; 4400 eb->lock_nested = 0;
4400 init_waitqueue_head(&eb->write_lock_wq); 4401 init_waitqueue_head(&eb->write_lock_wq);
4401 init_waitqueue_head(&eb->read_lock_wq); 4402 init_waitqueue_head(&eb->read_lock_wq);
4402 4403
4403 btrfs_leak_debug_add(&eb->leak_list, &buffers); 4404 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4404 4405
4405 spin_lock_init(&eb->refs_lock); 4406 spin_lock_init(&eb->refs_lock);
4406 atomic_set(&eb->refs, 1); 4407 atomic_set(&eb->refs, 1);
4407 atomic_set(&eb->io_pages, 0); 4408 atomic_set(&eb->io_pages, 0);
4408 4409
4409 /* 4410 /*
4410 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages 4411 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4411 */ 4412 */
4412 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE 4413 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4413 > MAX_INLINE_EXTENT_BUFFER_SIZE); 4414 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4414 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE); 4415 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4415 4416
4416 return eb; 4417 return eb;
4417 } 4418 }
4418 4419
4419 struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src) 4420 struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4420 { 4421 {
4421 unsigned long i; 4422 unsigned long i;
4422 struct page *p; 4423 struct page *p;
4423 struct extent_buffer *new; 4424 struct extent_buffer *new;
4424 unsigned long num_pages = num_extent_pages(src->start, src->len); 4425 unsigned long num_pages = num_extent_pages(src->start, src->len);
4425 4426
4426 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS); 4427 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS);
4427 if (new == NULL) 4428 if (new == NULL)
4428 return NULL; 4429 return NULL;
4429 4430
4430 for (i = 0; i < num_pages; i++) { 4431 for (i = 0; i < num_pages; i++) {
4431 p = alloc_page(GFP_NOFS); 4432 p = alloc_page(GFP_NOFS);
4432 if (!p) { 4433 if (!p) {
4433 btrfs_release_extent_buffer(new); 4434 btrfs_release_extent_buffer(new);
4434 return NULL; 4435 return NULL;
4435 } 4436 }
4436 attach_extent_buffer_page(new, p); 4437 attach_extent_buffer_page(new, p);
4437 WARN_ON(PageDirty(p)); 4438 WARN_ON(PageDirty(p));
4438 SetPageUptodate(p); 4439 SetPageUptodate(p);
4439 new->pages[i] = p; 4440 new->pages[i] = p;
4440 } 4441 }
4441 4442
4442 copy_extent_buffer(new, src, 0, 0, src->len); 4443 copy_extent_buffer(new, src, 0, 0, src->len);
4443 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags); 4444 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4444 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags); 4445 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4445 4446
4446 return new; 4447 return new;
4447 } 4448 }
4448 4449
4449 struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len) 4450 struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4450 { 4451 {
4451 struct extent_buffer *eb; 4452 struct extent_buffer *eb;
4452 unsigned long num_pages = num_extent_pages(0, len); 4453 unsigned long num_pages = num_extent_pages(0, len);
4453 unsigned long i; 4454 unsigned long i;
4454 4455
4455 eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS); 4456 eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS);
4456 if (!eb) 4457 if (!eb)
4457 return NULL; 4458 return NULL;
4458 4459
4459 for (i = 0; i < num_pages; i++) { 4460 for (i = 0; i < num_pages; i++) {
4460 eb->pages[i] = alloc_page(GFP_NOFS); 4461 eb->pages[i] = alloc_page(GFP_NOFS);
4461 if (!eb->pages[i]) 4462 if (!eb->pages[i])
4462 goto err; 4463 goto err;
4463 } 4464 }
4464 set_extent_buffer_uptodate(eb); 4465 set_extent_buffer_uptodate(eb);
4465 btrfs_set_header_nritems(eb, 0); 4466 btrfs_set_header_nritems(eb, 0);
4466 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags); 4467 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4467 4468
4468 return eb; 4469 return eb;
4469 err: 4470 err:
4470 for (; i > 0; i--) 4471 for (; i > 0; i--)
4471 __free_page(eb->pages[i - 1]); 4472 __free_page(eb->pages[i - 1]);
4472 __free_extent_buffer(eb); 4473 __free_extent_buffer(eb);
4473 return NULL; 4474 return NULL;
4474 } 4475 }
4475 4476
4476 static void check_buffer_tree_ref(struct extent_buffer *eb) 4477 static void check_buffer_tree_ref(struct extent_buffer *eb)
4477 { 4478 {
4478 int refs; 4479 int refs;
4479 /* the ref bit is tricky. We have to make sure it is set 4480 /* the ref bit is tricky. We have to make sure it is set
4480 * if we have the buffer dirty. Otherwise the 4481 * if we have the buffer dirty. Otherwise the
4481 * code to free a buffer can end up dropping a dirty 4482 * code to free a buffer can end up dropping a dirty
4482 * page 4483 * page
4483 * 4484 *
4484 * Once the ref bit is set, it won't go away while the 4485 * Once the ref bit is set, it won't go away while the
4485 * buffer is dirty or in writeback, and it also won't 4486 * buffer is dirty or in writeback, and it also won't
4486 * go away while we have the reference count on the 4487 * go away while we have the reference count on the
4487 * eb bumped. 4488 * eb bumped.
4488 * 4489 *
4489 * We can't just set the ref bit without bumping the 4490 * We can't just set the ref bit without bumping the
4490 * ref on the eb because free_extent_buffer might 4491 * ref on the eb because free_extent_buffer might
4491 * see the ref bit and try to clear it. If this happens 4492 * see the ref bit and try to clear it. If this happens
4492 * free_extent_buffer might end up dropping our original 4493 * free_extent_buffer might end up dropping our original
4493 * ref by mistake and freeing the page before we are able 4494 * ref by mistake and freeing the page before we are able
4494 * to add one more ref. 4495 * to add one more ref.
4495 * 4496 *
4496 * So bump the ref count first, then set the bit. If someone 4497 * So bump the ref count first, then set the bit. If someone
4497 * beat us to it, drop the ref we added. 4498 * beat us to it, drop the ref we added.
4498 */ 4499 */
4499 refs = atomic_read(&eb->refs); 4500 refs = atomic_read(&eb->refs);
4500 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 4501 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4501 return; 4502 return;
4502 4503
4503 spin_lock(&eb->refs_lock); 4504 spin_lock(&eb->refs_lock);
4504 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 4505 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4505 atomic_inc(&eb->refs); 4506 atomic_inc(&eb->refs);
4506 spin_unlock(&eb->refs_lock); 4507 spin_unlock(&eb->refs_lock);
4507 } 4508 }
4508 4509
4509 static void mark_extent_buffer_accessed(struct extent_buffer *eb) 4510 static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4510 { 4511 {
4511 unsigned long num_pages, i; 4512 unsigned long num_pages, i;
4512 4513
4513 check_buffer_tree_ref(eb); 4514 check_buffer_tree_ref(eb);
4514 4515
4515 num_pages = num_extent_pages(eb->start, eb->len); 4516 num_pages = num_extent_pages(eb->start, eb->len);
4516 for (i = 0; i < num_pages; i++) { 4517 for (i = 0; i < num_pages; i++) {
4517 struct page *p = extent_buffer_page(eb, i); 4518 struct page *p = extent_buffer_page(eb, i);
4518 mark_page_accessed(p); 4519 mark_page_accessed(p);
4519 } 4520 }
4520 } 4521 }
4521 4522
4522 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, 4523 struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4523 u64 start) 4524 u64 start)
4524 { 4525 {
4525 struct extent_buffer *eb; 4526 struct extent_buffer *eb;
4526 4527
4527 rcu_read_lock(); 4528 rcu_read_lock();
4528 eb = radix_tree_lookup(&fs_info->buffer_radix, 4529 eb = radix_tree_lookup(&fs_info->buffer_radix,
4529 start >> PAGE_CACHE_SHIFT); 4530 start >> PAGE_CACHE_SHIFT);
4530 if (eb && atomic_inc_not_zero(&eb->refs)) { 4531 if (eb && atomic_inc_not_zero(&eb->refs)) {
4531 rcu_read_unlock(); 4532 rcu_read_unlock();
4532 mark_extent_buffer_accessed(eb); 4533 mark_extent_buffer_accessed(eb);
4533 return eb; 4534 return eb;
4534 } 4535 }
4535 rcu_read_unlock(); 4536 rcu_read_unlock();
4536 4537
4537 return NULL; 4538 return NULL;
4538 } 4539 }
4539 4540
4540 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, 4541 struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
4541 u64 start, unsigned long len) 4542 u64 start, unsigned long len)
4542 { 4543 {
4543 unsigned long num_pages = num_extent_pages(start, len); 4544 unsigned long num_pages = num_extent_pages(start, len);
4544 unsigned long i; 4545 unsigned long i;
4545 unsigned long index = start >> PAGE_CACHE_SHIFT; 4546 unsigned long index = start >> PAGE_CACHE_SHIFT;
4546 struct extent_buffer *eb; 4547 struct extent_buffer *eb;
4547 struct extent_buffer *exists = NULL; 4548 struct extent_buffer *exists = NULL;
4548 struct page *p; 4549 struct page *p;
4549 struct address_space *mapping = fs_info->btree_inode->i_mapping; 4550 struct address_space *mapping = fs_info->btree_inode->i_mapping;
4550 int uptodate = 1; 4551 int uptodate = 1;
4551 int ret; 4552 int ret;
4552 4553
4553 eb = find_extent_buffer(fs_info, start); 4554 eb = find_extent_buffer(fs_info, start);
4554 if (eb) 4555 if (eb)
4555 return eb; 4556 return eb;
4556 4557
4557 eb = __alloc_extent_buffer(fs_info, start, len, GFP_NOFS); 4558 eb = __alloc_extent_buffer(fs_info, start, len, GFP_NOFS);
4558 if (!eb) 4559 if (!eb)
4559 return NULL; 4560 return NULL;
4560 4561
4561 for (i = 0; i < num_pages; i++, index++) { 4562 for (i = 0; i < num_pages; i++, index++) {
4562 p = find_or_create_page(mapping, index, GFP_NOFS); 4563 p = find_or_create_page(mapping, index, GFP_NOFS);
4563 if (!p) 4564 if (!p)
4564 goto free_eb; 4565 goto free_eb;
4565 4566
4566 spin_lock(&mapping->private_lock); 4567 spin_lock(&mapping->private_lock);
4567 if (PagePrivate(p)) { 4568 if (PagePrivate(p)) {
4568 /* 4569 /*
4569 * We could have already allocated an eb for this page 4570 * We could have already allocated an eb for this page
4570 * and attached one so lets see if we can get a ref on 4571 * and attached one so lets see if we can get a ref on
4571 * the existing eb, and if we can we know it's good and 4572 * the existing eb, and if we can we know it's good and
4572 * we can just return that one, else we know we can just 4573 * we can just return that one, else we know we can just
4573 * overwrite page->private. 4574 * overwrite page->private.
4574 */ 4575 */
4575 exists = (struct extent_buffer *)p->private; 4576 exists = (struct extent_buffer *)p->private;
4576 if (atomic_inc_not_zero(&exists->refs)) { 4577 if (atomic_inc_not_zero(&exists->refs)) {
4577 spin_unlock(&mapping->private_lock); 4578 spin_unlock(&mapping->private_lock);
4578 unlock_page(p); 4579 unlock_page(p);
4579 page_cache_release(p); 4580 page_cache_release(p);
4580 mark_extent_buffer_accessed(exists); 4581 mark_extent_buffer_accessed(exists);
4581 goto free_eb; 4582 goto free_eb;
4582 } 4583 }
4583 4584
4584 /* 4585 /*
4585 * Do this so attach doesn't complain and we need to 4586 * Do this so attach doesn't complain and we need to
4586 * drop the ref the old guy had. 4587 * drop the ref the old guy had.
4587 */ 4588 */
4588 ClearPagePrivate(p); 4589 ClearPagePrivate(p);
4589 WARN_ON(PageDirty(p)); 4590 WARN_ON(PageDirty(p));
4590 page_cache_release(p); 4591 page_cache_release(p);
4591 } 4592 }
4592 attach_extent_buffer_page(eb, p); 4593 attach_extent_buffer_page(eb, p);
4593 spin_unlock(&mapping->private_lock); 4594 spin_unlock(&mapping->private_lock);
4594 WARN_ON(PageDirty(p)); 4595 WARN_ON(PageDirty(p));
4595 mark_page_accessed(p); 4596 mark_page_accessed(p);
4596 eb->pages[i] = p; 4597 eb->pages[i] = p;
4597 if (!PageUptodate(p)) 4598 if (!PageUptodate(p))
4598 uptodate = 0; 4599 uptodate = 0;
4599 4600
4600 /* 4601 /*
4601 * see below about how we avoid a nasty race with release page 4602 * see below about how we avoid a nasty race with release page
4602 * and why we unlock later 4603 * and why we unlock later
4603 */ 4604 */
4604 } 4605 }
4605 if (uptodate) 4606 if (uptodate)
4606 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 4607 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4607 again: 4608 again:
4608 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM); 4609 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4609 if (ret) 4610 if (ret)
4610 goto free_eb; 4611 goto free_eb;
4611 4612
4612 spin_lock(&fs_info->buffer_lock); 4613 spin_lock(&fs_info->buffer_lock);
4613 ret = radix_tree_insert(&fs_info->buffer_radix, 4614 ret = radix_tree_insert(&fs_info->buffer_radix,
4614 start >> PAGE_CACHE_SHIFT, eb); 4615 start >> PAGE_CACHE_SHIFT, eb);
4615 spin_unlock(&fs_info->buffer_lock); 4616 spin_unlock(&fs_info->buffer_lock);
4616 radix_tree_preload_end(); 4617 radix_tree_preload_end();
4617 if (ret == -EEXIST) { 4618 if (ret == -EEXIST) {
4618 exists = find_extent_buffer(fs_info, start); 4619 exists = find_extent_buffer(fs_info, start);
4619 if (exists) 4620 if (exists)
4620 goto free_eb; 4621 goto free_eb;
4621 else 4622 else
4622 goto again; 4623 goto again;
4623 } 4624 }
4624 /* add one reference for the tree */ 4625 /* add one reference for the tree */
4625 check_buffer_tree_ref(eb); 4626 check_buffer_tree_ref(eb);
4626 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags); 4627 set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4627 4628
4628 /* 4629 /*
4629 * there is a race where release page may have 4630 * there is a race where release page may have
4630 * tried to find this extent buffer in the radix 4631 * tried to find this extent buffer in the radix
4631 * but failed. It will tell the VM it is safe to 4632 * but failed. It will tell the VM it is safe to
4632 * reclaim the, and it will clear the page private bit. 4633 * reclaim the, and it will clear the page private bit.
4633 * We must make sure to set the page private bit properly 4634 * We must make sure to set the page private bit properly
4634 * after the extent buffer is in the radix tree so 4635 * after the extent buffer is in the radix tree so
4635 * it doesn't get lost 4636 * it doesn't get lost
4636 */ 4637 */
4637 SetPageChecked(eb->pages[0]); 4638 SetPageChecked(eb->pages[0]);
4638 for (i = 1; i < num_pages; i++) { 4639 for (i = 1; i < num_pages; i++) {
4639 p = extent_buffer_page(eb, i); 4640 p = extent_buffer_page(eb, i);
4640 ClearPageChecked(p); 4641 ClearPageChecked(p);
4641 unlock_page(p); 4642 unlock_page(p);
4642 } 4643 }
4643 unlock_page(eb->pages[0]); 4644 unlock_page(eb->pages[0]);
4644 return eb; 4645 return eb;
4645 4646
4646 free_eb: 4647 free_eb:
4647 for (i = 0; i < num_pages; i++) { 4648 for (i = 0; i < num_pages; i++) {
4648 if (eb->pages[i]) 4649 if (eb->pages[i])
4649 unlock_page(eb->pages[i]); 4650 unlock_page(eb->pages[i]);
4650 } 4651 }
4651 4652
4652 WARN_ON(!atomic_dec_and_test(&eb->refs)); 4653 WARN_ON(!atomic_dec_and_test(&eb->refs));
4653 btrfs_release_extent_buffer(eb); 4654 btrfs_release_extent_buffer(eb);
4654 return exists; 4655 return exists;
4655 } 4656 }
4656 4657
4657 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head) 4658 static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4658 { 4659 {
4659 struct extent_buffer *eb = 4660 struct extent_buffer *eb =
4660 container_of(head, struct extent_buffer, rcu_head); 4661 container_of(head, struct extent_buffer, rcu_head);
4661 4662
4662 __free_extent_buffer(eb); 4663 __free_extent_buffer(eb);
4663 } 4664 }
4664 4665
4665 /* Expects to have eb->eb_lock already held */ 4666 /* Expects to have eb->eb_lock already held */
4666 static int release_extent_buffer(struct extent_buffer *eb) 4667 static int release_extent_buffer(struct extent_buffer *eb)
4667 { 4668 {
4668 WARN_ON(atomic_read(&eb->refs) == 0); 4669 WARN_ON(atomic_read(&eb->refs) == 0);
4669 if (atomic_dec_and_test(&eb->refs)) { 4670 if (atomic_dec_and_test(&eb->refs)) {
4670 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) { 4671 if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
4671 struct btrfs_fs_info *fs_info = eb->fs_info; 4672 struct btrfs_fs_info *fs_info = eb->fs_info;
4672 4673
4673 spin_unlock(&eb->refs_lock); 4674 spin_unlock(&eb->refs_lock);
4674 4675
4675 spin_lock(&fs_info->buffer_lock); 4676 spin_lock(&fs_info->buffer_lock);
4676 radix_tree_delete(&fs_info->buffer_radix, 4677 radix_tree_delete(&fs_info->buffer_radix,
4677 eb->start >> PAGE_CACHE_SHIFT); 4678 eb->start >> PAGE_CACHE_SHIFT);
4678 spin_unlock(&fs_info->buffer_lock); 4679 spin_unlock(&fs_info->buffer_lock);
4679 } else { 4680 } else {
4680 spin_unlock(&eb->refs_lock); 4681 spin_unlock(&eb->refs_lock);
4681 } 4682 }
4682 4683
4683 /* Should be safe to release our pages at this point */ 4684 /* Should be safe to release our pages at this point */
4684 btrfs_release_extent_buffer_page(eb, 0); 4685 btrfs_release_extent_buffer_page(eb, 0);
4685 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu); 4686 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4686 return 1; 4687 return 1;
4687 } 4688 }
4688 spin_unlock(&eb->refs_lock); 4689 spin_unlock(&eb->refs_lock);
4689 4690
4690 return 0; 4691 return 0;
4691 } 4692 }
4692 4693
4693 void free_extent_buffer(struct extent_buffer *eb) 4694 void free_extent_buffer(struct extent_buffer *eb)
4694 { 4695 {
4695 int refs; 4696 int refs;
4696 int old; 4697 int old;
4697 if (!eb) 4698 if (!eb)
4698 return; 4699 return;
4699 4700
4700 while (1) { 4701 while (1) {
4701 refs = atomic_read(&eb->refs); 4702 refs = atomic_read(&eb->refs);
4702 if (refs <= 3) 4703 if (refs <= 3)
4703 break; 4704 break;
4704 old = atomic_cmpxchg(&eb->refs, refs, refs - 1); 4705 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4705 if (old == refs) 4706 if (old == refs)
4706 return; 4707 return;
4707 } 4708 }
4708 4709
4709 spin_lock(&eb->refs_lock); 4710 spin_lock(&eb->refs_lock);
4710 if (atomic_read(&eb->refs) == 2 && 4711 if (atomic_read(&eb->refs) == 2 &&
4711 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) 4712 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4712 atomic_dec(&eb->refs); 4713 atomic_dec(&eb->refs);
4713 4714
4714 if (atomic_read(&eb->refs) == 2 && 4715 if (atomic_read(&eb->refs) == 2 &&
4715 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && 4716 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4716 !extent_buffer_under_io(eb) && 4717 !extent_buffer_under_io(eb) &&
4717 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 4718 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4718 atomic_dec(&eb->refs); 4719 atomic_dec(&eb->refs);
4719 4720
4720 /* 4721 /*
4721 * I know this is terrible, but it's temporary until we stop tracking 4722 * I know this is terrible, but it's temporary until we stop tracking
4722 * the uptodate bits and such for the extent buffers. 4723 * the uptodate bits and such for the extent buffers.
4723 */ 4724 */
4724 release_extent_buffer(eb); 4725 release_extent_buffer(eb);
4725 } 4726 }
4726 4727
4727 void free_extent_buffer_stale(struct extent_buffer *eb) 4728 void free_extent_buffer_stale(struct extent_buffer *eb)
4728 { 4729 {
4729 if (!eb) 4730 if (!eb)
4730 return; 4731 return;
4731 4732
4732 spin_lock(&eb->refs_lock); 4733 spin_lock(&eb->refs_lock);
4733 set_bit(EXTENT_BUFFER_STALE, &eb->bflags); 4734 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4734 4735
4735 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) && 4736 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4736 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) 4737 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4737 atomic_dec(&eb->refs); 4738 atomic_dec(&eb->refs);
4738 release_extent_buffer(eb); 4739 release_extent_buffer(eb);
4739 } 4740 }
4740 4741
4741 void clear_extent_buffer_dirty(struct extent_buffer *eb) 4742 void clear_extent_buffer_dirty(struct extent_buffer *eb)
4742 { 4743 {
4743 unsigned long i; 4744 unsigned long i;
4744 unsigned long num_pages; 4745 unsigned long num_pages;
4745 struct page *page; 4746 struct page *page;
4746 4747
4747 num_pages = num_extent_pages(eb->start, eb->len); 4748 num_pages = num_extent_pages(eb->start, eb->len);
4748 4749
4749 for (i = 0; i < num_pages; i++) { 4750 for (i = 0; i < num_pages; i++) {
4750 page = extent_buffer_page(eb, i); 4751 page = extent_buffer_page(eb, i);
4751 if (!PageDirty(page)) 4752 if (!PageDirty(page))
4752 continue; 4753 continue;
4753 4754
4754 lock_page(page); 4755 lock_page(page);
4755 WARN_ON(!PagePrivate(page)); 4756 WARN_ON(!PagePrivate(page));
4756 4757
4757 clear_page_dirty_for_io(page); 4758 clear_page_dirty_for_io(page);
4758 spin_lock_irq(&page->mapping->tree_lock); 4759 spin_lock_irq(&page->mapping->tree_lock);
4759 if (!PageDirty(page)) { 4760 if (!PageDirty(page)) {
4760 radix_tree_tag_clear(&page->mapping->page_tree, 4761 radix_tree_tag_clear(&page->mapping->page_tree,
4761 page_index(page), 4762 page_index(page),
4762 PAGECACHE_TAG_DIRTY); 4763 PAGECACHE_TAG_DIRTY);
4763 } 4764 }
4764 spin_unlock_irq(&page->mapping->tree_lock); 4765 spin_unlock_irq(&page->mapping->tree_lock);
4765 ClearPageError(page); 4766 ClearPageError(page);
4766 unlock_page(page); 4767 unlock_page(page);
4767 } 4768 }
4768 WARN_ON(atomic_read(&eb->refs) == 0); 4769 WARN_ON(atomic_read(&eb->refs) == 0);
4769 } 4770 }
4770 4771
4771 int set_extent_buffer_dirty(struct extent_buffer *eb) 4772 int set_extent_buffer_dirty(struct extent_buffer *eb)
4772 { 4773 {
4773 unsigned long i; 4774 unsigned long i;
4774 unsigned long num_pages; 4775 unsigned long num_pages;
4775 int was_dirty = 0; 4776 int was_dirty = 0;
4776 4777
4777 check_buffer_tree_ref(eb); 4778 check_buffer_tree_ref(eb);
4778 4779
4779 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); 4780 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
4780 4781
4781 num_pages = num_extent_pages(eb->start, eb->len); 4782 num_pages = num_extent_pages(eb->start, eb->len);
4782 WARN_ON(atomic_read(&eb->refs) == 0); 4783 WARN_ON(atomic_read(&eb->refs) == 0);
4783 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)); 4784 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4784 4785
4785 for (i = 0; i < num_pages; i++) 4786 for (i = 0; i < num_pages; i++)
4786 set_page_dirty(extent_buffer_page(eb, i)); 4787 set_page_dirty(extent_buffer_page(eb, i));
4787 return was_dirty; 4788 return was_dirty;
4788 } 4789 }
4789 4790
4790 int clear_extent_buffer_uptodate(struct extent_buffer *eb) 4791 int clear_extent_buffer_uptodate(struct extent_buffer *eb)
4791 { 4792 {
4792 unsigned long i; 4793 unsigned long i;
4793 struct page *page; 4794 struct page *page;
4794 unsigned long num_pages; 4795 unsigned long num_pages;
4795 4796
4796 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 4797 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4797 num_pages = num_extent_pages(eb->start, eb->len); 4798 num_pages = num_extent_pages(eb->start, eb->len);
4798 for (i = 0; i < num_pages; i++) { 4799 for (i = 0; i < num_pages; i++) {
4799 page = extent_buffer_page(eb, i); 4800 page = extent_buffer_page(eb, i);
4800 if (page) 4801 if (page)
4801 ClearPageUptodate(page); 4802 ClearPageUptodate(page);
4802 } 4803 }
4803 return 0; 4804 return 0;
4804 } 4805 }
4805 4806
4806 int set_extent_buffer_uptodate(struct extent_buffer *eb) 4807 int set_extent_buffer_uptodate(struct extent_buffer *eb)
4807 { 4808 {
4808 unsigned long i; 4809 unsigned long i;
4809 struct page *page; 4810 struct page *page;
4810 unsigned long num_pages; 4811 unsigned long num_pages;
4811 4812
4812 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 4813 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4813 num_pages = num_extent_pages(eb->start, eb->len); 4814 num_pages = num_extent_pages(eb->start, eb->len);
4814 for (i = 0; i < num_pages; i++) { 4815 for (i = 0; i < num_pages; i++) {
4815 page = extent_buffer_page(eb, i); 4816 page = extent_buffer_page(eb, i);
4816 SetPageUptodate(page); 4817 SetPageUptodate(page);
4817 } 4818 }
4818 return 0; 4819 return 0;
4819 } 4820 }
4820 4821
4821 int extent_buffer_uptodate(struct extent_buffer *eb) 4822 int extent_buffer_uptodate(struct extent_buffer *eb)
4822 { 4823 {
4823 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 4824 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4824 } 4825 }
4825 4826
4826 int read_extent_buffer_pages(struct extent_io_tree *tree, 4827 int read_extent_buffer_pages(struct extent_io_tree *tree,
4827 struct extent_buffer *eb, u64 start, int wait, 4828 struct extent_buffer *eb, u64 start, int wait,
4828 get_extent_t *get_extent, int mirror_num) 4829 get_extent_t *get_extent, int mirror_num)
4829 { 4830 {
4830 unsigned long i; 4831 unsigned long i;
4831 unsigned long start_i; 4832 unsigned long start_i;
4832 struct page *page; 4833 struct page *page;
4833 int err; 4834 int err;
4834 int ret = 0; 4835 int ret = 0;
4835 int locked_pages = 0; 4836 int locked_pages = 0;
4836 int all_uptodate = 1; 4837 int all_uptodate = 1;
4837 unsigned long num_pages; 4838 unsigned long num_pages;
4838 unsigned long num_reads = 0; 4839 unsigned long num_reads = 0;
4839 struct bio *bio = NULL; 4840 struct bio *bio = NULL;
4840 unsigned long bio_flags = 0; 4841 unsigned long bio_flags = 0;
4841 4842
4842 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) 4843 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4843 return 0; 4844 return 0;
4844 4845
4845 if (start) { 4846 if (start) {
4846 WARN_ON(start < eb->start); 4847 WARN_ON(start < eb->start);
4847 start_i = (start >> PAGE_CACHE_SHIFT) - 4848 start_i = (start >> PAGE_CACHE_SHIFT) -
4848 (eb->start >> PAGE_CACHE_SHIFT); 4849 (eb->start >> PAGE_CACHE_SHIFT);
4849 } else { 4850 } else {
4850 start_i = 0; 4851 start_i = 0;
4851 } 4852 }
4852 4853
4853 num_pages = num_extent_pages(eb->start, eb->len); 4854 num_pages = num_extent_pages(eb->start, eb->len);
4854 for (i = start_i; i < num_pages; i++) { 4855 for (i = start_i; i < num_pages; i++) {
4855 page = extent_buffer_page(eb, i); 4856 page = extent_buffer_page(eb, i);
4856 if (wait == WAIT_NONE) { 4857 if (wait == WAIT_NONE) {
4857 if (!trylock_page(page)) 4858 if (!trylock_page(page))
4858 goto unlock_exit; 4859 goto unlock_exit;
4859 } else { 4860 } else {
4860 lock_page(page); 4861 lock_page(page);
4861 } 4862 }
4862 locked_pages++; 4863 locked_pages++;
4863 if (!PageUptodate(page)) { 4864 if (!PageUptodate(page)) {
4864 num_reads++; 4865 num_reads++;
4865 all_uptodate = 0; 4866 all_uptodate = 0;
4866 } 4867 }
4867 } 4868 }
4868 if (all_uptodate) { 4869 if (all_uptodate) {
4869 if (start_i == 0) 4870 if (start_i == 0)
4870 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 4871 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4871 goto unlock_exit; 4872 goto unlock_exit;
4872 } 4873 }
4873 4874
4874 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags); 4875 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
4875 eb->read_mirror = 0; 4876 eb->read_mirror = 0;
4876 atomic_set(&eb->io_pages, num_reads); 4877 atomic_set(&eb->io_pages, num_reads);
4877 for (i = start_i; i < num_pages; i++) { 4878 for (i = start_i; i < num_pages; i++) {
4878 page = extent_buffer_page(eb, i); 4879 page = extent_buffer_page(eb, i);
4879 if (!PageUptodate(page)) { 4880 if (!PageUptodate(page)) {
4880 ClearPageError(page); 4881 ClearPageError(page);
4881 err = __extent_read_full_page(tree, page, 4882 err = __extent_read_full_page(tree, page,
4882 get_extent, &bio, 4883 get_extent, &bio,
4883 mirror_num, &bio_flags, 4884 mirror_num, &bio_flags,
4884 READ | REQ_META); 4885 READ | REQ_META);
4885 if (err) 4886 if (err)
4886 ret = err; 4887 ret = err;
4887 } else { 4888 } else {
4888 unlock_page(page); 4889 unlock_page(page);
4889 } 4890 }
4890 } 4891 }
4891 4892
4892 if (bio) { 4893 if (bio) {
4893 err = submit_one_bio(READ | REQ_META, bio, mirror_num, 4894 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
4894 bio_flags); 4895 bio_flags);
4895 if (err) 4896 if (err)
4896 return err; 4897 return err;
4897 } 4898 }
4898 4899
4899 if (ret || wait != WAIT_COMPLETE) 4900 if (ret || wait != WAIT_COMPLETE)
4900 return ret; 4901 return ret;
4901 4902
4902 for (i = start_i; i < num_pages; i++) { 4903 for (i = start_i; i < num_pages; i++) {
4903 page = extent_buffer_page(eb, i); 4904 page = extent_buffer_page(eb, i);
4904 wait_on_page_locked(page); 4905 wait_on_page_locked(page);
4905 if (!PageUptodate(page)) 4906 if (!PageUptodate(page))
4906 ret = -EIO; 4907 ret = -EIO;
4907 } 4908 }
4908 4909
4909 return ret; 4910 return ret;
4910 4911
4911 unlock_exit: 4912 unlock_exit:
4912 i = start_i; 4913 i = start_i;
4913 while (locked_pages > 0) { 4914 while (locked_pages > 0) {
4914 page = extent_buffer_page(eb, i); 4915 page = extent_buffer_page(eb, i);
4915 i++; 4916 i++;
4916 unlock_page(page); 4917 unlock_page(page);
4917 locked_pages--; 4918 locked_pages--;
4918 } 4919 }
4919 return ret; 4920 return ret;
4920 } 4921 }
4921 4922
4922 void read_extent_buffer(struct extent_buffer *eb, void *dstv, 4923 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4923 unsigned long start, 4924 unsigned long start,
4924 unsigned long len) 4925 unsigned long len)
4925 { 4926 {
4926 size_t cur; 4927 size_t cur;
4927 size_t offset; 4928 size_t offset;
4928 struct page *page; 4929 struct page *page;
4929 char *kaddr; 4930 char *kaddr;
4930 char *dst = (char *)dstv; 4931 char *dst = (char *)dstv;
4931 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 4932 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4932 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 4933 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4933 4934
4934 WARN_ON(start > eb->len); 4935 WARN_ON(start > eb->len);
4935 WARN_ON(start + len > eb->start + eb->len); 4936 WARN_ON(start + len > eb->start + eb->len);
4936 4937
4937 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1); 4938 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4938 4939
4939 while (len > 0) { 4940 while (len > 0) {
4940 page = extent_buffer_page(eb, i); 4941 page = extent_buffer_page(eb, i);
4941 4942
4942 cur = min(len, (PAGE_CACHE_SIZE - offset)); 4943 cur = min(len, (PAGE_CACHE_SIZE - offset));
4943 kaddr = page_address(page); 4944 kaddr = page_address(page);
4944 memcpy(dst, kaddr + offset, cur); 4945 memcpy(dst, kaddr + offset, cur);
4945 4946
4946 dst += cur; 4947 dst += cur;
4947 len -= cur; 4948 len -= cur;
4948 offset = 0; 4949 offset = 0;
4949 i++; 4950 i++;
4950 } 4951 }
4951 } 4952 }
4952 4953
4953 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start, 4954 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
4954 unsigned long min_len, char **map, 4955 unsigned long min_len, char **map,
4955 unsigned long *map_start, 4956 unsigned long *map_start,
4956 unsigned long *map_len) 4957 unsigned long *map_len)
4957 { 4958 {
4958 size_t offset = start & (PAGE_CACHE_SIZE - 1); 4959 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4959 char *kaddr; 4960 char *kaddr;
4960 struct page *p; 4961 struct page *p;
4961 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 4962 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4962 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 4963 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4963 unsigned long end_i = (start_offset + start + min_len - 1) >> 4964 unsigned long end_i = (start_offset + start + min_len - 1) >>
4964 PAGE_CACHE_SHIFT; 4965 PAGE_CACHE_SHIFT;
4965 4966
4966 if (i != end_i) 4967 if (i != end_i)
4967 return -EINVAL; 4968 return -EINVAL;
4968 4969
4969 if (i == 0) { 4970 if (i == 0) {
4970 offset = start_offset; 4971 offset = start_offset;
4971 *map_start = 0; 4972 *map_start = 0;
4972 } else { 4973 } else {
4973 offset = 0; 4974 offset = 0;
4974 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset; 4975 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4975 } 4976 }
4976 4977
4977 if (start + min_len > eb->len) { 4978 if (start + min_len > eb->len) {
4978 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, " 4979 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4979 "wanted %lu %lu\n", 4980 "wanted %lu %lu\n",
4980 eb->start, eb->len, start, min_len); 4981 eb->start, eb->len, start, min_len);
4981 return -EINVAL; 4982 return -EINVAL;
4982 } 4983 }
4983 4984
4984 p = extent_buffer_page(eb, i); 4985 p = extent_buffer_page(eb, i);
4985 kaddr = page_address(p); 4986 kaddr = page_address(p);
4986 *map = kaddr + offset; 4987 *map = kaddr + offset;
4987 *map_len = PAGE_CACHE_SIZE - offset; 4988 *map_len = PAGE_CACHE_SIZE - offset;
4988 return 0; 4989 return 0;
4989 } 4990 }
4990 4991
4991 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv, 4992 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4992 unsigned long start, 4993 unsigned long start,
4993 unsigned long len) 4994 unsigned long len)
4994 { 4995 {
4995 size_t cur; 4996 size_t cur;
4996 size_t offset; 4997 size_t offset;
4997 struct page *page; 4998 struct page *page;
4998 char *kaddr; 4999 char *kaddr;
4999 char *ptr = (char *)ptrv; 5000 char *ptr = (char *)ptrv;
5000 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 5001 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5001 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 5002 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5002 int ret = 0; 5003 int ret = 0;
5003 5004
5004 WARN_ON(start > eb->len); 5005 WARN_ON(start > eb->len);
5005 WARN_ON(start + len > eb->start + eb->len); 5006 WARN_ON(start + len > eb->start + eb->len);
5006 5007
5007 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1); 5008 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5008 5009
5009 while (len > 0) { 5010 while (len > 0) {
5010 page = extent_buffer_page(eb, i); 5011 page = extent_buffer_page(eb, i);
5011 5012
5012 cur = min(len, (PAGE_CACHE_SIZE - offset)); 5013 cur = min(len, (PAGE_CACHE_SIZE - offset));
5013 5014
5014 kaddr = page_address(page); 5015 kaddr = page_address(page);
5015 ret = memcmp(ptr, kaddr + offset, cur); 5016 ret = memcmp(ptr, kaddr + offset, cur);
5016 if (ret) 5017 if (ret)
5017 break; 5018 break;
5018 5019
5019 ptr += cur; 5020 ptr += cur;
5020 len -= cur; 5021 len -= cur;
5021 offset = 0; 5022 offset = 0;
5022 i++; 5023 i++;
5023 } 5024 }
5024 return ret; 5025 return ret;
5025 } 5026 }
5026 5027
5027 void write_extent_buffer(struct extent_buffer *eb, const void *srcv, 5028 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5028 unsigned long start, unsigned long len) 5029 unsigned long start, unsigned long len)
5029 { 5030 {
5030 size_t cur; 5031 size_t cur;
5031 size_t offset; 5032 size_t offset;
5032 struct page *page; 5033 struct page *page;
5033 char *kaddr; 5034 char *kaddr;
5034 char *src = (char *)srcv; 5035 char *src = (char *)srcv;
5035 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 5036 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5036 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 5037 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5037 5038
5038 WARN_ON(start > eb->len); 5039 WARN_ON(start > eb->len);
5039 WARN_ON(start + len > eb->start + eb->len); 5040 WARN_ON(start + len > eb->start + eb->len);
5040 5041
5041 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1); 5042 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5042 5043
5043 while (len > 0) { 5044 while (len > 0) {
5044 page = extent_buffer_page(eb, i); 5045 page = extent_buffer_page(eb, i);
5045 WARN_ON(!PageUptodate(page)); 5046 WARN_ON(!PageUptodate(page));
5046 5047
5047 cur = min(len, PAGE_CACHE_SIZE - offset); 5048 cur = min(len, PAGE_CACHE_SIZE - offset);
5048 kaddr = page_address(page); 5049 kaddr = page_address(page);
5049 memcpy(kaddr + offset, src, cur); 5050 memcpy(kaddr + offset, src, cur);
5050 5051
5051 src += cur; 5052 src += cur;
5052 len -= cur; 5053 len -= cur;
5053 offset = 0; 5054 offset = 0;
5054 i++; 5055 i++;
5055 } 5056 }
5056 } 5057 }
5057 5058
5058 void memset_extent_buffer(struct extent_buffer *eb, char c, 5059 void memset_extent_buffer(struct extent_buffer *eb, char c,
5059 unsigned long start, unsigned long len) 5060 unsigned long start, unsigned long len)
5060 { 5061 {
5061 size_t cur; 5062 size_t cur;
5062 size_t offset; 5063 size_t offset;
5063 struct page *page; 5064 struct page *page;
5064 char *kaddr; 5065 char *kaddr;
5065 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1); 5066 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5066 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT; 5067 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5067 5068
5068 WARN_ON(start > eb->len); 5069 WARN_ON(start > eb->len);
5069 WARN_ON(start + len > eb->start + eb->len); 5070 WARN_ON(start + len > eb->start + eb->len);
5070 5071
5071 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1); 5072 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5072 5073
5073 while (len > 0) { 5074 while (len > 0) {
5074 page = extent_buffer_page(eb, i); 5075 page = extent_buffer_page(eb, i);
5075 WARN_ON(!PageUptodate(page)); 5076 WARN_ON(!PageUptodate(page));
5076 5077
5077 cur = min(len, PAGE_CACHE_SIZE - offset); 5078 cur = min(len, PAGE_CACHE_SIZE - offset);
5078 kaddr = page_address(page); 5079 kaddr = page_address(page);
5079 memset(kaddr + offset, c, cur); 5080 memset(kaddr + offset, c, cur);
5080 5081
5081 len -= cur; 5082 len -= cur;
5082 offset = 0; 5083 offset = 0;
5083 i++; 5084 i++;
5084 } 5085 }
5085 } 5086 }
5086 5087
5087 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, 5088 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5088 unsigned long dst_offset, unsigned long src_offset, 5089 unsigned long dst_offset, unsigned long src_offset,
5089 unsigned long len) 5090 unsigned long len)
5090 { 5091 {
5091 u64 dst_len = dst->len; 5092 u64 dst_len = dst->len;
5092 size_t cur; 5093 size_t cur;
5093 size_t offset; 5094 size_t offset;
5094 struct page *page; 5095 struct page *page;
5095 char *kaddr; 5096 char *kaddr;
5096 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 5097 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5097 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT; 5098 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5098 5099
5099 WARN_ON(src->len != dst_len); 5100 WARN_ON(src->len != dst_len);
5100 5101
5101 offset = (start_offset + dst_offset) & 5102 offset = (start_offset + dst_offset) &
5102 (PAGE_CACHE_SIZE - 1); 5103 (PAGE_CACHE_SIZE - 1);
5103 5104
5104 while (len > 0) { 5105 while (len > 0) {
5105 page = extent_buffer_page(dst, i); 5106 page = extent_buffer_page(dst, i);
5106 WARN_ON(!PageUptodate(page)); 5107 WARN_ON(!PageUptodate(page));
5107 5108
5108 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset)); 5109 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5109 5110
5110 kaddr = page_address(page); 5111 kaddr = page_address(page);
5111 read_extent_buffer(src, kaddr + offset, src_offset, cur); 5112 read_extent_buffer(src, kaddr + offset, src_offset, cur);
5112 5113
5113 src_offset += cur; 5114 src_offset += cur;
5114 len -= cur; 5115 len -= cur;
5115 offset = 0; 5116 offset = 0;
5116 i++; 5117 i++;
5117 } 5118 }
5118 } 5119 }
5119 5120
5120 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len) 5121 static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5121 { 5122 {
5122 unsigned long distance = (src > dst) ? src - dst : dst - src; 5123 unsigned long distance = (src > dst) ? src - dst : dst - src;
5123 return distance < len; 5124 return distance < len;
5124 } 5125 }
5125 5126
5126 static void copy_pages(struct page *dst_page, struct page *src_page, 5127 static void copy_pages(struct page *dst_page, struct page *src_page,
5127 unsigned long dst_off, unsigned long src_off, 5128 unsigned long dst_off, unsigned long src_off,
5128 unsigned long len) 5129 unsigned long len)
5129 { 5130 {
5130 char *dst_kaddr = page_address(dst_page); 5131 char *dst_kaddr = page_address(dst_page);
5131 char *src_kaddr; 5132 char *src_kaddr;
5132 int must_memmove = 0; 5133 int must_memmove = 0;
5133 5134
5134 if (dst_page != src_page) { 5135 if (dst_page != src_page) {
5135 src_kaddr = page_address(src_page); 5136 src_kaddr = page_address(src_page);
5136 } else { 5137 } else {
5137 src_kaddr = dst_kaddr; 5138 src_kaddr = dst_kaddr;
5138 if (areas_overlap(src_off, dst_off, len)) 5139 if (areas_overlap(src_off, dst_off, len))
5139 must_memmove = 1; 5140 must_memmove = 1;
5140 } 5141 }
5141 5142
5142 if (must_memmove) 5143 if (must_memmove)
5143 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len); 5144 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5144 else 5145 else
5145 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len); 5146 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
5146 } 5147 }
5147 5148
5148 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 5149 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5149 unsigned long src_offset, unsigned long len) 5150 unsigned long src_offset, unsigned long len)
5150 { 5151 {
5151 size_t cur; 5152 size_t cur;
5152 size_t dst_off_in_page; 5153 size_t dst_off_in_page;
5153 size_t src_off_in_page; 5154 size_t src_off_in_page;
5154 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 5155 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5155 unsigned long dst_i; 5156 unsigned long dst_i;
5156 unsigned long src_i; 5157 unsigned long src_i;
5157 5158
5158 if (src_offset + len > dst->len) { 5159 if (src_offset + len > dst->len) {
5159 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move " 5160 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
5160 "len %lu dst len %lu\n", src_offset, len, dst->len); 5161 "len %lu dst len %lu\n", src_offset, len, dst->len);
5161 BUG_ON(1); 5162 BUG_ON(1);
5162 } 5163 }
5163 if (dst_offset + len > dst->len) { 5164 if (dst_offset + len > dst->len) {
5164 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move " 5165 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
5165 "len %lu dst len %lu\n", dst_offset, len, dst->len); 5166 "len %lu dst len %lu\n", dst_offset, len, dst->len);
5166 BUG_ON(1); 5167 BUG_ON(1);
5167 } 5168 }
5168 5169
5169 while (len > 0) { 5170 while (len > 0) {
5170 dst_off_in_page = (start_offset + dst_offset) & 5171 dst_off_in_page = (start_offset + dst_offset) &
5171 (PAGE_CACHE_SIZE - 1); 5172 (PAGE_CACHE_SIZE - 1);
5172 src_off_in_page = (start_offset + src_offset) & 5173 src_off_in_page = (start_offset + src_offset) &
5173 (PAGE_CACHE_SIZE - 1); 5174 (PAGE_CACHE_SIZE - 1);
5174 5175
5175 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT; 5176 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5176 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT; 5177 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5177 5178
5178 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - 5179 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5179 src_off_in_page)); 5180 src_off_in_page));
5180 cur = min_t(unsigned long, cur, 5181 cur = min_t(unsigned long, cur,
5181 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page)); 5182 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5182 5183
5183 copy_pages(extent_buffer_page(dst, dst_i), 5184 copy_pages(extent_buffer_page(dst, dst_i),
5184 extent_buffer_page(dst, src_i), 5185 extent_buffer_page(dst, src_i),
5185 dst_off_in_page, src_off_in_page, cur); 5186 dst_off_in_page, src_off_in_page, cur);
5186 5187
5187 src_offset += cur; 5188 src_offset += cur;
5188 dst_offset += cur; 5189 dst_offset += cur;
5189 len -= cur; 5190 len -= cur;
5190 } 5191 }
5191 } 5192 }
5192 5193
5193 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, 5194 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5194 unsigned long src_offset, unsigned long len) 5195 unsigned long src_offset, unsigned long len)
5195 { 5196 {
5196 size_t cur; 5197 size_t cur;
5197 size_t dst_off_in_page; 5198 size_t dst_off_in_page;
5198 size_t src_off_in_page; 5199 size_t src_off_in_page;
5199 unsigned long dst_end = dst_offset + len - 1; 5200 unsigned long dst_end = dst_offset + len - 1;
5200 unsigned long src_end = src_offset + len - 1; 5201 unsigned long src_end = src_offset + len - 1;
5201 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1); 5202 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5202 unsigned long dst_i; 5203 unsigned long dst_i;
5203 unsigned long src_i; 5204 unsigned long src_i;
5204 5205
5205 if (src_offset + len > dst->len) { 5206 if (src_offset + len > dst->len) {
5206 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move " 5207 printk(KERN_ERR "BTRFS: memmove bogus src_offset %lu move "
5207 "len %lu len %lu\n", src_offset, len, dst->len); 5208 "len %lu len %lu\n", src_offset, len, dst->len);
5208 BUG_ON(1); 5209 BUG_ON(1);
5209 } 5210 }
5210 if (dst_offset + len > dst->len) { 5211 if (dst_offset + len > dst->len) {
5211 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move " 5212 printk(KERN_ERR "BTRFS: memmove bogus dst_offset %lu move "
5212 "len %lu len %lu\n", dst_offset, len, dst->len); 5213 "len %lu len %lu\n", dst_offset, len, dst->len);
5213 BUG_ON(1); 5214 BUG_ON(1);
5214 } 5215 }
5215 if (dst_offset < src_offset) { 5216 if (dst_offset < src_offset) {
5216 memcpy_extent_buffer(dst, dst_offset, src_offset, len); 5217 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5217 return; 5218 return;
5218 } 5219 }
5219 while (len > 0) { 5220 while (len > 0) {
5220 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT; 5221 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5221 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT; 5222 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5222 5223
5223 dst_off_in_page = (start_offset + dst_end) & 5224 dst_off_in_page = (start_offset + dst_end) &
5224 (PAGE_CACHE_SIZE - 1); 5225 (PAGE_CACHE_SIZE - 1);
5225 src_off_in_page = (start_offset + src_end) & 5226 src_off_in_page = (start_offset + src_end) &
5226 (PAGE_CACHE_SIZE - 1); 5227 (PAGE_CACHE_SIZE - 1);
5227 5228
5228 cur = min_t(unsigned long, len, src_off_in_page + 1); 5229 cur = min_t(unsigned long, len, src_off_in_page + 1);
5229 cur = min(cur, dst_off_in_page + 1); 5230 cur = min(cur, dst_off_in_page + 1);
5230 copy_pages(extent_buffer_page(dst, dst_i), 5231 copy_pages(extent_buffer_page(dst, dst_i),
5231 extent_buffer_page(dst, src_i), 5232 extent_buffer_page(dst, src_i),
5232 dst_off_in_page - cur + 1, 5233 dst_off_in_page - cur + 1,
5233 src_off_in_page - cur + 1, cur); 5234 src_off_in_page - cur + 1, cur);
5234 5235
5235 dst_end -= cur; 5236 dst_end -= cur;
5236 src_end -= cur; 5237 src_end -= cur;
5237 len -= cur; 5238 len -= cur;
5238 } 5239 }
5239 } 5240 }
5240 5241
5241 int try_release_extent_buffer(struct page *page) 5242 int try_release_extent_buffer(struct page *page)
5242 { 5243 {
5243 struct extent_buffer *eb; 5244 struct extent_buffer *eb;
5244 5245
5245 /* 5246 /*
5246 * We need to make sure noboody is attaching this page to an eb right 5247 * We need to make sure noboody is attaching this page to an eb right
5247 * now. 5248 * now.
5248 */ 5249 */
5249 spin_lock(&page->mapping->private_lock); 5250 spin_lock(&page->mapping->private_lock);
5250 if (!PagePrivate(page)) { 5251 if (!PagePrivate(page)) {
5251 spin_unlock(&page->mapping->private_lock); 5252 spin_unlock(&page->mapping->private_lock);
5252 return 1; 5253 return 1;
5253 } 5254 }
5254 5255
5255 eb = (struct extent_buffer *)page->private; 5256 eb = (struct extent_buffer *)page->private;
5256 BUG_ON(!eb); 5257 BUG_ON(!eb);
5257 5258
5258 /* 5259 /*
5259 * This is a little awful but should be ok, we need to make sure that 5260 * This is a little awful but should be ok, we need to make sure that
5260 * the eb doesn't disappear out from under us while we're looking at 5261 * the eb doesn't disappear out from under us while we're looking at
5261 * this page. 5262 * this page.
5262 */ 5263 */
5263 spin_lock(&eb->refs_lock); 5264 spin_lock(&eb->refs_lock);
5264 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { 5265 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5265 spin_unlock(&eb->refs_lock); 5266 spin_unlock(&eb->refs_lock);
5266 spin_unlock(&page->mapping->private_lock); 5267 spin_unlock(&page->mapping->private_lock);
5267 return 0; 5268 return 0;
5268 } 5269 }
5269 spin_unlock(&page->mapping->private_lock); 5270 spin_unlock(&page->mapping->private_lock);
5270 5271
5271 /* 5272 /*
5272 * If tree ref isn't set then we know the ref on this eb is a real ref, 5273 * If tree ref isn't set then we know the ref on this eb is a real ref,
5273 * so just return, this page will likely be freed soon anyway. 5274 * so just return, this page will likely be freed soon anyway.
5274 */ 5275 */
5275 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { 5276 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5276 spin_unlock(&eb->refs_lock); 5277 spin_unlock(&eb->refs_lock);
5277 return 0; 5278 return 0;
5278 } 5279 }
5279 5280
5280 return release_extent_buffer(eb); 5281 return release_extent_buffer(eb);
5281 } 5282 }
5282 5283