Merge tag 'for-5.1-part2-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave...
[muen/linux.git] / fs / btrfs / extent_io.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/bitops.h>
4 #include <linux/slab.h>
5 #include <linux/bio.h>
6 #include <linux/mm.h>
7 #include <linux/pagemap.h>
8 #include <linux/page-flags.h>
9 #include <linux/spinlock.h>
10 #include <linux/blkdev.h>
11 #include <linux/swap.h>
12 #include <linux/writeback.h>
13 #include <linux/pagevec.h>
14 #include <linux/prefetch.h>
15 #include <linux/cleancache.h>
16 #include "extent_io.h"
17 #include "extent_map.h"
18 #include "ctree.h"
19 #include "btrfs_inode.h"
20 #include "volumes.h"
21 #include "check-integrity.h"
22 #include "locking.h"
23 #include "rcu-string.h"
24 #include "backref.h"
25 #include "disk-io.h"
26
27 static struct kmem_cache *extent_state_cache;
28 static struct kmem_cache *extent_buffer_cache;
29 static struct bio_set btrfs_bioset;
30
31 static inline bool extent_state_in_tree(const struct extent_state *state)
32 {
33         return !RB_EMPTY_NODE(&state->rb_node);
34 }
35
36 #ifdef CONFIG_BTRFS_DEBUG
37 static LIST_HEAD(buffers);
38 static LIST_HEAD(states);
39
40 static DEFINE_SPINLOCK(leak_lock);
41
42 static inline
43 void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
44 {
45         unsigned long flags;
46
47         spin_lock_irqsave(&leak_lock, flags);
48         list_add(new, head);
49         spin_unlock_irqrestore(&leak_lock, flags);
50 }
51
52 static inline
53 void btrfs_leak_debug_del(struct list_head *entry)
54 {
55         unsigned long flags;
56
57         spin_lock_irqsave(&leak_lock, flags);
58         list_del(entry);
59         spin_unlock_irqrestore(&leak_lock, flags);
60 }
61
62 static inline
63 void btrfs_leak_debug_check(void)
64 {
65         struct extent_state *state;
66         struct extent_buffer *eb;
67
68         while (!list_empty(&states)) {
69                 state = list_entry(states.next, struct extent_state, leak_list);
70                 pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
71                        state->start, state->end, state->state,
72                        extent_state_in_tree(state),
73                        refcount_read(&state->refs));
74                 list_del(&state->leak_list);
75                 kmem_cache_free(extent_state_cache, state);
76         }
77
78         while (!list_empty(&buffers)) {
79                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
80                 pr_err("BTRFS: buffer leak start %llu len %lu refs %d bflags %lu\n",
81                        eb->start, eb->len, atomic_read(&eb->refs), eb->bflags);
82                 list_del(&eb->leak_list);
83                 kmem_cache_free(extent_buffer_cache, eb);
84         }
85 }
86
87 #define btrfs_debug_check_extent_io_range(tree, start, end)             \
88         __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
89 static inline void __btrfs_debug_check_extent_io_range(const char *caller,
90                 struct extent_io_tree *tree, u64 start, u64 end)
91 {
92         struct inode *inode = tree->private_data;
93         u64 isize;
94
95         if (!inode || !is_data_inode(inode))
96                 return;
97
98         isize = i_size_read(inode);
99         if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
100                 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
101                     "%s: ino %llu isize %llu odd range [%llu,%llu]",
102                         caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
103         }
104 }
105 #else
106 #define btrfs_leak_debug_add(new, head) do {} while (0)
107 #define btrfs_leak_debug_del(entry)     do {} while (0)
108 #define btrfs_leak_debug_check()        do {} while (0)
109 #define btrfs_debug_check_extent_io_range(c, s, e)      do {} while (0)
110 #endif
111
112 #define BUFFER_LRU_MAX 64
113
114 struct tree_entry {
115         u64 start;
116         u64 end;
117         struct rb_node rb_node;
118 };
119
120 struct extent_page_data {
121         struct bio *bio;
122         struct extent_io_tree *tree;
123         /* tells writepage not to lock the state bits for this range
124          * it still does the unlocking
125          */
126         unsigned int extent_locked:1;
127
128         /* tells the submit_bio code to use REQ_SYNC */
129         unsigned int sync_io:1;
130 };
131
132 static int add_extent_changeset(struct extent_state *state, unsigned bits,
133                                  struct extent_changeset *changeset,
134                                  int set)
135 {
136         int ret;
137
138         if (!changeset)
139                 return 0;
140         if (set && (state->state & bits) == bits)
141                 return 0;
142         if (!set && (state->state & bits) == 0)
143                 return 0;
144         changeset->bytes_changed += state->end - state->start + 1;
145         ret = ulist_add(&changeset->range_changed, state->start, state->end,
146                         GFP_ATOMIC);
147         return ret;
148 }
149
150 static int __must_check submit_one_bio(struct bio *bio, int mirror_num,
151                                        unsigned long bio_flags)
152 {
153         blk_status_t ret = 0;
154         struct bio_vec *bvec = bio_last_bvec_all(bio);
155         struct bio_vec bv;
156         struct extent_io_tree *tree = bio->bi_private;
157         u64 start;
158
159         mp_bvec_last_segment(bvec, &bv);
160         start = page_offset(bv.bv_page) + bv.bv_offset;
161
162         bio->bi_private = NULL;
163
164         if (tree->ops)
165                 ret = tree->ops->submit_bio_hook(tree->private_data, bio,
166                                            mirror_num, bio_flags, start);
167         else
168                 btrfsic_submit_bio(bio);
169
170         return blk_status_to_errno(ret);
171 }
172
173 static void flush_write_bio(struct extent_page_data *epd)
174 {
175         if (epd->bio) {
176                 int ret;
177
178                 ret = submit_one_bio(epd->bio, 0, 0);
179                 BUG_ON(ret < 0); /* -ENOMEM */
180                 epd->bio = NULL;
181         }
182 }
183
184 int __init extent_io_init(void)
185 {
186         extent_state_cache = kmem_cache_create("btrfs_extent_state",
187                         sizeof(struct extent_state), 0,
188                         SLAB_MEM_SPREAD, NULL);
189         if (!extent_state_cache)
190                 return -ENOMEM;
191
192         extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
193                         sizeof(struct extent_buffer), 0,
194                         SLAB_MEM_SPREAD, NULL);
195         if (!extent_buffer_cache)
196                 goto free_state_cache;
197
198         if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE,
199                         offsetof(struct btrfs_io_bio, bio),
200                         BIOSET_NEED_BVECS))
201                 goto free_buffer_cache;
202
203         if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE))
204                 goto free_bioset;
205
206         return 0;
207
208 free_bioset:
209         bioset_exit(&btrfs_bioset);
210
211 free_buffer_cache:
212         kmem_cache_destroy(extent_buffer_cache);
213         extent_buffer_cache = NULL;
214
215 free_state_cache:
216         kmem_cache_destroy(extent_state_cache);
217         extent_state_cache = NULL;
218         return -ENOMEM;
219 }
220
221 void __cold extent_io_exit(void)
222 {
223         btrfs_leak_debug_check();
224
225         /*
226          * Make sure all delayed rcu free are flushed before we
227          * destroy caches.
228          */
229         rcu_barrier();
230         kmem_cache_destroy(extent_state_cache);
231         kmem_cache_destroy(extent_buffer_cache);
232         bioset_exit(&btrfs_bioset);
233 }
234
235 void extent_io_tree_init(struct extent_io_tree *tree,
236                          void *private_data)
237 {
238         tree->state = RB_ROOT;
239         tree->ops = NULL;
240         tree->dirty_bytes = 0;
241         spin_lock_init(&tree->lock);
242         tree->private_data = private_data;
243 }
244
245 static struct extent_state *alloc_extent_state(gfp_t mask)
246 {
247         struct extent_state *state;
248
249         /*
250          * The given mask might be not appropriate for the slab allocator,
251          * drop the unsupported bits
252          */
253         mask &= ~(__GFP_DMA32|__GFP_HIGHMEM);
254         state = kmem_cache_alloc(extent_state_cache, mask);
255         if (!state)
256                 return state;
257         state->state = 0;
258         state->failrec = NULL;
259         RB_CLEAR_NODE(&state->rb_node);
260         btrfs_leak_debug_add(&state->leak_list, &states);
261         refcount_set(&state->refs, 1);
262         init_waitqueue_head(&state->wq);
263         trace_alloc_extent_state(state, mask, _RET_IP_);
264         return state;
265 }
266
267 void free_extent_state(struct extent_state *state)
268 {
269         if (!state)
270                 return;
271         if (refcount_dec_and_test(&state->refs)) {
272                 WARN_ON(extent_state_in_tree(state));
273                 btrfs_leak_debug_del(&state->leak_list);
274                 trace_free_extent_state(state, _RET_IP_);
275                 kmem_cache_free(extent_state_cache, state);
276         }
277 }
278
279 static struct rb_node *tree_insert(struct rb_root *root,
280                                    struct rb_node *search_start,
281                                    u64 offset,
282                                    struct rb_node *node,
283                                    struct rb_node ***p_in,
284                                    struct rb_node **parent_in)
285 {
286         struct rb_node **p;
287         struct rb_node *parent = NULL;
288         struct tree_entry *entry;
289
290         if (p_in && parent_in) {
291                 p = *p_in;
292                 parent = *parent_in;
293                 goto do_insert;
294         }
295
296         p = search_start ? &search_start : &root->rb_node;
297         while (*p) {
298                 parent = *p;
299                 entry = rb_entry(parent, struct tree_entry, rb_node);
300
301                 if (offset < entry->start)
302                         p = &(*p)->rb_left;
303                 else if (offset > entry->end)
304                         p = &(*p)->rb_right;
305                 else
306                         return parent;
307         }
308
309 do_insert:
310         rb_link_node(node, parent, p);
311         rb_insert_color(node, root);
312         return NULL;
313 }
314
315 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
316                                       struct rb_node **next_ret,
317                                       struct rb_node **prev_ret,
318                                       struct rb_node ***p_ret,
319                                       struct rb_node **parent_ret)
320 {
321         struct rb_root *root = &tree->state;
322         struct rb_node **n = &root->rb_node;
323         struct rb_node *prev = NULL;
324         struct rb_node *orig_prev = NULL;
325         struct tree_entry *entry;
326         struct tree_entry *prev_entry = NULL;
327
328         while (*n) {
329                 prev = *n;
330                 entry = rb_entry(prev, struct tree_entry, rb_node);
331                 prev_entry = entry;
332
333                 if (offset < entry->start)
334                         n = &(*n)->rb_left;
335                 else if (offset > entry->end)
336                         n = &(*n)->rb_right;
337                 else
338                         return *n;
339         }
340
341         if (p_ret)
342                 *p_ret = n;
343         if (parent_ret)
344                 *parent_ret = prev;
345
346         if (next_ret) {
347                 orig_prev = prev;
348                 while (prev && offset > prev_entry->end) {
349                         prev = rb_next(prev);
350                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
351                 }
352                 *next_ret = prev;
353                 prev = orig_prev;
354         }
355
356         if (prev_ret) {
357                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
358                 while (prev && offset < prev_entry->start) {
359                         prev = rb_prev(prev);
360                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
361                 }
362                 *prev_ret = prev;
363         }
364         return NULL;
365 }
366
367 static inline struct rb_node *
368 tree_search_for_insert(struct extent_io_tree *tree,
369                        u64 offset,
370                        struct rb_node ***p_ret,
371                        struct rb_node **parent_ret)
372 {
373         struct rb_node *next= NULL;
374         struct rb_node *ret;
375
376         ret = __etree_search(tree, offset, &next, NULL, p_ret, parent_ret);
377         if (!ret)
378                 return next;
379         return ret;
380 }
381
382 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
383                                           u64 offset)
384 {
385         return tree_search_for_insert(tree, offset, NULL, NULL);
386 }
387
388 /*
389  * utility function to look for merge candidates inside a given range.
390  * Any extents with matching state are merged together into a single
391  * extent in the tree.  Extents with EXTENT_IO in their state field
392  * are not merged because the end_io handlers need to be able to do
393  * operations on them without sleeping (or doing allocations/splits).
394  *
395  * This should be called with the tree lock held.
396  */
397 static void merge_state(struct extent_io_tree *tree,
398                         struct extent_state *state)
399 {
400         struct extent_state *other;
401         struct rb_node *other_node;
402
403         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
404                 return;
405
406         other_node = rb_prev(&state->rb_node);
407         if (other_node) {
408                 other = rb_entry(other_node, struct extent_state, rb_node);
409                 if (other->end == state->start - 1 &&
410                     other->state == state->state) {
411                         if (tree->private_data &&
412                             is_data_inode(tree->private_data))
413                                 btrfs_merge_delalloc_extent(tree->private_data,
414                                                             state, other);
415                         state->start = other->start;
416                         rb_erase(&other->rb_node, &tree->state);
417                         RB_CLEAR_NODE(&other->rb_node);
418                         free_extent_state(other);
419                 }
420         }
421         other_node = rb_next(&state->rb_node);
422         if (other_node) {
423                 other = rb_entry(other_node, struct extent_state, rb_node);
424                 if (other->start == state->end + 1 &&
425                     other->state == state->state) {
426                         if (tree->private_data &&
427                             is_data_inode(tree->private_data))
428                                 btrfs_merge_delalloc_extent(tree->private_data,
429                                                             state, other);
430                         state->end = other->end;
431                         rb_erase(&other->rb_node, &tree->state);
432                         RB_CLEAR_NODE(&other->rb_node);
433                         free_extent_state(other);
434                 }
435         }
436 }
437
438 static void set_state_bits(struct extent_io_tree *tree,
439                            struct extent_state *state, unsigned *bits,
440                            struct extent_changeset *changeset);
441
442 /*
443  * insert an extent_state struct into the tree.  'bits' are set on the
444  * struct before it is inserted.
445  *
446  * This may return -EEXIST if the extent is already there, in which case the
447  * state struct is freed.
448  *
449  * The tree lock is not taken internally.  This is a utility function and
450  * probably isn't what you want to call (see set/clear_extent_bit).
451  */
452 static int insert_state(struct extent_io_tree *tree,
453                         struct extent_state *state, u64 start, u64 end,
454                         struct rb_node ***p,
455                         struct rb_node **parent,
456                         unsigned *bits, struct extent_changeset *changeset)
457 {
458         struct rb_node *node;
459
460         if (end < start)
461                 WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
462                        end, start);
463         state->start = start;
464         state->end = end;
465
466         set_state_bits(tree, state, bits, changeset);
467
468         node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
469         if (node) {
470                 struct extent_state *found;
471                 found = rb_entry(node, struct extent_state, rb_node);
472                 pr_err("BTRFS: found node %llu %llu on insert of %llu %llu\n",
473                        found->start, found->end, start, end);
474                 return -EEXIST;
475         }
476         merge_state(tree, state);
477         return 0;
478 }
479
480 /*
481  * split a given extent state struct in two, inserting the preallocated
482  * struct 'prealloc' as the newly created second half.  'split' indicates an
483  * offset inside 'orig' where it should be split.
484  *
485  * Before calling,
486  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
487  * are two extent state structs in the tree:
488  * prealloc: [orig->start, split - 1]
489  * orig: [ split, orig->end ]
490  *
491  * The tree locks are not taken by this function. They need to be held
492  * by the caller.
493  */
494 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
495                        struct extent_state *prealloc, u64 split)
496 {
497         struct rb_node *node;
498
499         if (tree->private_data && is_data_inode(tree->private_data))
500                 btrfs_split_delalloc_extent(tree->private_data, orig, split);
501
502         prealloc->start = orig->start;
503         prealloc->end = split - 1;
504         prealloc->state = orig->state;
505         orig->start = split;
506
507         node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
508                            &prealloc->rb_node, NULL, NULL);
509         if (node) {
510                 free_extent_state(prealloc);
511                 return -EEXIST;
512         }
513         return 0;
514 }
515
516 static struct extent_state *next_state(struct extent_state *state)
517 {
518         struct rb_node *next = rb_next(&state->rb_node);
519         if (next)
520                 return rb_entry(next, struct extent_state, rb_node);
521         else
522                 return NULL;
523 }
524
525 /*
526  * utility function to clear some bits in an extent state struct.
527  * it will optionally wake up anyone waiting on this state (wake == 1).
528  *
529  * If no bits are set on the state struct after clearing things, the
530  * struct is freed and removed from the tree
531  */
532 static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
533                                             struct extent_state *state,
534                                             unsigned *bits, int wake,
535                                             struct extent_changeset *changeset)
536 {
537         struct extent_state *next;
538         unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
539         int ret;
540
541         if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
542                 u64 range = state->end - state->start + 1;
543                 WARN_ON(range > tree->dirty_bytes);
544                 tree->dirty_bytes -= range;
545         }
546
547         if (tree->private_data && is_data_inode(tree->private_data))
548                 btrfs_clear_delalloc_extent(tree->private_data, state, bits);
549
550         ret = add_extent_changeset(state, bits_to_clear, changeset, 0);
551         BUG_ON(ret < 0);
552         state->state &= ~bits_to_clear;
553         if (wake)
554                 wake_up(&state->wq);
555         if (state->state == 0) {
556                 next = next_state(state);
557                 if (extent_state_in_tree(state)) {
558                         rb_erase(&state->rb_node, &tree->state);
559                         RB_CLEAR_NODE(&state->rb_node);
560                         free_extent_state(state);
561                 } else {
562                         WARN_ON(1);
563                 }
564         } else {
565                 merge_state(tree, state);
566                 next = next_state(state);
567         }
568         return next;
569 }
570
571 static struct extent_state *
572 alloc_extent_state_atomic(struct extent_state *prealloc)
573 {
574         if (!prealloc)
575                 prealloc = alloc_extent_state(GFP_ATOMIC);
576
577         return prealloc;
578 }
579
580 static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
581 {
582         struct inode *inode = tree->private_data;
583
584         btrfs_panic(btrfs_sb(inode->i_sb), err,
585         "locking error: extent tree was modified by another thread while locked");
586 }
587
588 /*
589  * clear some bits on a range in the tree.  This may require splitting
590  * or inserting elements in the tree, so the gfp mask is used to
591  * indicate which allocations or sleeping are allowed.
592  *
593  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
594  * the given range from the tree regardless of state (ie for truncate).
595  *
596  * the range [start, end] is inclusive.
597  *
598  * This takes the tree lock, and returns 0 on success and < 0 on error.
599  */
600 int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
601                               unsigned bits, int wake, int delete,
602                               struct extent_state **cached_state,
603                               gfp_t mask, struct extent_changeset *changeset)
604 {
605         struct extent_state *state;
606         struct extent_state *cached;
607         struct extent_state *prealloc = NULL;
608         struct rb_node *node;
609         u64 last_end;
610         int err;
611         int clear = 0;
612
613         btrfs_debug_check_extent_io_range(tree, start, end);
614
615         if (bits & EXTENT_DELALLOC)
616                 bits |= EXTENT_NORESERVE;
617
618         if (delete)
619                 bits |= ~EXTENT_CTLBITS;
620
621         if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
622                 clear = 1;
623 again:
624         if (!prealloc && gfpflags_allow_blocking(mask)) {
625                 /*
626                  * Don't care for allocation failure here because we might end
627                  * up not needing the pre-allocated extent state at all, which
628                  * is the case if we only have in the tree extent states that
629                  * cover our input range and don't cover too any other range.
630                  * If we end up needing a new extent state we allocate it later.
631                  */
632                 prealloc = alloc_extent_state(mask);
633         }
634
635         spin_lock(&tree->lock);
636         if (cached_state) {
637                 cached = *cached_state;
638
639                 if (clear) {
640                         *cached_state = NULL;
641                         cached_state = NULL;
642                 }
643
644                 if (cached && extent_state_in_tree(cached) &&
645                     cached->start <= start && cached->end > start) {
646                         if (clear)
647                                 refcount_dec(&cached->refs);
648                         state = cached;
649                         goto hit_next;
650                 }
651                 if (clear)
652                         free_extent_state(cached);
653         }
654         /*
655          * this search will find the extents that end after
656          * our range starts
657          */
658         node = tree_search(tree, start);
659         if (!node)
660                 goto out;
661         state = rb_entry(node, struct extent_state, rb_node);
662 hit_next:
663         if (state->start > end)
664                 goto out;
665         WARN_ON(state->end < start);
666         last_end = state->end;
667
668         /* the state doesn't have the wanted bits, go ahead */
669         if (!(state->state & bits)) {
670                 state = next_state(state);
671                 goto next;
672         }
673
674         /*
675          *     | ---- desired range ---- |
676          *  | state | or
677          *  | ------------- state -------------- |
678          *
679          * We need to split the extent we found, and may flip
680          * bits on second half.
681          *
682          * If the extent we found extends past our range, we
683          * just split and search again.  It'll get split again
684          * the next time though.
685          *
686          * If the extent we found is inside our range, we clear
687          * the desired bit on it.
688          */
689
690         if (state->start < start) {
691                 prealloc = alloc_extent_state_atomic(prealloc);
692                 BUG_ON(!prealloc);
693                 err = split_state(tree, state, prealloc, start);
694                 if (err)
695                         extent_io_tree_panic(tree, err);
696
697                 prealloc = NULL;
698                 if (err)
699                         goto out;
700                 if (state->end <= end) {
701                         state = clear_state_bit(tree, state, &bits, wake,
702                                                 changeset);
703                         goto next;
704                 }
705                 goto search_again;
706         }
707         /*
708          * | ---- desired range ---- |
709          *                        | state |
710          * We need to split the extent, and clear the bit
711          * on the first half
712          */
713         if (state->start <= end && state->end > end) {
714                 prealloc = alloc_extent_state_atomic(prealloc);
715                 BUG_ON(!prealloc);
716                 err = split_state(tree, state, prealloc, end + 1);
717                 if (err)
718                         extent_io_tree_panic(tree, err);
719
720                 if (wake)
721                         wake_up(&state->wq);
722
723                 clear_state_bit(tree, prealloc, &bits, wake, changeset);
724
725                 prealloc = NULL;
726                 goto out;
727         }
728
729         state = clear_state_bit(tree, state, &bits, wake, changeset);
730 next:
731         if (last_end == (u64)-1)
732                 goto out;
733         start = last_end + 1;
734         if (start <= end && state && !need_resched())
735                 goto hit_next;
736
737 search_again:
738         if (start > end)
739                 goto out;
740         spin_unlock(&tree->lock);
741         if (gfpflags_allow_blocking(mask))
742                 cond_resched();
743         goto again;
744
745 out:
746         spin_unlock(&tree->lock);
747         if (prealloc)
748                 free_extent_state(prealloc);
749
750         return 0;
751
752 }
753
754 static void wait_on_state(struct extent_io_tree *tree,
755                           struct extent_state *state)
756                 __releases(tree->lock)
757                 __acquires(tree->lock)
758 {
759         DEFINE_WAIT(wait);
760         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
761         spin_unlock(&tree->lock);
762         schedule();
763         spin_lock(&tree->lock);
764         finish_wait(&state->wq, &wait);
765 }
766
767 /*
768  * waits for one or more bits to clear on a range in the state tree.
769  * The range [start, end] is inclusive.
770  * The tree lock is taken by this function
771  */
772 static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
773                             unsigned long bits)
774 {
775         struct extent_state *state;
776         struct rb_node *node;
777
778         btrfs_debug_check_extent_io_range(tree, start, end);
779
780         spin_lock(&tree->lock);
781 again:
782         while (1) {
783                 /*
784                  * this search will find all the extents that end after
785                  * our range starts
786                  */
787                 node = tree_search(tree, start);
788 process_node:
789                 if (!node)
790                         break;
791
792                 state = rb_entry(node, struct extent_state, rb_node);
793
794                 if (state->start > end)
795                         goto out;
796
797                 if (state->state & bits) {
798                         start = state->start;
799                         refcount_inc(&state->refs);
800                         wait_on_state(tree, state);
801                         free_extent_state(state);
802                         goto again;
803                 }
804                 start = state->end + 1;
805
806                 if (start > end)
807                         break;
808
809                 if (!cond_resched_lock(&tree->lock)) {
810                         node = rb_next(node);
811                         goto process_node;
812                 }
813         }
814 out:
815         spin_unlock(&tree->lock);
816 }
817
818 static void set_state_bits(struct extent_io_tree *tree,
819                            struct extent_state *state,
820                            unsigned *bits, struct extent_changeset *changeset)
821 {
822         unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
823         int ret;
824
825         if (tree->private_data && is_data_inode(tree->private_data))
826                 btrfs_set_delalloc_extent(tree->private_data, state, bits);
827
828         if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
829                 u64 range = state->end - state->start + 1;
830                 tree->dirty_bytes += range;
831         }
832         ret = add_extent_changeset(state, bits_to_set, changeset, 1);
833         BUG_ON(ret < 0);
834         state->state |= bits_to_set;
835 }
836
837 static void cache_state_if_flags(struct extent_state *state,
838                                  struct extent_state **cached_ptr,
839                                  unsigned flags)
840 {
841         if (cached_ptr && !(*cached_ptr)) {
842                 if (!flags || (state->state & flags)) {
843                         *cached_ptr = state;
844                         refcount_inc(&state->refs);
845                 }
846         }
847 }
848
849 static void cache_state(struct extent_state *state,
850                         struct extent_state **cached_ptr)
851 {
852         return cache_state_if_flags(state, cached_ptr,
853                                     EXTENT_IOBITS | EXTENT_BOUNDARY);
854 }
855
856 /*
857  * set some bits on a range in the tree.  This may require allocations or
858  * sleeping, so the gfp mask is used to indicate what is allowed.
859  *
860  * If any of the exclusive bits are set, this will fail with -EEXIST if some
861  * part of the range already has the desired bits set.  The start of the
862  * existing range is returned in failed_start in this case.
863  *
864  * [start, end] is inclusive This takes the tree lock.
865  */
866
867 static int __must_check
868 __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
869                  unsigned bits, unsigned exclusive_bits,
870                  u64 *failed_start, struct extent_state **cached_state,
871                  gfp_t mask, struct extent_changeset *changeset)
872 {
873         struct extent_state *state;
874         struct extent_state *prealloc = NULL;
875         struct rb_node *node;
876         struct rb_node **p;
877         struct rb_node *parent;
878         int err = 0;
879         u64 last_start;
880         u64 last_end;
881
882         btrfs_debug_check_extent_io_range(tree, start, end);
883
884 again:
885         if (!prealloc && gfpflags_allow_blocking(mask)) {
886                 /*
887                  * Don't care for allocation failure here because we might end
888                  * up not needing the pre-allocated extent state at all, which
889                  * is the case if we only have in the tree extent states that
890                  * cover our input range and don't cover too any other range.
891                  * If we end up needing a new extent state we allocate it later.
892                  */
893                 prealloc = alloc_extent_state(mask);
894         }
895
896         spin_lock(&tree->lock);
897         if (cached_state && *cached_state) {
898                 state = *cached_state;
899                 if (state->start <= start && state->end > start &&
900                     extent_state_in_tree(state)) {
901                         node = &state->rb_node;
902                         goto hit_next;
903                 }
904         }
905         /*
906          * this search will find all the extents that end after
907          * our range starts.
908          */
909         node = tree_search_for_insert(tree, start, &p, &parent);
910         if (!node) {
911                 prealloc = alloc_extent_state_atomic(prealloc);
912                 BUG_ON(!prealloc);
913                 err = insert_state(tree, prealloc, start, end,
914                                    &p, &parent, &bits, changeset);
915                 if (err)
916                         extent_io_tree_panic(tree, err);
917
918                 cache_state(prealloc, cached_state);
919                 prealloc = NULL;
920                 goto out;
921         }
922         state = rb_entry(node, struct extent_state, rb_node);
923 hit_next:
924         last_start = state->start;
925         last_end = state->end;
926
927         /*
928          * | ---- desired range ---- |
929          * | state |
930          *
931          * Just lock what we found and keep going
932          */
933         if (state->start == start && state->end <= end) {
934                 if (state->state & exclusive_bits) {
935                         *failed_start = state->start;
936                         err = -EEXIST;
937                         goto out;
938                 }
939
940                 set_state_bits(tree, state, &bits, changeset);
941                 cache_state(state, cached_state);
942                 merge_state(tree, state);
943                 if (last_end == (u64)-1)
944                         goto out;
945                 start = last_end + 1;
946                 state = next_state(state);
947                 if (start < end && state && state->start == start &&
948                     !need_resched())
949                         goto hit_next;
950                 goto search_again;
951         }
952
953         /*
954          *     | ---- desired range ---- |
955          * | state |
956          *   or
957          * | ------------- state -------------- |
958          *
959          * We need to split the extent we found, and may flip bits on
960          * second half.
961          *
962          * If the extent we found extends past our
963          * range, we just split and search again.  It'll get split
964          * again the next time though.
965          *
966          * If the extent we found is inside our range, we set the
967          * desired bit on it.
968          */
969         if (state->start < start) {
970                 if (state->state & exclusive_bits) {
971                         *failed_start = start;
972                         err = -EEXIST;
973                         goto out;
974                 }
975
976                 prealloc = alloc_extent_state_atomic(prealloc);
977                 BUG_ON(!prealloc);
978                 err = split_state(tree, state, prealloc, start);
979                 if (err)
980                         extent_io_tree_panic(tree, err);
981
982                 prealloc = NULL;
983                 if (err)
984                         goto out;
985                 if (state->end <= end) {
986                         set_state_bits(tree, state, &bits, changeset);
987                         cache_state(state, cached_state);
988                         merge_state(tree, state);
989                         if (last_end == (u64)-1)
990                                 goto out;
991                         start = last_end + 1;
992                         state = next_state(state);
993                         if (start < end && state && state->start == start &&
994                             !need_resched())
995                                 goto hit_next;
996                 }
997                 goto search_again;
998         }
999         /*
1000          * | ---- desired range ---- |
1001          *     | state | or               | state |
1002          *
1003          * There's a hole, we need to insert something in it and
1004          * ignore the extent we found.
1005          */
1006         if (state->start > start) {
1007                 u64 this_end;
1008                 if (end < last_start)
1009                         this_end = end;
1010                 else
1011                         this_end = last_start - 1;
1012
1013                 prealloc = alloc_extent_state_atomic(prealloc);
1014                 BUG_ON(!prealloc);
1015
1016                 /*
1017                  * Avoid to free 'prealloc' if it can be merged with
1018                  * the later extent.
1019                  */
1020                 err = insert_state(tree, prealloc, start, this_end,
1021                                    NULL, NULL, &bits, changeset);
1022                 if (err)
1023                         extent_io_tree_panic(tree, err);
1024
1025                 cache_state(prealloc, cached_state);
1026                 prealloc = NULL;
1027                 start = this_end + 1;
1028                 goto search_again;
1029         }
1030         /*
1031          * | ---- desired range ---- |
1032          *                        | state |
1033          * We need to split the extent, and set the bit
1034          * on the first half
1035          */
1036         if (state->start <= end && state->end > end) {
1037                 if (state->state & exclusive_bits) {
1038                         *failed_start = start;
1039                         err = -EEXIST;
1040                         goto out;
1041                 }
1042
1043                 prealloc = alloc_extent_state_atomic(prealloc);
1044                 BUG_ON(!prealloc);
1045                 err = split_state(tree, state, prealloc, end + 1);
1046                 if (err)
1047                         extent_io_tree_panic(tree, err);
1048
1049                 set_state_bits(tree, prealloc, &bits, changeset);
1050                 cache_state(prealloc, cached_state);
1051                 merge_state(tree, prealloc);
1052                 prealloc = NULL;
1053                 goto out;
1054         }
1055
1056 search_again:
1057         if (start > end)
1058                 goto out;
1059         spin_unlock(&tree->lock);
1060         if (gfpflags_allow_blocking(mask))
1061                 cond_resched();
1062         goto again;
1063
1064 out:
1065         spin_unlock(&tree->lock);
1066         if (prealloc)
1067                 free_extent_state(prealloc);
1068
1069         return err;
1070
1071 }
1072
1073 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1074                    unsigned bits, u64 * failed_start,
1075                    struct extent_state **cached_state, gfp_t mask)
1076 {
1077         return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1078                                 cached_state, mask, NULL);
1079 }
1080
1081
1082 /**
1083  * convert_extent_bit - convert all bits in a given range from one bit to
1084  *                      another
1085  * @tree:       the io tree to search
1086  * @start:      the start offset in bytes
1087  * @end:        the end offset in bytes (inclusive)
1088  * @bits:       the bits to set in this range
1089  * @clear_bits: the bits to clear in this range
1090  * @cached_state:       state that we're going to cache
1091  *
1092  * This will go through and set bits for the given range.  If any states exist
1093  * already in this range they are set with the given bit and cleared of the
1094  * clear_bits.  This is only meant to be used by things that are mergeable, ie
1095  * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1096  * boundary bits like LOCK.
1097  *
1098  * All allocations are done with GFP_NOFS.
1099  */
1100 int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1101                        unsigned bits, unsigned clear_bits,
1102                        struct extent_state **cached_state)
1103 {
1104         struct extent_state *state;
1105         struct extent_state *prealloc = NULL;
1106         struct rb_node *node;
1107         struct rb_node **p;
1108         struct rb_node *parent;
1109         int err = 0;
1110         u64 last_start;
1111         u64 last_end;
1112         bool first_iteration = true;
1113
1114         btrfs_debug_check_extent_io_range(tree, start, end);
1115
1116 again:
1117         if (!prealloc) {
1118                 /*
1119                  * Best effort, don't worry if extent state allocation fails
1120                  * here for the first iteration. We might have a cached state
1121                  * that matches exactly the target range, in which case no
1122                  * extent state allocations are needed. We'll only know this
1123                  * after locking the tree.
1124                  */
1125                 prealloc = alloc_extent_state(GFP_NOFS);
1126                 if (!prealloc && !first_iteration)
1127                         return -ENOMEM;
1128         }
1129
1130         spin_lock(&tree->lock);
1131         if (cached_state && *cached_state) {
1132                 state = *cached_state;
1133                 if (state->start <= start && state->end > start &&
1134                     extent_state_in_tree(state)) {
1135                         node = &state->rb_node;
1136                         goto hit_next;
1137                 }
1138         }
1139
1140         /*
1141          * this search will find all the extents that end after
1142          * our range starts.
1143          */
1144         node = tree_search_for_insert(tree, start, &p, &parent);
1145         if (!node) {
1146                 prealloc = alloc_extent_state_atomic(prealloc);
1147                 if (!prealloc) {
1148                         err = -ENOMEM;
1149                         goto out;
1150                 }
1151                 err = insert_state(tree, prealloc, start, end,
1152                                    &p, &parent, &bits, NULL);
1153                 if (err)
1154                         extent_io_tree_panic(tree, err);
1155                 cache_state(prealloc, cached_state);
1156                 prealloc = NULL;
1157                 goto out;
1158         }
1159         state = rb_entry(node, struct extent_state, rb_node);
1160 hit_next:
1161         last_start = state->start;
1162         last_end = state->end;
1163
1164         /*
1165          * | ---- desired range ---- |
1166          * | state |
1167          *
1168          * Just lock what we found and keep going
1169          */
1170         if (state->start == start && state->end <= end) {
1171                 set_state_bits(tree, state, &bits, NULL);
1172                 cache_state(state, cached_state);
1173                 state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1174                 if (last_end == (u64)-1)
1175                         goto out;
1176                 start = last_end + 1;
1177                 if (start < end && state && state->start == start &&
1178                     !need_resched())
1179                         goto hit_next;
1180                 goto search_again;
1181         }
1182
1183         /*
1184          *     | ---- desired range ---- |
1185          * | state |
1186          *   or
1187          * | ------------- state -------------- |
1188          *
1189          * We need to split the extent we found, and may flip bits on
1190          * second half.
1191          *
1192          * If the extent we found extends past our
1193          * range, we just split and search again.  It'll get split
1194          * again the next time though.
1195          *
1196          * If the extent we found is inside our range, we set the
1197          * desired bit on it.
1198          */
1199         if (state->start < start) {
1200                 prealloc = alloc_extent_state_atomic(prealloc);
1201                 if (!prealloc) {
1202                         err = -ENOMEM;
1203                         goto out;
1204                 }
1205                 err = split_state(tree, state, prealloc, start);
1206                 if (err)
1207                         extent_io_tree_panic(tree, err);
1208                 prealloc = NULL;
1209                 if (err)
1210                         goto out;
1211                 if (state->end <= end) {
1212                         set_state_bits(tree, state, &bits, NULL);
1213                         cache_state(state, cached_state);
1214                         state = clear_state_bit(tree, state, &clear_bits, 0,
1215                                                 NULL);
1216                         if (last_end == (u64)-1)
1217                                 goto out;
1218                         start = last_end + 1;
1219                         if (start < end && state && state->start == start &&
1220                             !need_resched())
1221                                 goto hit_next;
1222                 }
1223                 goto search_again;
1224         }
1225         /*
1226          * | ---- desired range ---- |
1227          *     | state | or               | state |
1228          *
1229          * There's a hole, we need to insert something in it and
1230          * ignore the extent we found.
1231          */
1232         if (state->start > start) {
1233                 u64 this_end;
1234                 if (end < last_start)
1235                         this_end = end;
1236                 else
1237                         this_end = last_start - 1;
1238
1239                 prealloc = alloc_extent_state_atomic(prealloc);
1240                 if (!prealloc) {
1241                         err = -ENOMEM;
1242                         goto out;
1243                 }
1244
1245                 /*
1246                  * Avoid to free 'prealloc' if it can be merged with
1247                  * the later extent.
1248                  */
1249                 err = insert_state(tree, prealloc, start, this_end,
1250                                    NULL, NULL, &bits, NULL);
1251                 if (err)
1252                         extent_io_tree_panic(tree, err);
1253                 cache_state(prealloc, cached_state);
1254                 prealloc = NULL;
1255                 start = this_end + 1;
1256                 goto search_again;
1257         }
1258         /*
1259          * | ---- desired range ---- |
1260          *                        | state |
1261          * We need to split the extent, and set the bit
1262          * on the first half
1263          */
1264         if (state->start <= end && state->end > end) {
1265                 prealloc = alloc_extent_state_atomic(prealloc);
1266                 if (!prealloc) {
1267                         err = -ENOMEM;
1268                         goto out;
1269                 }
1270
1271                 err = split_state(tree, state, prealloc, end + 1);
1272                 if (err)
1273                         extent_io_tree_panic(tree, err);
1274
1275                 set_state_bits(tree, prealloc, &bits, NULL);
1276                 cache_state(prealloc, cached_state);
1277                 clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1278                 prealloc = NULL;
1279                 goto out;
1280         }
1281
1282 search_again:
1283         if (start > end)
1284                 goto out;
1285         spin_unlock(&tree->lock);
1286         cond_resched();
1287         first_iteration = false;
1288         goto again;
1289
1290 out:
1291         spin_unlock(&tree->lock);
1292         if (prealloc)
1293                 free_extent_state(prealloc);
1294
1295         return err;
1296 }
1297
1298 /* wrappers around set/clear extent bit */
1299 int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1300                            unsigned bits, struct extent_changeset *changeset)
1301 {
1302         /*
1303          * We don't support EXTENT_LOCKED yet, as current changeset will
1304          * record any bits changed, so for EXTENT_LOCKED case, it will
1305          * either fail with -EEXIST or changeset will record the whole
1306          * range.
1307          */
1308         BUG_ON(bits & EXTENT_LOCKED);
1309
1310         return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS,
1311                                 changeset);
1312 }
1313
1314 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1315                      unsigned bits, int wake, int delete,
1316                      struct extent_state **cached)
1317 {
1318         return __clear_extent_bit(tree, start, end, bits, wake, delete,
1319                                   cached, GFP_NOFS, NULL);
1320 }
1321
1322 int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1323                 unsigned bits, struct extent_changeset *changeset)
1324 {
1325         /*
1326          * Don't support EXTENT_LOCKED case, same reason as
1327          * set_record_extent_bits().
1328          */
1329         BUG_ON(bits & EXTENT_LOCKED);
1330
1331         return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS,
1332                                   changeset);
1333 }
1334
1335 /*
1336  * either insert or lock state struct between start and end use mask to tell
1337  * us if waiting is desired.
1338  */
1339 int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1340                      struct extent_state **cached_state)
1341 {
1342         int err;
1343         u64 failed_start;
1344
1345         while (1) {
1346                 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1347                                        EXTENT_LOCKED, &failed_start,
1348                                        cached_state, GFP_NOFS, NULL);
1349                 if (err == -EEXIST) {
1350                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1351                         start = failed_start;
1352                 } else
1353                         break;
1354                 WARN_ON(start > end);
1355         }
1356         return err;
1357 }
1358
1359 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1360 {
1361         int err;
1362         u64 failed_start;
1363
1364         err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1365                                &failed_start, NULL, GFP_NOFS, NULL);
1366         if (err == -EEXIST) {
1367                 if (failed_start > start)
1368                         clear_extent_bit(tree, start, failed_start - 1,
1369                                          EXTENT_LOCKED, 1, 0, NULL);
1370                 return 0;
1371         }
1372         return 1;
1373 }
1374
1375 void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1376 {
1377         unsigned long index = start >> PAGE_SHIFT;
1378         unsigned long end_index = end >> PAGE_SHIFT;
1379         struct page *page;
1380
1381         while (index <= end_index) {
1382                 page = find_get_page(inode->i_mapping, index);
1383                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1384                 clear_page_dirty_for_io(page);
1385                 put_page(page);
1386                 index++;
1387         }
1388 }
1389
1390 void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1391 {
1392         unsigned long index = start >> PAGE_SHIFT;
1393         unsigned long end_index = end >> PAGE_SHIFT;
1394         struct page *page;
1395
1396         while (index <= end_index) {
1397                 page = find_get_page(inode->i_mapping, index);
1398                 BUG_ON(!page); /* Pages should be in the extent_io_tree */
1399                 __set_page_dirty_nobuffers(page);
1400                 account_page_redirty(page);
1401                 put_page(page);
1402                 index++;
1403         }
1404 }
1405
1406 /* find the first state struct with 'bits' set after 'start', and
1407  * return it.  tree->lock must be held.  NULL will returned if
1408  * nothing was found after 'start'
1409  */
1410 static struct extent_state *
1411 find_first_extent_bit_state(struct extent_io_tree *tree,
1412                             u64 start, unsigned bits)
1413 {
1414         struct rb_node *node;
1415         struct extent_state *state;
1416
1417         /*
1418          * this search will find all the extents that end after
1419          * our range starts.
1420          */
1421         node = tree_search(tree, start);
1422         if (!node)
1423                 goto out;
1424
1425         while (1) {
1426                 state = rb_entry(node, struct extent_state, rb_node);
1427                 if (state->end >= start && (state->state & bits))
1428                         return state;
1429
1430                 node = rb_next(node);
1431                 if (!node)
1432                         break;
1433         }
1434 out:
1435         return NULL;
1436 }
1437
1438 /*
1439  * find the first offset in the io tree with 'bits' set. zero is
1440  * returned if we find something, and *start_ret and *end_ret are
1441  * set to reflect the state struct that was found.
1442  *
1443  * If nothing was found, 1 is returned. If found something, return 0.
1444  */
1445 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1446                           u64 *start_ret, u64 *end_ret, unsigned bits,
1447                           struct extent_state **cached_state)
1448 {
1449         struct extent_state *state;
1450         int ret = 1;
1451
1452         spin_lock(&tree->lock);
1453         if (cached_state && *cached_state) {
1454                 state = *cached_state;
1455                 if (state->end == start - 1 && extent_state_in_tree(state)) {
1456                         while ((state = next_state(state)) != NULL) {
1457                                 if (state->state & bits)
1458                                         goto got_it;
1459                         }
1460                         free_extent_state(*cached_state);
1461                         *cached_state = NULL;
1462                         goto out;
1463                 }
1464                 free_extent_state(*cached_state);
1465                 *cached_state = NULL;
1466         }
1467
1468         state = find_first_extent_bit_state(tree, start, bits);
1469 got_it:
1470         if (state) {
1471                 cache_state_if_flags(state, cached_state, 0);
1472                 *start_ret = state->start;
1473                 *end_ret = state->end;
1474                 ret = 0;
1475         }
1476 out:
1477         spin_unlock(&tree->lock);
1478         return ret;
1479 }
1480
1481 /*
1482  * find a contiguous range of bytes in the file marked as delalloc, not
1483  * more than 'max_bytes'.  start and end are used to return the range,
1484  *
1485  * true is returned if we find something, false if nothing was in the tree
1486  */
1487 static noinline bool find_delalloc_range(struct extent_io_tree *tree,
1488                                         u64 *start, u64 *end, u64 max_bytes,
1489                                         struct extent_state **cached_state)
1490 {
1491         struct rb_node *node;
1492         struct extent_state *state;
1493         u64 cur_start = *start;
1494         bool found = false;
1495         u64 total_bytes = 0;
1496
1497         spin_lock(&tree->lock);
1498
1499         /*
1500          * this search will find all the extents that end after
1501          * our range starts.
1502          */
1503         node = tree_search(tree, cur_start);
1504         if (!node) {
1505                 *end = (u64)-1;
1506                 goto out;
1507         }
1508
1509         while (1) {
1510                 state = rb_entry(node, struct extent_state, rb_node);
1511                 if (found && (state->start != cur_start ||
1512                               (state->state & EXTENT_BOUNDARY))) {
1513                         goto out;
1514                 }
1515                 if (!(state->state & EXTENT_DELALLOC)) {
1516                         if (!found)
1517                                 *end = state->end;
1518                         goto out;
1519                 }
1520                 if (!found) {
1521                         *start = state->start;
1522                         *cached_state = state;
1523                         refcount_inc(&state->refs);
1524                 }
1525                 found = true;
1526                 *end = state->end;
1527                 cur_start = state->end + 1;
1528                 node = rb_next(node);
1529                 total_bytes += state->end - state->start + 1;
1530                 if (total_bytes >= max_bytes)
1531                         break;
1532                 if (!node)
1533                         break;
1534         }
1535 out:
1536         spin_unlock(&tree->lock);
1537         return found;
1538 }
1539
1540 static int __process_pages_contig(struct address_space *mapping,
1541                                   struct page *locked_page,
1542                                   pgoff_t start_index, pgoff_t end_index,
1543                                   unsigned long page_ops, pgoff_t *index_ret);
1544
1545 static noinline void __unlock_for_delalloc(struct inode *inode,
1546                                            struct page *locked_page,
1547                                            u64 start, u64 end)
1548 {
1549         unsigned long index = start >> PAGE_SHIFT;
1550         unsigned long end_index = end >> PAGE_SHIFT;
1551
1552         ASSERT(locked_page);
1553         if (index == locked_page->index && end_index == index)
1554                 return;
1555
1556         __process_pages_contig(inode->i_mapping, locked_page, index, end_index,
1557                                PAGE_UNLOCK, NULL);
1558 }
1559
1560 static noinline int lock_delalloc_pages(struct inode *inode,
1561                                         struct page *locked_page,
1562                                         u64 delalloc_start,
1563                                         u64 delalloc_end)
1564 {
1565         unsigned long index = delalloc_start >> PAGE_SHIFT;
1566         unsigned long index_ret = index;
1567         unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1568         int ret;
1569
1570         ASSERT(locked_page);
1571         if (index == locked_page->index && index == end_index)
1572                 return 0;
1573
1574         ret = __process_pages_contig(inode->i_mapping, locked_page, index,
1575                                      end_index, PAGE_LOCK, &index_ret);
1576         if (ret == -EAGAIN)
1577                 __unlock_for_delalloc(inode, locked_page, delalloc_start,
1578                                       (u64)index_ret << PAGE_SHIFT);
1579         return ret;
1580 }
1581
1582 /*
1583  * Find and lock a contiguous range of bytes in the file marked as delalloc, no
1584  * more than @max_bytes.  @Start and @end are used to return the range,
1585  *
1586  * Return: true if we find something
1587  *         false if nothing was in the tree
1588  */
1589 EXPORT_FOR_TESTS
1590 noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
1591                                     struct extent_io_tree *tree,
1592                                     struct page *locked_page, u64 *start,
1593                                     u64 *end)
1594 {
1595         u64 max_bytes = BTRFS_MAX_EXTENT_SIZE;
1596         u64 delalloc_start;
1597         u64 delalloc_end;
1598         bool found;
1599         struct extent_state *cached_state = NULL;
1600         int ret;
1601         int loops = 0;
1602
1603 again:
1604         /* step one, find a bunch of delalloc bytes starting at start */
1605         delalloc_start = *start;
1606         delalloc_end = 0;
1607         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1608                                     max_bytes, &cached_state);
1609         if (!found || delalloc_end <= *start) {
1610                 *start = delalloc_start;
1611                 *end = delalloc_end;
1612                 free_extent_state(cached_state);
1613                 return false;
1614         }
1615
1616         /*
1617          * start comes from the offset of locked_page.  We have to lock
1618          * pages in order, so we can't process delalloc bytes before
1619          * locked_page
1620          */
1621         if (delalloc_start < *start)
1622                 delalloc_start = *start;
1623
1624         /*
1625          * make sure to limit the number of pages we try to lock down
1626          */
1627         if (delalloc_end + 1 - delalloc_start > max_bytes)
1628                 delalloc_end = delalloc_start + max_bytes - 1;
1629
1630         /* step two, lock all the pages after the page that has start */
1631         ret = lock_delalloc_pages(inode, locked_page,
1632                                   delalloc_start, delalloc_end);
1633         ASSERT(!ret || ret == -EAGAIN);
1634         if (ret == -EAGAIN) {
1635                 /* some of the pages are gone, lets avoid looping by
1636                  * shortening the size of the delalloc range we're searching
1637                  */
1638                 free_extent_state(cached_state);
1639                 cached_state = NULL;
1640                 if (!loops) {
1641                         max_bytes = PAGE_SIZE;
1642                         loops = 1;
1643                         goto again;
1644                 } else {
1645                         found = false;
1646                         goto out_failed;
1647                 }
1648         }
1649
1650         /* step three, lock the state bits for the whole range */
1651         lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1652
1653         /* then test to make sure it is all still delalloc */
1654         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1655                              EXTENT_DELALLOC, 1, cached_state);
1656         if (!ret) {
1657                 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1658                                      &cached_state);
1659                 __unlock_for_delalloc(inode, locked_page,
1660                               delalloc_start, delalloc_end);
1661                 cond_resched();
1662                 goto again;
1663         }
1664         free_extent_state(cached_state);
1665         *start = delalloc_start;
1666         *end = delalloc_end;
1667 out_failed:
1668         return found;
1669 }
1670
1671 static int __process_pages_contig(struct address_space *mapping,
1672                                   struct page *locked_page,
1673                                   pgoff_t start_index, pgoff_t end_index,
1674                                   unsigned long page_ops, pgoff_t *index_ret)
1675 {
1676         unsigned long nr_pages = end_index - start_index + 1;
1677         unsigned long pages_locked = 0;
1678         pgoff_t index = start_index;
1679         struct page *pages[16];
1680         unsigned ret;
1681         int err = 0;
1682         int i;
1683
1684         if (page_ops & PAGE_LOCK) {
1685                 ASSERT(page_ops == PAGE_LOCK);
1686                 ASSERT(index_ret && *index_ret == start_index);
1687         }
1688
1689         if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1690                 mapping_set_error(mapping, -EIO);
1691
1692         while (nr_pages > 0) {
1693                 ret = find_get_pages_contig(mapping, index,
1694                                      min_t(unsigned long,
1695                                      nr_pages, ARRAY_SIZE(pages)), pages);
1696                 if (ret == 0) {
1697                         /*
1698                          * Only if we're going to lock these pages,
1699                          * can we find nothing at @index.
1700                          */
1701                         ASSERT(page_ops & PAGE_LOCK);
1702                         err = -EAGAIN;
1703                         goto out;
1704                 }
1705
1706                 for (i = 0; i < ret; i++) {
1707                         if (page_ops & PAGE_SET_PRIVATE2)
1708                                 SetPagePrivate2(pages[i]);
1709
1710                         if (pages[i] == locked_page) {
1711                                 put_page(pages[i]);
1712                                 pages_locked++;
1713                                 continue;
1714                         }
1715                         if (page_ops & PAGE_CLEAR_DIRTY)
1716                                 clear_page_dirty_for_io(pages[i]);
1717                         if (page_ops & PAGE_SET_WRITEBACK)
1718                                 set_page_writeback(pages[i]);
1719                         if (page_ops & PAGE_SET_ERROR)
1720                                 SetPageError(pages[i]);
1721                         if (page_ops & PAGE_END_WRITEBACK)
1722                                 end_page_writeback(pages[i]);
1723                         if (page_ops & PAGE_UNLOCK)
1724                                 unlock_page(pages[i]);
1725                         if (page_ops & PAGE_LOCK) {
1726                                 lock_page(pages[i]);
1727                                 if (!PageDirty(pages[i]) ||
1728                                     pages[i]->mapping != mapping) {
1729                                         unlock_page(pages[i]);
1730                                         put_page(pages[i]);
1731                                         err = -EAGAIN;
1732                                         goto out;
1733                                 }
1734                         }
1735                         put_page(pages[i]);
1736                         pages_locked++;
1737                 }
1738                 nr_pages -= ret;
1739                 index += ret;
1740                 cond_resched();
1741         }
1742 out:
1743         if (err && index_ret)
1744                 *index_ret = start_index + pages_locked - 1;
1745         return err;
1746 }
1747
1748 void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1749                                  u64 delalloc_end, struct page *locked_page,
1750                                  unsigned clear_bits,
1751                                  unsigned long page_ops)
1752 {
1753         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits, 1, 0,
1754                          NULL);
1755
1756         __process_pages_contig(inode->i_mapping, locked_page,
1757                                start >> PAGE_SHIFT, end >> PAGE_SHIFT,
1758                                page_ops, NULL);
1759 }
1760
1761 /*
1762  * count the number of bytes in the tree that have a given bit(s)
1763  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1764  * cached.  The total number found is returned.
1765  */
1766 u64 count_range_bits(struct extent_io_tree *tree,
1767                      u64 *start, u64 search_end, u64 max_bytes,
1768                      unsigned bits, int contig)
1769 {
1770         struct rb_node *node;
1771         struct extent_state *state;
1772         u64 cur_start = *start;
1773         u64 total_bytes = 0;
1774         u64 last = 0;
1775         int found = 0;
1776
1777         if (WARN_ON(search_end <= cur_start))
1778                 return 0;
1779
1780         spin_lock(&tree->lock);
1781         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1782                 total_bytes = tree->dirty_bytes;
1783                 goto out;
1784         }
1785         /*
1786          * this search will find all the extents that end after
1787          * our range starts.
1788          */
1789         node = tree_search(tree, cur_start);
1790         if (!node)
1791                 goto out;
1792
1793         while (1) {
1794                 state = rb_entry(node, struct extent_state, rb_node);
1795                 if (state->start > search_end)
1796                         break;
1797                 if (contig && found && state->start > last + 1)
1798                         break;
1799                 if (state->end >= cur_start && (state->state & bits) == bits) {
1800                         total_bytes += min(search_end, state->end) + 1 -
1801                                        max(cur_start, state->start);
1802                         if (total_bytes >= max_bytes)
1803                                 break;
1804                         if (!found) {
1805                                 *start = max(cur_start, state->start);
1806                                 found = 1;
1807                         }
1808                         last = state->end;
1809                 } else if (contig && found) {
1810                         break;
1811                 }
1812                 node = rb_next(node);
1813                 if (!node)
1814                         break;
1815         }
1816 out:
1817         spin_unlock(&tree->lock);
1818         return total_bytes;
1819 }
1820
1821 /*
1822  * set the private field for a given byte offset in the tree.  If there isn't
1823  * an extent_state there already, this does nothing.
1824  */
1825 static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
1826                 struct io_failure_record *failrec)
1827 {
1828         struct rb_node *node;
1829         struct extent_state *state;
1830         int ret = 0;
1831
1832         spin_lock(&tree->lock);
1833         /*
1834          * this search will find all the extents that end after
1835          * our range starts.
1836          */
1837         node = tree_search(tree, start);
1838         if (!node) {
1839                 ret = -ENOENT;
1840                 goto out;
1841         }
1842         state = rb_entry(node, struct extent_state, rb_node);
1843         if (state->start != start) {
1844                 ret = -ENOENT;
1845                 goto out;
1846         }
1847         state->failrec = failrec;
1848 out:
1849         spin_unlock(&tree->lock);
1850         return ret;
1851 }
1852
1853 static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
1854                 struct io_failure_record **failrec)
1855 {
1856         struct rb_node *node;
1857         struct extent_state *state;
1858         int ret = 0;
1859
1860         spin_lock(&tree->lock);
1861         /*
1862          * this search will find all the extents that end after
1863          * our range starts.
1864          */
1865         node = tree_search(tree, start);
1866         if (!node) {
1867                 ret = -ENOENT;
1868                 goto out;
1869         }
1870         state = rb_entry(node, struct extent_state, rb_node);
1871         if (state->start != start) {
1872                 ret = -ENOENT;
1873                 goto out;
1874         }
1875         *failrec = state->failrec;
1876 out:
1877         spin_unlock(&tree->lock);
1878         return ret;
1879 }
1880
1881 /*
1882  * searches a range in the state tree for a given mask.
1883  * If 'filled' == 1, this returns 1 only if every extent in the tree
1884  * has the bits set.  Otherwise, 1 is returned if any bit in the
1885  * range is found set.
1886  */
1887 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1888                    unsigned bits, int filled, struct extent_state *cached)
1889 {
1890         struct extent_state *state = NULL;
1891         struct rb_node *node;
1892         int bitset = 0;
1893
1894         spin_lock(&tree->lock);
1895         if (cached && extent_state_in_tree(cached) && cached->start <= start &&
1896             cached->end > start)
1897                 node = &cached->rb_node;
1898         else
1899                 node = tree_search(tree, start);
1900         while (node && start <= end) {
1901                 state = rb_entry(node, struct extent_state, rb_node);
1902
1903                 if (filled && state->start > start) {
1904                         bitset = 0;
1905                         break;
1906                 }
1907
1908                 if (state->start > end)
1909                         break;
1910
1911                 if (state->state & bits) {
1912                         bitset = 1;
1913                         if (!filled)
1914                                 break;
1915                 } else if (filled) {
1916                         bitset = 0;
1917                         break;
1918                 }
1919
1920                 if (state->end == (u64)-1)
1921                         break;
1922
1923                 start = state->end + 1;
1924                 if (start > end)
1925                         break;
1926                 node = rb_next(node);
1927                 if (!node) {
1928                         if (filled)
1929                                 bitset = 0;
1930                         break;
1931                 }
1932         }
1933         spin_unlock(&tree->lock);
1934         return bitset;
1935 }
1936
1937 /*
1938  * helper function to set a given page up to date if all the
1939  * extents in the tree for that page are up to date
1940  */
1941 static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1942 {
1943         u64 start = page_offset(page);
1944         u64 end = start + PAGE_SIZE - 1;
1945         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1946                 SetPageUptodate(page);
1947 }
1948
1949 int free_io_failure(struct extent_io_tree *failure_tree,
1950                     struct extent_io_tree *io_tree,
1951                     struct io_failure_record *rec)
1952 {
1953         int ret;
1954         int err = 0;
1955
1956         set_state_failrec(failure_tree, rec->start, NULL);
1957         ret = clear_extent_bits(failure_tree, rec->start,
1958                                 rec->start + rec->len - 1,
1959                                 EXTENT_LOCKED | EXTENT_DIRTY);
1960         if (ret)
1961                 err = ret;
1962
1963         ret = clear_extent_bits(io_tree, rec->start,
1964                                 rec->start + rec->len - 1,
1965                                 EXTENT_DAMAGED);
1966         if (ret && !err)
1967                 err = ret;
1968
1969         kfree(rec);
1970         return err;
1971 }
1972
1973 /*
1974  * this bypasses the standard btrfs submit functions deliberately, as
1975  * the standard behavior is to write all copies in a raid setup. here we only
1976  * want to write the one bad copy. so we do the mapping for ourselves and issue
1977  * submit_bio directly.
1978  * to avoid any synchronization issues, wait for the data after writing, which
1979  * actually prevents the read that triggered the error from finishing.
1980  * currently, there can be no more than two copies of every data bit. thus,
1981  * exactly one rewrite is required.
1982  */
1983 int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start,
1984                       u64 length, u64 logical, struct page *page,
1985                       unsigned int pg_offset, int mirror_num)
1986 {
1987         struct bio *bio;
1988         struct btrfs_device *dev;
1989         u64 map_length = 0;
1990         u64 sector;
1991         struct btrfs_bio *bbio = NULL;
1992         int ret;
1993
1994         ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));
1995         BUG_ON(!mirror_num);
1996
1997         bio = btrfs_io_bio_alloc(1);
1998         bio->bi_iter.bi_size = 0;
1999         map_length = length;
2000
2001         /*
2002          * Avoid races with device replace and make sure our bbio has devices
2003          * associated to its stripes that don't go away while we are doing the
2004          * read repair operation.
2005          */
2006         btrfs_bio_counter_inc_blocked(fs_info);
2007         if (btrfs_is_parity_mirror(fs_info, logical, length)) {
2008                 /*
2009                  * Note that we don't use BTRFS_MAP_WRITE because it's supposed
2010                  * to update all raid stripes, but here we just want to correct
2011                  * bad stripe, thus BTRFS_MAP_READ is abused to only get the bad
2012                  * stripe's dev and sector.
2013                  */
2014                 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical,
2015                                       &map_length, &bbio, 0);
2016                 if (ret) {
2017                         btrfs_bio_counter_dec(fs_info);
2018                         bio_put(bio);
2019                         return -EIO;
2020                 }
2021                 ASSERT(bbio->mirror_num == 1);
2022         } else {
2023                 ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical,
2024                                       &map_length, &bbio, mirror_num);
2025                 if (ret) {
2026                         btrfs_bio_counter_dec(fs_info);
2027                         bio_put(bio);
2028                         return -EIO;
2029                 }
2030                 BUG_ON(mirror_num != bbio->mirror_num);
2031         }
2032
2033         sector = bbio->stripes[bbio->mirror_num - 1].physical >> 9;
2034         bio->bi_iter.bi_sector = sector;
2035         dev = bbio->stripes[bbio->mirror_num - 1].dev;
2036         btrfs_put_bbio(bbio);
2037         if (!dev || !dev->bdev ||
2038             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
2039                 btrfs_bio_counter_dec(fs_info);
2040                 bio_put(bio);
2041                 return -EIO;
2042         }
2043         bio_set_dev(bio, dev->bdev);
2044         bio->bi_opf = REQ_OP_WRITE | REQ_SYNC;
2045         bio_add_page(bio, page, length, pg_offset);
2046
2047         if (btrfsic_submit_bio_wait(bio)) {
2048                 /* try to remap that extent elsewhere? */
2049                 btrfs_bio_counter_dec(fs_info);
2050                 bio_put(bio);
2051                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2052                 return -EIO;
2053         }
2054
2055         btrfs_info_rl_in_rcu(fs_info,
2056                 "read error corrected: ino %llu off %llu (dev %s sector %llu)",
2057                                   ino, start,
2058                                   rcu_str_deref(dev->name), sector);
2059         btrfs_bio_counter_dec(fs_info);
2060         bio_put(bio);
2061         return 0;
2062 }
2063
2064 int repair_eb_io_failure(struct btrfs_fs_info *fs_info,
2065                          struct extent_buffer *eb, int mirror_num)
2066 {
2067         u64 start = eb->start;
2068         int i, num_pages = num_extent_pages(eb);
2069         int ret = 0;
2070
2071         if (sb_rdonly(fs_info->sb))
2072                 return -EROFS;
2073
2074         for (i = 0; i < num_pages; i++) {
2075                 struct page *p = eb->pages[i];
2076
2077                 ret = repair_io_failure(fs_info, 0, start, PAGE_SIZE, start, p,
2078                                         start - page_offset(p), mirror_num);
2079                 if (ret)
2080                         break;
2081                 start += PAGE_SIZE;
2082         }
2083
2084         return ret;
2085 }
2086
2087 /*
2088  * each time an IO finishes, we do a fast check in the IO failure tree
2089  * to see if we need to process or clean up an io_failure_record
2090  */
2091 int clean_io_failure(struct btrfs_fs_info *fs_info,
2092                      struct extent_io_tree *failure_tree,
2093                      struct extent_io_tree *io_tree, u64 start,
2094                      struct page *page, u64 ino, unsigned int pg_offset)
2095 {
2096         u64 private;
2097         struct io_failure_record *failrec;
2098         struct extent_state *state;
2099         int num_copies;
2100         int ret;
2101
2102         private = 0;
2103         ret = count_range_bits(failure_tree, &private, (u64)-1, 1,
2104                                EXTENT_DIRTY, 0);
2105         if (!ret)
2106                 return 0;
2107
2108         ret = get_state_failrec(failure_tree, start, &failrec);
2109         if (ret)
2110                 return 0;
2111
2112         BUG_ON(!failrec->this_mirror);
2113
2114         if (failrec->in_validation) {
2115                 /* there was no real error, just free the record */
2116                 btrfs_debug(fs_info,
2117                         "clean_io_failure: freeing dummy error at %llu",
2118                         failrec->start);
2119                 goto out;
2120         }
2121         if (sb_rdonly(fs_info->sb))
2122                 goto out;
2123
2124         spin_lock(&io_tree->lock);
2125         state = find_first_extent_bit_state(io_tree,
2126                                             failrec->start,
2127                                             EXTENT_LOCKED);
2128         spin_unlock(&io_tree->lock);
2129
2130         if (state && state->start <= failrec->start &&
2131             state->end >= failrec->start + failrec->len - 1) {
2132                 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2133                                               failrec->len);
2134                 if (num_copies > 1)  {
2135                         repair_io_failure(fs_info, ino, start, failrec->len,
2136                                           failrec->logical, page, pg_offset,
2137                                           failrec->failed_mirror);
2138                 }
2139         }
2140
2141 out:
2142         free_io_failure(failure_tree, io_tree, failrec);
2143
2144         return 0;
2145 }
2146
2147 /*
2148  * Can be called when
2149  * - hold extent lock
2150  * - under ordered extent
2151  * - the inode is freeing
2152  */
2153 void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end)
2154 {
2155         struct extent_io_tree *failure_tree = &inode->io_failure_tree;
2156         struct io_failure_record *failrec;
2157         struct extent_state *state, *next;
2158
2159         if (RB_EMPTY_ROOT(&failure_tree->state))
2160                 return;
2161
2162         spin_lock(&failure_tree->lock);
2163         state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2164         while (state) {
2165                 if (state->start > end)
2166                         break;
2167
2168                 ASSERT(state->end <= end);
2169
2170                 next = next_state(state);
2171
2172                 failrec = state->failrec;
2173                 free_extent_state(state);
2174                 kfree(failrec);
2175
2176                 state = next;
2177         }
2178         spin_unlock(&failure_tree->lock);
2179 }
2180
2181 int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2182                 struct io_failure_record **failrec_ret)
2183 {
2184         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2185         struct io_failure_record *failrec;
2186         struct extent_map *em;
2187         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2188         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2189         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2190         int ret;
2191         u64 logical;
2192
2193         ret = get_state_failrec(failure_tree, start, &failrec);
2194         if (ret) {
2195                 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2196                 if (!failrec)
2197                         return -ENOMEM;
2198
2199                 failrec->start = start;
2200                 failrec->len = end - start + 1;
2201                 failrec->this_mirror = 0;
2202                 failrec->bio_flags = 0;
2203                 failrec->in_validation = 0;
2204
2205                 read_lock(&em_tree->lock);
2206                 em = lookup_extent_mapping(em_tree, start, failrec->len);
2207                 if (!em) {
2208                         read_unlock(&em_tree->lock);
2209                         kfree(failrec);
2210                         return -EIO;
2211                 }
2212
2213                 if (em->start > start || em->start + em->len <= start) {
2214                         free_extent_map(em);
2215                         em = NULL;
2216                 }
2217                 read_unlock(&em_tree->lock);
2218                 if (!em) {
2219                         kfree(failrec);
2220                         return -EIO;
2221                 }
2222
2223                 logical = start - em->start;
2224                 logical = em->block_start + logical;
2225                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2226                         logical = em->block_start;
2227                         failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2228                         extent_set_compress_type(&failrec->bio_flags,
2229                                                  em->compress_type);
2230                 }
2231
2232                 btrfs_debug(fs_info,
2233                         "Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu",
2234                         logical, start, failrec->len);
2235
2236                 failrec->logical = logical;
2237                 free_extent_map(em);
2238
2239                 /* set the bits in the private failure tree */
2240                 ret = set_extent_bits(failure_tree, start, end,
2241                                         EXTENT_LOCKED | EXTENT_DIRTY);
2242                 if (ret >= 0)
2243                         ret = set_state_failrec(failure_tree, start, failrec);
2244                 /* set the bits in the inode's tree */
2245                 if (ret >= 0)
2246                         ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED);
2247                 if (ret < 0) {
2248                         kfree(failrec);
2249                         return ret;
2250                 }
2251         } else {
2252                 btrfs_debug(fs_info,
2253                         "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d",
2254                         failrec->logical, failrec->start, failrec->len,
2255                         failrec->in_validation);
2256                 /*
2257                  * when data can be on disk more than twice, add to failrec here
2258                  * (e.g. with a list for failed_mirror) to make
2259                  * clean_io_failure() clean all those errors at once.
2260                  */
2261         }
2262
2263         *failrec_ret = failrec;
2264
2265         return 0;
2266 }
2267
2268 bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages,
2269                            struct io_failure_record *failrec, int failed_mirror)
2270 {
2271         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2272         int num_copies;
2273
2274         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
2275         if (num_copies == 1) {
2276                 /*
2277                  * we only have a single copy of the data, so don't bother with
2278                  * all the retry and error correction code that follows. no
2279                  * matter what the error is, it is very likely to persist.
2280                  */
2281                 btrfs_debug(fs_info,
2282                         "Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
2283                         num_copies, failrec->this_mirror, failed_mirror);
2284                 return false;
2285         }
2286
2287         /*
2288          * there are two premises:
2289          *      a) deliver good data to the caller
2290          *      b) correct the bad sectors on disk
2291          */
2292         if (failed_bio_pages > 1) {
2293                 /*
2294                  * to fulfill b), we need to know the exact failing sectors, as
2295                  * we don't want to rewrite any more than the failed ones. thus,
2296                  * we need separate read requests for the failed bio
2297                  *
2298                  * if the following BUG_ON triggers, our validation request got
2299                  * merged. we need separate requests for our algorithm to work.
2300                  */
2301                 BUG_ON(failrec->in_validation);
2302                 failrec->in_validation = 1;
2303                 failrec->this_mirror = failed_mirror;
2304         } else {
2305                 /*
2306                  * we're ready to fulfill a) and b) alongside. get a good copy
2307                  * of the failed sector and if we succeed, we have setup
2308                  * everything for repair_io_failure to do the rest for us.
2309                  */
2310                 if (failrec->in_validation) {
2311                         BUG_ON(failrec->this_mirror != failed_mirror);
2312                         failrec->in_validation = 0;
2313                         failrec->this_mirror = 0;
2314                 }
2315                 failrec->failed_mirror = failed_mirror;
2316                 failrec->this_mirror++;
2317                 if (failrec->this_mirror == failed_mirror)
2318                         failrec->this_mirror++;
2319         }
2320
2321         if (failrec->this_mirror > num_copies) {
2322                 btrfs_debug(fs_info,
2323                         "Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
2324                         num_copies, failrec->this_mirror, failed_mirror);
2325                 return false;
2326         }
2327
2328         return true;
2329 }
2330
2331
2332 struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2333                                     struct io_failure_record *failrec,
2334                                     struct page *page, int pg_offset, int icsum,
2335                                     bio_end_io_t *endio_func, void *data)
2336 {
2337         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2338         struct bio *bio;
2339         struct btrfs_io_bio *btrfs_failed_bio;
2340         struct btrfs_io_bio *btrfs_bio;
2341
2342         bio = btrfs_io_bio_alloc(1);
2343         bio->bi_end_io = endio_func;
2344         bio->bi_iter.bi_sector = failrec->logical >> 9;
2345         bio_set_dev(bio, fs_info->fs_devices->latest_bdev);
2346         bio->bi_iter.bi_size = 0;
2347         bio->bi_private = data;
2348
2349         btrfs_failed_bio = btrfs_io_bio(failed_bio);
2350         if (btrfs_failed_bio->csum) {
2351                 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2352
2353                 btrfs_bio = btrfs_io_bio(bio);
2354                 btrfs_bio->csum = btrfs_bio->csum_inline;
2355                 icsum *= csum_size;
2356                 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2357                        csum_size);
2358         }
2359
2360         bio_add_page(bio, page, failrec->len, pg_offset);
2361
2362         return bio;
2363 }
2364
2365 /*
2366  * This is a generic handler for readpage errors. If other copies exist, read
2367  * those and write back good data to the failed position. Does not investigate
2368  * in remapping the failed extent elsewhere, hoping the device will be smart
2369  * enough to do this as needed
2370  */
2371 static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2372                               struct page *page, u64 start, u64 end,
2373                               int failed_mirror)
2374 {
2375         struct io_failure_record *failrec;
2376         struct inode *inode = page->mapping->host;
2377         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2378         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2379         struct bio *bio;
2380         int read_mode = 0;
2381         blk_status_t status;
2382         int ret;
2383         unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT;
2384
2385         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
2386
2387         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2388         if (ret)
2389                 return ret;
2390
2391         if (!btrfs_check_repairable(inode, failed_bio_pages, failrec,
2392                                     failed_mirror)) {
2393                 free_io_failure(failure_tree, tree, failrec);
2394                 return -EIO;
2395         }
2396
2397         if (failed_bio_pages > 1)
2398                 read_mode |= REQ_FAILFAST_DEV;
2399
2400         phy_offset >>= inode->i_sb->s_blocksize_bits;
2401         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2402                                       start - page_offset(page),
2403                                       (int)phy_offset, failed_bio->bi_end_io,
2404                                       NULL);
2405         bio->bi_opf = REQ_OP_READ | read_mode;
2406
2407         btrfs_debug(btrfs_sb(inode->i_sb),
2408                 "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d",
2409                 read_mode, failrec->this_mirror, failrec->in_validation);
2410
2411         status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror,
2412                                          failrec->bio_flags, 0);
2413         if (status) {
2414                 free_io_failure(failure_tree, tree, failrec);
2415                 bio_put(bio);
2416                 ret = blk_status_to_errno(status);
2417         }
2418
2419         return ret;
2420 }
2421
2422 /* lots and lots of room for performance fixes in the end_bio funcs */
2423
2424 void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2425 {
2426         int uptodate = (err == 0);
2427         int ret = 0;
2428
2429         btrfs_writepage_endio_finish_ordered(page, start, end, uptodate);
2430
2431         if (!uptodate) {
2432                 ClearPageUptodate(page);
2433                 SetPageError(page);
2434                 ret = err < 0 ? err : -EIO;
2435                 mapping_set_error(page->mapping, ret);
2436         }
2437 }
2438
2439 /*
2440  * after a writepage IO is done, we need to:
2441  * clear the uptodate bits on error
2442  * clear the writeback bits in the extent tree for this IO
2443  * end_page_writeback if the page has no more pending IO
2444  *
2445  * Scheduling is not allowed, so the extent state tree is expected
2446  * to have one and only one object corresponding to this IO.
2447  */
2448 static void end_bio_extent_writepage(struct bio *bio)
2449 {
2450         int error = blk_status_to_errno(bio->bi_status);
2451         struct bio_vec *bvec;
2452         u64 start;
2453         u64 end;
2454         int i;
2455         struct bvec_iter_all iter_all;
2456
2457         ASSERT(!bio_flagged(bio, BIO_CLONED));
2458         bio_for_each_segment_all(bvec, bio, i, iter_all) {
2459                 struct page *page = bvec->bv_page;
2460                 struct inode *inode = page->mapping->host;
2461                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2462
2463                 /* We always issue full-page reads, but if some block
2464                  * in a page fails to read, blk_update_request() will
2465                  * advance bv_offset and adjust bv_len to compensate.
2466                  * Print a warning for nonzero offsets, and an error
2467                  * if they don't add up to a full page.  */
2468                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2469                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2470                                 btrfs_err(fs_info,
2471                                    "partial page write in btrfs with offset %u and length %u",
2472                                         bvec->bv_offset, bvec->bv_len);
2473                         else
2474                                 btrfs_info(fs_info,
2475                                    "incomplete page write in btrfs with offset %u and length %u",
2476                                         bvec->bv_offset, bvec->bv_len);
2477                 }
2478
2479                 start = page_offset(page);
2480                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2481
2482                 end_extent_writepage(page, error, start, end);
2483                 end_page_writeback(page);
2484         }
2485
2486         bio_put(bio);
2487 }
2488
2489 static void
2490 endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2491                               int uptodate)
2492 {
2493         struct extent_state *cached = NULL;
2494         u64 end = start + len - 1;
2495
2496         if (uptodate && tree->track_uptodate)
2497                 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2498         unlock_extent_cached_atomic(tree, start, end, &cached);
2499 }
2500
2501 /*
2502  * after a readpage IO is done, we need to:
2503  * clear the uptodate bits on error
2504  * set the uptodate bits if things worked
2505  * set the page up to date if all extents in the tree are uptodate
2506  * clear the lock bit in the extent tree
2507  * unlock the page if there are no other extents locked for it
2508  *
2509  * Scheduling is not allowed, so the extent state tree is expected
2510  * to have one and only one object corresponding to this IO.
2511  */
2512 static void end_bio_extent_readpage(struct bio *bio)
2513 {
2514         struct bio_vec *bvec;
2515         int uptodate = !bio->bi_status;
2516         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2517         struct extent_io_tree *tree, *failure_tree;
2518         u64 offset = 0;
2519         u64 start;
2520         u64 end;
2521         u64 len;
2522         u64 extent_start = 0;
2523         u64 extent_len = 0;
2524         int mirror;
2525         int ret;
2526         int i;
2527         struct bvec_iter_all iter_all;
2528
2529         ASSERT(!bio_flagged(bio, BIO_CLONED));
2530         bio_for_each_segment_all(bvec, bio, i, iter_all) {
2531                 struct page *page = bvec->bv_page;
2532                 struct inode *inode = page->mapping->host;
2533                 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2534                 bool data_inode = btrfs_ino(BTRFS_I(inode))
2535                         != BTRFS_BTREE_INODE_OBJECTID;
2536
2537                 btrfs_debug(fs_info,
2538                         "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u",
2539                         (u64)bio->bi_iter.bi_sector, bio->bi_status,
2540                         io_bio->mirror_num);
2541                 tree = &BTRFS_I(inode)->io_tree;
2542                 failure_tree = &BTRFS_I(inode)->io_failure_tree;
2543
2544                 /* We always issue full-page reads, but if some block
2545                  * in a page fails to read, blk_update_request() will
2546                  * advance bv_offset and adjust bv_len to compensate.
2547                  * Print a warning for nonzero offsets, and an error
2548                  * if they don't add up to a full page.  */
2549                 if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2550                         if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2551                                 btrfs_err(fs_info,
2552                                         "partial page read in btrfs with offset %u and length %u",
2553                                         bvec->bv_offset, bvec->bv_len);
2554                         else
2555                                 btrfs_info(fs_info,
2556                                         "incomplete page read in btrfs with offset %u and length %u",
2557                                         bvec->bv_offset, bvec->bv_len);
2558                 }
2559
2560                 start = page_offset(page);
2561                 end = start + bvec->bv_offset + bvec->bv_len - 1;
2562                 len = bvec->bv_len;
2563
2564                 mirror = io_bio->mirror_num;
2565                 if (likely(uptodate)) {
2566                         ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2567                                                               page, start, end,
2568                                                               mirror);
2569                         if (ret)
2570                                 uptodate = 0;
2571                         else
2572                                 clean_io_failure(BTRFS_I(inode)->root->fs_info,
2573                                                  failure_tree, tree, start,
2574                                                  page,
2575                                                  btrfs_ino(BTRFS_I(inode)), 0);
2576                 }
2577
2578                 if (likely(uptodate))
2579                         goto readpage_ok;
2580
2581                 if (data_inode) {
2582
2583                         /*
2584                          * The generic bio_readpage_error handles errors the
2585                          * following way: If possible, new read requests are
2586                          * created and submitted and will end up in
2587                          * end_bio_extent_readpage as well (if we're lucky,
2588                          * not in the !uptodate case). In that case it returns
2589                          * 0 and we just go on with the next page in our bio.
2590                          * If it can't handle the error it will return -EIO and
2591                          * we remain responsible for that page.
2592                          */
2593                         ret = bio_readpage_error(bio, offset, page, start, end,
2594                                                  mirror);
2595                         if (ret == 0) {
2596                                 uptodate = !bio->bi_status;
2597                                 offset += len;
2598                                 continue;
2599                         }
2600                 } else {
2601                         struct extent_buffer *eb;
2602
2603                         eb = (struct extent_buffer *)page->private;
2604                         set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
2605                         eb->read_mirror = mirror;
2606                         atomic_dec(&eb->io_pages);
2607                         if (test_and_clear_bit(EXTENT_BUFFER_READAHEAD,
2608                                                &eb->bflags))
2609                                 btree_readahead_hook(eb, -EIO);
2610
2611                         ret = -EIO;
2612                 }
2613 readpage_ok:
2614                 if (likely(uptodate)) {
2615                         loff_t i_size = i_size_read(inode);
2616                         pgoff_t end_index = i_size >> PAGE_SHIFT;
2617                         unsigned off;
2618
2619                         /* Zero out the end if this page straddles i_size */
2620                         off = offset_in_page(i_size);
2621                         if (page->index == end_index && off)
2622                                 zero_user_segment(page, off, PAGE_SIZE);
2623                         SetPageUptodate(page);
2624                 } else {
2625                         ClearPageUptodate(page);
2626                         SetPageError(page);
2627                 }
2628                 unlock_page(page);
2629                 offset += len;
2630
2631                 if (unlikely(!uptodate)) {
2632                         if (extent_len) {
2633                                 endio_readpage_release_extent(tree,
2634                                                               extent_start,
2635                                                               extent_len, 1);
2636                                 extent_start = 0;
2637                                 extent_len = 0;
2638                         }
2639                         endio_readpage_release_extent(tree, start,
2640                                                       end - start + 1, 0);
2641                 } else if (!extent_len) {
2642                         extent_start = start;
2643                         extent_len = end + 1 - start;
2644                 } else if (extent_start + extent_len == start) {
2645                         extent_len += end + 1 - start;
2646                 } else {
2647                         endio_readpage_release_extent(tree, extent_start,
2648                                                       extent_len, uptodate);
2649                         extent_start = start;
2650                         extent_len = end + 1 - start;
2651                 }
2652         }
2653
2654         if (extent_len)
2655                 endio_readpage_release_extent(tree, extent_start, extent_len,
2656                                               uptodate);
2657         btrfs_io_bio_free_csum(io_bio);
2658         bio_put(bio);
2659 }
2660
2661 /*
2662  * Initialize the members up to but not including 'bio'. Use after allocating a
2663  * new bio by bio_alloc_bioset as it does not initialize the bytes outside of
2664  * 'bio' because use of __GFP_ZERO is not supported.
2665  */
2666 static inline void btrfs_io_bio_init(struct btrfs_io_bio *btrfs_bio)
2667 {
2668         memset(btrfs_bio, 0, offsetof(struct btrfs_io_bio, bio));
2669 }
2670
2671 /*
2672  * The following helpers allocate a bio. As it's backed by a bioset, it'll
2673  * never fail.  We're returning a bio right now but you can call btrfs_io_bio
2674  * for the appropriate container_of magic
2675  */
2676 struct bio *btrfs_bio_alloc(struct block_device *bdev, u64 first_byte)
2677 {
2678         struct bio *bio;
2679
2680         bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, &btrfs_bioset);
2681         bio_set_dev(bio, bdev);
2682         bio->bi_iter.bi_sector = first_byte >> 9;
2683         btrfs_io_bio_init(btrfs_io_bio(bio));
2684         return bio;
2685 }
2686
2687 struct bio *btrfs_bio_clone(struct bio *bio)
2688 {
2689         struct btrfs_io_bio *btrfs_bio;
2690         struct bio *new;
2691
2692         /* Bio allocation backed by a bioset does not fail */
2693         new = bio_clone_fast(bio, GFP_NOFS, &btrfs_bioset);
2694         btrfs_bio = btrfs_io_bio(new);
2695         btrfs_io_bio_init(btrfs_bio);
2696         btrfs_bio->iter = bio->bi_iter;
2697         return new;
2698 }
2699
2700 struct bio *btrfs_io_bio_alloc(unsigned int nr_iovecs)
2701 {
2702         struct bio *bio;
2703
2704         /* Bio allocation backed by a bioset does not fail */
2705         bio = bio_alloc_bioset(GFP_NOFS, nr_iovecs, &btrfs_bioset);
2706         btrfs_io_bio_init(btrfs_io_bio(bio));
2707         return bio;
2708 }
2709
2710 struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size)
2711 {
2712         struct bio *bio;
2713         struct btrfs_io_bio *btrfs_bio;
2714
2715         /* this will never fail when it's backed by a bioset */
2716         bio = bio_clone_fast(orig, GFP_NOFS, &btrfs_bioset);
2717         ASSERT(bio);
2718
2719         btrfs_bio = btrfs_io_bio(bio);
2720         btrfs_io_bio_init(btrfs_bio);
2721
2722         bio_trim(bio, offset >> 9, size >> 9);
2723         btrfs_bio->iter = bio->bi_iter;
2724         return bio;
2725 }
2726
2727 /*
2728  * @opf:        bio REQ_OP_* and REQ_* flags as one value
2729  * @tree:       tree so we can call our merge_bio hook
2730  * @wbc:        optional writeback control for io accounting
2731  * @page:       page to add to the bio
2732  * @pg_offset:  offset of the new bio or to check whether we are adding
2733  *              a contiguous page to the previous one
2734  * @size:       portion of page that we want to write
2735  * @offset:     starting offset in the page
2736  * @bdev:       attach newly created bios to this bdev
2737  * @bio_ret:    must be valid pointer, newly allocated bio will be stored there
2738  * @end_io_func:     end_io callback for new bio
2739  * @mirror_num:      desired mirror to read/write
2740  * @prev_bio_flags:  flags of previous bio to see if we can merge the current one
2741  * @bio_flags:  flags of the current bio to see if we can merge them
2742  */
2743 static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree,
2744                               struct writeback_control *wbc,
2745                               struct page *page, u64 offset,
2746                               size_t size, unsigned long pg_offset,
2747                               struct block_device *bdev,
2748                               struct bio **bio_ret,
2749                               bio_end_io_t end_io_func,
2750                               int mirror_num,
2751                               unsigned long prev_bio_flags,
2752                               unsigned long bio_flags,
2753                               bool force_bio_submit)
2754 {
2755         int ret = 0;
2756         struct bio *bio;
2757         size_t page_size = min_t(size_t, size, PAGE_SIZE);
2758         sector_t sector = offset >> 9;
2759
2760         ASSERT(bio_ret);
2761
2762         if (*bio_ret) {
2763                 bool contig;
2764                 bool can_merge = true;
2765
2766                 bio = *bio_ret;
2767                 if (prev_bio_flags & EXTENT_BIO_COMPRESSED)
2768                         contig = bio->bi_iter.bi_sector == sector;
2769                 else
2770                         contig = bio_end_sector(bio) == sector;
2771
2772                 ASSERT(tree->ops);
2773                 if (btrfs_bio_fits_in_stripe(page, page_size, bio, bio_flags))
2774                         can_merge = false;
2775
2776                 if (prev_bio_flags != bio_flags || !contig || !can_merge ||
2777                     force_bio_submit ||
2778                     bio_add_page(bio, page, page_size, pg_offset) < page_size) {
2779                         ret = submit_one_bio(bio, mirror_num, prev_bio_flags);
2780                         if (ret < 0) {
2781                                 *bio_ret = NULL;
2782                                 return ret;
2783                         }
2784                         bio = NULL;
2785                 } else {
2786                         if (wbc)
2787                                 wbc_account_io(wbc, page, page_size);
2788                         return 0;
2789                 }
2790         }
2791
2792         bio = btrfs_bio_alloc(bdev, offset);
2793         bio_add_page(bio, page, page_size, pg_offset);
2794         bio->bi_end_io = end_io_func;
2795         bio->bi_private = tree;
2796         bio->bi_write_hint = page->mapping->host->i_write_hint;
2797         bio->bi_opf = opf;
2798         if (wbc) {
2799                 wbc_init_bio(wbc, bio);
2800                 wbc_account_io(wbc, page, page_size);
2801         }
2802
2803         *bio_ret = bio;
2804
2805         return ret;
2806 }
2807
2808 static void attach_extent_buffer_page(struct extent_buffer *eb,
2809                                       struct page *page)
2810 {
2811         if (!PagePrivate(page)) {
2812                 SetPagePrivate(page);
2813                 get_page(page);
2814                 set_page_private(page, (unsigned long)eb);
2815         } else {
2816                 WARN_ON(page->private != (unsigned long)eb);
2817         }
2818 }
2819
2820 void set_page_extent_mapped(struct page *page)
2821 {
2822         if (!PagePrivate(page)) {
2823                 SetPagePrivate(page);
2824                 get_page(page);
2825                 set_page_private(page, EXTENT_PAGE_PRIVATE);
2826         }
2827 }
2828
2829 static struct extent_map *
2830 __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2831                  u64 start, u64 len, get_extent_t *get_extent,
2832                  struct extent_map **em_cached)
2833 {
2834         struct extent_map *em;
2835
2836         if (em_cached && *em_cached) {
2837                 em = *em_cached;
2838                 if (extent_map_in_tree(em) && start >= em->start &&
2839                     start < extent_map_end(em)) {
2840                         refcount_inc(&em->refs);
2841                         return em;
2842                 }
2843
2844                 free_extent_map(em);
2845                 *em_cached = NULL;
2846         }
2847
2848         em = get_extent(BTRFS_I(inode), page, pg_offset, start, len, 0);
2849         if (em_cached && !IS_ERR_OR_NULL(em)) {
2850                 BUG_ON(*em_cached);
2851                 refcount_inc(&em->refs);
2852                 *em_cached = em;
2853         }
2854         return em;
2855 }
2856 /*
2857  * basic readpage implementation.  Locked extent state structs are inserted
2858  * into the tree that are removed when the IO is done (by the end_io
2859  * handlers)
2860  * XXX JDM: This needs looking at to ensure proper page locking
2861  * return 0 on success, otherwise return error
2862  */
2863 static int __do_readpage(struct extent_io_tree *tree,
2864                          struct page *page,
2865                          get_extent_t *get_extent,
2866                          struct extent_map **em_cached,
2867                          struct bio **bio, int mirror_num,
2868                          unsigned long *bio_flags, unsigned int read_flags,
2869                          u64 *prev_em_start)
2870 {
2871         struct inode *inode = page->mapping->host;
2872         u64 start = page_offset(page);
2873         const u64 end = start + PAGE_SIZE - 1;
2874         u64 cur = start;
2875         u64 extent_offset;
2876         u64 last_byte = i_size_read(inode);
2877         u64 block_start;
2878         u64 cur_end;
2879         struct extent_map *em;
2880         struct block_device *bdev;
2881         int ret = 0;
2882         int nr = 0;
2883         size_t pg_offset = 0;
2884         size_t iosize;
2885         size_t disk_io_size;
2886         size_t blocksize = inode->i_sb->s_blocksize;
2887         unsigned long this_bio_flag = 0;
2888
2889         set_page_extent_mapped(page);
2890
2891         if (!PageUptodate(page)) {
2892                 if (cleancache_get_page(page) == 0) {
2893                         BUG_ON(blocksize != PAGE_SIZE);
2894                         unlock_extent(tree, start, end);
2895                         goto out;
2896                 }
2897         }
2898
2899         if (page->index == last_byte >> PAGE_SHIFT) {
2900                 char *userpage;
2901                 size_t zero_offset = offset_in_page(last_byte);
2902
2903                 if (zero_offset) {
2904                         iosize = PAGE_SIZE - zero_offset;
2905                         userpage = kmap_atomic(page);
2906                         memset(userpage + zero_offset, 0, iosize);
2907                         flush_dcache_page(page);
2908                         kunmap_atomic(userpage);
2909                 }
2910         }
2911         while (cur <= end) {
2912                 bool force_bio_submit = false;
2913                 u64 offset;
2914
2915                 if (cur >= last_byte) {
2916                         char *userpage;
2917                         struct extent_state *cached = NULL;
2918
2919                         iosize = PAGE_SIZE - pg_offset;
2920                         userpage = kmap_atomic(page);
2921                         memset(userpage + pg_offset, 0, iosize);
2922                         flush_dcache_page(page);
2923                         kunmap_atomic(userpage);
2924                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2925                                             &cached, GFP_NOFS);
2926                         unlock_extent_cached(tree, cur,
2927                                              cur + iosize - 1, &cached);
2928                         break;
2929                 }
2930                 em = __get_extent_map(inode, page, pg_offset, cur,
2931                                       end - cur + 1, get_extent, em_cached);
2932                 if (IS_ERR_OR_NULL(em)) {
2933                         SetPageError(page);
2934                         unlock_extent(tree, cur, end);
2935                         break;
2936                 }
2937                 extent_offset = cur - em->start;
2938                 BUG_ON(extent_map_end(em) <= cur);
2939                 BUG_ON(end < cur);
2940
2941                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2942                         this_bio_flag |= EXTENT_BIO_COMPRESSED;
2943                         extent_set_compress_type(&this_bio_flag,
2944                                                  em->compress_type);
2945                 }
2946
2947                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2948                 cur_end = min(extent_map_end(em) - 1, end);
2949                 iosize = ALIGN(iosize, blocksize);
2950                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2951                         disk_io_size = em->block_len;
2952                         offset = em->block_start;
2953                 } else {
2954                         offset = em->block_start + extent_offset;
2955                         disk_io_size = iosize;
2956                 }
2957                 bdev = em->bdev;
2958                 block_start = em->block_start;
2959                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2960                         block_start = EXTENT_MAP_HOLE;
2961
2962                 /*
2963                  * If we have a file range that points to a compressed extent
2964                  * and it's followed by a consecutive file range that points to
2965                  * to the same compressed extent (possibly with a different
2966                  * offset and/or length, so it either points to the whole extent
2967                  * or only part of it), we must make sure we do not submit a
2968                  * single bio to populate the pages for the 2 ranges because
2969                  * this makes the compressed extent read zero out the pages
2970                  * belonging to the 2nd range. Imagine the following scenario:
2971                  *
2972                  *  File layout
2973                  *  [0 - 8K]                     [8K - 24K]
2974                  *    |                               |
2975                  *    |                               |
2976                  * points to extent X,         points to extent X,
2977                  * offset 4K, length of 8K     offset 0, length 16K
2978                  *
2979                  * [extent X, compressed length = 4K uncompressed length = 16K]
2980                  *
2981                  * If the bio to read the compressed extent covers both ranges,
2982                  * it will decompress extent X into the pages belonging to the
2983                  * first range and then it will stop, zeroing out the remaining
2984                  * pages that belong to the other range that points to extent X.
2985                  * So here we make sure we submit 2 bios, one for the first
2986                  * range and another one for the third range. Both will target
2987                  * the same physical extent from disk, but we can't currently
2988                  * make the compressed bio endio callback populate the pages
2989                  * for both ranges because each compressed bio is tightly
2990                  * coupled with a single extent map, and each range can have
2991                  * an extent map with a different offset value relative to the
2992                  * uncompressed data of our extent and different lengths. This
2993                  * is a corner case so we prioritize correctness over
2994                  * non-optimal behavior (submitting 2 bios for the same extent).
2995                  */
2996                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
2997                     prev_em_start && *prev_em_start != (u64)-1 &&
2998                     *prev_em_start != em->start)
2999                         force_bio_submit = true;
3000
3001                 if (prev_em_start)
3002                         *prev_em_start = em->start;
3003
3004                 free_extent_map(em);
3005                 em = NULL;
3006
3007                 /* we've found a hole, just zero and go on */
3008                 if (block_start == EXTENT_MAP_HOLE) {
3009                         char *userpage;
3010                         struct extent_state *cached = NULL;
3011
3012                         userpage = kmap_atomic(page);
3013                         memset(userpage + pg_offset, 0, iosize);
3014                         flush_dcache_page(page);
3015                         kunmap_atomic(userpage);
3016
3017                         set_extent_uptodate(tree, cur, cur + iosize - 1,
3018                                             &cached, GFP_NOFS);
3019                         unlock_extent_cached(tree, cur,
3020                                              cur + iosize - 1, &cached);
3021                         cur = cur + iosize;
3022                         pg_offset += iosize;
3023                         continue;
3024                 }
3025                 /* the get_extent function already copied into the page */
3026                 if (test_range_bit(tree, cur, cur_end,
3027                                    EXTENT_UPTODATE, 1, NULL)) {
3028                         check_page_uptodate(tree, page);
3029                         unlock_extent(tree, cur, cur + iosize - 1);
3030                         cur = cur + iosize;
3031                         pg_offset += iosize;
3032                         continue;
3033                 }
3034                 /* we have an inline extent but it didn't get marked up
3035                  * to date.  Error out
3036                  */
3037                 if (block_start == EXTENT_MAP_INLINE) {
3038                         SetPageError(page);
3039                         unlock_extent(tree, cur, cur + iosize - 1);
3040                         cur = cur + iosize;
3041                         pg_offset += iosize;
3042                         continue;
3043                 }
3044
3045                 ret = submit_extent_page(REQ_OP_READ | read_flags, tree, NULL,
3046                                          page, offset, disk_io_size,
3047                                          pg_offset, bdev, bio,
3048                                          end_bio_extent_readpage, mirror_num,
3049                                          *bio_flags,
3050                                          this_bio_flag,
3051                                          force_bio_submit);
3052                 if (!ret) {
3053                         nr++;
3054                         *bio_flags = this_bio_flag;
3055                 } else {
3056                         SetPageError(page);
3057                         unlock_extent(tree, cur, cur + iosize - 1);
3058                         goto out;
3059                 }
3060                 cur = cur + iosize;
3061                 pg_offset += iosize;
3062         }
3063 out:
3064         if (!nr) {
3065                 if (!PageError(page))
3066                         SetPageUptodate(page);
3067                 unlock_page(page);
3068         }
3069         return ret;
3070 }
3071
3072 static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3073                                              struct page *pages[], int nr_pages,
3074                                              u64 start, u64 end,
3075                                              struct extent_map **em_cached,
3076                                              struct bio **bio,
3077                                              unsigned long *bio_flags,
3078                                              u64 *prev_em_start)
3079 {
3080         struct inode *inode;
3081         struct btrfs_ordered_extent *ordered;
3082         int index;
3083
3084         inode = pages[0]->mapping->host;
3085         while (1) {
3086                 lock_extent(tree, start, end);
3087                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
3088                                                      end - start + 1);
3089                 if (!ordered)
3090                         break;
3091                 unlock_extent(tree, start, end);
3092                 btrfs_start_ordered_extent(inode, ordered, 1);
3093                 btrfs_put_ordered_extent(ordered);
3094         }
3095
3096         for (index = 0; index < nr_pages; index++) {
3097                 __do_readpage(tree, pages[index], btrfs_get_extent, em_cached,
3098                                 bio, 0, bio_flags, REQ_RAHEAD, prev_em_start);
3099                 put_page(pages[index]);
3100         }
3101 }
3102
3103 static void __extent_readpages(struct extent_io_tree *tree,
3104                                struct page *pages[],
3105                                int nr_pages,
3106                                struct extent_map **em_cached,
3107                                struct bio **bio, unsigned long *bio_flags,
3108                                u64 *prev_em_start)
3109 {
3110         u64 start = 0;
3111         u64 end = 0;
3112         u64 page_start;
3113         int index;
3114         int first_index = 0;
3115
3116         for (index = 0; index < nr_pages; index++) {
3117                 page_start = page_offset(pages[index]);
3118                 if (!end) {
3119                         start = page_start;
3120                         end = start + PAGE_SIZE - 1;
3121                         first_index = index;
3122                 } else if (end + 1 == page_start) {
3123                         end += PAGE_SIZE;
3124                 } else {
3125                         __do_contiguous_readpages(tree, &pages[first_index],
3126                                                   index - first_index, start,
3127                                                   end, em_cached,
3128                                                   bio, bio_flags,
3129                                                   prev_em_start);
3130                         start = page_start;
3131                         end = start + PAGE_SIZE - 1;
3132                         first_index = index;
3133                 }
3134         }
3135
3136         if (end)
3137                 __do_contiguous_readpages(tree, &pages[first_index],
3138                                           index - first_index, start,
3139                                           end, em_cached, bio,
3140                                           bio_flags, prev_em_start);
3141 }
3142
3143 static int __extent_read_full_page(struct extent_io_tree *tree,
3144                                    struct page *page,
3145                                    get_extent_t *get_extent,
3146                                    struct bio **bio, int mirror_num,
3147                                    unsigned long *bio_flags,
3148                                    unsigned int read_flags)
3149 {
3150         struct inode *inode = page->mapping->host;
3151         struct btrfs_ordered_extent *ordered;
3152         u64 start = page_offset(page);
3153         u64 end = start + PAGE_SIZE - 1;
3154         int ret;
3155
3156         while (1) {
3157                 lock_extent(tree, start, end);
3158                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
3159                                                 PAGE_SIZE);
3160                 if (!ordered)
3161                         break;
3162                 unlock_extent(tree, start, end);
3163                 btrfs_start_ordered_extent(inode, ordered, 1);
3164                 btrfs_put_ordered_extent(ordered);
3165         }
3166
3167         ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3168                             bio_flags, read_flags, NULL);
3169         return ret;
3170 }
3171
3172 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
3173                             get_extent_t *get_extent, int mirror_num)
3174 {
3175         struct bio *bio = NULL;
3176         unsigned long bio_flags = 0;
3177         int ret;
3178
3179         ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
3180                                       &bio_flags, 0);
3181         if (bio)
3182                 ret = submit_one_bio(bio, mirror_num, bio_flags);
3183         return ret;
3184 }
3185
3186 static void update_nr_written(struct writeback_control *wbc,
3187                               unsigned long nr_written)
3188 {
3189         wbc->nr_to_write -= nr_written;
3190 }
3191
3192 /*
3193  * helper for __extent_writepage, doing all of the delayed allocation setup.
3194  *
3195  * This returns 1 if btrfs_run_delalloc_range function did all the work required
3196  * to write the page (copy into inline extent).  In this case the IO has
3197  * been started and the page is already unlocked.
3198  *
3199  * This returns 0 if all went well (page still locked)
3200  * This returns < 0 if there were errors (page still locked)
3201  */
3202 static noinline_for_stack int writepage_delalloc(struct inode *inode,
3203                 struct page *page, struct writeback_control *wbc,
3204                 u64 delalloc_start, unsigned long *nr_written)
3205 {
3206         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
3207         u64 page_end = delalloc_start + PAGE_SIZE - 1;
3208         bool found;
3209         u64 delalloc_to_write = 0;
3210         u64 delalloc_end = 0;
3211         int ret;
3212         int page_started = 0;
3213
3214
3215         while (delalloc_end < page_end) {
3216                 found = find_lock_delalloc_range(inode, tree,
3217                                                page,
3218                                                &delalloc_start,
3219                                                &delalloc_end);
3220                 if (!found) {
3221                         delalloc_start = delalloc_end + 1;
3222                         continue;
3223                 }
3224                 ret = btrfs_run_delalloc_range(inode, page, delalloc_start,
3225                                 delalloc_end, &page_started, nr_written, wbc);
3226                 /* File system has been set read-only */
3227                 if (ret) {
3228                         SetPageError(page);
3229                         /*
3230                          * btrfs_run_delalloc_range should return < 0 for error
3231                          * but just in case, we use > 0 here meaning the IO is
3232                          * started, so we don't want to return > 0 unless
3233                          * things are going well.
3234                          */
3235                         ret = ret < 0 ? ret : -EIO;
3236                         goto done;
3237                 }
3238                 /*
3239                  * delalloc_end is already one less than the total length, so
3240                  * we don't subtract one from PAGE_SIZE
3241                  */
3242                 delalloc_to_write += (delalloc_end - delalloc_start +
3243                                       PAGE_SIZE) >> PAGE_SHIFT;
3244                 delalloc_start = delalloc_end + 1;
3245         }
3246         if (wbc->nr_to_write < delalloc_to_write) {
3247                 int thresh = 8192;
3248
3249                 if (delalloc_to_write < thresh * 2)
3250                         thresh = delalloc_to_write;
3251                 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3252                                          thresh);
3253         }
3254
3255         /* did the fill delalloc function already unlock and start
3256          * the IO?
3257          */
3258         if (page_started) {
3259                 /*
3260                  * we've unlocked the page, so we can't update
3261                  * the mapping's writeback index, just update
3262                  * nr_to_write.
3263                  */
3264                 wbc->nr_to_write -= *nr_written;
3265                 return 1;
3266         }
3267
3268         ret = 0;
3269
3270 done:
3271         return ret;
3272 }
3273