4188a488114814aadf7df7b4c2ba07240b8e0408
[muen/linux.git] / drivers / md / raid5.c
1 /*
2  * raid5.c : Multiple Devices driver for Linux
3  *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
4  *         Copyright (C) 1999, 2000 Ingo Molnar
5  *         Copyright (C) 2002, 2003 H. Peter Anvin
6  *
7  * RAID-4/5/6 management functions.
8  * Thanks to Penguin Computing for making the RAID-6 development possible
9  * by donating a test server!
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2, or (at your option)
14  * any later version.
15  *
16  * You should have received a copy of the GNU General Public License
17  * (for example /usr/src/linux/COPYING); if not, write to the Free
18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  */
20
21 /*
22  * BITMAP UNPLUGGING:
23  *
24  * The sequencing for updating the bitmap reliably is a little
25  * subtle (and I got it wrong the first time) so it deserves some
26  * explanation.
27  *
28  * We group bitmap updates into batches.  Each batch has a number.
29  * We may write out several batches at once, but that isn't very important.
30  * conf->seq_write is the number of the last batch successfully written.
31  * conf->seq_flush is the number of the last batch that was closed to
32  *    new additions.
33  * When we discover that we will need to write to any block in a stripe
34  * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35  * the number of the batch it will be in. This is seq_flush+1.
36  * When we are ready to do a write, if that batch hasn't been written yet,
37  *   we plug the array and queue the stripe for later.
38  * When an unplug happens, we increment bm_flush, thus closing the current
39  *   batch.
40  * When we notice that bm_flush > bm_write, we write out all pending updates
41  * to the bitmap, and advance bm_write to where bm_flush was.
42  * This may occasionally write a bit out twice, but is sure never to
43  * miss any bits.
44  */
45
46 #include <linux/blkdev.h>
47 #include <linux/kthread.h>
48 #include <linux/raid/pq.h>
49 #include <linux/async_tx.h>
50 #include <linux/module.h>
51 #include <linux/async.h>
52 #include <linux/seq_file.h>
53 #include <linux/cpu.h>
54 #include <linux/slab.h>
55 #include <linux/ratelimit.h>
56 #include <linux/nodemask.h>
57 #include <linux/flex_array.h>
58 #include <linux/sched/signal.h>
59
60 #include <trace/events/block.h>
61 #include <linux/list_sort.h>
62
63 #include "md.h"
64 #include "raid5.h"
65 #include "raid0.h"
66 #include "bitmap.h"
67 #include "raid5-log.h"
68
69 #define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)
70
71 #define cpu_to_group(cpu) cpu_to_node(cpu)
72 #define ANY_GROUP NUMA_NO_NODE
73
74 static bool devices_handle_discard_safely = false;
75 module_param(devices_handle_discard_safely, bool, 0644);
76 MODULE_PARM_DESC(devices_handle_discard_safely,
77                  "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
78 static struct workqueue_struct *raid5_wq;
79
80 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
81 {
82         int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
83         return &conf->stripe_hashtbl[hash];
84 }
85
86 static inline int stripe_hash_locks_hash(sector_t sect)
87 {
88         return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
89 }
90
91 static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
92 {
93         spin_lock_irq(conf->hash_locks + hash);
94         spin_lock(&conf->device_lock);
95 }
96
97 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
98 {
99         spin_unlock(&conf->device_lock);
100         spin_unlock_irq(conf->hash_locks + hash);
101 }
102
103 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
104 {
105         int i;
106         spin_lock_irq(conf->hash_locks);
107         for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
108                 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
109         spin_lock(&conf->device_lock);
110 }
111
112 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
113 {
114         int i;
115         spin_unlock(&conf->device_lock);
116         for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--)
117                 spin_unlock(conf->hash_locks + i);
118         spin_unlock_irq(conf->hash_locks);
119 }
120
121 /* Find first data disk in a raid6 stripe */
122 static inline int raid6_d0(struct stripe_head *sh)
123 {
124         if (sh->ddf_layout)
125                 /* ddf always start from first device */
126                 return 0;
127         /* md starts just after Q block */
128         if (sh->qd_idx == sh->disks - 1)
129                 return 0;
130         else
131                 return sh->qd_idx + 1;
132 }
133 static inline int raid6_next_disk(int disk, int raid_disks)
134 {
135         disk++;
136         return (disk < raid_disks) ? disk : 0;
137 }
138
139 /* When walking through the disks in a raid5, starting at raid6_d0,
140  * We need to map each disk to a 'slot', where the data disks are slot
141  * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
142  * is raid_disks-1.  This help does that mapping.
143  */
144 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
145                              int *count, int syndrome_disks)
146 {
147         int slot = *count;
148
149         if (sh->ddf_layout)
150                 (*count)++;
151         if (idx == sh->pd_idx)
152                 return syndrome_disks;
153         if (idx == sh->qd_idx)
154                 return syndrome_disks + 1;
155         if (!sh->ddf_layout)
156                 (*count)++;
157         return slot;
158 }
159
160 static void print_raid5_conf (struct r5conf *conf);
161
162 static int stripe_operations_active(struct stripe_head *sh)
163 {
164         return sh->check_state || sh->reconstruct_state ||
165                test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
166                test_bit(STRIPE_COMPUTE_RUN, &sh->state);
167 }
168
169 static bool stripe_is_lowprio(struct stripe_head *sh)
170 {
171         return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
172                 test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
173                !test_bit(STRIPE_R5C_CACHING, &sh->state);
174 }
175
176 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
177 {
178         struct r5conf *conf = sh->raid_conf;
179         struct r5worker_group *group;
180         int thread_cnt;
181         int i, cpu = sh->cpu;
182
183         if (!cpu_online(cpu)) {
184                 cpu = cpumask_any(cpu_online_mask);
185                 sh->cpu = cpu;
186         }
187
188         if (list_empty(&sh->lru)) {
189                 struct r5worker_group *group;
190                 group = conf->worker_groups + cpu_to_group(cpu);
191                 if (stripe_is_lowprio(sh))
192                         list_add_tail(&sh->lru, &group->loprio_list);
193                 else
194                         list_add_tail(&sh->lru, &group->handle_list);
195                 group->stripes_cnt++;
196                 sh->group = group;
197         }
198
199         if (conf->worker_cnt_per_group == 0) {
200                 md_wakeup_thread(conf->mddev->thread);
201                 return;
202         }
203
204         group = conf->worker_groups + cpu_to_group(sh->cpu);
205
206         group->workers[0].working = true;
207         /* at least one worker should run to avoid race */
208         queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
209
210         thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
211         /* wakeup more workers */
212         for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
213                 if (group->workers[i].working == false) {
214                         group->workers[i].working = true;
215                         queue_work_on(sh->cpu, raid5_wq,
216                                       &group->workers[i].work);
217                         thread_cnt--;
218                 }
219         }
220 }
221
222 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
223                               struct list_head *temp_inactive_list)
224 {
225         int i;
226         int injournal = 0;      /* number of date pages with R5_InJournal */
227
228         BUG_ON(!list_empty(&sh->lru));
229         BUG_ON(atomic_read(&conf->active_stripes)==0);
230
231         if (r5c_is_writeback(conf->log))
232                 for (i = sh->disks; i--; )
233                         if (test_bit(R5_InJournal, &sh->dev[i].flags))
234                                 injournal++;
235         /*
236          * In the following cases, the stripe cannot be released to cached
237          * lists. Therefore, we make the stripe write out and set
238          * STRIPE_HANDLE:
239          *   1. when quiesce in r5c write back;
240          *   2. when resync is requested fot the stripe.
241          */
242         if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
243             (conf->quiesce && r5c_is_writeback(conf->log) &&
244              !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
245                 if (test_bit(STRIPE_R5C_CACHING, &sh->state))
246                         r5c_make_stripe_write_out(sh);
247                 set_bit(STRIPE_HANDLE, &sh->state);
248         }
249
250         if (test_bit(STRIPE_HANDLE, &sh->state)) {
251                 if (test_bit(STRIPE_DELAYED, &sh->state) &&
252                     !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
253                         list_add_tail(&sh->lru, &conf->delayed_list);
254                 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
255                            sh->bm_seq - conf->seq_write > 0)
256                         list_add_tail(&sh->lru, &conf->bitmap_list);
257                 else {
258                         clear_bit(STRIPE_DELAYED, &sh->state);
259                         clear_bit(STRIPE_BIT_DELAY, &sh->state);
260                         if (conf->worker_cnt_per_group == 0) {
261                                 if (stripe_is_lowprio(sh))
262                                         list_add_tail(&sh->lru,
263                                                         &conf->loprio_list);
264                                 else
265                                         list_add_tail(&sh->lru,
266                                                         &conf->handle_list);
267                         } else {
268                                 raid5_wakeup_stripe_thread(sh);
269                                 return;
270                         }
271                 }
272                 md_wakeup_thread(conf->mddev->thread);
273         } else {
274                 BUG_ON(stripe_operations_active(sh));
275                 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
276                         if (atomic_dec_return(&conf->preread_active_stripes)
277                             < IO_THRESHOLD)
278                                 md_wakeup_thread(conf->mddev->thread);
279                 atomic_dec(&conf->active_stripes);
280                 if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
281                         if (!r5c_is_writeback(conf->log))
282                                 list_add_tail(&sh->lru, temp_inactive_list);
283                         else {
284                                 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
285                                 if (injournal == 0)
286                                         list_add_tail(&sh->lru, temp_inactive_list);
287                                 else if (injournal == conf->raid_disks - conf->max_degraded) {
288                                         /* full stripe */
289                                         if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
290                                                 atomic_inc(&conf->r5c_cached_full_stripes);
291                                         if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
292                                                 atomic_dec(&conf->r5c_cached_partial_stripes);
293                                         list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
294                                         r5c_check_cached_full_stripe(conf);
295                                 } else
296                                         /*
297                                          * STRIPE_R5C_PARTIAL_STRIPE is set in
298                                          * r5c_try_caching_write(). No need to
299                                          * set it again.
300                                          */
301                                         list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
302                         }
303                 }
304         }
305 }
306
307 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
308                              struct list_head *temp_inactive_list)
309 {
310         if (atomic_dec_and_test(&sh->count))
311                 do_release_stripe(conf, sh, temp_inactive_list);
312 }
313
314 /*
315  * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
316  *
317  * Be careful: Only one task can add/delete stripes from temp_inactive_list at
318  * given time. Adding stripes only takes device lock, while deleting stripes
319  * only takes hash lock.
320  */
321 static void release_inactive_stripe_list(struct r5conf *conf,
322                                          struct list_head *temp_inactive_list,
323                                          int hash)
324 {
325         int size;
326         bool do_wakeup = false;
327         unsigned long flags;
328
329         if (hash == NR_STRIPE_HASH_LOCKS) {
330                 size = NR_STRIPE_HASH_LOCKS;
331                 hash = NR_STRIPE_HASH_LOCKS - 1;
332         } else
333                 size = 1;
334         while (size) {
335                 struct list_head *list = &temp_inactive_list[size - 1];
336
337                 /*
338                  * We don't hold any lock here yet, raid5_get_active_stripe() might
339                  * remove stripes from the list
340                  */
341                 if (!list_empty_careful(list)) {
342                         spin_lock_irqsave(conf->hash_locks + hash, flags);
343                         if (list_empty(conf->inactive_list + hash) &&
344                             !list_empty(list))
345                                 atomic_dec(&conf->empty_inactive_list_nr);
346                         list_splice_tail_init(list, conf->inactive_list + hash);
347                         do_wakeup = true;
348                         spin_unlock_irqrestore(conf->hash_locks + hash, flags);
349                 }
350                 size--;
351                 hash--;
352         }
353
354         if (do_wakeup) {
355                 wake_up(&conf->wait_for_stripe);
356                 if (atomic_read(&conf->active_stripes) == 0)
357                         wake_up(&conf->wait_for_quiescent);
358                 if (conf->retry_read_aligned)
359                         md_wakeup_thread(conf->mddev->thread);
360         }
361 }
362
363 /* should hold conf->device_lock already */
364 static int release_stripe_list(struct r5conf *conf,
365                                struct list_head *temp_inactive_list)
366 {
367         struct stripe_head *sh, *t;
368         int count = 0;
369         struct llist_node *head;
370
371         head = llist_del_all(&conf->released_stripes);
372         head = llist_reverse_order(head);
373         llist_for_each_entry_safe(sh, t, head, release_list) {
374                 int hash;
375
376                 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
377                 smp_mb();
378                 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
379                 /*
380                  * Don't worry the bit is set here, because if the bit is set
381                  * again, the count is always > 1. This is true for
382                  * STRIPE_ON_UNPLUG_LIST bit too.
383                  */
384                 hash = sh->hash_lock_index;
385                 __release_stripe(conf, sh, &temp_inactive_list[hash]);
386                 count++;
387         }
388
389         return count;
390 }
391
392 void raid5_release_stripe(struct stripe_head *sh)
393 {
394         struct r5conf *conf = sh->raid_conf;
395         unsigned long flags;
396         struct list_head list;
397         int hash;
398         bool wakeup;
399
400         /* Avoid release_list until the last reference.
401          */
402         if (atomic_add_unless(&sh->count, -1, 1))
403                 return;
404
405         if (unlikely(!conf->mddev->thread) ||
406                 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
407                 goto slow_path;
408         wakeup = llist_add(&sh->release_list, &conf->released_stripes);
409         if (wakeup)
410                 md_wakeup_thread(conf->mddev->thread);
411         return;
412 slow_path:
413         local_irq_save(flags);
414         /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
415         if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
416                 INIT_LIST_HEAD(&list);
417                 hash = sh->hash_lock_index;
418                 do_release_stripe(conf, sh, &list);
419                 spin_unlock(&conf->device_lock);
420                 release_inactive_stripe_list(conf, &list, hash);
421         }
422         local_irq_restore(flags);
423 }
424
425 static inline void remove_hash(struct stripe_head *sh)
426 {
427         pr_debug("remove_hash(), stripe %llu\n",
428                 (unsigned long long)sh->sector);
429
430         hlist_del_init(&sh->hash);
431 }
432
433 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
434 {
435         struct hlist_head *hp = stripe_hash(conf, sh->sector);
436
437         pr_debug("insert_hash(), stripe %llu\n",
438                 (unsigned long long)sh->sector);
439
440         hlist_add_head(&sh->hash, hp);
441 }
442
443 /* find an idle stripe, make sure it is unhashed, and return it. */
444 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
445 {
446         struct stripe_head *sh = NULL;
447         struct list_head *first;
448
449         if (list_empty(conf->inactive_list + hash))
450                 goto out;
451         first = (conf->inactive_list + hash)->next;
452         sh = list_entry(first, struct stripe_head, lru);
453         list_del_init(first);
454         remove_hash(sh);
455         atomic_inc(&conf->active_stripes);
456         BUG_ON(hash != sh->hash_lock_index);
457         if (list_empty(conf->inactive_list + hash))
458                 atomic_inc(&conf->empty_inactive_list_nr);
459 out:
460         return sh;
461 }
462
463 static void shrink_buffers(struct stripe_head *sh)
464 {
465         struct page *p;
466         int i;
467         int num = sh->raid_conf->pool_size;
468
469         for (i = 0; i < num ; i++) {
470                 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
471                 p = sh->dev[i].page;
472                 if (!p)
473                         continue;
474                 sh->dev[i].page = NULL;
475                 put_page(p);
476         }
477 }
478
479 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
480 {
481         int i;
482         int num = sh->raid_conf->pool_size;
483
484         for (i = 0; i < num; i++) {
485                 struct page *page;
486
487                 if (!(page = alloc_page(gfp))) {
488                         return 1;
489                 }
490                 sh->dev[i].page = page;
491                 sh->dev[i].orig_page = page;
492         }
493
494         return 0;
495 }
496
497 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
498                             struct stripe_head *sh);
499
500 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
501 {
502         struct r5conf *conf = sh->raid_conf;
503         int i, seq;
504
505         BUG_ON(atomic_read(&sh->count) != 0);
506         BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
507         BUG_ON(stripe_operations_active(sh));
508         BUG_ON(sh->batch_head);
509
510         pr_debug("init_stripe called, stripe %llu\n",
511                 (unsigned long long)sector);
512 retry:
513         seq = read_seqcount_begin(&conf->gen_lock);
514         sh->generation = conf->generation - previous;
515         sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
516         sh->sector = sector;
517         stripe_set_idx(sector, conf, previous, sh);
518         sh->state = 0;
519
520         for (i = sh->disks; i--; ) {
521                 struct r5dev *dev = &sh->dev[i];
522
523                 if (dev->toread || dev->read || dev->towrite || dev->written ||
524                     test_bit(R5_LOCKED, &dev->flags)) {
525                         pr_err("sector=%llx i=%d %p %p %p %p %d\n",
526                                (unsigned long long)sh->sector, i, dev->toread,
527                                dev->read, dev->towrite, dev->written,
528                                test_bit(R5_LOCKED, &dev->flags));
529                         WARN_ON(1);
530                 }
531                 dev->flags = 0;
532                 dev->sector = raid5_compute_blocknr(sh, i, previous);
533         }
534         if (read_seqcount_retry(&conf->gen_lock, seq))
535                 goto retry;
536         sh->overwrite_disks = 0;
537         insert_hash(conf, sh);
538         sh->cpu = smp_processor_id();
539         set_bit(STRIPE_BATCH_READY, &sh->state);
540 }
541
542 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
543                                          short generation)
544 {
545         struct stripe_head *sh;
546
547         pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
548         hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
549                 if (sh->sector == sector && sh->generation == generation)
550                         return sh;
551         pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
552         return NULL;
553 }
554
555 /*
556  * Need to check if array has failed when deciding whether to:
557  *  - start an array
558  *  - remove non-faulty devices
559  *  - add a spare
560  *  - allow a reshape
561  * This determination is simple when no reshape is happening.
562  * However if there is a reshape, we need to carefully check
563  * both the before and after sections.
564  * This is because some failed devices may only affect one
565  * of the two sections, and some non-in_sync devices may
566  * be insync in the section most affected by failed devices.
567  */
568 int raid5_calc_degraded(struct r5conf *conf)
569 {
570         int degraded, degraded2;
571         int i;
572
573         rcu_read_lock();
574         degraded = 0;
575         for (i = 0; i < conf->previous_raid_disks; i++) {
576                 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
577                 if (rdev && test_bit(Faulty, &rdev->flags))
578                         rdev = rcu_dereference(conf->disks[i].replacement);
579                 if (!rdev || test_bit(Faulty, &rdev->flags))
580                         degraded++;
581                 else if (test_bit(In_sync, &rdev->flags))
582                         ;
583                 else
584                         /* not in-sync or faulty.
585                          * If the reshape increases the number of devices,
586                          * this is being recovered by the reshape, so
587                          * this 'previous' section is not in_sync.
588                          * If the number of devices is being reduced however,
589                          * the device can only be part of the array if
590                          * we are reverting a reshape, so this section will
591                          * be in-sync.
592                          */
593                         if (conf->raid_disks >= conf->previous_raid_disks)
594                                 degraded++;
595         }
596         rcu_read_unlock();
597         if (conf->raid_disks == conf->previous_raid_disks)
598                 return degraded;
599         rcu_read_lock();
600         degraded2 = 0;
601         for (i = 0; i < conf->raid_disks; i++) {
602                 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
603                 if (rdev && test_bit(Faulty, &rdev->flags))
604                         rdev = rcu_dereference(conf->disks[i].replacement);
605                 if (!rdev || test_bit(Faulty, &rdev->flags))
606                         degraded2++;
607                 else if (test_bit(In_sync, &rdev->flags))
608                         ;
609                 else
610                         /* not in-sync or faulty.
611                          * If reshape increases the number of devices, this
612                          * section has already been recovered, else it
613                          * almost certainly hasn't.
614                          */
615                         if (conf->raid_disks <= conf->previous_raid_disks)
616                                 degraded2++;
617         }
618         rcu_read_unlock();
619         if (degraded2 > degraded)
620                 return degraded2;
621         return degraded;
622 }
623
624 static int has_failed(struct r5conf *conf)
625 {
626         int degraded;
627
628         if (conf->mddev->reshape_position == MaxSector)
629                 return conf->mddev->degraded > conf->max_degraded;
630
631         degraded = raid5_calc_degraded(conf);
632         if (degraded > conf->max_degraded)
633                 return 1;
634         return 0;
635 }
636
637 struct stripe_head *
638 raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
639                         int previous, int noblock, int noquiesce)
640 {
641         struct stripe_head *sh;
642         int hash = stripe_hash_locks_hash(sector);
643         int inc_empty_inactive_list_flag;
644
645         pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
646
647         spin_lock_irq(conf->hash_locks + hash);
648
649         do {
650                 wait_event_lock_irq(conf->wait_for_quiescent,
651                                     conf->quiesce == 0 || noquiesce,
652                                     *(conf->hash_locks + hash));
653                 sh = __find_stripe(conf, sector, conf->generation - previous);
654                 if (!sh) {
655                         if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
656                                 sh = get_free_stripe(conf, hash);
657                                 if (!sh && !test_bit(R5_DID_ALLOC,
658                                                      &conf->cache_state))
659                                         set_bit(R5_ALLOC_MORE,
660                                                 &conf->cache_state);
661                         }
662                         if (noblock && sh == NULL)
663                                 break;
664
665                         r5c_check_stripe_cache_usage(conf);
666                         if (!sh) {
667                                 set_bit(R5_INACTIVE_BLOCKED,
668                                         &conf->cache_state);
669                                 r5l_wake_reclaim(conf->log, 0);
670                                 wait_event_lock_irq(
671                                         conf->wait_for_stripe,
672                                         !list_empty(conf->inactive_list + hash) &&
673                                         (atomic_read(&conf->active_stripes)
674                                          < (conf->max_nr_stripes * 3 / 4)
675                                          || !test_bit(R5_INACTIVE_BLOCKED,
676                                                       &conf->cache_state)),
677                                         *(conf->hash_locks + hash));
678                                 clear_bit(R5_INACTIVE_BLOCKED,
679                                           &conf->cache_state);
680                         } else {
681                                 init_stripe(sh, sector, previous);
682                                 atomic_inc(&sh->count);
683                         }
684                 } else if (!atomic_inc_not_zero(&sh->count)) {
685                         spin_lock(&conf->device_lock);
686                         if (!atomic_read(&sh->count)) {
687                                 if (!test_bit(STRIPE_HANDLE, &sh->state))
688                                         atomic_inc(&conf->active_stripes);
689                                 BUG_ON(list_empty(&sh->lru) &&
690                                        !test_bit(STRIPE_EXPANDING, &sh->state));
691                                 inc_empty_inactive_list_flag = 0;
692                                 if (!list_empty(conf->inactive_list + hash))
693                                         inc_empty_inactive_list_flag = 1;
694                                 list_del_init(&sh->lru);
695                                 if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
696                                         atomic_inc(&conf->empty_inactive_list_nr);
697                                 if (sh->group) {
698                                         sh->group->stripes_cnt--;
699                                         sh->group = NULL;
700                                 }
701                         }
702                         atomic_inc(&sh->count);
703                         spin_unlock(&conf->device_lock);
704                 }
705         } while (sh == NULL);
706
707         spin_unlock_irq(conf->hash_locks + hash);
708         return sh;
709 }
710
711 static bool is_full_stripe_write(struct stripe_head *sh)
712 {
713         BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
714         return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
715 }
716
717 static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
718 {
719         if (sh1 > sh2) {
720                 spin_lock_irq(&sh2->stripe_lock);
721                 spin_lock_nested(&sh1->stripe_lock, 1);
722         } else {
723                 spin_lock_irq(&sh1->stripe_lock);
724                 spin_lock_nested(&sh2->stripe_lock, 1);
725         }
726 }
727
728 static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
729 {
730         spin_unlock(&sh1->stripe_lock);
731         spin_unlock_irq(&sh2->stripe_lock);
732 }
733
734 /* Only freshly new full stripe normal write stripe can be added to a batch list */
735 static bool stripe_can_batch(struct stripe_head *sh)
736 {
737         struct r5conf *conf = sh->raid_conf;
738
739         if (conf->log || raid5_has_ppl(conf))
740                 return false;
741         return test_bit(STRIPE_BATCH_READY, &sh->state) &&
742                 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
743                 is_full_stripe_write(sh);
744 }
745
746 /* we only do back search */
747 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
748 {
749         struct stripe_head *head;
750         sector_t head_sector, tmp_sec;
751         int hash;
752         int dd_idx;
753         int inc_empty_inactive_list_flag;
754
755         /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
756         tmp_sec = sh->sector;
757         if (!sector_div(tmp_sec, conf->chunk_sectors))
758                 return;
759         head_sector = sh->sector - STRIPE_SECTORS;
760
761         hash = stripe_hash_locks_hash(head_sector);
762         spin_lock_irq(conf->hash_locks + hash);
763         head = __find_stripe(conf, head_sector, conf->generation);
764         if (head && !atomic_inc_not_zero(&head->count)) {
765                 spin_lock(&conf->device_lock);
766                 if (!atomic_read(&head->count)) {
767                         if (!test_bit(STRIPE_HANDLE, &head->state))
768                                 atomic_inc(&conf->active_stripes);
769                         BUG_ON(list_empty(&head->lru) &&
770                                !test_bit(STRIPE_EXPANDING, &head->state));
771                         inc_empty_inactive_list_flag = 0;
772                         if (!list_empty(conf->inactive_list + hash))
773                                 inc_empty_inactive_list_flag = 1;
774                         list_del_init(&head->lru);
775                         if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
776                                 atomic_inc(&conf->empty_inactive_list_nr);
777                         if (head->group) {
778                                 head->group->stripes_cnt--;
779                                 head->group = NULL;
780                         }
781                 }
782                 atomic_inc(&head->count);
783                 spin_unlock(&conf->device_lock);
784         }
785         spin_unlock_irq(conf->hash_locks + hash);
786
787         if (!head)
788                 return;
789         if (!stripe_can_batch(head))
790                 goto out;
791
792         lock_two_stripes(head, sh);
793         /* clear_batch_ready clear the flag */
794         if (!stripe_can_batch(head) || !stripe_can_batch(sh))
795                 goto unlock_out;
796
797         if (sh->batch_head)
798                 goto unlock_out;
799
800         dd_idx = 0;
801         while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
802                 dd_idx++;
803         if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
804             bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
805                 goto unlock_out;
806
807         if (head->batch_head) {
808                 spin_lock(&head->batch_head->batch_lock);
809                 /* This batch list is already running */
810                 if (!stripe_can_batch(head)) {
811                         spin_unlock(&head->batch_head->batch_lock);
812                         goto unlock_out;
813                 }
814
815                 /*
816                  * at this point, head's BATCH_READY could be cleared, but we
817                  * can still add the stripe to batch list
818                  */
819                 list_add(&sh->batch_list, &head->batch_list);
820                 spin_unlock(&head->batch_head->batch_lock);
821
822                 sh->batch_head = head->batch_head;
823         } else {
824                 head->batch_head = head;
825                 sh->batch_head = head->batch_head;
826                 spin_lock(&head->batch_lock);
827                 list_add_tail(&sh->batch_list, &head->batch_list);
828                 spin_unlock(&head->batch_lock);
829         }
830
831         if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
832                 if (atomic_dec_return(&conf->preread_active_stripes)
833                     < IO_THRESHOLD)
834                         md_wakeup_thread(conf->mddev->thread);
835
836         if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
837                 int seq = sh->bm_seq;
838                 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
839                     sh->batch_head->bm_seq > seq)
840                         seq = sh->batch_head->bm_seq;
841                 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
842                 sh->batch_head->bm_seq = seq;
843         }
844
845         atomic_inc(&sh->count);
846 unlock_out:
847         unlock_two_stripes(head, sh);
848 out:
849         raid5_release_stripe(head);
850 }
851
852 /* Determine if 'data_offset' or 'new_data_offset' should be used
853  * in this stripe_head.
854  */
855 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
856 {
857         sector_t progress = conf->reshape_progress;
858         /* Need a memory barrier to make sure we see the value
859          * of conf->generation, or ->data_offset that was set before
860          * reshape_progress was updated.
861          */
862         smp_rmb();
863         if (progress == MaxSector)
864                 return 0;
865         if (sh->generation == conf->generation - 1)
866                 return 0;
867         /* We are in a reshape, and this is a new-generation stripe,
868          * so use new_data_offset.
869          */
870         return 1;
871 }
872
873 static void dispatch_bio_list(struct bio_list *tmp)
874 {
875         struct bio *bio;
876
877         while ((bio = bio_list_pop(tmp)))
878                 generic_make_request(bio);
879 }
880
881 static int cmp_stripe(void *priv, struct list_head *a, struct list_head *b)
882 {
883         const struct r5pending_data *da = list_entry(a,
884                                 struct r5pending_data, sibling);
885         const struct r5pending_data *db = list_entry(b,
886                                 struct r5pending_data, sibling);
887         if (da->sector > db->sector)
888                 return 1;
889         if (da->sector < db->sector)
890                 return -1;
891         return 0;
892 }
893
894 static void dispatch_defer_bios(struct r5conf *conf, int target,
895                                 struct bio_list *list)
896 {
897         struct r5pending_data *data;
898         struct list_head *first, *next = NULL;
899         int cnt = 0;
900
901         if (conf->pending_data_cnt == 0)
902                 return;
903
904         list_sort(NULL, &conf->pending_list, cmp_stripe);
905
906         first = conf->pending_list.next;
907
908         /* temporarily move the head */
909         if (conf->next_pending_data)
910                 list_move_tail(&conf->pending_list,
911                                 &conf->next_pending_data->sibling);
912
913         while (!list_empty(&conf->pending_list)) {
914                 data = list_first_entry(&conf->pending_list,
915                         struct r5pending_data, sibling);
916                 if (&data->sibling == first)
917                         first = data->sibling.next;
918                 next = data->sibling.next;
919
920                 bio_list_merge(list, &data->bios);
921                 list_move(&data->sibling, &conf->free_list);
922                 cnt++;
923                 if (cnt >= target)
924                         break;
925         }
926         conf->pending_data_cnt -= cnt;
927         BUG_ON(conf->pending_data_cnt < 0 || cnt < target);
928
929         if (next != &conf->pending_list)
930                 conf->next_pending_data = list_entry(next,
931                                 struct r5pending_data, sibling);
932         else
933                 conf->next_pending_data = NULL;
934         /* list isn't empty */
935         if (first != &conf->pending_list)
936                 list_move_tail(&conf->pending_list, first);
937 }
938
939 static void flush_deferred_bios(struct r5conf *conf)
940 {
941         struct bio_list tmp = BIO_EMPTY_LIST;
942
943         if (conf->pending_data_cnt == 0)
944                 return;
945
946         spin_lock(&conf->pending_bios_lock);
947         dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);
948         BUG_ON(conf->pending_data_cnt != 0);
949         spin_unlock(&conf->pending_bios_lock);
950
951         dispatch_bio_list(&tmp);
952 }
953
954 static void defer_issue_bios(struct r5conf *conf, sector_t sector,
955                                 struct bio_list *bios)
956 {
957         struct bio_list tmp = BIO_EMPTY_LIST;
958         struct r5pending_data *ent;
959
960         spin_lock(&conf->pending_bios_lock);
961         ent = list_first_entry(&conf->free_list, struct r5pending_data,
962                                                         sibling);
963         list_move_tail(&ent->sibling, &conf->pending_list);
964         ent->sector = sector;
965         bio_list_init(&ent->bios);
966         bio_list_merge(&ent->bios, bios);
967         conf->pending_data_cnt++;
968         if (conf->pending_data_cnt >= PENDING_IO_MAX)
969                 dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);
970
971         spin_unlock(&conf->pending_bios_lock);
972
973         dispatch_bio_list(&tmp);
974 }
975
976 static void
977 raid5_end_read_request(struct bio *bi);
978 static void
979 raid5_end_write_request(struct bio *bi);
980
981 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
982 {
983         struct r5conf *conf = sh->raid_conf;
984         int i, disks = sh->disks;
985         struct stripe_head *head_sh = sh;
986         struct bio_list pending_bios = BIO_EMPTY_LIST;
987         bool should_defer;
988
989         might_sleep();
990
991         if (log_stripe(sh, s) == 0)
992                 return;
993
994         should_defer = conf->batch_bio_dispatch && conf->group_cnt;
995
996         for (i = disks; i--; ) {
997                 int op, op_flags = 0;
998                 int replace_only = 0;
999                 struct bio *bi, *rbi;
1000                 struct md_rdev *rdev, *rrdev = NULL;
1001
1002                 sh = head_sh;
1003                 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
1004                         op = REQ_OP_WRITE;
1005                         if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
1006                                 op_flags = REQ_FUA;
1007                         if (test_bit(R5_Discard, &sh->dev[i].flags))
1008                                 op = REQ_OP_DISCARD;
1009                 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
1010                         op = REQ_OP_READ;
1011                 else if (test_and_clear_bit(R5_WantReplace,
1012                                             &sh->dev[i].flags)) {
1013                         op = REQ_OP_WRITE;
1014                         replace_only = 1;
1015                 } else
1016                         continue;
1017                 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
1018                         op_flags |= REQ_SYNC;
1019
1020 again:
1021                 bi = &sh->dev[i].req;
1022                 rbi = &sh->dev[i].rreq; /* For writing to replacement */
1023
1024                 rcu_read_lock();
1025                 rrdev = rcu_dereference(conf->disks[i].replacement);
1026                 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
1027                 rdev = rcu_dereference(conf->disks[i].rdev);
1028                 if (!rdev) {
1029                         rdev = rrdev;
1030                         rrdev = NULL;
1031                 }
1032                 if (op_is_write(op)) {
1033                         if (replace_only)
1034                                 rdev = NULL;
1035                         if (rdev == rrdev)
1036                                 /* We raced and saw duplicates */
1037                                 rrdev = NULL;
1038                 } else {
1039                         if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
1040                                 rdev = rrdev;
1041                         rrdev = NULL;
1042                 }
1043
1044                 if (rdev && test_bit(Faulty, &rdev->flags))
1045                         rdev = NULL;
1046                 if (rdev)
1047                         atomic_inc(&rdev->nr_pending);
1048                 if (rrdev && test_bit(Faulty, &rrdev->flags))
1049                         rrdev = NULL;
1050                 if (rrdev)
1051                         atomic_inc(&rrdev->nr_pending);
1052                 rcu_read_unlock();
1053
1054                 /* We have already checked bad blocks for reads.  Now
1055                  * need to check for writes.  We never accept write errors
1056                  * on the replacement, so we don't to check rrdev.
1057                  */
1058                 while (op_is_write(op) && rdev &&
1059                        test_bit(WriteErrorSeen, &rdev->flags)) {
1060                         sector_t first_bad;
1061                         int bad_sectors;
1062                         int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
1063                                               &first_bad, &bad_sectors);
1064                         if (!bad)
1065                                 break;
1066
1067                         if (bad < 0) {
1068                                 set_bit(BlockedBadBlocks, &rdev->flags);
1069                                 if (!conf->mddev->external &&
1070                                     conf->mddev->sb_flags) {
1071                                         /* It is very unlikely, but we might
1072                                          * still need to write out the
1073                                          * bad block log - better give it
1074                                          * a chance*/
1075                                         md_check_recovery(conf->mddev);
1076                                 }
1077                                 /*
1078                                  * Because md_wait_for_blocked_rdev
1079                                  * will dec nr_pending, we must
1080                                  * increment it first.
1081                                  */
1082                                 atomic_inc(&rdev->nr_pending);
1083                                 md_wait_for_blocked_rdev(rdev, conf->mddev);
1084                         } else {
1085                                 /* Acknowledged bad block - skip the write */
1086                                 rdev_dec_pending(rdev, conf->mddev);
1087                                 rdev = NULL;
1088                         }
1089                 }
1090
1091                 if (rdev) {
1092                         if (s->syncing || s->expanding || s->expanded
1093                             || s->replacing)
1094                                 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
1095
1096                         set_bit(STRIPE_IO_STARTED, &sh->state);
1097
1098                         bio_set_dev(bi, rdev->bdev);
1099                         bio_set_op_attrs(bi, op, op_flags);
1100                         bi->bi_end_io = op_is_write(op)
1101                                 ? raid5_end_write_request
1102                                 : raid5_end_read_request;
1103                         bi->bi_private = sh;
1104
1105                         pr_debug("%s: for %llu schedule op %d on disc %d\n",
1106                                 __func__, (unsigned long long)sh->sector,
1107                                 bi->bi_opf, i);
1108                         atomic_inc(&sh->count);
1109                         if (sh != head_sh)
1110                                 atomic_inc(&head_sh->count);
1111                         if (use_new_offset(conf, sh))
1112                                 bi->bi_iter.bi_sector = (sh->sector
1113                                                  + rdev->new_data_offset);
1114                         else
1115                                 bi->bi_iter.bi_sector = (sh->sector
1116                                                  + rdev->data_offset);
1117                         if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
1118                                 bi->bi_opf |= REQ_NOMERGE;
1119
1120                         if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1121                                 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1122
1123                         if (!op_is_write(op) &&
1124                             test_bit(R5_InJournal, &sh->dev[i].flags))
1125                                 /*
1126                                  * issuing read for a page in journal, this
1127                                  * must be preparing for prexor in rmw; read
1128                                  * the data into orig_page
1129                                  */
1130                                 sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
1131                         else
1132                                 sh->dev[i].vec.bv_page = sh->dev[i].page;
1133                         bi->bi_vcnt = 1;
1134                         bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1135                         bi->bi_io_vec[0].bv_offset = 0;
1136                         bi->bi_iter.bi_size = STRIPE_SIZE;
1137                         /*
1138                          * If this is discard request, set bi_vcnt 0. We don't
1139                          * want to confuse SCSI because SCSI will replace payload
1140                          */
1141                         if (op == REQ_OP_DISCARD)
1142                                 bi->bi_vcnt = 0;
1143                         if (rrdev)
1144                                 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
1145
1146                         if (conf->mddev->gendisk)
1147                                 trace_block_bio_remap(bi->bi_disk->queue,
1148                                                       bi, disk_devt(conf->mddev->gendisk),
1149                                                       sh->dev[i].sector);
1150                         if (should_defer && op_is_write(op))
1151                                 bio_list_add(&pending_bios, bi);
1152                         else
1153                                 generic_make_request(bi);
1154                 }
1155                 if (rrdev) {
1156                         if (s->syncing || s->expanding || s->expanded
1157                             || s->replacing)
1158                                 md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
1159
1160                         set_bit(STRIPE_IO_STARTED, &sh->state);
1161
1162                         bio_set_dev(rbi, rrdev->bdev);
1163                         bio_set_op_attrs(rbi, op, op_flags);
1164                         BUG_ON(!op_is_write(op));
1165                         rbi->bi_end_io = raid5_end_write_request;
1166                         rbi->bi_private = sh;
1167
1168                         pr_debug("%s: for %llu schedule op %d on "
1169                                  "replacement disc %d\n",
1170                                 __func__, (unsigned long long)sh->sector,
1171                                 rbi->bi_opf, i);
1172                         atomic_inc(&sh->count);
1173                         if (sh != head_sh)
1174                                 atomic_inc(&head_sh->count);
1175                         if (use_new_offset(conf, sh))
1176                                 rbi->bi_iter.bi_sector = (sh->sector
1177                                                   + rrdev->new_data_offset);
1178                         else
1179                                 rbi->bi_iter.bi_sector = (sh->sector
1180                                                   + rrdev->data_offset);
1181                         if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1182                                 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1183                         sh->dev[i].rvec.bv_page = sh->dev[i].page;
1184                         rbi->bi_vcnt = 1;
1185                         rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1186                         rbi->bi_io_vec[0].bv_offset = 0;
1187                         rbi->bi_iter.bi_size = STRIPE_SIZE;
1188                         /*
1189                          * If this is discard request, set bi_vcnt 0. We don't
1190                          * want to confuse SCSI because SCSI will replace payload
1191                          */
1192                         if (op == REQ_OP_DISCARD)
1193                                 rbi->bi_vcnt = 0;
1194                         if (conf->mddev->gendisk)
1195                                 trace_block_bio_remap(rbi->bi_disk->queue,
1196                                                       rbi, disk_devt(conf->mddev->gendisk),
1197                                                       sh->dev[i].sector);
1198                         if (should_defer && op_is_write(op))
1199                                 bio_list_add(&pending_bios, rbi);
1200                         else
1201                                 generic_make_request(rbi);
1202                 }
1203                 if (!rdev && !rrdev) {
1204                         if (op_is_write(op))
1205                                 set_bit(STRIPE_DEGRADED, &sh->state);
1206                         pr_debug("skip op %d on disc %d for sector %llu\n",
1207                                 bi->bi_opf, i, (unsigned long long)sh->sector);
1208                         clear_bit(R5_LOCKED, &sh->dev[i].flags);
1209                         set_bit(STRIPE_HANDLE, &sh->state);
1210                 }
1211
1212                 if (!head_sh->batch_head)
1213                         continue;
1214                 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1215                                       batch_list);
1216                 if (sh != head_sh)
1217                         goto again;
1218         }
1219
1220         if (should_defer && !bio_list_empty(&pending_bios))
1221                 defer_issue_bios(conf, head_sh->sector, &pending_bios);
1222 }
1223
1224 static struct dma_async_tx_descriptor *
1225 async_copy_data(int frombio, struct bio *bio, struct page **page,
1226         sector_t sector, struct dma_async_tx_descriptor *tx,
1227         struct stripe_head *sh, int no_skipcopy)
1228 {
1229         struct bio_vec bvl;
1230         struct bvec_iter iter;
1231         struct page *bio_page;
1232         int page_offset;
1233         struct async_submit_ctl submit;
1234         enum async_tx_flags flags = 0;
1235
1236         if (bio->bi_iter.bi_sector >= sector)
1237                 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
1238         else
1239                 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
1240
1241         if (frombio)
1242                 flags |= ASYNC_TX_FENCE;
1243         init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
1244
1245         bio_for_each_segment(bvl, bio, iter) {
1246                 int len = bvl.bv_len;
1247                 int clen;
1248                 int b_offset = 0;
1249
1250                 if (page_offset < 0) {
1251                         b_offset = -page_offset;
1252                         page_offset += b_offset;
1253                         len -= b_offset;
1254                 }
1255
1256                 if (len > 0 && page_offset + len > STRIPE_SIZE)
1257                         clen = STRIPE_SIZE - page_offset;
1258                 else
1259                         clen = len;
1260
1261                 if (clen > 0) {
1262                         b_offset += bvl.bv_offset;
1263                         bio_page = bvl.bv_page;
1264                         if (frombio) {
1265                                 if (sh->raid_conf->skip_copy &&
1266                                     b_offset == 0 && page_offset == 0 &&
1267                                     clen == STRIPE_SIZE &&
1268                                     !no_skipcopy)
1269                                         *page = bio_page;
1270                                 else
1271                                         tx = async_memcpy(*page, bio_page, page_offset,
1272                                                   b_offset, clen, &submit);
1273                         } else
1274                                 tx = async_memcpy(bio_page, *page, b_offset,
1275                                                   page_offset, clen, &submit);
1276                 }
1277                 /* chain the operations */
1278                 submit.depend_tx = tx;
1279
1280                 if (clen < len) /* hit end of page */
1281                         break;
1282                 page_offset +=  len;
1283         }
1284
1285         return tx;
1286 }
1287
1288 static void ops_complete_biofill(void *stripe_head_ref)
1289 {
1290         struct stripe_head *sh = stripe_head_ref;
1291         int i;
1292
1293         pr_debug("%s: stripe %llu\n", __func__,
1294                 (unsigned long long)sh->sector);
1295
1296         /* clear completed biofills */
1297         for (i = sh->disks; i--; ) {
1298                 struct r5dev *dev = &sh->dev[i];
1299
1300                 /* acknowledge completion of a biofill operation */
1301                 /* and check if we need to reply to a read request,
1302                  * new R5_Wantfill requests are held off until
1303                  * !STRIPE_BIOFILL_RUN
1304                  */
1305                 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
1306                         struct bio *rbi, *rbi2;
1307
1308                         BUG_ON(!dev->read);
1309                         rbi = dev->read;
1310                         dev->read = NULL;
1311                         while (rbi && rbi->bi_iter.bi_sector <
1312                                 dev->sector + STRIPE_SECTORS) {
1313                                 rbi2 = r5_next_bio(rbi, dev->sector);
1314                                 bio_endio(rbi);
1315                                 rbi = rbi2;
1316                         }
1317                 }
1318         }
1319         clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1320
1321         set_bit(STRIPE_HANDLE, &sh->state);
1322         raid5_release_stripe(sh);
1323 }
1324
1325 static void ops_run_biofill(struct stripe_head *sh)
1326 {
1327         struct dma_async_tx_descriptor *tx = NULL;
1328         struct async_submit_ctl submit;
1329         int i;
1330
1331         BUG_ON(sh->batch_head);
1332         pr_debug("%s: stripe %llu\n", __func__,
1333                 (unsigned long long)sh->sector);
1334
1335         for (i = sh->disks; i--; ) {
1336                 struct r5dev *dev = &sh->dev[i];
1337                 if (test_bit(R5_Wantfill, &dev->flags)) {
1338                         struct bio *rbi;
1339                         spin_lock_irq(&sh->stripe_lock);
1340                         dev->read = rbi = dev->toread;
1341                         dev->toread = NULL;
1342                         spin_unlock_irq(&sh->stripe_lock);
1343                         while (rbi && rbi->bi_iter.bi_sector <
1344                                 dev->sector + STRIPE_SECTORS) {
1345                                 tx = async_copy_data(0, rbi, &dev->page,
1346                                                      dev->sector, tx, sh, 0);
1347                                 rbi = r5_next_bio(rbi, dev->sector);
1348                         }
1349                 }
1350         }
1351
1352         atomic_inc(&sh->count);
1353         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1354         async_trigger_callback(&submit);
1355 }
1356
1357 static void mark_target_uptodate(struct stripe_head *sh, int target)
1358 {
1359         struct r5dev *tgt;
1360
1361         if (target < 0)
1362                 return;
1363
1364         tgt = &sh->dev[target];
1365         set_bit(R5_UPTODATE, &tgt->flags);
1366         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1367         clear_bit(R5_Wantcompute, &tgt->flags);
1368 }
1369
1370 static void ops_complete_compute(void *stripe_head_ref)
1371 {
1372         struct stripe_head *sh = stripe_head_ref;
1373
1374         pr_debug("%s: stripe %llu\n", __func__,
1375                 (unsigned long long)sh->sector);
1376
1377         /* mark the computed target(s) as uptodate */
1378         mark_target_uptodate(sh, sh->ops.target);
1379         mark_target_uptodate(sh, sh->ops.target2);
1380
1381         clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1382         if (sh->check_state == check_state_compute_run)
1383                 sh->check_state = check_state_compute_result;
1384         set_bit(STRIPE_HANDLE, &sh->state);
1385         raid5_release_stripe(sh);
1386 }
1387
1388 /* return a pointer to the address conversion region of the scribble buffer */
1389 static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1390                                  struct raid5_percpu *percpu, int i)
1391 {
1392         void *addr;
1393
1394         addr = flex_array_get(percpu->scribble, i);
1395         return addr + sizeof(struct page *) * (sh->disks + 2);
1396 }
1397
1398 /* return a pointer to the address conversion region of the scribble buffer */
1399 static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
1400 {
1401         void *addr;
1402
1403         addr = flex_array_get(percpu->scribble, i);
1404         return addr;
1405 }
1406
1407 static struct dma_async_tx_descriptor *
1408 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1409 {
1410         int disks = sh->disks;
1411         struct page **xor_srcs = to_addr_page(percpu, 0);
1412         int target = sh->ops.target;
1413         struct r5dev *tgt = &sh->dev[target];
1414         struct page *xor_dest = tgt->page;
1415         int count = 0;
1416         struct dma_async_tx_descriptor *tx;
1417         struct async_submit_ctl submit;
1418         int i;
1419
1420         BUG_ON(sh->batch_head);
1421
1422         pr_debug("%s: stripe %llu block: %d\n",
1423                 __func__, (unsigned long long)sh->sector, target);
1424         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1425
1426         for (i = disks; i--; )
1427                 if (i != target)
1428                         xor_srcs[count++] = sh->dev[i].page;
1429
1430         atomic_inc(&sh->count);
1431
1432         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1433                           ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1434         if (unlikely(count == 1))
1435                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1436         else
1437                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1438
1439         return tx;
1440 }
1441
1442 /* set_syndrome_sources - populate source buffers for gen_syndrome
1443  * @srcs - (struct page *) array of size sh->disks
1444  * @sh - stripe_head to parse
1445  *
1446  * Populates srcs in proper layout order for the stripe and returns the
1447  * 'count' of sources to be used in a call to async_gen_syndrome.  The P
1448  * destination buffer is recorded in srcs[count] and the Q destination
1449  * is recorded in srcs[count+1]].
1450  */
1451 static int set_syndrome_sources(struct page **srcs,
1452                                 struct stripe_head *sh,
1453                                 int srctype)
1454 {
1455         int disks = sh->disks;
1456         int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1457         int d0_idx = raid6_d0(sh);
1458         int count;
1459         int i;
1460
1461         for (i = 0; i < disks; i++)
1462                 srcs[i] = NULL;
1463
1464         count = 0;
1465         i = d0_idx;
1466         do {
1467                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1468                 struct r5dev *dev = &sh->dev[i];
1469
1470                 if (i == sh->qd_idx || i == sh->pd_idx ||
1471                     (srctype == SYNDROME_SRC_ALL) ||
1472                     (srctype == SYNDROME_SRC_WANT_DRAIN &&
1473                      (test_bit(R5_Wantdrain, &dev->flags) ||
1474                       test_bit(R5_InJournal, &dev->flags))) ||
1475                     (srctype == SYNDROME_SRC_WRITTEN &&
1476                      (dev->written ||
1477                       test_bit(R5_InJournal, &dev->flags)))) {
1478                         if (test_bit(R5_InJournal, &dev->flags))
1479                                 srcs[slot] = sh->dev[i].orig_page;
1480                         else
1481                                 srcs[slot] = sh->dev[i].page;
1482                 }
1483                 i = raid6_next_disk(i, disks);
1484         } while (i != d0_idx);
1485
1486         return syndrome_disks;
1487 }
1488
1489 static struct dma_async_tx_descriptor *
1490 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1491 {
1492         int disks = sh->disks;
1493         struct page **blocks = to_addr_page(percpu, 0);
1494         int target;
1495         int qd_idx = sh->qd_idx;
1496         struct dma_async_tx_descriptor *tx;
1497         struct async_submit_ctl submit;
1498         struct r5dev *tgt;
1499         struct page *dest;
1500         int i;
1501         int count;
1502
1503         BUG_ON(sh->batch_head);
1504         if (sh->ops.target < 0)
1505                 target = sh->ops.target2;
1506         else if (sh->ops.target2 < 0)
1507                 target = sh->ops.target;
1508         else
1509                 /* we should only have one valid target */
1510                 BUG();
1511         BUG_ON(target < 0);
1512         pr_debug("%s: stripe %llu block: %d\n",
1513                 __func__, (unsigned long long)sh->sector, target);
1514
1515         tgt = &sh->dev[target];
1516         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1517         dest = tgt->page;
1518
1519         atomic_inc(&sh->count);
1520
1521         if (target == qd_idx) {
1522                 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
1523                 blocks[count] = NULL; /* regenerating p is not necessary */
1524                 BUG_ON(blocks[count+1] != dest); /* q should already be set */
1525                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1526                                   ops_complete_compute, sh,
1527                                   to_addr_conv(sh, percpu, 0));
1528                 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1529         } else {
1530                 /* Compute any data- or p-drive using XOR */
1531                 count = 0;
1532                 for (i = disks; i-- ; ) {
1533                         if (i == target || i == qd_idx)
1534                                 continue;
1535                         blocks[count++] = sh->dev[i].page;
1536                 }
1537
1538                 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1539                                   NULL, ops_complete_compute, sh,
1540                                   to_addr_conv(sh, percpu, 0));
1541                 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
1542         }
1543
1544         return tx;
1545 }
1546
1547 static struct dma_async_tx_descriptor *
1548 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1549 {
1550         int i, count, disks = sh->disks;
1551         int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1552         int d0_idx = raid6_d0(sh);
1553         int faila = -1, failb = -1;
1554         int target = sh->ops.target;
1555         int target2 = sh->ops.target2;
1556         struct r5dev *tgt = &sh->dev[target];
1557         struct r5dev *tgt2 = &sh->dev[target2];
1558         struct dma_async_tx_descriptor *tx;
1559         struct page **blocks = to_addr_page(percpu, 0);
1560         struct async_submit_ctl submit;
1561
1562         BUG_ON(sh->batch_head);
1563         pr_debug("%s: stripe %llu block1: %d block2: %d\n",
1564                  __func__, (unsigned long long)sh->sector, target, target2);
1565         BUG_ON(target < 0 || target2 < 0);
1566         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1567         BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
1568
1569         /* we need to open-code set_syndrome_sources to handle the
1570          * slot number conversion for 'faila' and 'failb'
1571          */
1572         for (i = 0; i < disks ; i++)
1573                 blocks[i] = NULL;
1574         count = 0;
1575         i = d0_idx;
1576         do {
1577                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1578
1579                 blocks[slot] = sh->dev[i].page;
1580
1581                 if (i == target)
1582                         faila = slot;
1583                 if (i == target2)
1584                         failb = slot;
1585                 i = raid6_next_disk(i, disks);
1586         } while (i != d0_idx);
1587
1588         BUG_ON(faila == failb);
1589         if (failb < faila)
1590                 swap(faila, failb);
1591         pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
1592                  __func__, (unsigned long long)sh->sector, faila, failb);
1593
1594         atomic_inc(&sh->count);
1595
1596         if (failb == syndrome_disks+1) {
1597                 /* Q disk is one of the missing disks */
1598                 if (faila == syndrome_disks) {
1599                         /* Missing P+Q, just recompute */
1600                         init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1601                                           ops_complete_compute, sh,
1602                                           to_addr_conv(sh, percpu, 0));
1603                         return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1604                                                   STRIPE_SIZE, &submit);
1605                 } else {
1606                         struct page *dest;
1607                         int data_target;
1608                         int qd_idx = sh->qd_idx;
1609
1610                         /* Missing D+Q: recompute D from P, then recompute Q */
1611                         if (target == qd_idx)
1612                                 data_target = target2;
1613                         else
1614                                 data_target = target;
1615
1616                         count = 0;
1617                         for (i = disks; i-- ; ) {
1618                                 if (i == data_target || i == qd_idx)
1619                                         continue;
1620                                 blocks[count++] = sh->dev[i].page;
1621                         }
1622                         dest = sh->dev[data_target].page;
1623                         init_async_submit(&submit,
1624                                           ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1625                                           NULL, NULL, NULL,
1626                                           to_addr_conv(sh, percpu, 0));
1627                         tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
1628                                        &submit);
1629
1630                         count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
1631                         init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1632                                           ops_complete_compute, sh,
1633                                           to_addr_conv(sh, percpu, 0));
1634                         return async_gen_syndrome(blocks, 0, count+2,
1635                                                   STRIPE_SIZE, &submit);
1636                 }
1637         } else {
1638                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1639                                   ops_complete_compute, sh,
1640                                   to_addr_conv(sh, percpu, 0));
1641                 if (failb == syndrome_disks) {
1642                         /* We're missing D+P. */
1643                         return async_raid6_datap_recov(syndrome_disks+2,
1644                                                        STRIPE_SIZE, faila,
1645                                                        blocks, &submit);
1646                 } else {
1647                         /* We're missing D+D. */
1648                         return async_raid6_2data_recov(syndrome_disks+2,
1649                                                        STRIPE_SIZE, faila, failb,
1650                                                        blocks, &submit);
1651                 }
1652         }
1653 }
1654
1655 static void ops_complete_prexor(void *stripe_head_ref)
1656 {
1657         struct stripe_head *sh = stripe_head_ref;
1658
1659         pr_debug("%s: stripe %llu\n", __func__,
1660                 (unsigned long long)sh->sector);
1661
1662         if (r5c_is_writeback(sh->raid_conf->log))
1663                 /*
1664                  * raid5-cache write back uses orig_page during prexor.
1665                  * After prexor, it is time to free orig_page
1666                  */
1667                 r5c_release_extra_page(sh);
1668 }
1669
1670 static struct dma_async_tx_descriptor *
1671 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
1672                 struct dma_async_tx_descriptor *tx)
1673 {
1674         int disks = sh->disks;
1675         struct page **xor_srcs = to_addr_page(percpu, 0);
1676         int count = 0, pd_idx = sh->pd_idx, i;
1677         struct async_submit_ctl submit;
1678
1679         /* existing parity data subtracted */
1680         struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1681
1682         BUG_ON(sh->batch_head);
1683         pr_debug("%s: stripe %llu\n", __func__,
1684                 (unsigned long long)sh->sector);
1685
1686         for (i = disks; i--; ) {
1687                 struct r5dev *dev = &sh->dev[i];
1688                 /* Only process blocks that are known to be uptodate */
1689                 if (test_bit(R5_InJournal, &dev->flags))
1690                         xor_srcs[count++] = dev->orig_page;
1691                 else if (test_bit(R5_Wantdrain, &dev->flags))
1692                         xor_srcs[count++] = dev->page;
1693         }
1694
1695         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1696                           ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1697         tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1698
1699         return tx;
1700 }
1701
1702 static struct dma_async_tx_descriptor *
1703 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
1704                 struct dma_async_tx_descriptor *tx)
1705 {
1706         struct page **blocks = to_addr_page(percpu, 0);
1707         int count;
1708         struct async_submit_ctl submit;
1709
1710         pr_debug("%s: stripe %llu\n", __func__,
1711                 (unsigned long long)sh->sector);
1712
1713         count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
1714
1715         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
1716                           ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1717         tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1718
1719         return tx;
1720 }
1721
1722 static struct dma_async_tx_descriptor *
1723 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1724 {
1725         struct r5conf *conf = sh->raid_conf;
1726         int disks = sh->disks;
1727         int i;
1728         struct stripe_head *head_sh = sh;
1729
1730         pr_debug("%s: stripe %llu\n", __func__,
1731                 (unsigned long long)sh->sector);
1732
1733         for (i = disks; i--; ) {
1734                 struct r5dev *dev;
1735                 struct bio *chosen;
1736
1737                 sh = head_sh;
1738                 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
1739                         struct bio *wbi;
1740
1741 again:
1742                         dev = &sh->dev[i];
1743                         /*
1744                          * clear R5_InJournal, so when rewriting a page in
1745                          * journal, it is not skipped by r5l_log_stripe()
1746                          */
1747                         clear_bit(R5_InJournal, &dev->flags);
1748                         spin_lock_irq(&sh->stripe_lock);
1749                         chosen = dev->towrite;
1750                         dev->towrite = NULL;
1751                         sh->overwrite_disks = 0;
1752                         BUG_ON(dev->written);
1753                         wbi = dev->written = chosen;
1754                         spin_unlock_irq(&sh->stripe_lock);
1755                         WARN_ON(dev->page != dev->orig_page);
1756
1757                         while (wbi && wbi->bi_iter.bi_sector <
1758                                 dev->sector + STRIPE_SECTORS) {
1759                                 if (wbi->bi_opf & REQ_FUA)
1760                                         set_bit(R5_WantFUA, &dev->flags);
1761                                 if (wbi->bi_opf & REQ_SYNC)
1762                                         set_bit(R5_SyncIO, &dev->flags);
1763                                 if (bio_op(wbi) == REQ_OP_DISCARD)
1764                                         set_bit(R5_Discard, &dev->flags);
1765                                 else {
1766                                         tx = async_copy_data(1, wbi, &dev->page,
1767                                                              dev->sector, tx, sh,
1768                                                              r5c_is_writeback(conf->log));
1769                                         if (dev->page != dev->orig_page &&
1770                                             !r5c_is_writeback(conf->log)) {
1771                                                 set_bit(R5_SkipCopy, &dev->flags);
1772                                                 clear_bit(R5_UPTODATE, &dev->flags);
1773                                                 clear_bit(R5_OVERWRITE, &dev->flags);
1774                                         }
1775                                 }
1776                                 wbi = r5_next_bio(wbi, dev->sector);
1777                         }
1778
1779                         if (head_sh->batch_head) {
1780                                 sh = list_first_entry(&sh->batch_list,
1781                                                       struct stripe_head,
1782                                                       batch_list);
1783                                 if (sh == head_sh)
1784                                         continue;
1785                                 goto again;
1786                         }
1787                 }
1788         }
1789
1790         return tx;
1791 }
1792
1793 static void ops_complete_reconstruct(void *stripe_head_ref)
1794 {
1795         struct stripe_head *sh = stripe_head_ref;
1796         int disks = sh->disks;
1797         int pd_idx = sh->pd_idx;
1798         int qd_idx = sh->qd_idx;
1799         int i;
1800         bool fua = false, sync = false, discard = false;
1801
1802         pr_debug("%s: stripe %llu\n", __func__,
1803                 (unsigned long long)sh->sector);
1804
1805         for (i = disks; i--; ) {
1806                 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
1807                 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1808                 discard |= test_bit(R5_Discard, &sh->dev[i].flags);
1809         }
1810
1811         for (i = disks; i--; ) {
1812                 struct r5dev *dev = &sh->dev[i];
1813
1814                 if (dev->written || i == pd_idx || i == qd_idx) {
1815                         if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
1816                                 set_bit(R5_UPTODATE, &dev->flags);
1817                         if (fua)
1818                                 set_bit(R5_WantFUA, &dev->flags);
1819                         if (sync)
1820                                 set_bit(R5_SyncIO, &dev->flags);
1821                 }
1822         }
1823
1824         if (sh->reconstruct_state == reconstruct_state_drain_run)
1825                 sh->reconstruct_state = reconstruct_state_drain_result;
1826         else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1827                 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1828         else {
1829                 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1830                 sh->reconstruct_state = reconstruct_state_result;
1831         }
1832
1833         set_bit(STRIPE_HANDLE, &sh->state);
1834         raid5_release_stripe(sh);
1835 }
1836
1837 static void
1838 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1839                      struct dma_async_tx_descriptor *tx)
1840 {
1841         int disks = sh->disks;
1842         struct page **xor_srcs;
1843         struct async_submit_ctl submit;
1844         int count, pd_idx = sh->pd_idx, i;
1845         struct page *xor_dest;
1846         int prexor = 0;
1847         unsigned long flags;
1848         int j = 0;
1849         struct stripe_head *head_sh = sh;
1850         int last_stripe;
1851
1852         pr_debug("%s: stripe %llu\n", __func__,
1853                 (unsigned long long)sh->sector);
1854
1855         for (i = 0; i < sh->disks; i++) {
1856                 if (pd_idx == i)
1857                         continue;
1858                 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1859                         break;
1860         }
1861         if (i >= sh->disks) {
1862                 atomic_inc(&sh->count);
1863                 set_bit(R5_Discard, &sh->dev[pd_idx].flags);
1864                 ops_complete_reconstruct(sh);
1865                 return;
1866         }
1867 again:
1868         count = 0;
1869         xor_srcs = to_addr_page(percpu, j);
1870         /* check if prexor is active which means only process blocks
1871          * that are part of a read-modify-write (written)
1872          */
1873         if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1874                 prexor = 1;
1875                 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1876                 for (i = disks; i--; ) {
1877                         struct r5dev *dev = &sh->dev[i];
1878                         if (head_sh->dev[i].written ||
1879                             test_bit(R5_InJournal, &head_sh->dev[i].flags))
1880                                 xor_srcs[count++] = dev->page;
1881                 }
1882         } else {
1883                 xor_dest = sh->dev[pd_idx].page;
1884                 for (i = disks; i--; ) {
1885                         struct r5dev *dev = &sh->dev[i];
1886                         if (i != pd_idx)
1887                                 xor_srcs[count++] = dev->page;
1888                 }
1889         }
1890
1891         /* 1/ if we prexor'd then the dest is reused as a source
1892          * 2/ if we did not prexor then we are redoing the parity
1893          * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1894          * for the synchronous xor case
1895          */
1896         last_stripe = !head_sh->batch_head ||
1897                 list_first_entry(&sh->batch_list,
1898                                  struct stripe_head, batch_list) == head_sh;
1899         if (last_stripe) {
1900                 flags = ASYNC_TX_ACK |
1901                         (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1902
1903                 atomic_inc(&head_sh->count);
1904                 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
1905                                   to_addr_conv(sh, percpu, j));
1906         } else {
1907                 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
1908                 init_async_submit(&submit, flags, tx, NULL, NULL,
1909                                   to_addr_conv(sh, percpu, j));
1910         }
1911
1912         if (unlikely(count == 1))
1913                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1914         else
1915                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1916         if (!last_stripe) {
1917                 j++;
1918                 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1919                                       batch_list);
1920                 goto again;
1921         }
1922 }
1923
1924 static void
1925 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1926                      struct dma_async_tx_descriptor *tx)
1927 {
1928         struct async_submit_ctl submit;
1929         struct page **blocks;
1930         int count, i, j = 0;
1931         struct stripe_head *head_sh = sh;
1932         int last_stripe;
1933         int synflags;
1934         unsigned long txflags;
1935
1936         pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1937
1938         for (i = 0; i < sh->disks; i++) {
1939                 if (sh->pd_idx == i || sh->qd_idx == i)
1940                         continue;
1941                 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1942                         break;
1943         }
1944         if (i >= sh->disks) {
1945                 atomic_inc(&sh->count);
1946                 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
1947                 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
1948                 ops_complete_reconstruct(sh);
1949                 return;
1950         }
1951
1952 again:
1953         blocks = to_addr_page(percpu, j);
1954
1955         if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1956                 synflags = SYNDROME_SRC_WRITTEN;
1957                 txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
1958         } else {
1959                 synflags = SYNDROME_SRC_ALL;
1960                 txflags = ASYNC_TX_ACK;
1961         }
1962
1963         count = set_syndrome_sources(blocks, sh, synflags);
1964         last_stripe = !head_sh->batch_head ||
1965                 list_first_entry(&sh->batch_list,
1966                                  struct stripe_head, batch_list) == head_sh;
1967
1968         if (last_stripe) {
1969                 atomic_inc(&head_sh->count);
1970                 init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
1971                                   head_sh, to_addr_conv(sh, percpu, j));
1972         } else
1973                 init_async_submit(&submit, 0, tx, NULL, NULL,
1974                                   to_addr_conv(sh, percpu, j));
1975         tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1976         if (!last_stripe) {
1977                 j++;
1978                 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1979                                       batch_list);
1980                 goto again;
1981         }
1982 }
1983
1984 static void ops_complete_check(void *stripe_head_ref)
1985 {
1986         struct stripe_head *sh = stripe_head_ref;
1987
1988         pr_debug("%s: stripe %llu\n", __func__,
1989                 (unsigned long long)sh->sector);
1990
1991         sh->check_state = check_state_check_result;
1992         set_bit(STRIPE_HANDLE, &sh->state);
1993         raid5_release_stripe(sh);
1994 }
1995
1996 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1997 {
1998         int disks = sh->disks;
1999         int pd_idx = sh->pd_idx;
2000         int qd_idx = sh->qd_idx;
2001         struct page *xor_dest;
2002         struct page **xor_srcs = to_addr_page(percpu, 0);
2003         struct dma_async_tx_descriptor *tx;
2004         struct async_submit_ctl submit;
2005         int count;
2006         int i;
2007
2008         pr_debug("%s: stripe %llu\n", __func__,
2009                 (unsigned long long)sh->sector);
2010
2011         BUG_ON(sh->batch_head);
2012         count = 0;
2013         xor_dest = sh->dev[pd_idx].page;
2014         xor_srcs[count++] = xor_dest;
2015         for (i = disks; i--; ) {
2016                 if (i == pd_idx || i == qd_idx)
2017                         continue;
2018                 xor_srcs[count++] = sh->dev[i].page;
2019         }
2020
2021         init_async_submit(&submit, 0, NULL, NULL, NULL,
2022                           to_addr_conv(sh, percpu, 0));
2023         tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
2024                            &sh->ops.zero_sum_result, &submit);
2025
2026         atomic_inc(&sh->count);
2027         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
2028         tx = async_trigger_callback(&submit);
2029 }
2030
2031 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
2032 {
2033         struct page **srcs = to_addr_page(percpu, 0);
2034         struct async_submit_ctl submit;
2035         int count;
2036
2037         pr_debug("%s: stripe %llu checkp: %d\n", __func__,
2038                 (unsigned long long)sh->sector, checkp);
2039
2040         BUG_ON(sh->batch_head);
2041         count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
2042         if (!checkp)
2043                 srcs[count] = NULL;
2044
2045         atomic_inc(&sh->count);
2046         init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
2047                           sh, to_addr_conv(sh, percpu, 0));
2048         async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
2049                            &sh->ops.zero_sum_result, percpu->spare_page, &submit);
2050 }
2051
2052 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
2053 {
2054         int overlap_clear = 0, i, disks = sh->disks;
2055         struct dma_async_tx_descriptor *tx = NULL;
2056         struct r5conf *conf = sh->raid_conf;
2057         int level = conf->level;
2058         struct raid5_percpu *percpu;
2059         unsigned long cpu;
2060
2061         cpu = get_cpu();
2062         percpu = per_cpu_ptr(conf->percpu, cpu);
2063         if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
2064                 ops_run_biofill(sh);
2065                 overlap_clear++;
2066         }
2067
2068         if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
2069                 if (level < 6)
2070                         tx = ops_run_compute5(sh, percpu);
2071                 else {
2072                         if (sh->ops.target2 < 0 || sh->ops.target < 0)
2073                                 tx = ops_run_compute6_1(sh, percpu);
2074                         else
2075                                 tx = ops_run_compute6_2(sh, percpu);
2076                 }
2077                 /* terminate the chain if reconstruct is not set to be run */
2078                 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
2079                         async_tx_ack(tx);
2080         }
2081
2082         if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
2083                 if (level < 6)
2084                         tx = ops_run_prexor5(sh, percpu, tx);
2085                 else
2086                         tx = ops_run_prexor6(sh, percpu, tx);
2087         }
2088
2089         if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
2090                 tx = ops_run_partial_parity(sh, percpu, tx);
2091
2092         if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
2093                 tx = ops_run_biodrain(sh, tx);
2094                 overlap_clear++;
2095         }
2096
2097         if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
2098                 if (level < 6)
2099                         ops_run_reconstruct5(sh, percpu, tx);
2100                 else
2101                         ops_run_reconstruct6(sh, percpu, tx);
2102         }
2103
2104         if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
2105                 if (sh->check_state == check_state_run)
2106                         ops_run_check_p(sh, percpu);
2107                 else if (sh->check_state == check_state_run_q)
2108                         ops_run_check_pq(sh, percpu, 0);
2109                 else if (sh->check_state == check_state_run_pq)
2110                         ops_run_check_pq(sh, percpu, 1);
2111                 else
2112                         BUG();
2113         }
2114
2115         if (overlap_clear && !sh->batch_head)
2116                 for (i = disks; i--; ) {
2117                         struct r5dev *dev = &sh->dev[i];
2118                         if (test_and_clear_bit(R5_Overlap, &dev->flags))
2119                                 wake_up(&sh->raid_conf->wait_for_overlap);
2120                 }
2121         put_cpu();
2122 }
2123
2124 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
2125 {
2126         if (sh->ppl_page)
2127                 __free_page(sh->ppl_page);
2128         kmem_cache_free(sc, sh);
2129 }
2130
2131 static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
2132         int disks, struct r5conf *conf)
2133 {
2134         struct stripe_head *sh;
2135         int i;
2136
2137         sh = kmem_cache_zalloc(sc, gfp);
2138         if (sh) {
2139                 spin_lock_init(&sh->stripe_lock);
2140                 spin_lock_init(&sh->batch_lock);
2141                 INIT_LIST_HEAD(&sh->batch_list);
2142                 INIT_LIST_HEAD(&sh->lru);
2143                 INIT_LIST_HEAD(&sh->r5c);
2144                 INIT_LIST_HEAD(&sh->log_list);
2145                 atomic_set(&sh->count, 1);
2146                 sh->raid_conf = conf;
2147                 sh->log_start = MaxSector;
2148                 for (i = 0; i < disks; i++) {
2149                         struct r5dev *dev = &sh->dev[i];
2150
2151                         bio_init(&dev->req, &dev->vec, 1);
2152                         bio_init(&dev->rreq, &dev->rvec, 1);
2153                 }
2154
2155                 if (raid5_has_ppl(conf)) {
2156                         sh->ppl_page = alloc_page(gfp);
2157                         if (!sh->ppl_page) {
2158                                 free_stripe(sc, sh);
2159                                 sh = NULL;
2160                         }
2161                 }
2162         }
2163         return sh;
2164 }
2165 static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
2166 {
2167         struct stripe_head *sh;
2168
2169         sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
2170         if (!sh)
2171                 return 0;
2172
2173         if (grow_buffers(sh, gfp)) {
2174                 shrink_buffers(sh);
2175                 free_stripe(conf->slab_cache, sh);
2176                 return 0;
2177         }
2178         sh->hash_lock_index =
2179                 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
2180         /* we just created an active stripe so... */
2181         atomic_inc(&conf->active_stripes);
2182
2183         raid5_release_stripe(sh);
2184         conf->max_nr_stripes++;
2185         return 1;
2186 }
2187
2188 static int grow_stripes(struct r5conf *conf, int num)
2189 {
2190         struct kmem_cache *sc;
2191         int devs = max(conf->raid_disks, conf->previous_raid_disks);
2192
2193         if (conf->mddev->gendisk)
2194                 sprintf(conf->cache_name[0],
2195                         "raid%d-%s", conf->level, mdname(conf->mddev));
2196         else
2197                 sprintf(conf->cache_name[0],
2198                         "raid%d-%p", conf->level, conf->mddev);
2199         sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
2200
2201         conf->active_name = 0;
2202         sc = kmem_cache_create(conf->cache_name[conf->active_name],
2203                                sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
2204                                0, 0, NULL);
2205         if (!sc)
2206                 return 1;
2207         conf->slab_cache = sc;
2208         conf->pool_size = devs;
2209         while (num--)
2210                 if (!grow_one_stripe(conf, GFP_KERNEL))
2211                         return 1;
2212
2213         return 0;
2214 }
2215
2216 /**
2217  * scribble_len - return the required size of the scribble region
2218  * @num - total number of disks in the array
2219  *
2220  * The size must be enough to contain:
2221  * 1/ a struct page pointer for each device in the array +2
2222  * 2/ room to convert each entry in (1) to its corresponding dma
2223  *    (dma_map_page()) or page (page_address()) address.
2224  *
2225  * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
2226  * calculate over all devices (not just the data blocks), using zeros in place
2227  * of the P and Q blocks.
2228  */
2229 static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
2230 {
2231         struct flex_array *ret;
2232         size_t len;
2233
2234         len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
2235         ret = flex_array_alloc(len, cnt, flags);
2236         if (!ret)
2237                 return NULL;
2238         /* always prealloc all elements, so no locking is required */
2239         if (flex_array_prealloc(ret, 0, cnt, flags)) {
2240                 flex_array_free(ret);
2241                 return NULL;
2242         }
2243         return ret;
2244 }
2245
2246 static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
2247 {
2248         unsigned long cpu;
2249         int err = 0;
2250
2251         /*
2252          * Never shrink. And mddev_suspend() could deadlock if this is called
2253          * from raid5d. In that case, scribble_disks and scribble_sectors
2254          * should equal to new_disks and new_sectors
2255          */
2256         if (conf->scribble_disks >= new_disks &&
2257             conf->scribble_sectors >= new_sectors)
2258                 return 0;
2259         mddev_suspend(conf->mddev);
2260         get_online_cpus();
2261         for_each_present_cpu(cpu) {
2262                 struct raid5_percpu *percpu;
2263                 struct flex_array *scribble;
2264
2265                 percpu = per_cpu_ptr(conf->percpu, cpu);
2266                 scribble = scribble_alloc(new_disks,
2267                                           new_sectors / STRIPE_SECTORS,
2268                                           GFP_NOIO);
2269
2270                 if (scribble) {
2271                         flex_array_free(percpu->scribble);
2272                         percpu->scribble = scribble;
2273                 } else {
2274                         err = -ENOMEM;
2275                         break;
2276                 }
2277         }
2278         put_online_cpus();
2279         mddev_resume(conf->mddev);
2280         if (!err) {
2281                 conf->scribble_disks = new_disks;
2282                 conf->scribble_sectors = new_sectors;
2283         }
2284         return err;
2285 }
2286
2287 static int resize_stripes(struct r5conf *conf, int newsize)
2288 {
2289         /* Make all the stripes able to hold 'newsize' devices.
2290          * New slots in each stripe get 'page' set to a new page.
2291          *
2292          * This happens in stages:
2293          * 1/ create a new kmem_cache and allocate the required number of
2294          *    stripe_heads.
2295          * 2/ gather all the old stripe_heads and transfer the pages across
2296          *    to the new stripe_heads.  This will have the side effect of
2297          *    freezing the array as once all stripe_heads have been collected,
2298          *    no IO will be possible.  Old stripe heads are freed once their
2299          *    pages have been transferred over, and the old kmem_cache is
2300          *    freed when all stripes are done.
2301          * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
2302          *    we simple return a failure status - no need to clean anything up.
2303          * 4/ allocate new pages for the new slots in the new stripe_heads.
2304          *    If this fails, we don't bother trying the shrink the
2305          *    stripe_heads down again, we just leave them as they are.
2306          *    As each stripe_head is processed the new one is released into
2307          *    active service.
2308          *
2309          * Once step2 is started, we cannot afford to wait for a write,
2310          * so we use GFP_NOIO allocations.
2311          */
2312         struct stripe_head *osh, *nsh;
2313         LIST_HEAD(newstripes);
2314         struct disk_info *ndisks;
2315         int err = 0;
2316         struct kmem_cache *sc;
2317         int i;
2318         int hash, cnt;
2319
2320         md_allow_write(conf->mddev);
2321
2322         /* Step 1 */
2323         sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
2324                                sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
2325                                0, 0, NULL);
2326         if (!sc)
2327                 return -ENOMEM;
2328
2329         /* Need to ensure auto-resizing doesn't interfere */
2330         mutex_lock(&conf->cache_size_mutex);
2331
2332         for (i = conf->max_nr_stripes; i; i--) {
2333                 nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);
2334                 if (!nsh)
2335                         break;
2336
2337                 list_add(&nsh->lru, &newstripes);
2338         }
2339         if (i) {
2340                 /* didn't get enough, give up */
2341                 while (!list_empty(&newstripes)) {
2342                         nsh = list_entry(newstripes.next, struct stripe_head, lru);
2343                         list_del(&nsh->lru);
2344                         free_stripe(sc, nsh);
2345                 }
2346                 kmem_cache_destroy(sc);
2347                 mutex_unlock(&conf->cache_size_mutex);
2348                 return -ENOMEM;
2349         }
2350         /* Step 2 - Must use GFP_NOIO now.
2351          * OK, we have enough stripes, start collecting inactive
2352          * stripes and copying them over
2353          */
2354         hash = 0;
2355         cnt = 0;
2356         list_for_each_entry(nsh, &newstripes, lru) {
2357                 lock_device_hash_lock(conf, hash);
2358                 wait_event_cmd(conf->wait_for_stripe,
2359                                     !list_empty(conf->inactive_list + hash),
2360                                     unlock_device_hash_lock(conf, hash),
2361                                     lock_device_hash_lock(conf, hash));
2362                 osh = get_free_stripe(conf, hash);
2363                 unlock_device_hash_lock(conf, hash);
2364
2365                 for(i=0; i<conf->pool_size; i++) {
2366                         nsh->dev[i].page = osh->dev[i].page;
2367                         nsh->dev[i].orig_page = osh->dev[i].page;
2368                 }
2369                 nsh->hash_lock_index = hash;
2370                 free_stripe(conf->slab_cache, osh);
2371                 cnt++;
2372                 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
2373                     !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
2374                         hash++;
2375                         cnt = 0;
2376                 }
2377         }
2378         kmem_cache_destroy(conf->slab_cache);
2379
2380         /* Step 3.
2381          * At this point, we are holding all the stripes so the array
2382          * is completely stalled, so now is a good time to resize
2383          * conf->disks and the scribble region
2384          */
2385         ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
2386         if (ndisks) {
2387                 for (i = 0; i < conf->pool_size; i++)
2388                         ndisks[i] = conf->disks[i];
2389
2390                 for (i = conf->pool_size; i < newsize; i++) {
2391                         ndisks[i].extra_page = alloc_page(GFP_NOIO);
2392                         if (!ndisks[i].extra_page)
2393                                 err = -ENOMEM;
2394                 }
2395
2396                 if (err) {
2397                         for (i = conf->pool_size; i < newsize; i++)
2398                                 if (ndisks[i].extra_page)
2399                                         put_page(ndisks[i].extra_page);
2400                         kfree(ndisks);
2401                 } else {
2402                         kfree(conf->disks);
2403                         conf->disks = ndisks;
2404                 }
2405         } else
2406                 err = -ENOMEM;
2407
2408         mutex_unlock(&conf->cache_size_mutex);
2409
2410         conf->slab_cache = sc;
2411         conf->active_name = 1-conf->active_name;
2412
2413         /* Step 4, return new stripes to service */
2414         while(!list_empty(&newstripes)) {
2415                 nsh = list_entry(newstripes.next, struct stripe_head, lru);
2416                 list_del_init(&nsh->lru);
2417
2418                 for (i=conf->raid_disks; i < newsize; i++)
2419                         if (nsh->dev[i].page == NULL) {
2420                                 struct page *p = alloc_page(GFP_NOIO);
2421                                 nsh->dev[i].page = p;
2422                                 nsh->dev[i].orig_page = p;
2423                                 if (!p)
2424                                         err = -ENOMEM;
2425                         }
2426                 raid5_release_stripe(nsh);
2427         }
2428         /* critical section pass, GFP_NOIO no longer needed */
2429
2430         if (!err)
2431                 conf->pool_size = newsize;
2432         return err;
2433 }
2434
2435 static int drop_one_stripe(struct r5conf *conf)
2436 {
2437         struct stripe_head *sh;
2438         int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
2439
2440         spin_lock_irq(conf->hash_locks + hash);
2441         sh = get_free_stripe(conf, hash);
2442         spin_unlock_irq(conf->hash_locks + hash);
2443         if (!sh)
2444                 return 0;
2445         BUG_ON(atomic_read(&sh->count));
2446         shrink_buffers(sh);
2447         free_stripe(conf->slab_cache, sh);
2448         atomic_dec(&conf->active_stripes);
2449         conf->max_nr_stripes--;
2450         return 1;
2451 }
2452
2453 static void shrink_stripes(struct r5conf *conf)
2454 {
2455         while (conf->max_nr_stripes &&
2456                drop_one_stripe(conf))
2457                 ;
2458
2459         kmem_cache_destroy(conf->slab_cache);
2460         conf->slab_cache = NULL;
2461 }
2462
2463 static void raid5_end_read_request(struct bio * bi)
2464 {
2465         struct stripe_head *sh = bi->bi_private;
2466         struct r5conf *conf = sh->raid_conf;
2467         int disks = sh->disks, i;
2468         char b[BDEVNAME_SIZE];
2469         struct md_rdev *rdev = NULL;
2470         sector_t s;
2471
2472         for (i=0 ; i<disks; i++)
2473                 if (bi == &sh->dev[i].req)
2474                         break;
2475
2476         pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
2477                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2478                 bi->bi_status);
2479         if (i == disks) {
2480                 bio_reset(bi);
2481                 BUG();
2482                 return;
2483         }
2484         if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2485                 /* If replacement finished while this request was outstanding,
2486                  * 'replacement' might be NULL already.
2487                  * In that case it moved down to 'rdev'.
2488                  * rdev is not removed until all requests are finished.
2489                  */
2490                 rdev = conf->disks[i].replacement;
2491         if (!rdev)
2492                 rdev = conf->disks[i].rdev;
2493
2494         if (use_new_offset(conf, sh))
2495                 s = sh->sector + rdev->new_data_offset;
2496         else
2497                 s = sh->sector + rdev->data_offset;
2498         if (!bi->bi_status) {
2499                 set_bit(R5_UPTODATE, &sh->dev[i].flags);
2500                 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2501                         /* Note that this cannot happen on a
2502                          * replacement device.  We just fail those on
2503                          * any error
2504                          */
2505                         pr_info_ratelimited(
2506                                 "md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
2507                                 mdname(conf->mddev), STRIPE_SECTORS,
2508                                 (unsigned long long)s,
2509                                 bdevname(rdev->bdev, b));
2510                         atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
2511                         clear_bit(R5_ReadError, &sh->dev[i].flags);
2512                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
2513                 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2514                         clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2515
2516                 if (test_bit(R5_InJournal, &sh->dev[i].flags))
2517                         /*
2518                          * end read for a page in journal, this
2519                          * must be preparing for prexor in rmw
2520                          */
2521                         set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2522
2523                 if (atomic_read(&rdev->read_errors))
2524                         atomic_set(&rdev->read_errors, 0);
2525         } else {
2526                 const char *bdn = bdevname(rdev->bdev, b);
2527                 int retry = 0;
2528                 int set_bad = 0;
2529
2530                 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2531                 atomic_inc(&rdev->read_errors);
2532                 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2533                         pr_warn_ratelimited(
2534                                 "md/raid:%s: read error on replacement device (sector %llu on %s).\n",
2535                                 mdname(conf->mddev),
2536                                 (unsigned long long)s,
2537                                 bdn);
2538                 else if (conf->mddev->degraded >= conf->max_degraded) {
2539                         set_bad = 1;
2540                         pr_warn_ratelimited(
2541                                 "md/raid:%s: read error not correctable (sector %llu on %s).\n",
2542                                 mdname(conf->mddev),
2543                                 (unsigned long long)s,
2544                                 bdn);
2545                 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2546                         /* Oh, no!!! */
2547                         set_bad = 1;
2548                         pr_warn_ratelimited(
2549                                 "md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n",
2550                                 mdname(conf->mddev),
2551                                 (unsigned long long)s,
2552                                 bdn);
2553                 } else if (atomic_read(&rdev->read_errors)
2554                          > conf->max_nr_stripes)
2555                         pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
2556                                mdname(conf->mddev), bdn);
2557                 else
2558                         retry = 1;
2559                 if (set_bad && test_bit(In_sync, &rdev->flags)
2560                     && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2561                         retry = 1;
2562                 if (retry)
2563                         if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
2564                                 set_bit(R5_ReadError, &sh->dev[i].flags);
2565                                 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2566                         } else
2567                                 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2568                 else {
2569                         clear_bit(R5_ReadError, &sh->dev[i].flags);
2570                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
2571                         if (!(set_bad
2572                               && test_bit(In_sync, &rdev->flags)
2573                               && rdev_set_badblocks(
2574                                       rdev, sh->sector, STRIPE_SECTORS, 0)))
2575                                 md_error(conf->mddev, rdev);
2576                 }
2577         }
2578         rdev_dec_pending(rdev, conf->mddev);
2579         bio_reset(bi);
2580         clear_bit(R5_LOCKED, &sh->dev[i].flags);
2581         set_bit(STRIPE_HANDLE, &sh->state);
2582         raid5_release_stripe(sh);
2583 }
2584
2585 static void raid5_end_write_request(struct bio *bi)
2586 {
2587         struct stripe_head *sh = bi->bi_private;
2588         struct r5conf *conf = sh->raid_conf;
2589         int disks = sh->disks, i;
2590         struct md_rdev *uninitialized_var(rdev);
2591         sector_t first_bad;
2592         int bad_sectors;
2593         int replacement = 0;
2594
2595         for (i = 0 ; i < disks; i++) {
2596                 if (bi == &sh->dev[i].req) {
2597                         rdev = conf->disks[i].rdev;
2598                         break;
2599                 }
2600                 if (bi == &sh->dev[i].rreq) {
2601                         rdev = conf->disks[i].replacement;
2602                         if (rdev)
2603                                 replacement = 1;
2604                         else
2605                                 /* rdev was removed and 'replacement'
2606                                  * replaced it.  rdev is not removed
2607                                  * until all requests are finished.
2608                                  */
2609                                 rdev = conf->disks[i].rdev;
2610                         break;
2611                 }
2612         }
2613         pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
2614                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2615                 bi->bi_status);
2616         if (i == disks) {
2617                 bio_reset(bi);
2618                 BUG();
2619                 return;
2620         }
2621
2622         if (replacement) {
2623                 if (bi->bi_status)
2624                         md_error(conf->mddev, rdev);
2625                 else if (is_badblock(rdev, sh->sector,
2626                                      STRIPE_SECTORS,
2627                                      &first_bad, &bad_sectors))
2628                         set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2629         } else {
2630                 if (bi->bi_status) {
2631                         set_bit(STRIPE_DEGRADED, &sh->state);
2632                         set_bit(WriteErrorSeen, &rdev->flags);
2633                         set_bit(R5_WriteError, &sh->dev[i].flags);
2634                         if (!test_and_set_bit(WantReplacement, &rdev->flags))
2635                                 set_bit(MD_RECOVERY_NEEDED,
2636                                         &rdev->mddev->recovery);
2637                 } else if (is_badblock(rdev, sh->sector,
2638                                        STRIPE_SECTORS,
2639                                        &first_bad, &bad_sectors)) {
2640                         set_bit(R5_MadeGood, &sh->dev[i].flags);
2641                         if (test_bit(R5_ReadError, &sh->dev[i].flags))
2642                                 /* That was a successful write so make
2643                                  * sure it looks like we already did
2644                                  * a re-write.
2645                                  */
2646                                 set_bit(R5_ReWrite, &sh->dev[i].flags);
2647                 }
2648         }
2649         rdev_dec_pending(rdev, conf->mddev);
2650
2651         if (sh->batch_head && bi->bi_status && !replacement)
2652                 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2653
2654         bio_reset(bi);
2655         if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2656                 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2657         set_bit(STRIPE_HANDLE, &sh->state);
2658         raid5_release_stripe(sh);
2659
2660         if (sh->batch_head && sh != sh->batch_head)
2661                 raid5_release_stripe(sh->batch_head);
2662 }
2663
2664 static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
2665 {
2666         char b[BDEVNAME_SIZE];
2667         struct r5conf *conf = mddev->private;
2668         unsigned long flags;
2669         pr_debug("raid456: error called\n");
2670
2671         spin_lock_irqsave(&conf->device_lock, flags);
2672         clear_bit(In_sync, &rdev->flags);
2673         mddev->degraded = raid5_calc_degraded(conf);
2674         spin_unlock_irqrestore(&conf->device_lock, flags);
2675         set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2676
2677         set_bit(Blocked, &rdev->flags);
2678         set_bit(Faulty, &rdev->flags);
2679         set_mask_bits(&mddev->sb_flags, 0,
2680                       BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
2681         pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
2682                 "md/raid:%s: Operation continuing on %d devices.\n",
2683                 mdname(mddev),
2684                 bdevname(rdev->bdev, b),
2685                 mdname(mddev),
2686                 conf->raid_disks - mddev->degraded);
2687         r5c_update_on_rdev_error(mddev, rdev);
2688 }
2689
2690 /*
2691  * Input: a 'big' sector number,
2692  * Output: index of the data and parity disk, and the sector # in them.
2693  */
2694 sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2695                               int previous, int *dd_idx,
2696                               struct stripe_head *sh)
2697 {
2698         sector_t stripe, stripe2;
2699         sector_t chunk_number;
2700         unsigned int chunk_offset;
2701         int pd_idx, qd_idx;
2702         int ddf_layout = 0;
2703         sector_t new_sector;
2704         int algorithm = previous ? conf->prev_algo
2705                                  : conf->algorithm;
2706         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2707                                          : conf->chunk_sectors;
2708         int raid_disks = previous ? conf->previous_raid_disks
2709                                   : conf->raid_disks;
2710         int data_disks = raid_disks - conf->max_degraded;
2711
2712         /* First compute the information on this sector */
2713
2714         /*
2715          * Compute the chunk number and the sector offset inside the chunk
2716          */
2717         chunk_offset = sector_div(r_sector, sectors_per_chunk);
2718         chunk_number = r_sector;
2719
2720         /*
2721          * Compute the stripe number
2722          */
2723         stripe = chunk_number;
2724         *dd_idx = sector_div(stripe, data_disks);
2725         stripe2 = stripe;
2726         /*
2727          * Select the parity disk based on the user selected algorithm.
2728          */
2729         pd_idx = qd_idx = -1;
2730         switch(conf->level) {
2731         case 4:
2732                 pd_idx = data_disks;
2733                 break;
2734         case 5:
2735                 switch (algorithm) {
2736                 case ALGORITHM_LEFT_ASYMMETRIC:
2737                         pd_idx = data_disks - sector_div(stripe2, raid_disks);
2738                         if (*dd_idx >= pd_idx)
2739                                 (*dd_idx)++;
2740                         break;
2741                 case ALGORITHM_RIGHT_ASYMMETRIC:
2742                         pd_idx = sector_div(stripe2, raid_disks);
2743                         if (*dd_idx >= pd_idx)
2744                                 (*dd_idx)++;
2745                         break;
2746                 case ALGORITHM_LEFT_SYMMETRIC:
2747                         pd_idx = data_disks - sector_div(stripe2, raid_disks);
2748                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2749                         break;
2750                 case ALGORITHM_RIGHT_SYMMETRIC:
2751                         pd_idx = sector_div(stripe2, raid_disks);
2752                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2753                         break;
2754                 case ALGORITHM_PARITY_0:
2755                         pd_idx = 0;
2756                         (*dd_idx)++;
2757                         break;
2758                 case ALGORITHM_PARITY_N:
2759                         pd_idx = data_disks;
2760                         break;
2761                 default:
2762                         BUG();
2763                 }
2764                 break;
2765         case 6:
2766
2767                 switch (algorithm) {
2768                 case ALGORITHM_LEFT_ASYMMETRIC:
2769                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2770                         qd_idx = pd_idx + 1;
2771                         if (pd_idx == raid_disks-1) {
2772                                 (*dd_idx)++;    /* Q D D D P */
2773                                 qd_idx = 0;
2774                         } else if (*dd_idx >= pd_idx)
2775                                 (*dd_idx) += 2; /* D D P Q D */
2776                         break;
2777                 case ALGORITHM_RIGHT_ASYMMETRIC:
2778                         pd_idx = sector_div(stripe2, raid_disks);
2779                         qd_idx = pd_idx + 1;
2780                         if (pd_idx == raid_disks-1) {
2781                                 (*dd_idx)++;    /* Q D D D P */
2782                                 qd_idx = 0;
2783                         } else if (*dd_idx >= pd_idx)
2784                                 (*dd_idx) += 2; /* D D P Q D */
2785                         break;
2786                 case ALGORITHM_LEFT_SYMMETRIC:
2787                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2788                         qd_idx = (pd_idx + 1) % raid_disks;
2789                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2790                         break;
2791                 case ALGORITHM_RIGHT_SYMMETRIC:
2792                         pd_idx = sector_div(stripe2, raid_disks);
2793                         qd_idx = (pd_idx + 1) % raid_disks;
2794                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2795                         break;
2796
2797                 case ALGORITHM_PARITY_0:
2798                         pd_idx = 0;
2799                         qd_idx = 1;
2800                         (*dd_idx) += 2;
2801                         break;
2802                 case ALGORITHM_PARITY_N:
2803                         pd_idx = data_disks;
2804                         qd_idx = data_disks + 1;
2805                         break;
2806
2807                 case ALGORITHM_ROTATING_ZERO_RESTART:
2808                         /* Exactly the same as RIGHT_ASYMMETRIC, but or
2809                          * of blocks for computing Q is different.
2810                          */
2811                         pd_idx = sector_div(stripe2, raid_disks);
2812                         qd_idx = pd_idx + 1;
2813                         if (pd_idx == raid_disks-1) {
2814                                 (*dd_idx)++;    /* Q D D D P */
2815                                 qd_idx = 0;
2816                         } else if (*dd_idx >= pd_idx)
2817                                 (*dd_idx) += 2; /* D D P Q D */
2818                         ddf_layout = 1;
2819                         break;
2820
2821                 case ALGORITHM_ROTATING_N_RESTART:
2822                         /* Same a left_asymmetric, by first stripe is
2823                          * D D D P Q  rather than
2824                          * Q D D D P
2825                          */
2826                         stripe2 += 1;
2827                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2828                         qd_idx = pd_idx + 1;
2829                         if (pd_idx == raid_disks-1) {
2830                                 (*dd_idx)++;    /* Q D D D P */
2831                                 qd_idx = 0;
2832                         } else if (*dd_idx >= pd_idx)
2833                                 (*dd_idx) += 2; /* D D P Q D */
2834                         ddf_layout = 1;
2835                         break;
2836
2837                 case ALGORITHM_ROTATING_N_CONTINUE:
2838                         /* Same as left_symmetric but Q is before P */
2839                         pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2840                         qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
2841                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2842                         ddf_layout = 1;
2843                         break;
2844
2845                 case ALGORITHM_LEFT_ASYMMETRIC_6:
2846                         /* RAID5 left_asymmetric, with Q on last device */
2847                         pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2848                         if (*dd_idx >= pd_idx)
2849                                 (*dd_idx)++;
2850                         qd_idx = raid_disks - 1;
2851                         break;
2852
2853                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
2854                         pd_idx = sector_div(stripe2, raid_disks-1);
2855                         if (*dd_idx >= pd_idx)
2856                                 (*dd_idx)++;
2857                         qd_idx = raid_disks - 1;
2858                         break;
2859
2860                 case ALGORITHM_LEFT_SYMMETRIC_6:
2861                         pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2862                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2863                         qd_idx = raid_disks - 1;
2864                         break;
2865
2866                 case ALGORITHM_RIGHT_SYMMETRIC_6:
2867                         pd_idx = sector_div(stripe2, raid_disks-1);
2868                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2869                         qd_idx = raid_disks - 1;
2870                         break;
2871
2872                 case ALGORITHM_PARITY_0_6:
2873                         pd_idx = 0;
2874                         (*dd_idx)++;
2875                         qd_idx = raid_disks - 1;
2876                         break;
2877
2878                 default:
2879                         BUG();
2880                 }
2881                 break;
2882         }
2883
2884         if (sh) {
2885                 sh->pd_idx = pd_idx;
2886                 sh->qd_idx = qd_idx;
2887                 sh->ddf_layout = ddf_layout;
2888         }
2889         /*
2890          * Finally, compute the new sector number
2891          */
2892         new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
2893         return new_sector;
2894 }
2895
2896 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
2897 {
2898         struct r5conf *conf = sh->raid_conf;
2899         int raid_disks = sh->disks;
2900         int data_disks = raid_disks - conf->max_degraded;
2901         sector_t new_sector = sh->sector, check;
2902         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2903                                          : conf->chunk_sectors;
2904         int algorithm = previous ? conf->prev_algo
2905                                  : conf->algorithm;
2906         sector_t stripe;
2907         int chunk_offset;
2908         sector_t chunk_number;
2909         int dummy1, dd_idx = i;
2910         sector_t r_sector;
2911         struct stripe_head sh2;
2912
2913         chunk_offset = sector_div(new_sector, sectors_per_chunk);
2914         stripe = new_sector;
2915
2916         if (i == sh->pd_idx)
2917                 return 0;
2918         switch(conf->level) {
2919         case 4: break;
2920         case 5:
2921                 switch (algorithm) {
2922                 case ALGORITHM_LEFT_ASYMMETRIC:
2923                 case ALGORITHM_RIGHT_ASYMMETRIC:
2924                         if (i > sh->pd_idx)
2925                                 i--;
2926                         break;
2927                 case ALGORITHM_LEFT_SYMMETRIC:
2928                 case ALGORITHM_RIGHT_SYMMETRIC:
2929                         if (i < sh->pd_idx)
2930                                 i += raid_disks;
2931                         i -= (sh->pd_idx + 1);
2932                         break;
2933                 case ALGORITHM_PARITY_0:
2934                         i -= 1;
2935                         break;
2936                 case ALGORITHM_PARITY_N:
2937                         break;
2938                 default:
2939                         BUG();
2940                 }
2941                 break;
2942         case 6:
2943                 if (i == sh->qd_idx)
2944                         return 0; /* It is the Q disk */
2945                 switch (algorithm) {
2946                 case ALGORITHM_LEFT_ASYMMETRIC:
2947                 case ALGORITHM_RIGHT_ASYMMETRIC:
2948                 case ALGORITHM_ROTATING_ZERO_RESTART:
2949                 case ALGORITHM_ROTATING_N_RESTART:
2950                         if (sh->pd_idx == raid_disks-1)
2951                                 i--;    /* Q D D D P */
2952                         else if (i > sh->pd_idx)
2953                                 i -= 2; /* D D P Q D */
2954                         break;
2955                 case ALGORITHM_LEFT_SYMMETRIC:
2956                 case ALGORITHM_RIGHT_SYMMETRIC:
2957                         if (sh->pd_idx == raid_disks-1)
2958                                 i--; /* Q D D D P */
2959                         else {
2960                                 /* D D P Q D */
2961                                 if (i < sh->pd_idx)
2962                                         i += raid_disks;
2963                                 i -= (sh->pd_idx + 2);
2964                         }
2965                         break;
2966                 case ALGORITHM_PARITY_0:
2967                         i -= 2;
2968                         break;
2969                 case ALGORITHM_PARITY_N:
2970                         break;
2971                 case ALGORITHM_ROTATING_N_CONTINUE:
2972                         /* Like left_symmetric, but P is before Q */
2973                         if (sh->pd_idx == 0)
2974                                 i--;    /* P D D D Q */
2975                         else {
2976                                 /* D D Q P D */
2977                                 if (i < sh->pd_idx)
2978                                         i += raid_disks;
2979                                 i -= (sh->pd_idx + 1);
2980                         }
2981                         break;
2982                 case ALGORITHM_LEFT_ASYMMETRIC_6:
2983                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
2984                         if (i > sh->pd_idx)
2985                                 i--;
2986                         break;
2987                 case ALGORITHM_LEFT_SYMMETRIC_6:
2988                 case ALGORITHM_RIGHT_SYMMETRIC_6:
2989                         if (i < sh->pd_idx)
2990                                 i += data_disks + 1;
2991                         i -= (sh->pd_idx + 1);
2992                         break;
2993                 case ALGORITHM_PARITY_0_6:
2994                         i -= 1;
2995                         break;
2996                 default:
2997                         BUG();
2998                 }
2999                 break;
3000         }
3001
3002         chunk_number = stripe * data_disks + i;
3003         r_sector = chunk_number * sectors_per_chunk + chunk_offset;
3004
3005         check = raid5_compute_sector(conf, r_sector,
3006                                      previous, &dummy1, &sh2);
3007         if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
3008                 || sh2.qd_idx != sh->qd_idx) {
3009                 pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
3010                         mdname(conf->mddev));
3011                 return 0;
3012         }
3013         return r_sector;
3014 }
3015
3016 /*
3017  * There are cases where we want handle_stripe_dirtying() and
3018  * schedule_reconstruction() to delay towrite to some dev of a stripe.
3019  *
3020  * This function checks whether we want to delay the towrite. Specifically,
3021  * we delay the towrite when:
3022  *
3023  *   1. degraded stripe has a non-overwrite to the missing dev, AND this
3024  *      stripe has data in journal (for other devices).
3025  *
3026  *      In this case, when reading data for the non-overwrite dev, it is
3027  *      necessary to handle complex rmw of write back cache (prexor with
3028  *      orig_page, and xor with page). To keep read path simple, we would
3029  *      like to flush data in journal to RAID disks first, so complex rmw
3030  *      is handled in the write patch (handle_stripe_dirtying).
3031  *
3032  *   2. when journal space is critical (R5C_LOG_CRITICAL=1)
3033  *
3034  *      It is important to be able to flush all stripes in raid5-cache.
3035  *      Therefore, we need reserve some space on the journal device for
3036  *      these flushes. If flush operation includes pending writes to the
3037  *      stripe, we need to reserve (conf->raid_disk + 1) pages per stripe
3038  *      for the flush out. If we exclude these pending writes from flush
3039  *      operation, we only need (conf->max_degraded + 1) pages per stripe.
3040  *      Therefore, excluding pending writes in these cases enables more
3041  *      efficient use of the journal device.
3042  *
3043  *      Note: To make sure the stripe makes progress, we only delay
3044  *      towrite for stripes with data already in journal (injournal > 0).
3045  *      When LOG_CRITICAL, stripes with injournal == 0 will be sent to
3046  *      no_space_stripes list.
3047  *
3048  *   3. during journal failure
3049  *      In journal failure, we try to flush all cached data to raid disks
3050  *      based on data in stripe cache. The array is read-only to upper
3051  *      layers, so we would skip all pending writes.
3052  *
3053  */
3054 static inline bool delay_towrite(struct r5conf *conf,
3055                                  struct r5dev *dev,
3056                                  struct stripe_head_state *s)
3057 {
3058         /* case 1 above */
3059         if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3060             !test_bit(R5_Insync, &dev->flags) && s->injournal)
3061                 return true;
3062         /* case 2 above */
3063         if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
3064             s->injournal > 0)
3065                 return true;
3066         /* case 3 above */
3067         if (s->log_failed && s->injournal)
3068                 return true;
3069         return false;
3070 }
3071
3072 static void
3073 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
3074                          int rcw, int expand)
3075 {
3076         int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
3077         struct r5conf *conf = sh->raid_conf;
3078         int level = conf->level;
3079
3080         if (rcw) {
3081                 /*
3082                  * In some cases, handle_stripe_dirtying initially decided to
3083                  * run rmw and allocates extra page for prexor. However, rcw is
3084                  * cheaper later on. We need to free the extra page now,
3085                  * because we won't be able to do that in ops_complete_prexor().
3086                  */
3087                 r5c_release_extra_page(sh);
3088
3089                 for (i = disks; i--; ) {
3090                         struct r5dev *dev = &sh->dev[i];
3091
3092                         if (dev->towrite && !delay_towrite(conf, dev, s)) {
3093                                 set_bit(R5_LOCKED, &dev->flags);
3094                                 set_bit(R5_Wantdrain, &dev->flags);
3095                                 if (!expand)
3096                                         clear_bit(R5_UPTODATE, &dev->flags);
3097                                 s->locked++;
3098                         } else if (test_bit(R5_InJournal, &dev->flags)) {
3099                                 set_bit(R5_LOCKED, &dev->flags);
3100                                 s->locked++;
3101                         }
3102                 }
3103                 /* if we are not expanding this is a proper write request, and
3104                  * there will be bios with new data to be drained into the
3105                  * stripe cache
3106                  */
3107                 if (!expand) {
3108                         if (!s->locked)
3109                                 /* False alarm, nothing to do */
3110                                 return;
3111                         sh->reconstruct_state = reconstruct_state_drain_run;
3112                         set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
3113                 } else
3114                         sh->reconstruct_state = reconstruct_state_run;
3115
3116                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
3117
3118                 if (s->locked + conf->max_degraded == disks)
3119                         if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
3120                                 atomic_inc(&conf->pending_full_writes);
3121         } else {
3122                 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
3123                         test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
3124                 BUG_ON(level == 6 &&
3125                         (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
3126                            test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
3127
3128                 for (i = disks; i--; ) {
3129                         struct r5dev *dev = &sh->dev[i];
3130                         if (i == pd_idx || i == qd_idx)
3131                                 continue;
3132
3133                         if (dev->towrite &&
3134                             (test_bit(R5_UPTODATE, &dev->flags) ||
3135                              test_bit(R5_Wantcompute, &dev->flags))) {
3136                                 set_bit(R5_Wantdrain, &dev->flags);
3137                                 set_bit(R5_LOCKED, &dev->flags);
3138                                 clear_bit(R5_UPTODATE, &dev->flags);
3139                                 s->locked++;
3140                         } else if (test_bit(R5_InJournal, &dev->flags)) {
3141                                 set_bit(R5_LOCKED, &dev->flags);
3142                                 s->locked++;
3143                         }
3144                 }
3145                 if (!s->locked)
3146                         /* False alarm - nothing to do */
3147                         return;
3148                 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
3149                 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
3150                 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
3151                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
3152         }
3153
3154         /* keep the parity disk(s) locked while asynchronous operations
3155          * are in flight
3156          */
3157         set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
3158         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3159         s->locked++;
3160
3161         if (level == 6) {
3162                 int qd_idx = sh->qd_idx;
3163                 struct r5dev *dev = &sh->dev[qd_idx];
3164
3165                 set_bit(R5_LOCKED, &dev->flags);
3166                 clear_bit(R5_UPTODATE, &dev->flags);
3167                 s->locked++;
3168         }
3169
3170         if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
3171             test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&
3172             !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
3173             test_bit(R5_Insync, &sh->dev[pd_idx].flags))
3174                 set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);
3175
3176         pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
3177                 __func__, (unsigned long long)sh->sector,
3178                 s->locked, s->ops_request);
3179