spufs: switch to ->free_inode()
[muen/linux.git] / block / blk-lib.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Functions related to generic helpers functions
4  */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/scatterlist.h>
10
11 #include "blk.h"
12
13 struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp)
14 {
15         struct bio *new = bio_alloc(gfp, nr_pages);
16
17         if (bio) {
18                 bio_chain(bio, new);
19                 submit_bio(bio);
20         }
21
22         return new;
23 }
24
25 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
26                 sector_t nr_sects, gfp_t gfp_mask, int flags,
27                 struct bio **biop)
28 {
29         struct request_queue *q = bdev_get_queue(bdev);
30         struct bio *bio = *biop;
31         unsigned int op;
32         sector_t bs_mask;
33
34         if (!q)
35                 return -ENXIO;
36
37         if (bdev_read_only(bdev))
38                 return -EPERM;
39
40         if (flags & BLKDEV_DISCARD_SECURE) {
41                 if (!blk_queue_secure_erase(q))
42                         return -EOPNOTSUPP;
43                 op = REQ_OP_SECURE_ERASE;
44         } else {
45                 if (!blk_queue_discard(q))
46                         return -EOPNOTSUPP;
47                 op = REQ_OP_DISCARD;
48         }
49
50         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
51         if ((sector | nr_sects) & bs_mask)
52                 return -EINVAL;
53
54         if (!nr_sects)
55                 return -EINVAL;
56
57         while (nr_sects) {
58                 sector_t req_sects = min_t(sector_t, nr_sects,
59                                 bio_allowed_max_sectors(q));
60
61                 WARN_ON_ONCE((req_sects << 9) > UINT_MAX);
62
63                 bio = blk_next_bio(bio, 0, gfp_mask);
64                 bio->bi_iter.bi_sector = sector;
65                 bio_set_dev(bio, bdev);
66                 bio_set_op_attrs(bio, op, 0);
67
68                 bio->bi_iter.bi_size = req_sects << 9;
69                 sector += req_sects;
70                 nr_sects -= req_sects;
71
72                 /*
73                  * We can loop for a long time in here, if someone does
74                  * full device discards (like mkfs). Be nice and allow
75                  * us to schedule out to avoid softlocking if preempt
76                  * is disabled.
77                  */
78                 cond_resched();
79         }
80
81         *biop = bio;
82         return 0;
83 }
84 EXPORT_SYMBOL(__blkdev_issue_discard);
85
86 /**
87  * blkdev_issue_discard - queue a discard
88  * @bdev:       blockdev to issue discard for
89  * @sector:     start sector
90  * @nr_sects:   number of sectors to discard
91  * @gfp_mask:   memory allocation flags (for bio_alloc)
92  * @flags:      BLKDEV_DISCARD_* flags to control behaviour
93  *
94  * Description:
95  *    Issue a discard request for the sectors in question.
96  */
97 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
98                 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
99 {
100         struct bio *bio = NULL;
101         struct blk_plug plug;
102         int ret;
103
104         blk_start_plug(&plug);
105         ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
106                         &bio);
107         if (!ret && bio) {
108                 ret = submit_bio_wait(bio);
109                 if (ret == -EOPNOTSUPP)
110                         ret = 0;
111                 bio_put(bio);
112         }
113         blk_finish_plug(&plug);
114
115         return ret;
116 }
117 EXPORT_SYMBOL(blkdev_issue_discard);
118
119 /**
120  * __blkdev_issue_write_same - generate number of bios with same page
121  * @bdev:       target blockdev
122  * @sector:     start sector
123  * @nr_sects:   number of sectors to write
124  * @gfp_mask:   memory allocation flags (for bio_alloc)
125  * @page:       page containing data to write
126  * @biop:       pointer to anchor bio
127  *
128  * Description:
129  *  Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page.
130  */
131 static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
132                 sector_t nr_sects, gfp_t gfp_mask, struct page *page,
133                 struct bio **biop)
134 {
135         struct request_queue *q = bdev_get_queue(bdev);
136         unsigned int max_write_same_sectors;
137         struct bio *bio = *biop;
138         sector_t bs_mask;
139
140         if (!q)
141                 return -ENXIO;
142
143         if (bdev_read_only(bdev))
144                 return -EPERM;
145
146         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
147         if ((sector | nr_sects) & bs_mask)
148                 return -EINVAL;
149
150         if (!bdev_write_same(bdev))
151                 return -EOPNOTSUPP;
152
153         /* Ensure that max_write_same_sectors doesn't overflow bi_size */
154         max_write_same_sectors = bio_allowed_max_sectors(q);
155
156         while (nr_sects) {
157                 bio = blk_next_bio(bio, 1, gfp_mask);
158                 bio->bi_iter.bi_sector = sector;
159                 bio_set_dev(bio, bdev);
160                 bio->bi_vcnt = 1;
161                 bio->bi_io_vec->bv_page = page;
162                 bio->bi_io_vec->bv_offset = 0;
163                 bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
164                 bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0);
165
166                 if (nr_sects > max_write_same_sectors) {
167                         bio->bi_iter.bi_size = max_write_same_sectors << 9;
168                         nr_sects -= max_write_same_sectors;
169                         sector += max_write_same_sectors;
170                 } else {
171                         bio->bi_iter.bi_size = nr_sects << 9;
172                         nr_sects = 0;
173                 }
174                 cond_resched();
175         }
176
177         *biop = bio;
178         return 0;
179 }
180
181 /**
182  * blkdev_issue_write_same - queue a write same operation
183  * @bdev:       target blockdev
184  * @sector:     start sector
185  * @nr_sects:   number of sectors to write
186  * @gfp_mask:   memory allocation flags (for bio_alloc)
187  * @page:       page containing data
188  *
189  * Description:
190  *    Issue a write same request for the sectors in question.
191  */
192 int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
193                                 sector_t nr_sects, gfp_t gfp_mask,
194                                 struct page *page)
195 {
196         struct bio *bio = NULL;
197         struct blk_plug plug;
198         int ret;
199
200         blk_start_plug(&plug);
201         ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page,
202                         &bio);
203         if (ret == 0 && bio) {
204                 ret = submit_bio_wait(bio);
205                 bio_put(bio);
206         }
207         blk_finish_plug(&plug);
208         return ret;
209 }
210 EXPORT_SYMBOL(blkdev_issue_write_same);
211
212 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
213                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
214                 struct bio **biop, unsigned flags)
215 {
216         struct bio *bio = *biop;
217         unsigned int max_write_zeroes_sectors;
218         struct request_queue *q = bdev_get_queue(bdev);
219
220         if (!q)
221                 return -ENXIO;
222
223         if (bdev_read_only(bdev))
224                 return -EPERM;
225
226         /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
227         max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
228
229         if (max_write_zeroes_sectors == 0)
230                 return -EOPNOTSUPP;
231
232         while (nr_sects) {
233                 bio = blk_next_bio(bio, 0, gfp_mask);
234                 bio->bi_iter.bi_sector = sector;
235                 bio_set_dev(bio, bdev);
236                 bio->bi_opf = REQ_OP_WRITE_ZEROES;
237                 if (flags & BLKDEV_ZERO_NOUNMAP)
238                         bio->bi_opf |= REQ_NOUNMAP;
239
240                 if (nr_sects > max_write_zeroes_sectors) {
241                         bio->bi_iter.bi_size = max_write_zeroes_sectors << 9;
242                         nr_sects -= max_write_zeroes_sectors;
243                         sector += max_write_zeroes_sectors;
244                 } else {
245                         bio->bi_iter.bi_size = nr_sects << 9;
246                         nr_sects = 0;
247                 }
248                 cond_resched();
249         }
250
251         *biop = bio;
252         return 0;
253 }
254
255 /*
256  * Convert a number of 512B sectors to a number of pages.
257  * The result is limited to a number of pages that can fit into a BIO.
258  * Also make sure that the result is always at least 1 (page) for the cases
259  * where nr_sects is lower than the number of sectors in a page.
260  */
261 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
262 {
263         sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
264
265         return min(pages, (sector_t)BIO_MAX_PAGES);
266 }
267
268 static int __blkdev_issue_zero_pages(struct block_device *bdev,
269                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
270                 struct bio **biop)
271 {
272         struct request_queue *q = bdev_get_queue(bdev);
273         struct bio *bio = *biop;
274         int bi_size = 0;
275         unsigned int sz;
276
277         if (!q)
278                 return -ENXIO;
279
280         if (bdev_read_only(bdev))
281                 return -EPERM;
282
283         while (nr_sects != 0) {
284                 bio = blk_next_bio(bio, __blkdev_sectors_to_bio_pages(nr_sects),
285                                    gfp_mask);
286                 bio->bi_iter.bi_sector = sector;
287                 bio_set_dev(bio, bdev);
288                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
289
290                 while (nr_sects != 0) {
291                         sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
292                         bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
293                         nr_sects -= bi_size >> 9;
294                         sector += bi_size >> 9;
295                         if (bi_size < sz)
296                                 break;
297                 }
298                 cond_resched();
299         }
300
301         *biop = bio;
302         return 0;
303 }
304
305 /**
306  * __blkdev_issue_zeroout - generate number of zero filed write bios
307  * @bdev:       blockdev to issue
308  * @sector:     start sector
309  * @nr_sects:   number of sectors to write
310  * @gfp_mask:   memory allocation flags (for bio_alloc)
311  * @biop:       pointer to anchor bio
312  * @flags:      controls detailed behavior
313  *
314  * Description:
315  *  Zero-fill a block range, either using hardware offload or by explicitly
316  *  writing zeroes to the device.
317  *
318  *  If a device is using logical block provisioning, the underlying space will
319  *  not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
320  *
321  *  If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
322  *  -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
323  */
324 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
325                 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
326                 unsigned flags)
327 {
328         int ret;
329         sector_t bs_mask;
330
331         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
332         if ((sector | nr_sects) & bs_mask)
333                 return -EINVAL;
334
335         ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
336                         biop, flags);
337         if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
338                 return ret;
339
340         return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
341                                          biop);
342 }
343 EXPORT_SYMBOL(__blkdev_issue_zeroout);
344
345 /**
346  * blkdev_issue_zeroout - zero-fill a block range
347  * @bdev:       blockdev to write
348  * @sector:     start sector
349  * @nr_sects:   number of sectors to write
350  * @gfp_mask:   memory allocation flags (for bio_alloc)
351  * @flags:      controls detailed behavior
352  *
353  * Description:
354  *  Zero-fill a block range, either using hardware offload or by explicitly
355  *  writing zeroes to the device.  See __blkdev_issue_zeroout() for the
356  *  valid values for %flags.
357  */
358 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
359                 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
360 {
361         int ret = 0;
362         sector_t bs_mask;
363         struct bio *bio;
364         struct blk_plug plug;
365         bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
366
367         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
368         if ((sector | nr_sects) & bs_mask)
369                 return -EINVAL;
370
371 retry:
372         bio = NULL;
373         blk_start_plug(&plug);
374         if (try_write_zeroes) {
375                 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
376                                                   gfp_mask, &bio, flags);
377         } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
378                 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
379                                                 gfp_mask, &bio);
380         } else {
381                 /* No zeroing offload support */
382                 ret = -EOPNOTSUPP;
383         }
384         if (ret == 0 && bio) {
385                 ret = submit_bio_wait(bio);
386                 bio_put(bio);
387         }
388         blk_finish_plug(&plug);
389         if (ret && try_write_zeroes) {
390                 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
391                         try_write_zeroes = false;
392                         goto retry;
393                 }
394                 if (!bdev_write_zeroes_sectors(bdev)) {
395                         /*
396                          * Zeroing offload support was indicated, but the
397                          * device reported ILLEGAL REQUEST (for some devices
398                          * there is no non-destructive way to verify whether
399                          * WRITE ZEROES is actually supported).
400                          */
401                         ret = -EOPNOTSUPP;
402                 }
403         }
404
405         return ret;
406 }
407 EXPORT_SYMBOL(blkdev_issue_zeroout);