e9a7ac74823de13b912c2896f3254e762d17a172
[muen/linux.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/mm.h>
32 #include <linux/sched/signal.h>
33 #include <linux/export.h>
34 #include <linux/swap.h>
35 #include <linux/uio.h>
36 #include <linux/khugepaged.h>
37 #include <linux/hugetlb.h>
38
39 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
40
41 static struct vfsmount *shm_mnt;
42
43 #ifdef CONFIG_SHMEM
44 /*
45  * This virtual memory filesystem is heavily based on the ramfs. It
46  * extends ramfs by the ability to use swap and honor resource limits
47  * which makes it a completely usable filesystem.
48  */
49
50 #include <linux/xattr.h>
51 #include <linux/exportfs.h>
52 #include <linux/posix_acl.h>
53 #include <linux/posix_acl_xattr.h>
54 #include <linux/mman.h>
55 #include <linux/string.h>
56 #include <linux/slab.h>
57 #include <linux/backing-dev.h>
58 #include <linux/shmem_fs.h>
59 #include <linux/writeback.h>
60 #include <linux/blkdev.h>
61 #include <linux/pagevec.h>
62 #include <linux/percpu_counter.h>
63 #include <linux/falloc.h>
64 #include <linux/splice.h>
65 #include <linux/security.h>
66 #include <linux/swapops.h>
67 #include <linux/mempolicy.h>
68 #include <linux/namei.h>
69 #include <linux/ctype.h>
70 #include <linux/migrate.h>
71 #include <linux/highmem.h>
72 #include <linux/seq_file.h>
73 #include <linux/magic.h>
74 #include <linux/syscalls.h>
75 #include <linux/fcntl.h>
76 #include <uapi/linux/memfd.h>
77 #include <linux/userfaultfd_k.h>
78 #include <linux/rmap.h>
79 #include <linux/uuid.h>
80
81 #include <linux/uaccess.h>
82 #include <asm/pgtable.h>
83
84 #include "internal.h"
85
86 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
87 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
88
89 /* Pretend that each entry is of this size in directory's i_size */
90 #define BOGO_DIRENT_SIZE 20
91
92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
93 #define SHORT_SYMLINK_LEN 128
94
95 /*
96  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
97  * inode->i_private (with i_mutex making sure that it has only one user at
98  * a time): we would prefer not to enlarge the shmem inode just for that.
99  */
100 struct shmem_falloc {
101         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
102         pgoff_t start;          /* start of range currently being fallocated */
103         pgoff_t next;           /* the next page offset to be fallocated */
104         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
105         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
106 };
107
108 #ifdef CONFIG_TMPFS
109 static unsigned long shmem_default_max_blocks(void)
110 {
111         return totalram_pages / 2;
112 }
113
114 static unsigned long shmem_default_max_inodes(void)
115 {
116         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
117 }
118 #endif
119
120 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
121 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
122                                 struct shmem_inode_info *info, pgoff_t index);
123 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
124                 struct page **pagep, enum sgp_type sgp,
125                 gfp_t gfp, struct vm_area_struct *vma,
126                 struct vm_fault *vmf, int *fault_type);
127
128 int shmem_getpage(struct inode *inode, pgoff_t index,
129                 struct page **pagep, enum sgp_type sgp)
130 {
131         return shmem_getpage_gfp(inode, index, pagep, sgp,
132                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
133 }
134
135 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
136 {
137         return sb->s_fs_info;
138 }
139
140 /*
141  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
142  * for shared memory and for shared anonymous (/dev/zero) mappings
143  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
144  * consistent with the pre-accounting of private mappings ...
145  */
146 static inline int shmem_acct_size(unsigned long flags, loff_t size)
147 {
148         return (flags & VM_NORESERVE) ?
149                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
150 }
151
152 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
153 {
154         if (!(flags & VM_NORESERVE))
155                 vm_unacct_memory(VM_ACCT(size));
156 }
157
158 static inline int shmem_reacct_size(unsigned long flags,
159                 loff_t oldsize, loff_t newsize)
160 {
161         if (!(flags & VM_NORESERVE)) {
162                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
163                         return security_vm_enough_memory_mm(current->mm,
164                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
165                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
166                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
167         }
168         return 0;
169 }
170
171 /*
172  * ... whereas tmpfs objects are accounted incrementally as
173  * pages are allocated, in order to allow large sparse files.
174  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
175  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
176  */
177 static inline int shmem_acct_block(unsigned long flags, long pages)
178 {
179         if (!(flags & VM_NORESERVE))
180                 return 0;
181
182         return security_vm_enough_memory_mm(current->mm,
183                         pages * VM_ACCT(PAGE_SIZE));
184 }
185
186 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
187 {
188         if (flags & VM_NORESERVE)
189                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
190 }
191
192 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
193 {
194         struct shmem_inode_info *info = SHMEM_I(inode);
195         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
196
197         if (shmem_acct_block(info->flags, pages))
198                 return false;
199
200         if (sbinfo->max_blocks) {
201                 if (percpu_counter_compare(&sbinfo->used_blocks,
202                                            sbinfo->max_blocks - pages) > 0)
203                         goto unacct;
204                 percpu_counter_add(&sbinfo->used_blocks, pages);
205         }
206
207         return true;
208
209 unacct:
210         shmem_unacct_blocks(info->flags, pages);
211         return false;
212 }
213
214 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
215 {
216         struct shmem_inode_info *info = SHMEM_I(inode);
217         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
218
219         if (sbinfo->max_blocks)
220                 percpu_counter_sub(&sbinfo->used_blocks, pages);
221         shmem_unacct_blocks(info->flags, pages);
222 }
223
224 static const struct super_operations shmem_ops;
225 static const struct address_space_operations shmem_aops;
226 static const struct file_operations shmem_file_operations;
227 static const struct inode_operations shmem_inode_operations;
228 static const struct inode_operations shmem_dir_inode_operations;
229 static const struct inode_operations shmem_special_inode_operations;
230 static const struct vm_operations_struct shmem_vm_ops;
231 static struct file_system_type shmem_fs_type;
232
233 bool vma_is_shmem(struct vm_area_struct *vma)
234 {
235         return vma->vm_ops == &shmem_vm_ops;
236 }
237
238 static LIST_HEAD(shmem_swaplist);
239 static DEFINE_MUTEX(shmem_swaplist_mutex);
240
241 static int shmem_reserve_inode(struct super_block *sb)
242 {
243         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
244         if (sbinfo->max_inodes) {
245                 spin_lock(&sbinfo->stat_lock);
246                 if (!sbinfo->free_inodes) {
247                         spin_unlock(&sbinfo->stat_lock);
248                         return -ENOSPC;
249                 }
250                 sbinfo->free_inodes--;
251                 spin_unlock(&sbinfo->stat_lock);
252         }
253         return 0;
254 }
255
256 static void shmem_free_inode(struct super_block *sb)
257 {
258         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
259         if (sbinfo->max_inodes) {
260                 spin_lock(&sbinfo->stat_lock);
261                 sbinfo->free_inodes++;
262                 spin_unlock(&sbinfo->stat_lock);
263         }
264 }
265
266 /**
267  * shmem_recalc_inode - recalculate the block usage of an inode
268  * @inode: inode to recalc
269  *
270  * We have to calculate the free blocks since the mm can drop
271  * undirtied hole pages behind our back.
272  *
273  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
274  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
275  *
276  * It has to be called with the spinlock held.
277  */
278 static void shmem_recalc_inode(struct inode *inode)
279 {
280         struct shmem_inode_info *info = SHMEM_I(inode);
281         long freed;
282
283         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
284         if (freed > 0) {
285                 info->alloced -= freed;
286                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
287                 shmem_inode_unacct_blocks(inode, freed);
288         }
289 }
290
291 bool shmem_charge(struct inode *inode, long pages)
292 {
293         struct shmem_inode_info *info = SHMEM_I(inode);
294         unsigned long flags;
295
296         if (!shmem_inode_acct_block(inode, pages))
297                 return false;
298
299         spin_lock_irqsave(&info->lock, flags);
300         info->alloced += pages;
301         inode->i_blocks += pages * BLOCKS_PER_PAGE;
302         shmem_recalc_inode(inode);
303         spin_unlock_irqrestore(&info->lock, flags);
304         inode->i_mapping->nrpages += pages;
305
306         return true;
307 }
308
309 void shmem_uncharge(struct inode *inode, long pages)
310 {
311         struct shmem_inode_info *info = SHMEM_I(inode);
312         unsigned long flags;
313
314         spin_lock_irqsave(&info->lock, flags);
315         info->alloced -= pages;
316         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
317         shmem_recalc_inode(inode);
318         spin_unlock_irqrestore(&info->lock, flags);
319
320         shmem_inode_unacct_blocks(inode, pages);
321 }
322
323 /*
324  * Replace item expected in radix tree by a new item, while holding tree lock.
325  */
326 static int shmem_radix_tree_replace(struct address_space *mapping,
327                         pgoff_t index, void *expected, void *replacement)
328 {
329         struct radix_tree_node *node;
330         void __rcu **pslot;
331         void *item;
332
333         VM_BUG_ON(!expected);
334         VM_BUG_ON(!replacement);
335         item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
336         if (!item)
337                 return -ENOENT;
338         if (item != expected)
339                 return -ENOENT;
340         __radix_tree_replace(&mapping->i_pages, node, pslot,
341                              replacement, NULL);
342         return 0;
343 }
344
345 /*
346  * Sometimes, before we decide whether to proceed or to fail, we must check
347  * that an entry was not already brought back from swap by a racing thread.
348  *
349  * Checking page is not enough: by the time a SwapCache page is locked, it
350  * might be reused, and again be SwapCache, using the same swap as before.
351  */
352 static bool shmem_confirm_swap(struct address_space *mapping,
353                                pgoff_t index, swp_entry_t swap)
354 {
355         void *item;
356
357         rcu_read_lock();
358         item = radix_tree_lookup(&mapping->i_pages, index);
359         rcu_read_unlock();
360         return item == swp_to_radix_entry(swap);
361 }
362
363 /*
364  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
365  *
366  * SHMEM_HUGE_NEVER:
367  *      disables huge pages for the mount;
368  * SHMEM_HUGE_ALWAYS:
369  *      enables huge pages for the mount;
370  * SHMEM_HUGE_WITHIN_SIZE:
371  *      only allocate huge pages if the page will be fully within i_size,
372  *      also respect fadvise()/madvise() hints;
373  * SHMEM_HUGE_ADVISE:
374  *      only allocate huge pages if requested with fadvise()/madvise();
375  */
376
377 #define SHMEM_HUGE_NEVER        0
378 #define SHMEM_HUGE_ALWAYS       1
379 #define SHMEM_HUGE_WITHIN_SIZE  2
380 #define SHMEM_HUGE_ADVISE       3
381
382 /*
383  * Special values.
384  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
385  *
386  * SHMEM_HUGE_DENY:
387  *      disables huge on shm_mnt and all mounts, for emergency use;
388  * SHMEM_HUGE_FORCE:
389  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
390  *
391  */
392 #define SHMEM_HUGE_DENY         (-1)
393 #define SHMEM_HUGE_FORCE        (-2)
394
395 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
396 /* ifdef here to avoid bloating shmem.o when not necessary */
397
398 static int shmem_huge __read_mostly;
399
400 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
401 static int shmem_parse_huge(const char *str)
402 {
403         if (!strcmp(str, "never"))
404                 return SHMEM_HUGE_NEVER;
405         if (!strcmp(str, "always"))
406                 return SHMEM_HUGE_ALWAYS;
407         if (!strcmp(str, "within_size"))
408                 return SHMEM_HUGE_WITHIN_SIZE;
409         if (!strcmp(str, "advise"))
410                 return SHMEM_HUGE_ADVISE;
411         if (!strcmp(str, "deny"))
412                 return SHMEM_HUGE_DENY;
413         if (!strcmp(str, "force"))
414                 return SHMEM_HUGE_FORCE;
415         return -EINVAL;
416 }
417
418 static const char *shmem_format_huge(int huge)
419 {
420         switch (huge) {
421         case SHMEM_HUGE_NEVER:
422                 return "never";
423         case SHMEM_HUGE_ALWAYS:
424                 return "always";
425         case SHMEM_HUGE_WITHIN_SIZE:
426                 return "within_size";
427         case SHMEM_HUGE_ADVISE:
428                 return "advise";
429         case SHMEM_HUGE_DENY:
430                 return "deny";
431         case SHMEM_HUGE_FORCE:
432                 return "force";
433         default:
434                 VM_BUG_ON(1);
435                 return "bad_val";
436         }
437 }
438 #endif
439
440 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
441                 struct shrink_control *sc, unsigned long nr_to_split)
442 {
443         LIST_HEAD(list), *pos, *next;
444         LIST_HEAD(to_remove);
445         struct inode *inode;
446         struct shmem_inode_info *info;
447         struct page *page;
448         unsigned long batch = sc ? sc->nr_to_scan : 128;
449         int removed = 0, split = 0;
450
451         if (list_empty(&sbinfo->shrinklist))
452                 return SHRINK_STOP;
453
454         spin_lock(&sbinfo->shrinklist_lock);
455         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
456                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
457
458                 /* pin the inode */
459                 inode = igrab(&info->vfs_inode);
460
461                 /* inode is about to be evicted */
462                 if (!inode) {
463                         list_del_init(&info->shrinklist);
464                         removed++;
465                         goto next;
466                 }
467
468                 /* Check if there's anything to gain */
469                 if (round_up(inode->i_size, PAGE_SIZE) ==
470                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
471                         list_move(&info->shrinklist, &to_remove);
472                         removed++;
473                         goto next;
474                 }
475
476                 list_move(&info->shrinklist, &list);
477 next:
478                 if (!--batch)
479                         break;
480         }
481         spin_unlock(&sbinfo->shrinklist_lock);
482
483         list_for_each_safe(pos, next, &to_remove) {
484                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
485                 inode = &info->vfs_inode;
486                 list_del_init(&info->shrinklist);
487                 iput(inode);
488         }
489
490         list_for_each_safe(pos, next, &list) {
491                 int ret;
492
493                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
494                 inode = &info->vfs_inode;
495
496                 if (nr_to_split && split >= nr_to_split)
497                         goto leave;
498
499                 page = find_get_page(inode->i_mapping,
500                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
501                 if (!page)
502                         goto drop;
503
504                 /* No huge page at the end of the file: nothing to split */
505                 if (!PageTransHuge(page)) {
506                         put_page(page);
507                         goto drop;
508                 }
509
510                 /*
511                  * Leave the inode on the list if we failed to lock
512                  * the page at this time.
513                  *
514                  * Waiting for the lock may lead to deadlock in the
515                  * reclaim path.
516                  */
517                 if (!trylock_page(page)) {
518                         put_page(page);
519                         goto leave;
520                 }
521
522                 ret = split_huge_page(page);
523                 unlock_page(page);
524                 put_page(page);
525
526                 /* If split failed leave the inode on the list */
527                 if (ret)
528                         goto leave;
529
530                 split++;
531 drop:
532                 list_del_init(&info->shrinklist);
533                 removed++;
534 leave:
535                 iput(inode);
536         }
537
538         spin_lock(&sbinfo->shrinklist_lock);
539         list_splice_tail(&list, &sbinfo->shrinklist);
540         sbinfo->shrinklist_len -= removed;
541         spin_unlock(&sbinfo->shrinklist_lock);
542
543         return split;
544 }
545
546 static long shmem_unused_huge_scan(struct super_block *sb,
547                 struct shrink_control *sc)
548 {
549         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
550
551         if (!READ_ONCE(sbinfo->shrinklist_len))
552                 return SHRINK_STOP;
553
554         return shmem_unused_huge_shrink(sbinfo, sc, 0);
555 }
556
557 static long shmem_unused_huge_count(struct super_block *sb,
558                 struct shrink_control *sc)
559 {
560         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
561         return READ_ONCE(sbinfo->shrinklist_len);
562 }
563 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
564
565 #define shmem_huge SHMEM_HUGE_DENY
566
567 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
568                 struct shrink_control *sc, unsigned long nr_to_split)
569 {
570         return 0;
571 }
572 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
573
574 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
575 {
576         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
577             (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
578             shmem_huge != SHMEM_HUGE_DENY)
579                 return true;
580         return false;
581 }
582
583 /*
584  * Like add_to_page_cache_locked, but error if expected item has gone.
585  */
586 static int shmem_add_to_page_cache(struct page *page,
587                                    struct address_space *mapping,
588                                    pgoff_t index, void *expected)
589 {
590         int error, nr = hpage_nr_pages(page);
591
592         VM_BUG_ON_PAGE(PageTail(page), page);
593         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
594         VM_BUG_ON_PAGE(!PageLocked(page), page);
595         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
596         VM_BUG_ON(expected && PageTransHuge(page));
597
598         page_ref_add(page, nr);
599         page->mapping = mapping;
600         page->index = index;
601
602         xa_lock_irq(&mapping->i_pages);
603         if (PageTransHuge(page)) {
604                 void __rcu **results;
605                 pgoff_t idx;
606                 int i;
607
608                 error = 0;
609                 if (radix_tree_gang_lookup_slot(&mapping->i_pages,
610                                         &results, &idx, index, 1) &&
611                                 idx < index + HPAGE_PMD_NR) {
612                         error = -EEXIST;
613                 }
614
615                 if (!error) {
616                         for (i = 0; i < HPAGE_PMD_NR; i++) {
617                                 error = radix_tree_insert(&mapping->i_pages,
618                                                 index + i, page + i);
619                                 VM_BUG_ON(error);
620                         }
621                         count_vm_event(THP_FILE_ALLOC);
622                 }
623         } else if (!expected) {
624                 error = radix_tree_insert(&mapping->i_pages, index, page);
625         } else {
626                 error = shmem_radix_tree_replace(mapping, index, expected,
627                                                                  page);
628         }
629
630         if (!error) {
631                 mapping->nrpages += nr;
632                 if (PageTransHuge(page))
633                         __inc_node_page_state(page, NR_SHMEM_THPS);
634                 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
635                 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
636                 xa_unlock_irq(&mapping->i_pages);
637         } else {
638                 page->mapping = NULL;
639                 xa_unlock_irq(&mapping->i_pages);
640                 page_ref_sub(page, nr);
641         }
642         return error;
643 }
644
645 /*
646  * Like delete_from_page_cache, but substitutes swap for page.
647  */
648 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
649 {
650         struct address_space *mapping = page->mapping;
651         int error;
652
653         VM_BUG_ON_PAGE(PageCompound(page), page);
654
655         xa_lock_irq(&mapping->i_pages);
656         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
657         page->mapping = NULL;
658         mapping->nrpages--;
659         __dec_node_page_state(page, NR_FILE_PAGES);
660         __dec_node_page_state(page, NR_SHMEM);
661         xa_unlock_irq(&mapping->i_pages);
662         put_page(page);
663         BUG_ON(error);
664 }
665
666 /*
667  * Remove swap entry from radix tree, free the swap and its page cache.
668  */
669 static int shmem_free_swap(struct address_space *mapping,
670                            pgoff_t index, void *radswap)
671 {
672         void *old;
673
674         xa_lock_irq(&mapping->i_pages);
675         old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
676         xa_unlock_irq(&mapping->i_pages);
677         if (old != radswap)
678                 return -ENOENT;
679         free_swap_and_cache(radix_to_swp_entry(radswap));
680         return 0;
681 }
682
683 /*
684  * Determine (in bytes) how many of the shmem object's pages mapped by the
685  * given offsets are swapped out.
686  *
687  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
688  * as long as the inode doesn't go away and racy results are not a problem.
689  */
690 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
691                                                 pgoff_t start, pgoff_t end)
692 {
693         struct radix_tree_iter iter;
694         void __rcu **slot;
695         struct page *page;
696         unsigned long swapped = 0;
697
698         rcu_read_lock();
699
700         radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
701                 if (iter.index >= end)
702                         break;
703
704                 page = radix_tree_deref_slot(slot);
705
706                 if (radix_tree_deref_retry(page)) {
707                         slot = radix_tree_iter_retry(&iter);
708                         continue;
709                 }
710
711                 if (radix_tree_exceptional_entry(page))
712                         swapped++;
713
714                 if (need_resched()) {
715                         slot = radix_tree_iter_resume(slot, &iter);
716                         cond_resched_rcu();
717                 }
718         }
719
720         rcu_read_unlock();
721
722         return swapped << PAGE_SHIFT;
723 }
724
725 /*
726  * Determine (in bytes) how many of the shmem object's pages mapped by the
727  * given vma is swapped out.
728  *
729  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
730  * as long as the inode doesn't go away and racy results are not a problem.
731  */
732 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
733 {
734         struct inode *inode = file_inode(vma->vm_file);
735         struct shmem_inode_info *info = SHMEM_I(inode);
736         struct address_space *mapping = inode->i_mapping;
737         unsigned long swapped;
738
739         /* Be careful as we don't hold info->lock */
740         swapped = READ_ONCE(info->swapped);
741
742         /*
743          * The easier cases are when the shmem object has nothing in swap, or
744          * the vma maps it whole. Then we can simply use the stats that we
745          * already track.
746          */
747         if (!swapped)
748                 return 0;
749
750         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
751                 return swapped << PAGE_SHIFT;
752
753         /* Here comes the more involved part */
754         return shmem_partial_swap_usage(mapping,
755                         linear_page_index(vma, vma->vm_start),
756                         linear_page_index(vma, vma->vm_end));
757 }
758
759 /*
760  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
761  */
762 void shmem_unlock_mapping(struct address_space *mapping)
763 {
764         struct pagevec pvec;
765         pgoff_t indices[PAGEVEC_SIZE];
766         pgoff_t index = 0;
767
768         pagevec_init(&pvec);
769         /*
770          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
771          */
772         while (!mapping_unevictable(mapping)) {
773                 /*
774                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
775                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
776                  */
777                 pvec.nr = find_get_entries(mapping, index,
778                                            PAGEVEC_SIZE, pvec.pages, indices);
779                 if (!pvec.nr)
780                         break;
781                 index = indices[pvec.nr - 1] + 1;
782                 pagevec_remove_exceptionals(&pvec);
783                 check_move_unevictable_pages(pvec.pages, pvec.nr);
784                 pagevec_release(&pvec);
785                 cond_resched();
786         }
787 }
788
789 /*
790  * Remove range of pages and swap entries from radix tree, and free them.
791  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
792  */
793 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
794                                                                  bool unfalloc)
795 {
796         struct address_space *mapping = inode->i_mapping;
797         struct shmem_inode_info *info = SHMEM_I(inode);
798         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
799         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
800         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
801         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
802         struct pagevec pvec;
803         pgoff_t indices[PAGEVEC_SIZE];
804         long nr_swaps_freed = 0;
805         pgoff_t index;
806         int i;
807
808         if (lend == -1)
809                 end = -1;       /* unsigned, so actually very big */
810
811         pagevec_init(&pvec);
812         index = start;
813         while (index < end) {
814                 pvec.nr = find_get_entries(mapping, index,
815                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
816                         pvec.pages, indices);
817                 if (!pvec.nr)
818                         break;
819                 for (i = 0; i < pagevec_count(&pvec); i++) {
820                         struct page *page = pvec.pages[i];
821
822                         index = indices[i];
823                         if (index >= end)
824                                 break;
825
826                         if (radix_tree_exceptional_entry(page)) {
827                                 if (unfalloc)
828                                         continue;
829                                 nr_swaps_freed += !shmem_free_swap(mapping,
830                                                                 index, page);
831                                 continue;
832                         }
833
834                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
835
836                         if (!trylock_page(page))
837                                 continue;
838
839                         if (PageTransTail(page)) {
840                                 /* Middle of THP: zero out the page */
841                                 clear_highpage(page);
842                                 unlock_page(page);
843                                 continue;
844                         } else if (PageTransHuge(page)) {
845                                 if (index == round_down(end, HPAGE_PMD_NR)) {
846                                         /*
847                                          * Range ends in the middle of THP:
848                                          * zero out the page
849                                          */
850                                         clear_highpage(page);
851                                         unlock_page(page);
852                                         continue;
853                                 }
854                                 index += HPAGE_PMD_NR - 1;
855                                 i += HPAGE_PMD_NR - 1;
856                         }
857
858                         if (!unfalloc || !PageUptodate(page)) {
859                                 VM_BUG_ON_PAGE(PageTail(page), page);
860                                 if (page_mapping(page) == mapping) {
861                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
862                                         truncate_inode_page(mapping, page);
863                                 }
864                         }
865                         unlock_page(page);
866                 }
867                 pagevec_remove_exceptionals(&pvec);
868                 pagevec_release(&pvec);
869                 cond_resched();
870                 index++;
871         }
872
873         if (partial_start) {
874                 struct page *page = NULL;
875                 shmem_getpage(inode, start - 1, &page, SGP_READ);
876                 if (page) {
877                         unsigned int top = PAGE_SIZE;
878                         if (start > end) {
879                                 top = partial_end;
880                                 partial_end = 0;
881                         }
882                         zero_user_segment(page, partial_start, top);
883                         set_page_dirty(page);
884                         unlock_page(page);
885                         put_page(page);
886                 }
887         }
888         if (partial_end) {
889                 struct page *page = NULL;
890                 shmem_getpage(inode, end, &page, SGP_READ);
891                 if (page) {
892                         zero_user_segment(page, 0, partial_end);
893                         set_page_dirty(page);
894                         unlock_page(page);
895                         put_page(page);
896                 }
897         }
898         if (start >= end)
899                 return;
900
901         index = start;
902         while (index < end) {
903                 cond_resched();
904
905                 pvec.nr = find_get_entries(mapping, index,
906                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
907                                 pvec.pages, indices);
908                 if (!pvec.nr) {
909                         /* If all gone or hole-punch or unfalloc, we're done */
910                         if (index == start || end != -1)
911                                 break;
912                         /* But if truncating, restart to make sure all gone */
913                         index = start;
914                         continue;
915                 }
916                 for (i = 0; i < pagevec_count(&pvec); i++) {
917                         struct page *page = pvec.pages[i];
918
919                         index = indices[i];
920                         if (index >= end)
921                                 break;
922
923                         if (radix_tree_exceptional_entry(page)) {
924                                 if (unfalloc)
925                                         continue;
926                                 if (shmem_free_swap(mapping, index, page)) {
927                                         /* Swap was replaced by page: retry */
928                                         index--;
929                                         break;
930                                 }
931                                 nr_swaps_freed++;
932                                 continue;
933                         }
934
935                         lock_page(page);
936
937                         if (PageTransTail(page)) {
938                                 /* Middle of THP: zero out the page */
939                                 clear_highpage(page);
940                                 unlock_page(page);
941                                 /*
942                                  * Partial thp truncate due 'start' in middle
943                                  * of THP: don't need to look on these pages
944                                  * again on !pvec.nr restart.
945                                  */
946                                 if (index != round_down(end, HPAGE_PMD_NR))
947                                         start++;
948                                 continue;
949                         } else if (PageTransHuge(page)) {
950                                 if (index == round_down(end, HPAGE_PMD_NR)) {
951                                         /*
952                                          * Range ends in the middle of THP:
953                                          * zero out the page
954                                          */
955                                         clear_highpage(page);
956                                         unlock_page(page);
957                                         continue;
958                                 }
959                                 index += HPAGE_PMD_NR - 1;
960                                 i += HPAGE_PMD_NR - 1;
961                         }
962
963                         if (!unfalloc || !PageUptodate(page)) {
964                                 VM_BUG_ON_PAGE(PageTail(page), page);
965                                 if (page_mapping(page) == mapping) {
966                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
967                                         truncate_inode_page(mapping, page);
968                                 } else {
969                                         /* Page was replaced by swap: retry */
970                                         unlock_page(page);
971                                         index--;
972                                         break;
973                                 }
974                         }
975                         unlock_page(page);
976                 }
977                 pagevec_remove_exceptionals(&pvec);
978                 pagevec_release(&pvec);
979                 index++;
980         }
981
982         spin_lock_irq(&info->lock);
983         info->swapped -= nr_swaps_freed;
984         shmem_recalc_inode(inode);
985         spin_unlock_irq(&info->lock);
986 }
987
988 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
989 {
990         shmem_undo_range(inode, lstart, lend, false);
991         inode->i_ctime = inode->i_mtime = current_time(inode);
992 }
993 EXPORT_SYMBOL_GPL(shmem_truncate_range);
994
995 static int shmem_getattr(const struct path *path, struct kstat *stat,
996                          u32 request_mask, unsigned int query_flags)
997 {
998         struct inode *inode = path->dentry->d_inode;
999         struct shmem_inode_info *info = SHMEM_I(inode);
1000         struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
1001
1002         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1003                 spin_lock_irq(&info->lock);
1004                 shmem_recalc_inode(inode);
1005                 spin_unlock_irq(&info->lock);
1006         }
1007         generic_fillattr(inode, stat);
1008
1009         if (is_huge_enabled(sb_info))
1010                 stat->blksize = HPAGE_PMD_SIZE;
1011
1012         return 0;
1013 }
1014
1015 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1016 {
1017         struct inode *inode = d_inode(dentry);
1018         struct shmem_inode_info *info = SHMEM_I(inode);
1019         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1020         int error;
1021
1022         error = setattr_prepare(dentry, attr);
1023         if (error)
1024                 return error;
1025
1026         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1027                 loff_t oldsize = inode->i_size;
1028                 loff_t newsize = attr->ia_size;
1029
1030                 /* protected by i_mutex */
1031                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1032                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1033                         return -EPERM;
1034
1035                 if (newsize != oldsize) {
1036                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1037                                         oldsize, newsize);
1038                         if (error)
1039                                 return error;
1040                         i_size_write(inode, newsize);
1041                         inode->i_ctime = inode->i_mtime = current_time(inode);
1042                 }
1043                 if (newsize <= oldsize) {
1044                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1045                         if (oldsize > holebegin)
1046                                 unmap_mapping_range(inode->i_mapping,
1047                                                         holebegin, 0, 1);
1048                         if (info->alloced)
1049                                 shmem_truncate_range(inode,
1050                                                         newsize, (loff_t)-1);
1051                         /* unmap again to remove racily COWed private pages */
1052                         if (oldsize > holebegin)
1053                                 unmap_mapping_range(inode->i_mapping,
1054                                                         holebegin, 0, 1);
1055
1056                         /*
1057                          * Part of the huge page can be beyond i_size: subject
1058                          * to shrink under memory pressure.
1059                          */
1060                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1061                                 spin_lock(&sbinfo->shrinklist_lock);
1062                                 /*
1063                                  * _careful to defend against unlocked access to
1064                                  * ->shrink_list in shmem_unused_huge_shrink()
1065                                  */
1066                                 if (list_empty_careful(&info->shrinklist)) {
1067                                         list_add_tail(&info->shrinklist,
1068                                                         &sbinfo->shrinklist);
1069                                         sbinfo->shrinklist_len++;
1070                                 }
1071                                 spin_unlock(&sbinfo->shrinklist_lock);
1072                         }
1073                 }
1074         }
1075
1076         setattr_copy(inode, attr);
1077         if (attr->ia_valid & ATTR_MODE)
1078                 error = posix_acl_chmod(inode, inode->i_mode);
1079         return error;
1080 }
1081
1082 static void shmem_evict_inode(struct inode *inode)
1083 {
1084         struct shmem_inode_info *info = SHMEM_I(inode);
1085         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1086
1087         if (inode->i_mapping->a_ops == &shmem_aops) {
1088                 shmem_unacct_size(info->flags, inode->i_size);
1089                 inode->i_size = 0;
1090                 shmem_truncate_range(inode, 0, (loff_t)-1);
1091                 if (!list_empty(&info->shrinklist)) {
1092                         spin_lock(&sbinfo->shrinklist_lock);
1093                         if (!list_empty(&info->shrinklist)) {
1094                                 list_del_init(&info->shrinklist);
1095                                 sbinfo->shrinklist_len--;
1096                         }
1097                         spin_unlock(&sbinfo->shrinklist_lock);
1098                 }
1099                 if (!list_empty(&info->swaplist)) {
1100                         mutex_lock(&shmem_swaplist_mutex);
1101                         list_del_init(&info->swaplist);
1102                         mutex_unlock(&shmem_swaplist_mutex);
1103                 }
1104         }
1105
1106         simple_xattrs_free(&info->xattrs);
1107         WARN_ON(inode->i_blocks);
1108         shmem_free_inode(inode->i_sb);
1109         clear_inode(inode);
1110 }
1111
1112 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
1113 {
1114         struct radix_tree_iter iter;
1115         void __rcu **slot;
1116         unsigned long found = -1;
1117         unsigned int checked = 0;
1118
1119         rcu_read_lock();
1120         radix_tree_for_each_slot(slot, root, &iter, 0) {
1121                 void *entry = radix_tree_deref_slot(slot);
1122
1123                 if (radix_tree_deref_retry(entry)) {
1124                         slot = radix_tree_iter_retry(&iter);
1125                         continue;
1126                 }
1127                 if (entry == item) {
1128                         found = iter.index;
1129                         break;
1130                 }
1131                 checked++;
1132                 if ((checked % 4096) != 0)
1133                         continue;
1134                 slot = radix_tree_iter_resume(slot, &iter);
1135                 cond_resched_rcu();
1136         }
1137
1138         rcu_read_unlock();
1139         return found;
1140 }
1141
1142 /*
1143  * If swap found in inode, free it and move page from swapcache to filecache.
1144  */
1145 static int shmem_unuse_inode(struct shmem_inode_info *info,
1146                              swp_entry_t swap, struct page **pagep)
1147 {
1148         struct address_space *mapping = info->vfs_inode.i_mapping;
1149         void *radswap;
1150         pgoff_t index;
1151         gfp_t gfp;
1152         int error = 0;
1153
1154         radswap = swp_to_radix_entry(swap);
1155         index = find_swap_entry(&mapping->i_pages, radswap);
1156         if (index == -1)
1157                 return -EAGAIN; /* tell shmem_unuse we found nothing */
1158
1159         /*
1160          * Move _head_ to start search for next from here.
1161          * But be careful: shmem_evict_inode checks list_empty without taking
1162          * mutex, and there's an instant in list_move_tail when info->swaplist
1163          * would appear empty, if it were the only one on shmem_swaplist.
1164          */
1165         if (shmem_swaplist.next != &info->swaplist)
1166                 list_move_tail(&shmem_swaplist, &info->swaplist);
1167
1168         gfp = mapping_gfp_mask(mapping);
1169         if (shmem_should_replace_page(*pagep, gfp)) {
1170                 mutex_unlock(&shmem_swaplist_mutex);
1171                 error = shmem_replace_page(pagep, gfp, info, index);
1172                 mutex_lock(&shmem_swaplist_mutex);
1173                 /*
1174                  * We needed to drop mutex to make that restrictive page
1175                  * allocation, but the inode might have been freed while we
1176                  * dropped it: although a racing shmem_evict_inode() cannot
1177                  * complete without emptying the radix_tree, our page lock
1178                  * on this swapcache page is not enough to prevent that -
1179                  * free_swap_and_cache() of our swap entry will only
1180                  * trylock_page(), removing swap from radix_tree whatever.
1181                  *
1182                  * We must not proceed to shmem_add_to_page_cache() if the
1183                  * inode has been freed, but of course we cannot rely on
1184                  * inode or mapping or info to check that.  However, we can
1185                  * safely check if our swap entry is still in use (and here
1186                  * it can't have got reused for another page): if it's still
1187                  * in use, then the inode cannot have been freed yet, and we
1188                  * can safely proceed (if it's no longer in use, that tells
1189                  * nothing about the inode, but we don't need to unuse swap).
1190                  */
1191                 if (!page_swapcount(*pagep))
1192                         error = -ENOENT;
1193         }
1194
1195         /*
1196          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
1197          * but also to hold up shmem_evict_inode(): so inode cannot be freed
1198          * beneath us (pagelock doesn't help until the page is in pagecache).
1199          */
1200         if (!error)
1201                 error = shmem_add_to_page_cache(*pagep, mapping, index,
1202                                                 radswap);
1203         if (error != -ENOMEM) {
1204                 /*
1205                  * Truncation and eviction use free_swap_and_cache(), which
1206                  * only does trylock page: if we raced, best clean up here.
1207                  */
1208                 delete_from_swap_cache(*pagep);
1209                 set_page_dirty(*pagep);
1210                 if (!error) {
1211                         spin_lock_irq(&info->lock);
1212                         info->swapped--;
1213                         spin_unlock_irq(&info->lock);
1214                         swap_free(swap);
1215                 }
1216         }
1217         return error;
1218 }
1219
1220 /*
1221  * Search through swapped inodes to find and replace swap by page.
1222  */
1223 int shmem_unuse(swp_entry_t swap, struct page *page)
1224 {
1225         struct list_head *this, *next;
1226         struct shmem_inode_info *info;
1227         struct mem_cgroup *memcg;
1228         int error = 0;
1229
1230         /*
1231          * There's a faint possibility that swap page was replaced before
1232          * caller locked it: caller will come back later with the right page.
1233          */
1234         if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
1235                 goto out;
1236
1237         /*
1238          * Charge page using GFP_KERNEL while we can wait, before taking
1239          * the shmem_swaplist_mutex which might hold up shmem_writepage().
1240          * Charged back to the user (not to caller) when swap account is used.
1241          */
1242         error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg,
1243                         false);
1244         if (error)
1245                 goto out;
1246         /* No radix_tree_preload: swap entry keeps a place for page in tree */
1247         error = -EAGAIN;
1248
1249         mutex_lock(&shmem_swaplist_mutex);
1250         list_for_each_safe(this, next, &shmem_swaplist) {
1251                 info = list_entry(this, struct shmem_inode_info, swaplist);
1252                 if (info->swapped)
1253                         error = shmem_unuse_inode(info, swap, &page);
1254                 else
1255                         list_del_init(&info->swaplist);
1256                 cond_resched();
1257                 if (error != -EAGAIN)
1258                         break;
1259                 /* found nothing in this: move on to search the next */
1260         }
1261         mutex_unlock(&shmem_swaplist_mutex);
1262
1263         if (error) {
1264                 if (error != -ENOMEM)
1265                         error = 0;
1266                 mem_cgroup_cancel_charge(page, memcg, false);
1267         } else
1268                 mem_cgroup_commit_charge(page, memcg, true, false);
1269 out:
1270         unlock_page(page);
1271         put_page(page);
1272         return error;
1273 }
1274
1275 /*
1276  * Move the page from the page cache to the swap cache.
1277  */
1278 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1279 {
1280         struct shmem_inode_info *info;
1281         struct address_space *mapping;
1282         struct inode *inode;
1283         swp_entry_t swap;
1284         pgoff_t index;
1285
1286         VM_BUG_ON_PAGE(PageCompound(page), page);
1287         BUG_ON(!PageLocked(page));
1288         mapping = page->mapping;
1289         index = page->index;
1290         inode = mapping->host;
1291         info = SHMEM_I(inode);
1292         if (info->flags & VM_LOCKED)
1293                 goto redirty;
1294         if (!total_swap_pages)
1295                 goto redirty;
1296
1297         /*
1298          * Our capabilities prevent regular writeback or sync from ever calling
1299          * shmem_writepage; but a stacking filesystem might use ->writepage of
1300          * its underlying filesystem, in which case tmpfs should write out to
1301          * swap only in response to memory pressure, and not for the writeback
1302          * threads or sync.
1303          */
1304         if (!wbc->for_reclaim) {
1305                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1306                 goto redirty;
1307         }
1308
1309         /*
1310          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1311          * value into swapfile.c, the only way we can correctly account for a
1312          * fallocated page arriving here is now to initialize it and write it.
1313          *
1314          * That's okay for a page already fallocated earlier, but if we have
1315          * not yet completed the fallocation, then (a) we want to keep track
1316          * of this page in case we have to undo it, and (b) it may not be a
1317          * good idea to continue anyway, once we're pushing into swap.  So
1318          * reactivate the page, and let shmem_fallocate() quit when too many.
1319          */
1320         if (!PageUptodate(page)) {
1321                 if (inode->i_private) {
1322                         struct shmem_falloc *shmem_falloc;
1323                         spin_lock(&inode->i_lock);
1324                         shmem_falloc = inode->i_private;
1325                         if (shmem_falloc &&
1326                             !shmem_falloc->waitq &&
1327                             index >= shmem_falloc->start &&
1328                             index < shmem_falloc->next)
1329                                 shmem_falloc->nr_unswapped++;
1330                         else
1331                                 shmem_falloc = NULL;
1332                         spin_unlock(&inode->i_lock);
1333                         if (shmem_falloc)
1334                                 goto redirty;
1335                 }
1336                 clear_highpage(page);
1337                 flush_dcache_page(page);
1338                 SetPageUptodate(page);
1339         }
1340
1341         swap = get_swap_page(page);
1342         if (!swap.val)
1343                 goto redirty;
1344
1345         /*
1346          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1347          * if it's not already there.  Do it now before the page is
1348          * moved to swap cache, when its pagelock no longer protects
1349          * the inode from eviction.  But don't unlock the mutex until
1350          * we've incremented swapped, because shmem_unuse_inode() will
1351          * prune a !swapped inode from the swaplist under this mutex.
1352          */
1353         mutex_lock(&shmem_swaplist_mutex);
1354         if (list_empty(&info->swaplist))
1355                 list_add_tail(&info->swaplist, &shmem_swaplist);
1356
1357         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1358                 spin_lock_irq(&info->lock);
1359                 shmem_recalc_inode(inode);
1360                 info->swapped++;
1361                 spin_unlock_irq(&info->lock);
1362
1363                 swap_shmem_alloc(swap);
1364                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1365
1366                 mutex_unlock(&shmem_swaplist_mutex);
1367                 BUG_ON(page_mapped(page));
1368                 swap_writepage(page, wbc);
1369                 return 0;
1370         }
1371
1372         mutex_unlock(&shmem_swaplist_mutex);
1373         put_swap_page(page, swap);
1374 redirty:
1375         set_page_dirty(page);
1376         if (wbc->for_reclaim)
1377                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1378         unlock_page(page);
1379         return 0;
1380 }
1381
1382 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1383 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1384 {
1385         char buffer[64];
1386
1387         if (!mpol || mpol->mode == MPOL_DEFAULT)
1388                 return;         /* show nothing */
1389
1390         mpol_to_str(buffer, sizeof(buffer), mpol);
1391
1392         seq_printf(seq, ",mpol=%s", buffer);
1393 }
1394
1395 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1396 {
1397         struct mempolicy *mpol = NULL;
1398         if (sbinfo->mpol) {
1399                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1400                 mpol = sbinfo->mpol;
1401                 mpol_get(mpol);
1402                 spin_unlock(&sbinfo->stat_lock);
1403         }
1404         return mpol;
1405 }
1406 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1407 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1408 {
1409 }
1410 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1411 {
1412         return NULL;
1413 }
1414 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1415 #ifndef CONFIG_NUMA
1416 #define vm_policy vm_private_data
1417 #endif
1418
1419 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1420                 struct shmem_inode_info *info, pgoff_t index)
1421 {
1422         /* Create a pseudo vma that just contains the policy */
1423         memset(vma, 0, sizeof(*vma));
1424         /* Bias interleave by inode number to distribute better across nodes */
1425         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1426         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1427 }
1428
1429 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1430 {
1431         /* Drop reference taken by mpol_shared_policy_lookup() */
1432         mpol_cond_put(vma->vm_policy);
1433 }
1434
1435 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1436                         struct shmem_inode_info *info, pgoff_t index)
1437 {
1438         struct vm_area_struct pvma;
1439         struct page *page;
1440         struct vm_fault vmf;
1441
1442         shmem_pseudo_vma_init(&pvma, info, index);
1443         vmf.vma = &pvma;
1444         vmf.address = 0;
1445         page = swap_cluster_readahead(swap, gfp, &vmf);
1446         shmem_pseudo_vma_destroy(&pvma);
1447
1448         return page;
1449 }
1450
1451 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1452                 struct shmem_inode_info *info, pgoff_t index)
1453 {
1454         struct vm_area_struct pvma;
1455         struct inode *inode = &info->vfs_inode;
1456         struct address_space *mapping = inode->i_mapping;
1457         pgoff_t idx, hindex;
1458         void __rcu **results;
1459         struct page *page;
1460
1461         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1462                 return NULL;
1463
1464         hindex = round_down(index, HPAGE_PMD_NR);
1465         rcu_read_lock();
1466         if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1467                                 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1468                 rcu_read_unlock();
1469                 return NULL;
1470         }
1471         rcu_read_unlock();
1472
1473         shmem_pseudo_vma_init(&pvma, info, hindex);
1474         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1475                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1476         shmem_pseudo_vma_destroy(&pvma);
1477         if (page)
1478                 prep_transhuge_page(page);
1479         return page;
1480 }
1481
1482 static struct page *shmem_alloc_page(gfp_t gfp,
1483                         struct shmem_inode_info *info, pgoff_t index)
1484 {
1485         struct vm_area_struct pvma;
1486         struct page *page;
1487
1488         shmem_pseudo_vma_init(&pvma, info, index);
1489         page = alloc_page_vma(gfp, &pvma, 0);
1490         shmem_pseudo_vma_destroy(&pvma);
1491
1492         return page;
1493 }
1494
1495 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1496                 struct inode *inode,
1497                 pgoff_t index, bool huge)
1498 {
1499         struct shmem_inode_info *info = SHMEM_I(inode);
1500         struct page *page;
1501         int nr;
1502         int err = -ENOSPC;
1503
1504         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1505                 huge = false;
1506         nr = huge ? HPAGE_PMD_NR : 1;
1507
1508         if (!shmem_inode_acct_block(inode, nr))
1509                 goto failed;
1510
1511         if (huge)
1512                 page = shmem_alloc_hugepage(gfp, info, index);
1513         else
1514                 page = shmem_alloc_page(gfp, info, index);
1515         if (page) {
1516                 __SetPageLocked(page);
1517                 __SetPageSwapBacked(page);
1518                 return page;
1519         }
1520
1521         err = -ENOMEM;
1522         shmem_inode_unacct_blocks(inode, nr);
1523 failed:
1524         return ERR_PTR(err);
1525 }
1526
1527 /*
1528  * When a page is moved from swapcache to shmem filecache (either by the
1529  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1530  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1531  * ignorance of the mapping it belongs to.  If that mapping has special
1532  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1533  * we may need to copy to a suitable page before moving to filecache.
1534  *
1535  * In a future release, this may well be extended to respect cpuset and
1536  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1537  * but for now it is a simple matter of zone.
1538  */
1539 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1540 {
1541         return page_zonenum(page) > gfp_zone(gfp);
1542 }
1543
1544 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1545                                 struct shmem_inode_info *info, pgoff_t index)
1546 {
1547         struct page *oldpage, *newpage;
1548         struct address_space *swap_mapping;
1549         pgoff_t swap_index;
1550         int error;
1551
1552         oldpage = *pagep;
1553         swap_index = page_private(oldpage);
1554         swap_mapping = page_mapping(oldpage);
1555
1556         /*
1557          * We have arrived here because our zones are constrained, so don't
1558          * limit chance of success by further cpuset and node constraints.
1559          */
1560         gfp &= ~GFP_CONSTRAINT_MASK;
1561         newpage = shmem_alloc_page(gfp, info, index);
1562         if (!newpage)
1563                 return -ENOMEM;
1564
1565         get_page(newpage);
1566         copy_highpage(newpage, oldpage);
1567         flush_dcache_page(newpage);
1568
1569         __SetPageLocked(newpage);
1570         __SetPageSwapBacked(newpage);
1571         SetPageUptodate(newpage);
1572         set_page_private(newpage, swap_index);
1573         SetPageSwapCache(newpage);
1574
1575         /*
1576          * Our caller will very soon move newpage out of swapcache, but it's
1577          * a nice clean interface for us to replace oldpage by newpage there.
1578          */
1579         xa_lock_irq(&swap_mapping->i_pages);
1580         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1581                                                                    newpage);
1582         if (!error) {
1583                 __inc_node_page_state(newpage, NR_FILE_PAGES);
1584                 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1585         }
1586         xa_unlock_irq(&swap_mapping->i_pages);
1587
1588         if (unlikely(error)) {
1589                 /*
1590                  * Is this possible?  I think not, now that our callers check
1591                  * both PageSwapCache and page_private after getting page lock;
1592                  * but be defensive.  Reverse old to newpage for clear and free.
1593                  */
1594                 oldpage = newpage;
1595         } else {
1596                 mem_cgroup_migrate(oldpage, newpage);
1597                 lru_cache_add_anon(newpage);
1598                 *pagep = newpage;
1599         }
1600
1601         ClearPageSwapCache(oldpage);
1602         set_page_private(oldpage, 0);
1603
1604         unlock_page(oldpage);
1605         put_page(oldpage);
1606         put_page(oldpage);
1607         return error;
1608 }
1609
1610 /*
1611  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1612  *
1613  * If we allocate a new one we do not mark it dirty. That's up to the
1614  * vm. If we swap it in we mark it dirty since we also free the swap
1615  * entry since a page cannot live in both the swap and page cache.
1616  *
1617  * fault_mm and fault_type are only supplied by shmem_fault:
1618  * otherwise they are NULL.
1619  */
1620 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1621         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1622         struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type)
1623 {
1624         struct address_space *mapping = inode->i_mapping;
1625         struct shmem_inode_info *info = SHMEM_I(inode);
1626         struct shmem_sb_info *sbinfo;
1627         struct mm_struct *charge_mm;
1628         struct mem_cgroup *memcg;
1629         struct page *page;
1630         swp_entry_t swap;
1631         enum sgp_type sgp_huge = sgp;
1632         pgoff_t hindex = index;
1633         int error;
1634         int once = 0;
1635         int alloced = 0;
1636
1637         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1638                 return -EFBIG;
1639         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1640                 sgp = SGP_CACHE;
1641 repeat:
1642         swap.val = 0;
1643         page = find_lock_entry(mapping, index);
1644         if (radix_tree_exceptional_entry(page)) {
1645                 swap = radix_to_swp_entry(page);
1646                 page = NULL;
1647         }
1648
1649         if (sgp <= SGP_CACHE &&
1650             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1651                 error = -EINVAL;
1652                 goto unlock;
1653         }
1654
1655         if (page && sgp == SGP_WRITE)
1656                 mark_page_accessed(page);
1657
1658         /* fallocated page? */
1659         if (page && !PageUptodate(page)) {
1660                 if (sgp != SGP_READ)
1661                         goto clear;
1662                 unlock_page(page);
1663                 put_page(page);
1664                 page = NULL;
1665         }
1666         if (page || (sgp == SGP_READ && !swap.val)) {
1667                 *pagep = page;
1668                 return 0;
1669         }
1670
1671         /*
1672          * Fast cache lookup did not find it:
1673          * bring it back from swap or allocate.
1674          */
1675         sbinfo = SHMEM_SB(inode->i_sb);
1676         charge_mm = vma ? vma->vm_mm : current->mm;
1677
1678         if (swap.val) {
1679                 /* Look it up and read it in.. */
1680                 page = lookup_swap_cache(swap, NULL, 0);
1681                 if (!page) {
1682                         /* Or update major stats only when swapin succeeds?? */
1683                         if (fault_type) {
1684                                 *fault_type |= VM_FAULT_MAJOR;
1685                                 count_vm_event(PGMAJFAULT);
1686                                 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1687                         }
1688                         /* Here we actually start the io */
1689                         page = shmem_swapin(swap, gfp, info, index);
1690                         if (!page) {
1691                                 error = -ENOMEM;
1692                                 goto failed;
1693                         }
1694                 }
1695
1696                 /* We have to do this with page locked to prevent races */
1697                 lock_page(page);
1698                 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1699                     !shmem_confirm_swap(mapping, index, swap)) {
1700                         error = -EEXIST;        /* try again */
1701                         goto unlock;
1702                 }
1703                 if (!PageUptodate(page)) {
1704                         error = -EIO;
1705                         goto failed;
1706                 }
1707                 wait_on_page_writeback(page);
1708
1709                 if (shmem_should_replace_page(page, gfp)) {
1710                         error = shmem_replace_page(&page, gfp, info, index);
1711                         if (error)
1712                                 goto failed;
1713                 }
1714
1715                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1716                                 false);
1717                 if (!error) {
1718                         error = shmem_add_to_page_cache(page, mapping, index,
1719                                                 swp_to_radix_entry(swap));
1720                         /*
1721                          * We already confirmed swap under page lock, and make
1722                          * no memory allocation here, so usually no possibility
1723                          * of error; but free_swap_and_cache() only trylocks a
1724                          * page, so it is just possible that the entry has been
1725                          * truncated or holepunched since swap was confirmed.
1726                          * shmem_undo_range() will have done some of the
1727                          * unaccounting, now delete_from_swap_cache() will do
1728                          * the rest.
1729                          * Reset swap.val? No, leave it so "failed" goes back to
1730                          * "repeat": reading a hole and writing should succeed.
1731                          */
1732                         if (error) {
1733                                 mem_cgroup_cancel_charge(page, memcg, false);
1734                                 delete_from_swap_cache(page);
1735                         }
1736                 }
1737                 if (error)
1738                         goto failed;
1739
1740                 mem_cgroup_commit_charge(page, memcg, true, false);
1741
1742                 spin_lock_irq(&info->lock);
1743                 info->swapped--;
1744                 shmem_recalc_inode(inode);
1745                 spin_unlock_irq(&info->lock);
1746
1747                 if (sgp == SGP_WRITE)
1748                         mark_page_accessed(page);
1749
1750                 delete_from_swap_cache(page);
1751                 set_page_dirty(page);
1752                 swap_free(swap);
1753
1754         } else {
1755                 if (vma && userfaultfd_missing(vma)) {
1756                         *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1757                         return 0;
1758                 }
1759
1760                 /* shmem_symlink() */
1761                 if (mapping->a_ops != &shmem_aops)
1762                         goto alloc_nohuge;
1763                 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1764                         goto alloc_nohuge;
1765                 if (shmem_huge == SHMEM_HUGE_FORCE)
1766                         goto alloc_huge;
1767                 switch (sbinfo->huge) {
1768                         loff_t i_size;
1769                         pgoff_t off;
1770                 case SHMEM_HUGE_NEVER:
1771                         goto alloc_nohuge;
1772                 case SHMEM_HUGE_WITHIN_SIZE:
1773                         off = round_up(index, HPAGE_PMD_NR);
1774                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
1775                         if (i_size >= HPAGE_PMD_SIZE &&
1776                                         i_size >> PAGE_SHIFT >= off)
1777                                 goto alloc_huge;
1778                         /* fallthrough */
1779                 case SHMEM_HUGE_ADVISE:
1780                         if (sgp_huge == SGP_HUGE)
1781                                 goto alloc_huge;
1782                         /* TODO: implement fadvise() hints */
1783                         goto alloc_nohuge;
1784                 }
1785
1786 alloc_huge:
1787                 page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1788                 if (IS_ERR(page)) {
1789 alloc_nohuge:           page = shmem_alloc_and_acct_page(gfp, inode,
1790                                         index, false);
1791                 }
1792                 if (IS_ERR(page)) {
1793                         int retry = 5;
1794                         error = PTR_ERR(page);
1795                         page = NULL;
1796                         if (error != -ENOSPC)
1797                                 goto failed;
1798                         /*
1799                          * Try to reclaim some spece by splitting a huge page
1800                          * beyond i_size on the filesystem.
1801                          */
1802                         while (retry--) {
1803                                 int ret;
1804                                 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1805                                 if (ret == SHRINK_STOP)
1806                                         break;
1807                                 if (ret)
1808                                         goto alloc_nohuge;
1809                         }
1810                         goto failed;
1811                 }
1812
1813                 if (PageTransHuge(page))
1814                         hindex = round_down(index, HPAGE_PMD_NR);
1815                 else
1816                         hindex = index;
1817
1818                 if (sgp == SGP_WRITE)
1819                         __SetPageReferenced(page);
1820
1821                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1822                                 PageTransHuge(page));
1823                 if (error)
1824                         goto unacct;
1825                 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK,
1826                                 compound_order(page));
1827                 if (!error) {
1828                         error = shmem_add_to_page_cache(page, mapping, hindex,
1829                                                         NULL);
1830                         radix_tree_preload_end();
1831                 }
1832                 if (error) {
1833                         mem_cgroup_cancel_charge(page, memcg,
1834                                         PageTransHuge(page));
1835                         goto unacct;
1836                 }
1837                 mem_cgroup_commit_charge(page, memcg, false,
1838                                 PageTransHuge(page));
1839                 lru_cache_add_anon(page);
1840
1841                 spin_lock_irq(&info->lock);
1842                 info->alloced += 1 << compound_order(page);
1843                 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1844                 shmem_recalc_inode(inode);
1845                 spin_unlock_irq(&info->lock);
1846                 alloced = true;
1847
1848                 if (PageTransHuge(page) &&
1849                                 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1850                                 hindex + HPAGE_PMD_NR - 1) {
1851                         /*
1852                          * Part of the huge page is beyond i_size: subject
1853                          * to shrink under memory pressure.
1854                          */
1855                         spin_lock(&sbinfo->shrinklist_lock);
1856                         /*
1857                          * _careful to defend against unlocked access to
1858                          * ->shrink_list in shmem_unused_huge_shrink()
1859                          */
1860                         if (list_empty_careful(&info->shrinklist)) {
1861                                 list_add_tail(&info->shrinklist,
1862                                                 &sbinfo->shrinklist);
1863                                 sbinfo->shrinklist_len++;
1864                         }
1865                         spin_unlock(&sbinfo->shrinklist_lock);
1866                 }
1867
1868                 /*
1869                  * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1870                  */
1871                 if (sgp == SGP_FALLOC)
1872                         sgp = SGP_WRITE;
1873 clear:
1874                 /*
1875                  * Let SGP_WRITE caller clear ends if write does not fill page;
1876                  * but SGP_FALLOC on a page fallocated earlier must initialize
1877                  * it now, lest undo on failure cancel our earlier guarantee.
1878                  */
1879                 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1880                         struct page *head = compound_head(page);
1881                         int i;
1882
1883                         for (i = 0; i < (1 << compound_order(head)); i++) {
1884                                 clear_highpage(head + i);
1885                                 flush_dcache_page(head + i);
1886                         }
1887                         SetPageUptodate(head);
1888                 }
1889         }
1890
1891         /* Perhaps the file has been truncated since we checked */
1892         if (sgp <= SGP_CACHE &&
1893             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1894                 if (alloced) {
1895                         ClearPageDirty(page);
1896                         delete_from_page_cache(page);
1897                         spin_lock_irq(&info->lock);
1898                         shmem_recalc_inode(inode);
1899                         spin_unlock_irq(&info->lock);
1900                 }
1901                 error = -EINVAL;
1902                 goto unlock;
1903         }
1904         *pagep = page + index - hindex;
1905         return 0;
1906
1907         /*
1908          * Error recovery.
1909          */
1910 unacct:
1911         shmem_inode_unacct_blocks(inode, 1 << compound_order(page));
1912
1913         if (PageTransHuge(page)) {
1914                 unlock_page(page);
1915                 put_page(page);
1916                 goto alloc_nohuge;
1917         }
1918 failed:
1919         if (swap.val && !shmem_confirm_swap(mapping, index, swap))
1920                 error = -EEXIST;
1921 unlock:
1922         if (page) {
1923                 unlock_page(page);
1924                 put_page(page);
1925         }
1926         if (error == -ENOSPC && !once++) {
1927                 spin_lock_irq(&info->lock);
1928                 shmem_recalc_inode(inode);
1929                 spin_unlock_irq(&info->lock);
1930                 goto repeat;
1931         }
1932         if (error == -EEXIST)   /* from above or from radix_tree_insert */
1933                 goto repeat;
1934         return error;
1935 }
1936
1937 /*
1938  * This is like autoremove_wake_function, but it removes the wait queue
1939  * entry unconditionally - even if something else had already woken the
1940  * target.
1941  */
1942 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1943 {
1944         int ret = default_wake_function(wait, mode, sync, key);
1945         list_del_init(&wait->entry);
1946         return ret;
1947 }
1948
1949 static vm_fault_t shmem_fault(struct vm_fault *vmf)
1950 {
1951         struct vm_area_struct *vma = vmf->vma;
1952         struct inode *inode = file_inode(vma->vm_file);
1953         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1954         enum sgp_type sgp;
1955         int err;
1956         vm_fault_t ret = VM_FAULT_LOCKED;
1957
1958         /*
1959          * Trinity finds that probing a hole which tmpfs is punching can
1960          * prevent the hole-punch from ever completing: which in turn
1961          * locks writers out with its hold on i_mutex.  So refrain from
1962          * faulting pages into the hole while it's being punched.  Although
1963          * shmem_undo_range() does remove the additions, it may be unable to
1964          * keep up, as each new page needs its own unmap_mapping_range() call,
1965          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1966          *
1967          * It does not matter if we sometimes reach this check just before the
1968          * hole-punch begins, so that one fault then races with the punch:
1969          * we just need to make racing faults a rare case.
1970          *
1971          * The implementation below would be much simpler if we just used a
1972          * standard mutex or completion: but we cannot take i_mutex in fault,
1973          * and bloating every shmem inode for this unlikely case would be sad.
1974          */
1975         if (unlikely(inode->i_private)) {
1976                 struct shmem_falloc *shmem_falloc;
1977
1978                 spin_lock(&inode->i_lock);
1979                 shmem_falloc = inode->i_private;
1980                 if (shmem_falloc &&
1981                     shmem_falloc->waitq &&
1982                     vmf->pgoff >= shmem_falloc->start &&
1983                     vmf->pgoff < shmem_falloc->next) {
1984                         wait_queue_head_t *shmem_falloc_waitq;
1985                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
1986
1987                         ret = VM_FAULT_NOPAGE;
1988                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1989                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1990                                 /* It's polite to up mmap_sem if we can */
1991                                 up_read(&vma->vm_mm->mmap_sem);
1992                                 ret = VM_FAULT_RETRY;
1993                         }
1994
1995                         shmem_falloc_waitq = shmem_falloc->waitq;
1996                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1997                                         TASK_UNINTERRUPTIBLE);
1998                         spin_unlock(&inode->i_lock);
1999                         schedule();
2000
2001                         /*
2002                          * shmem_falloc_waitq points into the shmem_fallocate()
2003                          * stack of the hole-punching task: shmem_falloc_waitq
2004                          * is usually invalid by the time we reach here, but
2005                          * finish_wait() does not dereference it in that case;
2006                          * though i_lock needed lest racing with wake_up_all().
2007                          */
2008                         spin_lock(&inode->i_lock);
2009                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2010                         spin_unlock(&inode->i_lock);
2011                         return ret;
2012                 }
2013                 spin_unlock(&inode->i_lock);
2014         }
2015
2016         sgp = SGP_CACHE;
2017
2018         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2019             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2020                 sgp = SGP_NOHUGE;
2021         else if (vma->vm_flags & VM_HUGEPAGE)
2022                 sgp = SGP_HUGE;
2023
2024         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2025                                   gfp, vma, vmf, &ret);
2026         if (err)
2027                 return vmf_error(err);
2028         return ret;
2029 }
2030
2031 unsigned long shmem_get_unmapped_area(struct file *file,
2032                                       unsigned long uaddr, unsigned long len,
2033                                       unsigned long pgoff, unsigned long flags)
2034 {
2035         unsigned long (*get_area)(struct file *,
2036                 unsigned long, unsigned long, unsigned long, unsigned long);
2037         unsigned long addr;
2038         unsigned long offset;
2039         unsigned long inflated_len;
2040         unsigned long inflated_addr;
2041         unsigned long inflated_offset;
2042
2043         if (len > TASK_SIZE)
2044                 return -ENOMEM;
2045
2046         get_area = current->mm->get_unmapped_area;
2047         addr = get_area(file, uaddr, len, pgoff, flags);
2048
2049         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2050                 return addr;
2051         if (IS_ERR_VALUE(addr))
2052                 return addr;
2053         if (addr & ~PAGE_MASK)
2054                 return addr;
2055         if (addr > TASK_SIZE - len)
2056                 return addr;
2057
2058         if (shmem_huge == SHMEM_HUGE_DENY)
2059                 return addr;
2060         if (len < HPAGE_PMD_SIZE)
2061                 return addr;
2062         if (flags & MAP_FIXED)
2063                 return addr;
2064         /*
2065          * Our priority is to support MAP_SHARED mapped hugely;
2066          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2067          * But if caller specified an address hint, respect that as before.
2068          */
2069         if (uaddr)
2070                 return addr;
2071
2072         if (shmem_huge != SHMEM_HUGE_FORCE) {
2073                 struct super_block *sb;
2074
2075                 if (file) {
2076                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2077                         sb = file_inode(file)->i_sb;
2078                 } else {
2079                         /*
2080                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2081                          * for "/dev/zero", to create a shared anonymous object.
2082                          */
2083                         if (IS_ERR(shm_mnt))
2084                                 return addr;
2085                         sb = shm_mnt->mnt_sb;
2086                 }
2087                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2088                         return addr;
2089         }
2090
2091         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2092         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2093                 return addr;
2094         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2095                 return addr;
2096
2097         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2098         if (inflated_len > TASK_SIZE)
2099                 return addr;
2100         if (inflated_len < len)
2101                 return addr;
2102
2103         inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2104         if (IS_ERR_VALUE(inflated_addr))
2105                 return addr;
2106         if (inflated_addr & ~PAGE_MASK)
2107                 return addr;
2108
2109         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2110         inflated_addr += offset - inflated_offset;
2111         if (inflated_offset > offset)
2112                 inflated_addr += HPAGE_PMD_SIZE;
2113
2114         if (inflated_addr > TASK_SIZE - len)
2115                 return addr;
2116         return inflated_addr;
2117 }
2118
2119 #ifdef CONFIG_NUMA
2120 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2121 {
2122         struct inode *inode = file_inode(vma->vm_file);
2123         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2124 }
2125
2126 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2127                                           unsigned long addr)
2128 {
2129         struct inode *inode = file_inode(vma->vm_file);
2130         pgoff_t index;
2131
2132         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2133         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2134 }
2135 #endif
2136
2137 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2138 {
2139         struct inode *inode = file_inode(file);
2140         struct shmem_inode_info *info = SHMEM_I(inode);
2141         int retval = -ENOMEM;
2142
2143         spin_lock_irq(&info->lock);
2144         if (lock && !(info->flags & VM_LOCKED)) {
2145                 if (!user_shm_lock(inode->i_size, user))
2146                         goto out_nomem;
2147                 info->flags |= VM_LOCKED;
2148                 mapping_set_unevictable(file->f_mapping);
2149         }
2150         if (!lock && (info->flags & VM_LOCKED) && user) {
2151                 user_shm_unlock(inode->i_size, user);
2152                 info->flags &= ~VM_LOCKED;
2153                 mapping_clear_unevictable(file->f_mapping);
2154         }
2155         retval = 0;
2156
2157 out_nomem:
2158         spin_unlock_irq(&info->lock);
2159         return retval;
2160 }
2161
2162 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2163 {
2164         file_accessed(file);
2165         vma->vm_ops = &shmem_vm_ops;
2166         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2167                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2168                         (vma->vm_end & HPAGE_PMD_MASK)) {
2169                 khugepaged_enter(vma, vma->vm_flags);
2170         }
2171         return 0;
2172 }
2173
2174 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2175                                      umode_t mode, dev_t dev, unsigned long flags)
2176 {
2177         struct inode *inode;
2178         struct shmem_inode_info *info;
2179         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2180
2181         if (shmem_reserve_inode(sb))
2182                 return NULL;
2183
2184         inode = new_inode(sb);
2185         if (inode) {
2186                 inode->i_ino = get_next_ino();
2187                 inode_init_owner(inode, dir, mode);
2188                 inode->i_blocks = 0;
2189                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2190                 inode->i_generation = get_seconds();
2191                 info = SHMEM_I(inode);
2192                 memset(info, 0, (char *)inode - (char *)info);
2193                 spin_lock_init(&info->lock);
2194                 info->seals = F_SEAL_SEAL;
2195                 info->flags = flags & VM_NORESERVE;
2196                 INIT_LIST_HEAD(&info->shrinklist);
2197                 INIT_LIST_HEAD(&info->swaplist);
2198                 simple_xattrs_init(&info->xattrs);
2199                 cache_no_acl(inode);
2200
2201                 switch (mode & S_IFMT) {
2202                 default:
2203                         inode->i_op = &shmem_special_inode_operations;
2204                         init_special_inode(inode, mode, dev);
2205                         break;
2206                 case S_IFREG:
2207                         inode->i_mapping->a_ops = &shmem_aops;
2208                         inode->i_op = &shmem_inode_operations;
2209                         inode->i_fop = &shmem_file_operations;
2210                         mpol_shared_policy_init(&info->policy,
2211                                                  shmem_get_sbmpol(sbinfo));
2212                         break;
2213                 case S_IFDIR:
2214                         inc_nlink(inode);
2215                         /* Some things misbehave if size == 0 on a directory */
2216                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2217                         inode->i_op = &shmem_dir_inode_operations;
2218                         inode->i_fop = &simple_dir_operations;
2219                         break;
2220                 case S_IFLNK:
2221                         /*
2222                          * Must not load anything in the rbtree,
2223                          * mpol_free_shared_policy will not be called.
2224                          */
2225                         mpol_shared_policy_init(&info->policy, NULL);
2226                         break;
2227                 }
2228         } else
2229                 shmem_free_inode(sb);
2230         return inode;
2231 }
2232
2233 bool shmem_mapping(struct address_space *mapping)
2234 {
2235         return mapping->a_ops == &shmem_aops;
2236 }
2237
2238 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2239                                   pmd_t *dst_pmd,
2240                                   struct vm_area_struct *dst_vma,
2241                                   unsigned long dst_addr,
2242                                   unsigned long src_addr,
2243                                   bool zeropage,
2244                                   struct page **pagep)
2245 {
2246         struct inode *inode = file_inode(dst_vma->vm_file);
2247         struct shmem_inode_info *info = SHMEM_I(inode);
2248         struct address_space *mapping = inode->i_mapping;
2249         gfp_t gfp = mapping_gfp_mask(mapping);
2250         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2251         struct mem_cgroup *memcg;
2252         spinlock_t *ptl;
2253         void *page_kaddr;
2254         struct page *page;
2255         pte_t _dst_pte, *dst_pte;
2256         int ret;
2257
2258         ret = -ENOMEM;
2259         if (!shmem_inode_acct_block(inode, 1))
2260                 goto out;
2261
2262         if (!*pagep) {
2263                 page = shmem_alloc_page(gfp, info, pgoff);
2264                 if (!page)
2265                         goto out_unacct_blocks;
2266
2267                 if (!zeropage) {        /* mcopy_atomic */
2268                         page_kaddr = kmap_atomic(page);
2269                         ret = copy_from_user(page_kaddr,
2270                                              (const void __user *)src_addr,
2271                                              PAGE_SIZE);
2272                         kunmap_atomic(page_kaddr);
2273
2274                         /* fallback to copy_from_user outside mmap_sem */
2275                         if (unlikely(ret)) {
2276                                 *pagep = page;
2277                                 shmem_inode_unacct_blocks(inode, 1);
2278                                 /* don't free the page */
2279                                 return -EFAULT;
2280                         }
2281                 } else {                /* mfill_zeropage_atomic */
2282                         clear_highpage(page);
2283                 }
2284         } else {
2285                 page = *pagep;
2286                 *pagep = NULL;
2287         }
2288
2289         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2290         __SetPageLocked(page);
2291         __SetPageSwapBacked(page);
2292         __SetPageUptodate(page);
2293
2294         ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false);
2295         if (ret)
2296                 goto out_release;
2297
2298         ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
2299         if (!ret) {
2300                 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL);
2301                 radix_tree_preload_end();
2302         }
2303         if (ret)
2304                 goto out_release_uncharge;
2305
2306         mem_cgroup_commit_charge(page, memcg, false, false);
2307
2308         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2309         if (dst_vma->vm_flags & VM_WRITE)
2310                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2311
2312         ret = -EEXIST;
2313         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2314         if (!pte_none(*dst_pte))
2315                 goto out_release_uncharge_unlock;
2316
2317         lru_cache_add_anon(page);
2318
2319         spin_lock(&info->lock);
2320         info->alloced++;
2321         inode->i_blocks += BLOCKS_PER_PAGE;
2322         shmem_recalc_inode(inode);
2323         spin_unlock(&info->lock);
2324
2325         inc_mm_counter(dst_mm, mm_counter_file(page));
2326         page_add_file_rmap(page, false);
2327         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2328
2329         /* No need to invalidate - it was non-present before */
2330         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2331         unlock_page(page);
2332         pte_unmap_unlock(dst_pte, ptl);
2333         ret = 0;
2334 out:
2335         return ret;
2336 out_release_uncharge_unlock:
2337         pte_unmap_unlock(dst_pte, ptl);
2338 out_release_uncharge:
2339         mem_cgroup_cancel_charge(page, memcg, false);
2340 out_release:
2341         unlock_page(page);
2342         put_page(page);
2343 out_unacct_blocks:
2344         shmem_inode_unacct_blocks(inode, 1);
2345         goto out;
2346 }
2347
2348 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2349                            pmd_t *dst_pmd,
2350                            struct vm_area_struct *dst_vma,
2351                            unsigned long dst_addr,
2352                            unsigned long src_addr,
2353                            struct page **pagep)
2354 {
2355         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2356                                       dst_addr, src_addr, false, pagep);
2357 }
2358
2359 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2360                              pmd_t *dst_pmd,
2361                              struct vm_area_struct *dst_vma,
2362                              unsigned long dst_addr)
2363 {
2364         struct page *page = NULL;
2365
2366         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2367                                       dst_addr, 0, true, &page);
2368 }
2369
2370 #ifdef CONFIG_TMPFS
2371 static const struct inode_operations shmem_symlink_inode_operations;
2372 static const struct inode_operations shmem_short_symlink_operations;
2373
2374 #ifdef CONFIG_TMPFS_XATTR
2375 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2376 #else
2377 #define shmem_initxattrs NULL
2378 #endif
2379
2380 static int
2381 shmem_write_begin(struct file *file, struct address_space *mapping,
2382                         loff_t pos, unsigned len, unsigned flags,
2383                         struct page **pagep, void **fsdata)
2384 {
2385         struct inode *inode = mapping->host;
2386         struct shmem_inode_info *info = SHMEM_I(inode);
2387         pgoff_t index = pos >> PAGE_SHIFT;
2388
2389         /* i_mutex is held by caller */
2390         if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) {
2391                 if (info->seals & F_SEAL_WRITE)
2392                         return -EPERM;
2393                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2394                         return -EPERM;
2395         }
2396
2397         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2398 }
2399
2400 static int
2401 shmem_write_end(struct file *file, struct address_space *mapping,
2402                         loff_t pos, unsigned len, unsigned copied,
2403                         struct page *page, void *fsdata)
2404 {
2405         struct inode *inode = mapping->host;
2406
2407         if (pos + copied > inode->i_size)
2408                 i_size_write(inode, pos + copied);
2409
2410         if (!PageUptodate(page)) {
2411                 struct page *head = compound_head(page);
2412                 if (PageTransCompound(page)) {
2413                         int i;
2414
2415                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2416                                 if (head + i == page)
2417                                         continue;
2418                                 clear_highpage(head + i);
2419                                 flush_dcache_page(head + i);
2420                         }
2421                 }
2422                 if (copied < PAGE_SIZE) {
2423                         unsigned from = pos & (PAGE_SIZE - 1);
2424                         zero_user_segments(page, 0, from,
2425                                         from + copied, PAGE_SIZE);
2426                 }
2427                 SetPageUptodate(head);
2428         }
2429         set_page_dirty(page);
2430         unlock_page(page);
2431         put_page(page);
2432
2433         return copied;
2434 }
2435
2436 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2437 {
2438         struct file *file = iocb->ki_filp;
2439         struct inode *inode = file_inode(file);
2440         struct address_space *mapping = inode->i_mapping;
2441         pgoff_t index;
2442         unsigned long offset;
2443         enum sgp_type sgp = SGP_READ;
2444         int error = 0;
2445         ssize_t retval = 0;
2446         loff_t *ppos = &iocb->ki_pos;
2447
2448         /*
2449          * Might this read be for a stacking filesystem?  Then when reading
2450          * holes of a sparse file, we actually need to allocate those pages,
2451          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2452          */
2453         if (!iter_is_iovec(to))
2454                 sgp = SGP_CACHE;
2455
2456         index = *ppos >> PAGE_SHIFT;
2457         offset = *ppos & ~PAGE_MASK;
2458
2459         for (;;) {
2460                 struct page *page = NULL;
2461                 pgoff_t end_index;
2462                 unsigned long nr, ret;
2463                 loff_t i_size = i_size_read(inode);
2464
2465                 end_index = i_size >> PAGE_SHIFT;
2466                 if (index > end_index)
2467                         break;
2468                 if (index == end_index) {
2469                         nr = i_size & ~PAGE_MASK;
2470                         if (nr <= offset)
2471                                 break;
2472                 }
2473
2474                 error = shmem_getpage(inode, index, &page, sgp);
2475                 if (error) {
2476                         if (error == -EINVAL)
2477                                 error = 0;
2478                         break;
2479                 }
2480                 if (page) {
2481                         if (sgp == SGP_CACHE)
2482                                 set_page_dirty(page);
2483                         unlock_page(page);
2484                 }
2485
2486                 /*
2487                  * We must evaluate after, since reads (unlike writes)
2488                  * are called without i_mutex protection against truncate
2489                  */
2490                 nr = PAGE_SIZE;
2491                 i_size = i_size_read(inode);
2492                 end_index = i_size >> PAGE_SHIFT;
2493                 if (index == end_index) {
2494                         nr = i_size & ~PAGE_MASK;
2495                         if (nr <= offset) {
2496                                 if (page)
2497                                         put_page(page);
2498                                 break;
2499                         }
2500                 }
2501                 nr -= offset;
2502
2503                 if (page) {
2504                         /*
2505                          * If users can be writing to this page using arbitrary
2506                          * virtual addresses, take care about potential aliasing
2507                          * before reading the page on the kernel side.
2508                          */
2509                         if (mapping_writably_mapped(mapping))
2510                                 flush_dcache_page(page);
2511                         /*
2512                          * Mark the page accessed if we read the beginning.
2513                          */
2514                         if (!offset)
2515                                 mark_page_accessed(page);
2516                 } else {
2517                         page = ZERO_PAGE(0);
2518                         get_page(page);
2519                 }
2520
2521                 /*
2522                  * Ok, we have the page, and it's up-to-date, so
2523                  * now we can copy it to user space...
2524                  */
2525                 ret = copy_page_to_iter(page, offset, nr, to);
2526                 retval += ret;
2527                 offset += ret;
2528                 index += offset >> PAGE_SHIFT;
2529                 offset &= ~PAGE_MASK;
2530
2531                 put_page(page);
2532                 if (!iov_iter_count(to))
2533                         break;
2534                 if (ret < nr) {
2535                         error = -EFAULT;
2536                         break;
2537                 }
2538                 cond_resched();
2539         }
2540
2541         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2542         file_accessed(file);
2543         return retval ? retval : error;
2544 }
2545
2546 /*
2547  * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
2548  */
2549 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2550                                     pgoff_t index, pgoff_t end, int whence)
2551 {
2552         struct page *page;
2553         struct pagevec pvec;
2554         pgoff_t indices[PAGEVEC_SIZE];
2555         bool done = false;
2556         int i;
2557
2558         pagevec_init(&pvec);
2559         pvec.nr = 1;            /* start small: we may be there already */
2560         while (!done) {
2561                 pvec.nr = find_get_entries(mapping, index,
2562                                         pvec.nr, pvec.pages, indices);
2563                 if (!pvec.nr) {
2564                         if (whence == SEEK_DATA)
2565                                 index = end;
2566                         break;
2567                 }
2568                 for (i = 0; i < pvec.nr; i++, index++) {
2569                         if (index < indices[i]) {
2570                                 if (whence == SEEK_HOLE) {
2571                                         done = true;
2572                                         break;
2573                                 }
2574                                 index = indices[i];
2575                         }
2576                         page = pvec.pages[i];
2577                         if (page && !radix_tree_exceptional_entry(page)) {
2578                                 if (!PageUptodate(page))
2579                                         page = NULL;
2580                         }
2581                         if (index >= end ||
2582                             (page && whence == SEEK_DATA) ||
2583                             (!page && whence == SEEK_HOLE)) {
2584                                 done = true;
2585                                 break;
2586                         }
2587                 }
2588                 pagevec_remove_exceptionals(&pvec);
2589                 pagevec_release(&pvec);
2590                 pvec.nr = PAGEVEC_SIZE;
2591                 cond_resched();
2592         }
2593         return index;
2594 }
2595
2596 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2597 {
2598         struct address_space *mapping = file->f_mapping;
2599         struct inode *inode = mapping->host;
2600         pgoff_t start, end;
2601         loff_t new_offset;
2602
2603         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2604                 return generic_file_llseek_size(file, offset, whence,
2605                                         MAX_LFS_FILESIZE, i_size_read(inode));
2606         inode_lock(inode);
2607         /* We're holding i_mutex so we can access i_size directly */
2608
2609         if (offset < 0)
2610                 offset = -EINVAL;
2611         else if (offset >= inode->i_size)
2612                 offset = -ENXIO;
2613         else {
2614                 start = offset >> PAGE_SHIFT;
2615                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2616                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2617                 new_offset <<= PAGE_SHIFT;
2618                 if (new_offset > offset) {
2619                         if (new_offset < inode->i_size)
2620                                 offset = new_offset;
2621                         else if (whence == SEEK_DATA)
2622                                 offset = -ENXIO;
2623                         else
2624                                 offset = inode->i_size;
2625                 }
2626         }
2627
2628         if (offset >= 0)
2629                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2630         inode_unlock(inode);
2631         return offset;
2632 }
2633
2634 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2635                                                          loff_t len)
2636 {
2637         struct inode *inode = file_inode(file);
2638         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2639         struct shmem_inode_info *info = SHMEM_I(inode);
2640         struct shmem_falloc shmem_falloc;
2641         pgoff_t start, index, end;
2642         int error;
2643
2644         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2645                 return -EOPNOTSUPP;
2646
2647         inode_lock(inode);
2648
2649         if (mode & FALLOC_FL_PUNCH_HOLE) {
2650                 struct address_space *mapping = file->f_mapping;
2651                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2652                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2653                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2654
2655                 /* protected by i_mutex */
2656                 if (info->seals & F_SEAL_WRITE) {
2657                         error = -EPERM;
2658                         goto out;
2659                 }
2660
2661                 shmem_falloc.waitq = &shmem_falloc_waitq;
2662                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2663                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2664                 spin_lock(&inode->i_lock);
2665                 inode->i_private = &shmem_falloc;
2666                 spin_unlock(&inode->i_lock);
2667
2668                 if ((u64)unmap_end > (u64)unmap_start)
2669                         unmap_mapping_range(mapping, unmap_start,
2670                                             1 + unmap_end - unmap_start, 0);
2671                 shmem_truncate_range(inode, offset, offset + len - 1);
2672                 /* No need to unmap again: hole-punching leaves COWed pages */
2673
2674                 spin_lock(&inode->i_lock);
2675                 inode->i_private = NULL;
2676                 wake_up_all(&shmem_falloc_waitq);
2677                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2678                 spin_unlock(&inode->i_lock);
2679                 error = 0;
2680                 goto out;
2681         }
2682
2683         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2684         error = inode_newsize_ok(inode, offset + len);
2685         if (error)
2686                 goto out;
2687
2688         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2689                 error = -EPERM;
2690                 goto out;
2691         }
2692
2693         start = offset >> PAGE_SHIFT;
2694         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2695         /* Try to avoid a swapstorm if len is impossible to satisfy */
2696         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2697                 error = -ENOSPC;
2698                 goto out;
2699         }
2700
2701         shmem_falloc.waitq = NULL;
2702         shmem_falloc.start = start;
2703         shmem_falloc.next  = start;
2704         shmem_falloc.nr_falloced = 0;
2705         shmem_falloc.nr_unswapped = 0;
2706         spin_lock(&inode->i_lock);
2707         inode->i_private = &shmem_falloc;
2708         spin_unlock(&inode->i_lock);
2709
2710         for (index = start; index < end; index++) {
2711                 struct page *page;
2712
2713                 /*
2714                  * Good, the fallocate(2) manpage permits EINTR: we may have
2715                  * been interrupted because we are using up too much memory.
2716                  */
2717                 if (signal_pending(current))
2718                         error = -EINTR;
2719                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2720                         error = -ENOMEM;
2721                 else
2722                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2723                 if (error) {
2724                         /* Remove the !PageUptodate pages we added */
2725                         if (index > start) {
2726                                 shmem_undo_range(inode,
2727                                     (loff_t)start << PAGE_SHIFT,
2728                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2729                         }
2730                         goto undone;
2731                 }
2732
2733                 /*
2734                  * Inform shmem_writepage() how far we have reached.
2735                  * No need for lock or barrier: we have the page lock.
2736                  */
2737                 shmem_falloc.next++;
2738                 if (!PageUptodate(page))
2739                         shmem_falloc.nr_falloced++;
2740
2741                 /*
2742                  * If !PageUptodate, leave it that way so that freeable pages
2743                  * can be recognized if we need to rollback on error later.
2744                  * But set_page_dirty so that memory pressure will swap rather
2745                  * than free the pages we are allocating (and SGP_CACHE pages
2746                  * might still be clean: we now need to mark those dirty too).
2747                  */
2748                 set_page_dirty(page);
2749                 unlock_page(page);
2750                 put_page(page);
2751                 cond_resched();
2752         }
2753
2754         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2755                 i_size_write(inode, offset + len);
2756         inode->i_ctime = current_time(inode);
2757 undone:
2758         spin_lock(&inode->i_lock);
2759         inode->i_private = NULL;
2760         spin_unlock(&inode->i_lock);
2761 out:
2762         inode_unlock(inode);
2763         return error;
2764 }
2765
2766 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2767 {
2768         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2769
2770         buf->f_type = TMPFS_MAGIC;
2771         buf->f_bsize = PAGE_SIZE;
2772         buf->f_namelen = NAME_MAX;
2773         if (sbinfo->max_blocks) {
2774                 buf->f_blocks = sbinfo->max_blocks;
2775                 buf->f_bavail =
2776                 buf->f_bfree  = sbinfo->max_blocks -
2777                                 percpu_counter_sum(&sbinfo->used_blocks);
2778         }
2779         if (sbinfo->max_inodes) {
2780                 buf->f_files = sbinfo->max_inodes;
2781                 buf->f_ffree = sbinfo->free_inodes;
2782         }
2783         /* else leave those fields 0 like simple_statfs */
2784         return 0;
2785 }
2786
2787 /*
2788  * File creation. Allocate an inode, and we're done..
2789  */
2790 static int
2791 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2792 {
2793         struct inode *inode;
2794         int error = -ENOSPC;
2795
2796         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2797         if (inode) {
2798                 error = simple_acl_create(dir, inode);
2799                 if (error)
2800                         goto out_iput;
2801                 error = security_inode_init_security(inode, dir,
2802                                                      &dentry->d_name,
2803                                                      shmem_initxattrs, NULL);
2804                 if (error && error != -EOPNOTSUPP)
2805                         goto out_iput;
2806
2807                 error = 0;
2808                 dir->i_size += BOGO_DIRENT_SIZE;
2809                 dir->i_ctime = dir->i_mtime = current_time(dir);
2810                 d_instantiate(dentry, inode);
2811                 dget(dentry); /* Extra count - pin the dentry in core */
2812         }
2813         return error;
2814 out_iput:
2815         iput(inode);
2816         return error;
2817 }
2818
2819 static int
2820 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2821 {
2822         struct inode *inode;
2823         int error = -ENOSPC;
2824
2825         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2826         if (inode) {
2827                 error = security_inode_init_security(inode, dir,
2828                                                      NULL,
2829                                                      shmem_initxattrs, NULL);
2830                 if (error && error != -EOPNOTSUPP)
2831                         goto out_iput;
2832                 error = simple_acl_create(dir, inode);
2833                 if (error)
2834                         goto out_iput;
2835                 d_tmpfile(dentry, inode);
2836         }
2837         return error;
2838 out_iput:
2839         iput(inode);
2840         return error;
2841 }
2842
2843 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2844 {
2845         int error;
2846
2847         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2848                 return error;
2849         inc_nlink(dir);
2850         return 0;
2851 }
2852
2853 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2854                 bool excl)
2855 {
2856         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2857 }
2858
2859 /*
2860  * Link a file..
2861  */
2862 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2863 {
2864         struct inode *inode = d_inode(old_dentry);
2865         int ret;
2866
2867         /*
2868          * No ordinary (disk based) filesystem counts links as inodes;
2869          * but each new link needs a new dentry, pinning lowmem, and
2870          * tmpfs dentries cannot be pruned until they are unlinked.
2871          */
2872         ret = shmem_reserve_inode(inode->i_sb);
2873         if (ret)
2874                 goto out;
2875
2876         dir->i_size += BOGO_DIRENT_SIZE;
2877         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2878         inc_nlink(inode);
2879         ihold(inode);   /* New dentry reference */
2880         dget(dentry);           /* Extra pinning count for the created dentry */
2881         d_instantiate(dentry, inode);
2882 out:
2883         return ret;
2884 }
2885
2886 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2887 {
2888         struct inode *inode = d_inode(dentry);
2889
2890         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2891                 shmem_free_inode(inode->i_sb);
2892
2893         dir->i_size -= BOGO_DIRENT_SIZE;
2894         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2895         drop_nlink(inode);
2896         dput(dentry);   /* Undo the count from "create" - this does all the work */
2897         return 0;
2898 }
2899
2900 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2901 {
2902         if (!simple_empty(dentry))
2903                 return -ENOTEMPTY;
2904
2905         drop_nlink(d_inode(dentry));
2906         drop_nlink(dir);
2907         return shmem_unlink(dir, dentry);
2908 }
2909
2910 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2911 {
2912         bool old_is_dir = d_is_dir(old_dentry);
2913         bool new_is_dir = d_is_dir(new_dentry);
2914
2915         if (old_dir != new_dir && old_is_dir != new_is_dir) {
2916                 if (old_is_dir) {
2917                         drop_nlink(old_dir);
2918                         inc_nlink(new_dir);
2919                 } else {
2920                         drop_nlink(new_dir);
2921                         inc_nlink(old_dir);
2922                 }
2923         }
2924         old_dir->i_ctime = old_dir->i_mtime =
2925         new_dir->i_ctime = new_dir->i_mtime =
2926         d_inode(old_dentry)->i_ctime =
2927         d_inode(new_dentry)->i_ctime = current_time(old_dir);
2928
2929         return 0;
2930 }
2931
2932 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2933 {
2934         struct dentry *whiteout;
2935         int error;
2936
2937         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
2938         if (!whiteout)
2939                 return -ENOMEM;
2940
2941         error = shmem_mknod(old_dir, whiteout,
2942                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
2943         dput(whiteout);
2944         if (error)
2945                 return error;
2946
2947         /*
2948          * Cheat and hash the whiteout while the old dentry is still in
2949          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
2950          *
2951          * d_lookup() will consistently find one of them at this point,
2952          * not sure which one, but that isn't even important.
2953          */
2954         d_rehash(whiteout);
2955         return 0;
2956 }
2957
2958 /*
2959  * The VFS layer already does all the dentry stuff for rename,
2960  * we just have to decrement the usage count for the target if
2961  * it exists so that the VFS layer correctly free's it when it
2962  * gets overwritten.
2963  */
2964 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
2965 {
2966         struct inode *inode = d_inode(old_dentry);
2967         int they_are_dirs = S_ISDIR(inode->i_mode);
2968
2969         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
2970                 return -EINVAL;
2971
2972         if (flags & RENAME_EXCHANGE)
2973                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
2974
2975         if (!simple_empty(new_dentry))
2976                 return -ENOTEMPTY;
2977
2978         if (flags & RENAME_WHITEOUT) {
2979                 int error;
2980
2981                 error = shmem_whiteout(old_dir, old_dentry);
2982                 if (error)
2983                         return error;
2984         }
2985
2986         if (d_really_is_positive(new_dentry)) {
2987                 (void) shmem_unlink(new_dir, new_dentry);
2988                 if (they_are_dirs) {
2989                         drop_nlink(d_inode(new_dentry));
2990                         drop_nlink(old_dir);
2991                 }
2992         } else if (they_are_dirs) {
2993                 drop_nlink(old_dir);
2994                 inc_nlink(new_dir);
2995         }
2996
2997         old_dir->i_size -= BOGO_DIRENT_SIZE;
2998         new_dir->i_size += BOGO_DIRENT_SIZE;
2999         old_dir->i_ctime = old_dir->i_mtime =
3000         new_dir->i_ctime = new_dir->i_mtime =
3001         inode->i_ctime = current_time(old_dir);
3002         return 0;
3003 }
3004
3005 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3006 {
3007         int error;
3008         int len;
3009         struct inode *inode;
3010         struct page *page;
3011
3012         len = strlen(symname) + 1;
3013         if (len > PAGE_SIZE)
3014                 return -ENAMETOOLONG;
3015
3016         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE);
3017         if (!inode)
3018                 return -ENOSPC;
3019
3020         error = security_inode_init_security(inode, dir, &dentry->d_name,
3021                                              shmem_initxattrs, NULL);
3022         if (error) {
3023                 if (error != -EOPNOTSUPP) {
3024                         iput(inode);
3025                         return error;
3026                 }
3027                 error = 0;
3028         }
3029
3030         inode->i_size = len-1;
3031         if (len <= SHORT_SYMLINK_LEN) {
3032                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3033                 if (!inode->i_link) {
3034                         iput(inode);
3035                         return -ENOMEM;
3036                 }
3037                 inode->i_op = &shmem_short_symlink_operations;
3038         } else {
3039                 inode_nohighmem(inode);
3040                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3041                 if (error) {
3042                         iput(inode);
3043                         return error;
3044                 }
3045                 inode->i_mapping->a_ops = &shmem_aops;
3046                 inode->i_op = &shmem_symlink_inode_operations;
3047                 memcpy(page_address(page), symname, len);
3048                 SetPageUptodate(page);
3049                 set_page_dirty(page);
3050                 unlock_page(page);
3051                 put_page(page);
3052         }
3053         dir->i_size += BOGO_DIRENT_SIZE;
3054         dir->i_ctime = dir->i_mtime = current_time(dir);
3055         d_instantiate(dentry, inode);
3056         dget(dentry);
3057         return 0;
3058 }
3059
3060 static void shmem_put_link(void *arg)
3061 {
3062         mark_page_accessed(arg);
3063         put_page(arg);
3064 }
3065
3066 static const char *shmem_get_link(struct dentry *dentry,
3067                                   struct inode *inode,
3068                                   struct delayed_call *done)
3069 {
3070         struct page *page = NULL;
3071         int error;
3072         if (!dentry) {
3073                 page = find_get_page(inode->i_mapping, 0);
3074                 if (!page)
3075                         return ERR_PTR(-ECHILD);
3076                 if (!PageUptodate(page)) {
3077                         put_page(page);
3078                         return ERR_PTR(-ECHILD);
3079                 }
3080         } else {
3081                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3082                 if (error)
3083                         return ERR_PTR(error);
3084                 unlock_page(page);
3085         }
3086         set_delayed_call(done, shmem_put_link, page);
3087         return page_address(page);
3088 }
3089
3090 #ifdef CONFIG_TMPFS_XATTR
3091 /*
3092  * Superblocks without xattr inode operations may get some security.* xattr
3093  * support from the LSM "for free". As soon as we have any other xattrs
3094  * like ACLs, we also need to implement the security.* handlers at
3095  * filesystem level, though.
3096  */
3097
3098 /*
3099  * Callback for security_inode_init_security() for acquiring xattrs.
3100  */
3101 static int shmem_initxattrs(struct inode *inode,
3102                             const struct xattr *xattr_array,
3103                             void *fs_info)
3104 {
3105         struct shmem_inode_info *info = SHMEM_I(inode);
3106         const struct xattr *xattr;
3107         struct simple_xattr *new_xattr;
3108         size_t len;
3109
3110         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3111                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3112                 if (!new_xattr)
3113                         return -ENOMEM;
3114
3115                 len = strlen(xattr->name) + 1;
3116                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3117                                           GFP_KERNEL);
3118                 if (!new_xattr->name) {
3119                         kfree(new_xattr);
3120                         return -ENOMEM;
3121                 }
3122
3123                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3124                        XATTR_SECURITY_PREFIX_LEN);
3125                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3126                        xattr->name, len);
3127
3128                 simple_xattr_list_add(&info->xattrs, new_xattr);
3129         }
3130
3131         return 0;
3132 }
3133
3134 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3135                                    struct dentry *unused, struct inode *inode,
3136                                    const char *name, void *buffer, size_t size)
3137 {
3138         struct shmem_inode_info *info = SHMEM_I(inode);
3139
3140         name = xattr_full_name(handler, name);
3141         return simple_xattr_get(&info->xattrs, name, buffer, size);
3142 }
3143
3144 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3145                                    struct dentry *unused, struct inode *inode,
3146                                    const char *name, const void *value,
3147                                    size_t size, int flags)
3148 {
3149         struct shmem_inode_info *info = SHMEM_I(inode);
3150
3151         name = xattr_full_name(handler, name);
3152         return simple_xattr_set(&info->xattrs, name, value, size, flags);
3153 }
3154
3155 static const struct xattr_handler shmem_security_xattr_handler = {
3156         .prefix = XATTR_SECURITY_PREFIX,
3157         .get = shmem_xattr_handler_get,
3158         .set = shmem_xattr_handler_set,
3159 };
3160
3161 static const struct xattr_handler shmem_trusted_xattr_handler = {
3162         .prefix = XATTR_TRUSTED_PREFIX,
3163         .get = shmem_xattr_handler_get,
3164         .set = shmem_xattr_handler_set,
3165 };
3166
3167 static const struct xattr_handler *shmem_xattr_handlers[] = {
3168 #ifdef CONFIG_TMPFS_POSIX_ACL
3169         &posix_acl_access_xattr_handler,
3170         &posix_acl_default_xattr_handler,
3171 #endif
3172         &shmem_security_xattr_handler,
3173         &shmem_trusted_xattr_handler,
3174         NULL
3175 };
3176
3177 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3178 {
3179         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3180         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3181 }
3182 #endif /* CONFIG_TMPFS_XATTR */
3183
3184 static const struct inode_operations shmem_short_symlink_operations = {
3185         .get_link       = simple_get_link,
3186 #ifdef CONFIG_TMPFS_XATTR
3187         .listxattr      = shmem_listxattr,
3188 #endif
3189 };
3190
3191 static const struct inode_operations shmem_symlink_inode_operations = {
3192         .get_link       = shmem_get_link,
3193 #ifdef CONFIG_TMPFS_XATTR
3194         .listxattr      = shmem_listxattr,
3195 #endif
3196 };
3197
3198 static struct dentry *shmem_get_parent(struct dentry *child)
3199 {
3200         return ERR_PTR(-ESTALE);
3201 }
3202
3203 static int shmem_match(struct inode *ino, void *vfh)
3204 {
3205         __u32 *fh = vfh;
3206         __u64 inum = fh[2];
3207         inum = (inum << 32) | fh[1];
3208         return ino->i_ino == inum && fh[0] == ino->i_generation;
3209 }
3210
3211 /* Find any alias of inode, but prefer a hashed alias */
3212 static struct dentry *shmem_find_alias(struct inode *inode)
3213 {
3214         struct dentry *alias = d_find_alias(inode);
3215
3216         return alias ?: d_find_any_alias(inode);
3217 }
3218
3219
3220 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3221                 struct fid *fid, int fh_len, int fh_type)
3222 {
3223         struct inode *inode;
3224         struct dentry *dentry = NULL;
3225         u64 inum;
3226
3227         if (fh_len < 3)
3228                 return NULL;
3229
3230         inum = fid->raw[2];
3231         inum = (inum << 32) | fid->raw[1];
3232
3233         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3234                         shmem_match, fid->raw);
3235         if (inode) {
3236                 dentry = shmem_find_alias(inode);
3237                 iput(inode);
3238         }
3239
3240         return dentry;
3241 }
3242
3243 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3244                                 struct inode *parent)
3245 {
3246         if (*len < 3) {
3247                 *len = 3;
3248                 return FILEID_INVALID;
3249         }
3250
3251         if (inode_unhashed(inode)) {
3252                 /* Unfortunately insert_inode_hash is not idempotent,
3253                  * so as we hash inodes here rather than at creation
3254                  * time, we need a lock to ensure we only try
3255                  * to do it once
3256                  */
3257                 static DEFINE_SPINLOCK(lock);
3258                 spin_lock(&lock);
3259                 if (inode_unhashed(inode))
3260                         __insert_inode_hash(inode,
3261                                             inode->i_ino + inode->i_generation);
3262                 spin_unlock(&lock);
3263         }
3264
3265         fh[0] = inode->i_generation;
3266         fh[1] = inode->i_ino;
3267         fh[2] = ((__u64)inode->i_ino) >> 32;
3268
3269         *len = 3;
3270         return 1;
3271 }
3272
3273 static const struct export_operations shmem_export_ops = {
3274         .get_parent     = shmem_get_parent,
3275         .encode_fh      = shmem_encode_fh,
3276         .fh_to_dentry   = shmem_fh_to_dentry,
3277 };
3278
3279 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
3280                                bool remount)
3281 {
3282         char *this_char, *value, *rest;
3283         struct mempolicy *mpol = NULL;
3284         uid_t uid;
3285         gid_t gid;
3286
3287         while (options != NULL) {
3288                 this_char = options;
3289                 for (;;) {
3290                         /*
3291                          * NUL-terminate this option: unfortunately,
3292                          * mount options form a comma-separated list,
3293                          * but mpol's nodelist may also contain commas.
3294                          */
3295                         options = strchr(options, ',');
3296                         if (options == NULL)
3297                                 break;
3298                         options++;
3299                         if (!isdigit(*options)) {
3300                                 options[-1] = '\0';
3301                                 break;
3302                         }
3303                 }
3304                 if (!*this_char)
3305                         continue;
3306                 if ((value = strchr(this_char,'=')) != NULL) {
3307                         *value++ = 0;
3308                 } else {
3309                         pr_err("tmpfs: No value for mount option '%s'\n",
3310                                this_char);
3311                         goto error;
3312                 }
3313
3314                 if (!strcmp(this_char,"size")) {
3315                         unsigned long long size;
3316                         size = memparse(value,&rest);
3317                         if (*rest == '%') {
3318                                 size <<= PAGE_SHIFT;
3319                                 size *= totalram_pages;
3320                                 do_div(size, 100);
3321                                 rest++;
3322                         }
3323                         if (*rest)
3324                                 goto bad_val;
3325                         sbinfo->max_blocks =
3326                                 DIV_ROUND_UP(size, PAGE_SIZE);
3327                 } else if (!strcmp(this_char,"nr_blocks")) {
3328                         sbinfo->max_blocks = memparse(value, &rest);
3329                         if (*rest)
3330                                 goto bad_val;
3331                 } else if (!strcmp(this_char,"nr_inodes")) {
3332                         sbinfo->max_inodes = memparse(value, &rest);
3333                         if (*rest)
3334                                 goto bad_val;
3335                 } else if (!strcmp(this_char,"mode")) {
3336                         if (remount)
3337                                 continue;
3338                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
3339                         if (*rest)
3340                                 goto bad_val;
3341                 } else if (!strcmp(this_char,"uid")) {
3342                         if (remount)
3343                                 continue;
3344                         uid = simple_strtoul(value, &rest, 0);
3345                         if (*rest)
3346                                 goto bad_val;
3347                         sbinfo->uid = make_kuid(current_user_ns(), uid);
3348                         if (!uid_valid(sbinfo->uid))
3349                                 goto bad_val;
3350                 } else if (!strcmp(this_char,"gid")) {
3351                         if (remount)
3352                                 continue;
3353                         gid = simple_strtoul(value, &rest, 0);
3354                         if (*rest)
3355                                 goto bad_val;
3356                         sbinfo->gid = make_kgid(current_user_ns(), gid);
3357                         if (!gid_valid(sbinfo->gid))
3358                                 goto bad_val;
3359 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3360                 } else if (!strcmp(this_char, "huge")) {
3361                         int huge;
3362                         huge = shmem_parse_huge(value);
3363                         if (huge < 0)
3364                                 goto bad_val;
3365                         if (!has_transparent_hugepage() &&
3366                                         huge != SHMEM_HUGE_NEVER)
3367                                 goto bad_val;
3368                         sbinfo->huge = huge;
3369 #endif
3370 #ifdef CONFIG_NUMA
3371                 } else if (!strcmp(this_char,"mpol")) {
3372                         mpol_put(mpol);
3373                         mpol = NULL;
3374                         if (mpol_parse_str(value, &mpol))
3375                                 goto bad_val;
3376 #endif
3377                 } else {
3378                         pr_err("tmpfs: Bad mount option %s\n", this_char);
3379                         goto error;
3380                 }
3381         }
3382         sbinfo->mpol = mpol;
3383         return 0;
3384
3385 bad_val:
3386         pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
3387                value, this_char);
3388 error:
3389         mpol_put(mpol);
3390         return 1;
3391
3392 }
3393
3394 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
3395 {
3396         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3397         struct shmem_sb_info config = *sbinfo;
3398         unsigned long inodes;
3399         int error = -EINVAL;
3400
3401         config.mpol = NULL;
3402         if (shmem_parse_options(data, &config, true))
3403                 return error;
3404
3405         spin_lock(&sbinfo->stat_lock);
3406         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3407         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
3408                 goto out;
3409         if (config.max_inodes < inodes)
3410                 goto out;
3411         /*
3412          * Those tests disallow limited->unlimited while any are in use;
3413          * but we must separately disallow unlimited->limited, because
3414          * in that case we have no record of how much is already in use.
3415          */
3416         if (config.max_blocks && !sbinfo->max_blocks)
3417                 goto out;
3418         if (config.max_inodes && !sbinfo->max_inodes)
3419                 goto out;
3420
3421         error = 0;
3422         sbinfo->huge = config.huge;
3423         sbinfo->max_blocks  = config.max_blocks;
3424         sbinfo->max_inodes  = config.max_inodes;
3425         sbinfo->free_inodes = config.max_inodes - inodes;
3426
3427         /*
3428          * Preserve previous mempolicy unless mpol remount option was specified.
3429          */
3430         if (config.mpol) {
3431                 mpol_put(sbinfo->mpol);
3432                 sbinfo->mpol = config.mpol;     /* transfers initial ref */
3433         }
3434 out:
3435         spin_unlock(&sbinfo->stat_lock);
3436         return error;
3437 }
3438
3439 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3440 {
3441         struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3442
3443         if (sbinfo->max_blocks != shmem_default_max_blocks())
3444                 seq_printf(seq, ",size=%luk",
3445                         sbinfo->max_blocks << (PAGE_SHIFT - 10));
3446         if (sbinfo->max_inodes != shmem_default_max_inodes())
3447                 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3448         if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
3449                 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3450         if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3451                 seq_printf(seq, ",uid=%u",
3452                                 from_kuid_munged(&init_user_ns, sbinfo->uid));
3453         if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3454                 seq_printf(seq, ",gid=%u",
3455                                 from_kgid_munged(&init_user_ns, sbinfo->gid));
3456 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3457         /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3458         if (sbinfo->huge)
3459                 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3460 #endif
3461         shmem_show_mpol(seq, sbinfo->mpol);
3462         return 0;
3463 }
3464
3465 #endif /* CONFIG_TMPFS */
3466
3467 static void shmem_put_super(struct super_block *sb)
3468 {
3469         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3470
3471         percpu_counter_destroy(&sbinfo->used_blocks);
3472         mpol_put(sbinfo->mpol);
3473         kfree(sbinfo);
3474         sb->s_fs_info = NULL;
3475 }
3476
3477 int shmem_fill_super(struct super_block *sb, void *data, int silent)
3478 {
3479         struct inode *inode;
3480         struct shmem_sb_info *sbinfo;
3481         int err = -ENOMEM;
3482
3483         /* Round up to L1_CACHE_BYTES to resist false sharing */
3484         sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3485                                 L1_CACHE_BYTES), GFP_KERNEL);
3486         if (!sbinfo)
3487                 return -ENOMEM;
3488
3489         sbinfo->mode = S_IRWXUGO | S_ISVTX;
3490         sbinfo->uid = current_fsuid();
3491         sbinfo->gid = current_fsgid();
3492         sb->s_fs_info = sbinfo;
3493
3494 #ifdef CONFIG_TMPFS
3495         /*
3496          * Per default we only allow half of the physical ram per
3497          * tmpfs instance, limiting inodes to one per page of lowmem;
3498          * but the internal instance is left unlimited.
3499          */
3500         if (!(sb->s_flags & SB_KERNMOUNT)) {
3501                 sbinfo->max_blocks = shmem_default_max_blocks();
3502                 sbinfo->max_inodes = shmem_default_max_inodes();
3503                 if (shmem_parse_options(data, sbinfo, false)) {
3504                         err = -EINVAL;
3505                         goto failed;
3506                 }
3507         } else {
3508                 sb->s_flags |= SB_NOUSER;
3509         }
3510         sb->s_export_op = &shmem_export_ops;
3511         sb->s_flags |= SB_NOSEC;
3512 #else
3513         sb->s_flags |= SB_NOUSER;
3514 #endif
3515
3516         spin_lock_init(&sbinfo->stat_lock);
3517         if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3518                 goto failed;
3519         sbinfo->free_inodes = sbinfo->max_inodes;
3520         spin_lock_init(&sbinfo->shrinklist_lock);
3521         INIT_LIST_HEAD(&sbinfo->shrinklist);
3522
3523         sb->s_maxbytes = MAX_LFS_FILESIZE;
3524         sb->s_blocksize = PAGE_SIZE;
3525         sb->s_blocksize_bits = PAGE_SHIFT;
3526         sb->s_magic = TMPFS_MAGIC;
3527         sb->s_op = &shmem_ops;
3528         sb->s_time_gran = 1;
3529 #ifdef CONFIG_TMPFS_XATTR
3530         sb->s_xattr = shmem_xattr_handlers;
3531 #endif
3532 #ifdef CONFIG_TMPFS_POSIX_ACL
3533         sb->s_flags |= SB_POSIXACL;
3534 #endif
3535         uuid_gen(&sb->s_uuid);
3536
3537         inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3538         if (!inode)
3539                 goto failed;
3540         inode->i_uid = sbinfo->uid;
3541         inode->i_gid = sbinfo->gid;
3542         sb->s_root = d_make_root(inode);
3543         if (!sb->s_root)
3544                 goto failed;
3545         return 0;
3546
3547 failed:
3548         shmem_put_super(sb);
3549         return err;
3550 }
3551
3552 static struct kmem_cache *shmem_inode_cachep;
3553
3554 static struct inode *shmem_alloc_inode(struct super_block *sb)
3555 {
3556         struct shmem_inode_info *info;
3557         info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
3558         if (!info)
3559                 return NULL;
3560         return &info->vfs_inode;
3561 }
3562
3563 static void shmem_destroy_callback(struct rcu_head *head)
3564 {
3565         struct inode *inode = container_of(head, struct inode, i_rcu);
3566         if (S_ISLNK(inode->i_mode))
3567                 kfree(inode->i_link);
3568         kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3569 }
3570
3571 static void shmem_destroy_inode(struct inode *inode)
3572 {
3573         if (S_ISREG(inode->i_mode))
3574                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3575         call_rcu(&inode->i_rcu, shmem_destroy_callback);
3576 }
3577
3578 static void shmem_init_inode(void *foo)
3579 {
3580         struct shmem_inode_info *info = foo;
3581         inode_init_once(&info->vfs_inode);
3582 }
3583
3584 static void shmem_init_inodecache(void)
3585 {
3586         shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3587                                 sizeof(struct shmem_inode_info),
3588                                 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3589 }
3590
3591 static void shmem_destroy_inodecache(void)
3592 {
3593         kmem_cache_destroy(shmem_inode_cachep);
3594 }
3595
3596 static const struct address_space_operations shmem_aops = {
3597         .writepage      = shmem_writepage,
3598         .set_page_dirty = __set_page_dirty_no_writeback,
3599 #ifdef CONFIG_TMPFS
3600         .write_begin    = shmem_write_begin,
3601         .write_end      = shmem_write_end,
3602 #endif
3603 #ifdef CONFIG_MIGRATION
3604         .migratepage    = migrate_page,
3605 #endif
3606         .error_remove_page = generic_error_remove_page,
3607 };
3608
3609 static const struct file_operations shmem_file_operations = {
3610         .mmap           = shmem_mmap,
3611         .get_unmapped_area = shmem_get_unmapped_area,
3612 #ifdef CONFIG_TMPFS
3613         .llseek         = shmem_file_llseek,
3614         .read_iter      = shmem_file_read_iter,
3615         .write_iter     = generic_file_write_iter,
3616         .fsync          = noop_fsync,
3617         .splice_read    = generic_file_splice_read,
3618         .splice_write   = iter_file_splice_write,
3619         .fallocate      = shmem_fallocate,
3620 #endif
3621 };
3622
3623 static const struct inode_operations shmem_inode_operations = {
3624         .getattr        = shmem_getattr,
3625         .setattr        = shmem_setattr,
3626 #ifdef CONFIG_TMPFS_XATTR
3627         .listxattr      = shmem_listxattr,
3628         .set_acl        = simple_set_acl,
3629 #endif
3630 };
3631
3632 static const struct inode_operations shmem_dir_inode_operations = {
3633 #ifdef CONFIG_TMPFS
3634         .create         = shmem_create,
3635         .lookup         = simple_lookup,
3636         .link           = shmem_link,
3637         .unlink         = shmem_unlink,
3638         .symlink        = shmem_symlink,
3639         .mkdir          = shmem_mkdir,
3640         .rmdir          = shmem_rmdir,
3641         .mknod          = shmem_mknod,
3642         .rename         = shmem_rename2,
3643         .tmpfile        = shmem_tmpfile,
3644 #endif
3645 #ifdef CONFIG_TMPFS_XATTR
3646         .listxattr      = shmem_listxattr,
3647 #endif
3648 #ifdef CONFIG_TMPFS_POSIX_ACL
3649         .setattr        = shmem_setattr,
3650         .set_acl        = simple_set_acl,
3651 #endif
3652 };
3653
3654 static const struct inode_operations shmem_special_inode_operations = {
3655 #ifdef CONFIG_TMPFS_XATTR
3656         .listxattr      = shmem_listxattr,
3657 #endif
3658 #ifdef CONFIG_TMPFS_POSIX_ACL
3659         .setattr        = shmem_setattr,
3660         .set_acl        = simple_set_acl,
3661 #endif
3662 };
3663
3664 static const struct super_operations shmem_ops = {
3665         .alloc_inode    = shmem_alloc_inode,
3666         .destroy_inode  = shmem_destroy_inode,
3667 #ifdef CONFIG_TMPFS
3668         .statfs         = shmem_statfs,
3669         .remount_fs     = shmem_remount_fs,
3670         .show_options   = shmem_show_options,
3671 #endif
3672         .evict_inode    = shmem_evict_inode,
3673         .drop_inode     = generic_delete_inode,
3674         .put_super      = shmem_put_super,
3675 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3676         .nr_cached_objects      = shmem_unused_huge_count,
3677         .free_cached_objects    = shmem_unused_huge_scan,
3678 #endif
3679 };
3680
3681 static const struct vm_operations_struct shmem_vm_ops = {
3682         .fault          = shmem_fault,
3683         .map_pages      = filemap_map_pages,
3684 #ifdef CONFIG_NUMA
3685         .set_policy     = shmem_set_policy,
3686         .get_policy     = shmem_get_policy,
3687 #endif
3688 };
3689
3690 static struct dentry *shmem_mount(struct file_system_type *fs_type,
3691         int flags, const char *dev_name, void *data)
3692 {
3693         return mount_nodev(fs_type, flags, data, shmem_fill_super);
3694 }
3695
3696 static struct file_system_type shmem_fs_type = {
3697         .owner          = THIS_MODULE,
3698         .name           = "tmpfs",
3699         .mount          = shmem_mount,
3700         .kill_sb        = kill_litter_super,
3701         .fs_flags       = FS_USERNS_MOUNT,
3702 };
3703
3704 int __init shmem_init(void)
3705 {
3706         int error;
3707
3708         /* If rootfs called this, don't re-init */
3709         if (shmem_inode_cachep)
3710                 return 0;
3711
3712         shmem_init_inodecache();
3713
3714         error = register_filesystem(&shmem_fs_type);
3715         if (error) {
3716                 pr_err("Could not register tmpfs\n");
3717                 goto out2;
3718         }
3719
3720         shm_mnt = kern_mount(&shmem_fs_type);
3721         if (IS_ERR(shm_mnt)) {
3722                 error = PTR_ERR(shm_mnt);
3723                 pr_err("Could not kern_mount tmpfs\n");
3724                 goto out1;
3725         }
3726
3727 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3728         if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3729                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3730         else
3731                 shmem_huge = 0; /* just in case it was patched */
3732 #endif
3733         return 0;
3734
3735 out1:
3736         unregister_filesystem(&shmem_fs_type);
3737 out2:
3738         shmem_destroy_inodecache();
3739         shm_mnt = ERR_PTR(error);
3740         return error;
3741 }
3742
3743 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS)
3744 static ssize_t shmem_enabled_show(struct kobject *kobj,
3745                 struct kobj_attribute *attr, char *buf)
3746 {
3747         int values[] = {
3748                 SHMEM_HUGE_ALWAYS,
3749                 SHMEM_HUGE_WITHIN_SIZE,
3750                 SHMEM_HUGE_ADVISE,
3751                 SHMEM_HUGE_NEVER,
3752                 SHMEM_HUGE_DENY,
3753                 SHMEM_HUGE_FORCE,
3754         };
3755         int i, count;
3756
3757         for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) {
3758                 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s ";
3759
3760                 count += sprintf(buf + count, fmt,
3761                                 shmem_format_huge(values[i]));
3762         }
3763         buf[count - 1] = '\n';
3764         return count;
3765 }
3766
3767 static ssize_t shmem_enabled_store(struct kobject *kobj,
3768                 struct kobj_attribute *attr, const char *buf, size_t count)
3769 {
3770         char tmp[16];
3771         int huge;
3772
3773         if (count + 1 > sizeof(tmp))
3774                 return -EINVAL;
3775         memcpy(tmp, buf, count);
3776         tmp[count] = '\0';
3777         if (count && tmp[count - 1] == '\n')
3778                 tmp[count - 1] = '\0';
3779
3780         huge = shmem_parse_huge(tmp);
3781         if (huge == -EINVAL)
3782                 return -EINVAL;
3783         if (!has_transparent_hugepage() &&
3784                         huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
3785                 return -EINVAL;
3786
3787         shmem_huge = huge;
3788         if (shmem_huge > SHMEM_HUGE_DENY)
3789                 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3790         return count;
3791 }
3792
3793 struct kobj_attribute shmem_enabled_attr =
3794         __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store);
3795 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */
3796
3797 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3798 bool shmem_huge_enabled(struct vm_area_struct *vma)
3799 {
3800         struct inode *inode = file_inode(vma->vm_file);
3801         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
3802         loff_t i_size;
3803         pgoff_t off;
3804
3805         if (shmem_huge == SHMEM_HUGE_FORCE)
3806                 return true;
3807         if (shmem_huge == SHMEM_HUGE_DENY)
3808                 return false;
3809         switch (sbinfo->huge) {
3810                 case SHMEM_HUGE_NEVER:
3811                         return false;
3812                 case SHMEM_HUGE_ALWAYS:
3813                         return true;
3814                 case SHMEM_HUGE_WITHIN_SIZE:
3815                         off = round_up(vma->vm_pgoff, HPAGE_PMD_NR);
3816                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
3817                         if (i_size >= HPAGE_PMD_SIZE &&
3818                                         i_size >> PAGE_SHIFT >= off)
3819                                 return true;
3820                         /* fall through */
3821                 case SHMEM_HUGE_ADVISE:
3822                         /* TODO: implement fadvise() hints */
3823                         return (vma->vm_flags & VM_HUGEPAGE);
3824                 default:
3825                         VM_BUG_ON(1);
3826                         return false;
3827         }
3828 }
3829 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
3830
3831 #else /* !CONFIG_SHMEM */
3832
3833 /*
3834  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3835  *
3836  * This is intended for small system where the benefits of the full
3837  * shmem code (swap-backed and resource-limited) are outweighed by
3838  * their complexity. On systems without swap this code should be
3839  * effectively equivalent, but much lighter weight.
3840  */
3841
3842 static struct file_system_type shmem_fs_type = {
3843         .name           = "tmpfs",
3844         .mount          = ramfs_mount,
3845         .kill_sb        = kill_litter_super,
3846         .fs_flags       = FS_USERNS_MOUNT,
3847 };
3848
3849 int __init shmem_init(void)
3850 {
3851         BUG_ON(register_filesystem(&shmem_fs_type) != 0);
3852
3853         shm_mnt = kern_mount(&shmem_fs_type);
3854         BUG_ON(IS_ERR(shm_mnt));
3855
3856         return 0;
3857 }
3858
3859 int shmem_unuse(swp_entry_t swap, struct page *page)
3860 {
3861         return 0;
3862 }
3863
3864 int shmem_lock(struct file *file, int lock, struct user_struct *user)
3865 {
3866         return 0;
3867 }
3868
3869 void shmem_unlock_mapping(struct address_space *mapping)
3870 {
3871 }
3872
3873 #ifdef CONFIG_MMU
3874 unsigned long shmem_get_unmapped_area(struct file *file,
3875                                       unsigned long addr, unsigned long len,
3876                                       unsigned long pgoff, unsigned long flags)
3877 {
3878         return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
3879 }
3880 #endif
3881
3882 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
3883 {
3884         truncate_inode_pages_range(inode->i_mapping, lstart, lend);
3885 }
3886 EXPORT_SYMBOL_GPL(shmem_truncate_range);
3887
3888 #define shmem_vm_ops                            generic_file_vm_ops
3889 #define shmem_file_operations                   ramfs_file_operations
3890 #define shmem_get_inode(sb, dir, mode, dev, flags)      ramfs_get_inode(sb, dir, mode, dev)
3891 #define shmem_acct_size(flags, size)            0
3892 #define shmem_unacct_size(flags, size)          do {} while (0)
3893
3894 #endif /* CONFIG_SHMEM */
3895
3896 /* common code */
3897
3898 static const struct dentry_operations anon_ops = {
3899         .d_dname = simple_dname
3900 };
3901
3902 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
3903                                        unsigned long flags, unsigned int i_flags)
3904 {
3905         struct file *res;
3906         struct inode *inode;
3907         struct path path;
3908         struct super_block *sb;
3909         struct qstr this;
3910
3911         if (IS_ERR(mnt))
3912                 return ERR_CAST(mnt);
3913
3914         if (size < 0 || size > MAX_LFS_FILESIZE)
3915                 return ERR_PTR(-EINVAL);
3916
3917         if (shmem_acct_size(flags, size))
3918                 return ERR_PTR(-ENOMEM);
3919
3920         res = ERR_PTR(-ENOMEM);
3921         this.name = name;
3922         this.len = strlen(name);
3923         this.hash = 0; /* will go */
3924         sb = mnt->mnt_sb;
3925         path.mnt = mntget(mnt);
3926         path.dentry = d_alloc_pseudo(sb, &this);
3927         if (!path.dentry)
3928                 goto put_memory;
3929         d_set_d_op(path.dentry, &anon_ops);
3930
3931         res = ERR_PTR(-ENOSPC);
3932         inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags);
3933         if (!inode)
3934                 goto put_memory;
3935
3936         inode->i_flags |= i_flags;
3937         d_instantiate(path.dentry, inode);
3938         inode->i_size = size;
3939         clear_nlink(inode);     /* It is unlinked */
3940         res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
3941         if (IS_ERR(res))
3942                 goto put_path;
3943
3944         res = alloc_file(&path, FMODE_WRITE | FMODE_READ,
3945                   &shmem_file_operations);
3946         if (IS_ERR(res))
3947                 goto put_path;
3948
3949         return res;
3950
3951 put_memory:
3952         shmem_unacct_size(flags, size);
3953 put_path:
3954         path_put(&path);
3955         return res;
3956 }
3957
3958 /**
3959  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
3960  *      kernel internal.  There will be NO LSM permission checks against the
3961  *      underlying inode.  So users of this interface must do LSM checks at a
3962  *      higher layer.  The users are the big_key and shm implementations.  LSM
3963  *      checks are provided at the key or shm level rather than the inode.
3964  * @name: name for dentry (to be seen in /proc/<pid>/maps
3965  * @size: size to be set for the file
3966  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
3967  */
3968 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
3969 {
3970         return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
3971 }
3972
3973 /**
3974  * shmem_file_setup - get an unlinked file living in tmpfs
3975  * @name: name for dentry (to be seen in /proc/<pid>/maps
3976  * @size: size to be set for the file
3977  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
3978  */
3979 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
3980 {
3981         return __shmem_file_setup(shm_mnt, name, size, flags, 0);
3982 }
3983 EXPORT_SYMBOL_GPL(shmem_file_setup);
3984
3985 /**
3986  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
3987  * @mnt: the tmpfs mount where the file will be created
3988  * @name: name for dentry (to be seen in /proc/<pid>/maps
3989  * @size: size to be set for the file
3990  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
3991  */
3992 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
3993                                        loff_t size, unsigned long flags)
3994 {
3995         return __shmem_file_setup(mnt, name, size, flags, 0);
3996 }
3997 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
3998
3999 /**
4000  * shmem_zero_setup - setup a shared anonymous mapping
4001  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
4002  */
4003 int shmem_zero_setup(struct vm_area_struct *vma)
4004 {
4005         struct file *file;
4006         loff_t size = vma->vm_end - vma->vm_start;
4007
4008         /*
4009          * Cloning a new file under mmap_sem leads to a lock ordering conflict
4010          * between XFS directory reading and selinux: since this file is only
4011          * accessible to the user through its mapping, use S_PRIVATE flag to
4012          * bypass file security, in the same way as shmem_kernel_file_setup().
4013          */
4014         file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4015         if (IS_ERR(file))
4016                 return PTR_ERR(file);
4017
4018         if (vma->vm_file)
4019                 fput(vma->vm_file);
4020         vma->vm_file = file;
4021         vma->vm_ops = &shmem_vm_ops;
4022
4023         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
4024                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4025                         (vma->vm_end & HPAGE_PMD_MASK)) {
4026                 khugepaged_enter(vma, vma->vm_flags);
4027         }
4028
4029         return 0;
4030 }
4031
4032 /**
4033  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4034  * @mapping:    the page's address_space
4035  * @index:      the page index
4036  * @gfp:        the page allocator flags to use if allocating
4037  *
4038  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4039  * with any new page allocations done using the specified allocation flags.
4040  * But read_cache_page_gfp() uses the ->readpage() method: which does not
4041  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4042  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4043  *
4044  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4045  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4046  */
4047 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4048                                          pgoff_t index, gfp_t gfp)
4049 {
4050 #ifdef CONFIG_SHMEM
4051         struct inode *inode = mapping->host;
4052         struct page *page;
4053         int error;
4054
4055         BUG_ON(mapping->a_ops != &shmem_aops);
4056         error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4057                                   gfp, NULL, NULL, NULL);
4058         if (error)
4059                 page = ERR_PTR(error);
4060         else
4061                 unlock_page(page);
4062         return page;