... and switch shmem_file_setup() to alloc_file_pseudo()
[muen/linux.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/mm.h>
32 #include <linux/sched/signal.h>
33 #include <linux/export.h>
34 #include <linux/swap.h>
35 #include <linux/uio.h>
36 #include <linux/khugepaged.h>
37 #include <linux/hugetlb.h>
38
39 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
40
41 static struct vfsmount *shm_mnt;
42
43 #ifdef CONFIG_SHMEM
44 /*
45  * This virtual memory filesystem is heavily based on the ramfs. It
46  * extends ramfs by the ability to use swap and honor resource limits
47  * which makes it a completely usable filesystem.
48  */
49
50 #include <linux/xattr.h>
51 #include <linux/exportfs.h>
52 #include <linux/posix_acl.h>
53 #include <linux/posix_acl_xattr.h>
54 #include <linux/mman.h>
55 #include <linux/string.h>
56 #include <linux/slab.h>
57 #include <linux/backing-dev.h>
58 #include <linux/shmem_fs.h>
59 #include <linux/writeback.h>
60 #include <linux/blkdev.h>
61 #include <linux/pagevec.h>
62 #include <linux/percpu_counter.h>
63 #include <linux/falloc.h>
64 #include <linux/splice.h>
65 #include <linux/security.h>
66 #include <linux/swapops.h>
67 #include <linux/mempolicy.h>
68 #include <linux/namei.h>
69 #include <linux/ctype.h>
70 #include <linux/migrate.h>
71 #include <linux/highmem.h>
72 #include <linux/seq_file.h>
73 #include <linux/magic.h>
74 #include <linux/syscalls.h>
75 #include <linux/fcntl.h>
76 #include <uapi/linux/memfd.h>
77 #include <linux/userfaultfd_k.h>
78 #include <linux/rmap.h>
79 #include <linux/uuid.h>
80
81 #include <linux/uaccess.h>
82 #include <asm/pgtable.h>
83
84 #include "internal.h"
85
86 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
87 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
88
89 /* Pretend that each entry is of this size in directory's i_size */
90 #define BOGO_DIRENT_SIZE 20
91
92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
93 #define SHORT_SYMLINK_LEN 128
94
95 /*
96  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
97  * inode->i_private (with i_mutex making sure that it has only one user at
98  * a time): we would prefer not to enlarge the shmem inode just for that.
99  */
100 struct shmem_falloc {
101         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
102         pgoff_t start;          /* start of range currently being fallocated */
103         pgoff_t next;           /* the next page offset to be fallocated */
104         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
105         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
106 };
107
108 #ifdef CONFIG_TMPFS
109 static unsigned long shmem_default_max_blocks(void)
110 {
111         return totalram_pages / 2;
112 }
113
114 static unsigned long shmem_default_max_inodes(void)
115 {
116         return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
117 }
118 #endif
119
120 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
121 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
122                                 struct shmem_inode_info *info, pgoff_t index);
123 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
124                 struct page **pagep, enum sgp_type sgp,
125                 gfp_t gfp, struct vm_area_struct *vma,
126                 struct vm_fault *vmf, int *fault_type);
127
128 int shmem_getpage(struct inode *inode, pgoff_t index,
129                 struct page **pagep, enum sgp_type sgp)
130 {
131         return shmem_getpage_gfp(inode, index, pagep, sgp,
132                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
133 }
134
135 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
136 {
137         return sb->s_fs_info;
138 }
139
140 /*
141  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
142  * for shared memory and for shared anonymous (/dev/zero) mappings
143  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
144  * consistent with the pre-accounting of private mappings ...
145  */
146 static inline int shmem_acct_size(unsigned long flags, loff_t size)
147 {
148         return (flags & VM_NORESERVE) ?
149                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
150 }
151
152 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
153 {
154         if (!(flags & VM_NORESERVE))
155                 vm_unacct_memory(VM_ACCT(size));
156 }
157
158 static inline int shmem_reacct_size(unsigned long flags,
159                 loff_t oldsize, loff_t newsize)
160 {
161         if (!(flags & VM_NORESERVE)) {
162                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
163                         return security_vm_enough_memory_mm(current->mm,
164                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
165                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
166                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
167         }
168         return 0;
169 }
170
171 /*
172  * ... whereas tmpfs objects are accounted incrementally as
173  * pages are allocated, in order to allow large sparse files.
174  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
175  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
176  */
177 static inline int shmem_acct_block(unsigned long flags, long pages)
178 {
179         if (!(flags & VM_NORESERVE))
180                 return 0;
181
182         return security_vm_enough_memory_mm(current->mm,
183                         pages * VM_ACCT(PAGE_SIZE));
184 }
185
186 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
187 {
188         if (flags & VM_NORESERVE)
189                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
190 }
191
192 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
193 {
194         struct shmem_inode_info *info = SHMEM_I(inode);
195         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
196
197         if (shmem_acct_block(info->flags, pages))
198                 return false;
199
200         if (sbinfo->max_blocks) {
201                 if (percpu_counter_compare(&sbinfo->used_blocks,
202                                            sbinfo->max_blocks - pages) > 0)
203                         goto unacct;
204                 percpu_counter_add(&sbinfo->used_blocks, pages);
205         }
206
207         return true;
208
209 unacct:
210         shmem_unacct_blocks(info->flags, pages);
211         return false;
212 }
213
214 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
215 {
216         struct shmem_inode_info *info = SHMEM_I(inode);
217         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
218
219         if (sbinfo->max_blocks)
220                 percpu_counter_sub(&sbinfo->used_blocks, pages);
221         shmem_unacct_blocks(info->flags, pages);
222 }
223
224 static const struct super_operations shmem_ops;
225 static const struct address_space_operations shmem_aops;
226 static const struct file_operations shmem_file_operations;
227 static const struct inode_operations shmem_inode_operations;
228 static const struct inode_operations shmem_dir_inode_operations;
229 static const struct inode_operations shmem_special_inode_operations;
230 static const struct vm_operations_struct shmem_vm_ops;
231 static struct file_system_type shmem_fs_type;
232
233 bool vma_is_shmem(struct vm_area_struct *vma)
234 {
235         return vma->vm_ops == &shmem_vm_ops;
236 }
237
238 static LIST_HEAD(shmem_swaplist);
239 static DEFINE_MUTEX(shmem_swaplist_mutex);
240
241 static int shmem_reserve_inode(struct super_block *sb)
242 {
243         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
244         if (sbinfo->max_inodes) {
245                 spin_lock(&sbinfo->stat_lock);
246                 if (!sbinfo->free_inodes) {
247                         spin_unlock(&sbinfo->stat_lock);
248                         return -ENOSPC;
249                 }
250                 sbinfo->free_inodes--;
251                 spin_unlock(&sbinfo->stat_lock);
252         }
253         return 0;
254 }
255
256 static void shmem_free_inode(struct super_block *sb)
257 {
258         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
259         if (sbinfo->max_inodes) {
260                 spin_lock(&sbinfo->stat_lock);
261                 sbinfo->free_inodes++;
262                 spin_unlock(&sbinfo->stat_lock);
263         }
264 }
265
266 /**
267  * shmem_recalc_inode - recalculate the block usage of an inode
268  * @inode: inode to recalc
269  *
270  * We have to calculate the free blocks since the mm can drop
271  * undirtied hole pages behind our back.
272  *
273  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
274  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
275  *
276  * It has to be called with the spinlock held.
277  */
278 static void shmem_recalc_inode(struct inode *inode)
279 {
280         struct shmem_inode_info *info = SHMEM_I(inode);
281         long freed;
282
283         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
284         if (freed > 0) {
285                 info->alloced -= freed;
286                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
287                 shmem_inode_unacct_blocks(inode, freed);
288         }
289 }
290
291 bool shmem_charge(struct inode *inode, long pages)
292 {
293         struct shmem_inode_info *info = SHMEM_I(inode);
294         unsigned long flags;
295
296         if (!shmem_inode_acct_block(inode, pages))
297                 return false;
298
299         spin_lock_irqsave(&info->lock, flags);
300         info->alloced += pages;
301         inode->i_blocks += pages * BLOCKS_PER_PAGE;
302         shmem_recalc_inode(inode);
303         spin_unlock_irqrestore(&info->lock, flags);
304         inode->i_mapping->nrpages += pages;
305
306         return true;
307 }
308
309 void shmem_uncharge(struct inode *inode, long pages)
310 {
311         struct shmem_inode_info *info = SHMEM_I(inode);
312         unsigned long flags;
313
314         spin_lock_irqsave(&info->lock, flags);
315         info->alloced -= pages;
316         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
317         shmem_recalc_inode(inode);
318         spin_unlock_irqrestore(&info->lock, flags);
319
320         shmem_inode_unacct_blocks(inode, pages);
321 }
322
323 /*
324  * Replace item expected in radix tree by a new item, while holding tree lock.
325  */
326 static int shmem_radix_tree_replace(struct address_space *mapping,
327                         pgoff_t index, void *expected, void *replacement)
328 {
329         struct radix_tree_node *node;
330         void __rcu **pslot;
331         void *item;
332
333         VM_BUG_ON(!expected);
334         VM_BUG_ON(!replacement);
335         item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
336         if (!item)
337                 return -ENOENT;
338         if (item != expected)
339                 return -ENOENT;
340         __radix_tree_replace(&mapping->i_pages, node, pslot,
341                              replacement, NULL);
342         return 0;
343 }
344
345 /*
346  * Sometimes, before we decide whether to proceed or to fail, we must check
347  * that an entry was not already brought back from swap by a racing thread.
348  *
349  * Checking page is not enough: by the time a SwapCache page is locked, it
350  * might be reused, and again be SwapCache, using the same swap as before.
351  */
352 static bool shmem_confirm_swap(struct address_space *mapping,
353                                pgoff_t index, swp_entry_t swap)
354 {
355         void *item;
356
357         rcu_read_lock();
358         item = radix_tree_lookup(&mapping->i_pages, index);
359         rcu_read_unlock();
360         return item == swp_to_radix_entry(swap);
361 }
362
363 /*
364  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
365  *
366  * SHMEM_HUGE_NEVER:
367  *      disables huge pages for the mount;
368  * SHMEM_HUGE_ALWAYS:
369  *      enables huge pages for the mount;
370  * SHMEM_HUGE_WITHIN_SIZE:
371  *      only allocate huge pages if the page will be fully within i_size,
372  *      also respect fadvise()/madvise() hints;
373  * SHMEM_HUGE_ADVISE:
374  *      only allocate huge pages if requested with fadvise()/madvise();
375  */
376
377 #define SHMEM_HUGE_NEVER        0
378 #define SHMEM_HUGE_ALWAYS       1
379 #define SHMEM_HUGE_WITHIN_SIZE  2
380 #define SHMEM_HUGE_ADVISE       3
381
382 /*
383  * Special values.
384  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
385  *
386  * SHMEM_HUGE_DENY:
387  *      disables huge on shm_mnt and all mounts, for emergency use;
388  * SHMEM_HUGE_FORCE:
389  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
390  *
391  */
392 #define SHMEM_HUGE_DENY         (-1)
393 #define SHMEM_HUGE_FORCE        (-2)
394
395 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
396 /* ifdef here to avoid bloating shmem.o when not necessary */
397
398 static int shmem_huge __read_mostly;
399
400 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
401 static int shmem_parse_huge(const char *str)
402 {
403         if (!strcmp(str, "never"))
404                 return SHMEM_HUGE_NEVER;
405         if (!strcmp(str, "always"))
406                 return SHMEM_HUGE_ALWAYS;
407         if (!strcmp(str, "within_size"))
408                 return SHMEM_HUGE_WITHIN_SIZE;
409         if (!strcmp(str, "advise"))
410                 return SHMEM_HUGE_ADVISE;
411         if (!strcmp(str, "deny"))
412                 return SHMEM_HUGE_DENY;
413         if (!strcmp(str, "force"))
414                 return SHMEM_HUGE_FORCE;
415         return -EINVAL;
416 }
417
418 static const char *shmem_format_huge(int huge)
419 {
420         switch (huge) {
421         case SHMEM_HUGE_NEVER:
422                 return "never";
423         case SHMEM_HUGE_ALWAYS:
424                 return "always";
425         case SHMEM_HUGE_WITHIN_SIZE:
426                 return "within_size";
427         case SHMEM_HUGE_ADVISE:
428                 return "advise";
429         case SHMEM_HUGE_DENY:
430                 return "deny";
431         case SHMEM_HUGE_FORCE:
432                 return "force";
433         default:
434                 VM_BUG_ON(1);
435                 return "bad_val";
436         }
437 }
438 #endif
439
440 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
441                 struct shrink_control *sc, unsigned long nr_to_split)
442 {
443         LIST_HEAD(list), *pos, *next;
444         LIST_HEAD(to_remove);
445         struct inode *inode;
446         struct shmem_inode_info *info;
447         struct page *page;
448         unsigned long batch = sc ? sc->nr_to_scan : 128;
449         int removed = 0, split = 0;
450
451         if (list_empty(&sbinfo->shrinklist))
452                 return SHRINK_STOP;
453
454         spin_lock(&sbinfo->shrinklist_lock);
455         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
456                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
457
458                 /* pin the inode */
459                 inode = igrab(&info->vfs_inode);
460
461                 /* inode is about to be evicted */
462                 if (!inode) {
463                         list_del_init(&info->shrinklist);
464                         removed++;
465                         goto next;
466                 }
467
468                 /* Check if there's anything to gain */
469                 if (round_up(inode->i_size, PAGE_SIZE) ==
470                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
471                         list_move(&info->shrinklist, &to_remove);
472                         removed++;
473                         goto next;
474                 }
475
476                 list_move(&info->shrinklist, &list);
477 next:
478                 if (!--batch)
479                         break;
480         }
481         spin_unlock(&sbinfo->shrinklist_lock);
482
483         list_for_each_safe(pos, next, &to_remove) {
484                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
485                 inode = &info->vfs_inode;
486                 list_del_init(&info->shrinklist);
487                 iput(inode);
488         }
489
490         list_for_each_safe(pos, next, &list) {
491                 int ret;
492
493                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
494                 inode = &info->vfs_inode;
495
496                 if (nr_to_split && split >= nr_to_split)
497                         goto leave;
498
499                 page = find_get_page(inode->i_mapping,
500                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
501                 if (!page)
502                         goto drop;
503
504                 /* No huge page at the end of the file: nothing to split */
505                 if (!PageTransHuge(page)) {
506                         put_page(page);
507                         goto drop;
508                 }
509
510                 /*
511                  * Leave the inode on the list if we failed to lock
512                  * the page at this time.
513                  *
514                  * Waiting for the lock may lead to deadlock in the
515                  * reclaim path.
516                  */
517                 if (!trylock_page(page)) {
518                         put_page(page);
519                         goto leave;
520                 }
521
522                 ret = split_huge_page(page);
523                 unlock_page(page);
524                 put_page(page);
525
526                 /* If split failed leave the inode on the list */
527                 if (ret)
528                         goto leave;
529
530                 split++;
531 drop:
532                 list_del_init(&info->shrinklist);
533                 removed++;
534 leave:
535                 iput(inode);
536         }
537
538         spin_lock(&sbinfo->shrinklist_lock);
539         list_splice_tail(&list, &sbinfo->shrinklist);
540         sbinfo->shrinklist_len -= removed;
541         spin_unlock(&sbinfo->shrinklist_lock);
542
543         return split;
544 }
545
546 static long shmem_unused_huge_scan(struct super_block *sb,
547                 struct shrink_control *sc)
548 {
549         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
550
551         if (!READ_ONCE(sbinfo->shrinklist_len))
552                 return SHRINK_STOP;
553
554         return shmem_unused_huge_shrink(sbinfo, sc, 0);
555 }
556
557 static long shmem_unused_huge_count(struct super_block *sb,
558                 struct shrink_control *sc)
559 {
560         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
561         return READ_ONCE(sbinfo->shrinklist_len);
562 }
563 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
564
565 #define shmem_huge SHMEM_HUGE_DENY
566
567 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
568                 struct shrink_control *sc, unsigned long nr_to_split)
569 {
570         return 0;
571 }
572 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
573
574 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
575 {
576         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
577             (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
578             shmem_huge != SHMEM_HUGE_DENY)
579                 return true;
580         return false;
581 }
582
583 /*
584  * Like add_to_page_cache_locked, but error if expected item has gone.
585  */
586 static int shmem_add_to_page_cache(struct page *page,
587                                    struct address_space *mapping,
588                                    pgoff_t index, void *expected)
589 {
590         int error, nr = hpage_nr_pages(page);
591
592         VM_BUG_ON_PAGE(PageTail(page), page);
593         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
594         VM_BUG_ON_PAGE(!PageLocked(page), page);
595         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
596         VM_BUG_ON(expected && PageTransHuge(page));
597
598         page_ref_add(page, nr);
599         page->mapping = mapping;
600         page->index = index;
601
602         xa_lock_irq(&mapping->i_pages);
603         if (PageTransHuge(page)) {
604                 void __rcu **results;
605                 pgoff_t idx;
606                 int i;
607
608                 error = 0;
609                 if (radix_tree_gang_lookup_slot(&mapping->i_pages,
610                                         &results, &idx, index, 1) &&
611                                 idx < index + HPAGE_PMD_NR) {
612                         error = -EEXIST;
613                 }
614
615                 if (!error) {
616                         for (i = 0; i < HPAGE_PMD_NR; i++) {
617                                 error = radix_tree_insert(&mapping->i_pages,
618                                                 index + i, page + i);
619                                 VM_BUG_ON(error);
620                         }
621                         count_vm_event(THP_FILE_ALLOC);
622                 }
623         } else if (!expected) {
624                 error = radix_tree_insert(&mapping->i_pages, index, page);
625         } else {
626                 error = shmem_radix_tree_replace(mapping, index, expected,
627                                                                  page);
628         }
629
630         if (!error) {
631                 mapping->nrpages += nr;
632                 if (PageTransHuge(page))
633                         __inc_node_page_state(page, NR_SHMEM_THPS);
634                 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
635                 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
636                 xa_unlock_irq(&mapping->i_pages);
637         } else {
638                 page->mapping = NULL;
639                 xa_unlock_irq(&mapping->i_pages);
640                 page_ref_sub(page, nr);
641         }
642         return error;
643 }
644
645 /*
646  * Like delete_from_page_cache, but substitutes swap for page.
647  */
648 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
649 {
650         struct address_space *mapping = page->mapping;
651         int error;
652
653         VM_BUG_ON_PAGE(PageCompound(page), page);
654
655         xa_lock_irq(&mapping->i_pages);
656         error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
657         page->mapping = NULL;
658         mapping->nrpages--;
659         __dec_node_page_state(page, NR_FILE_PAGES);
660         __dec_node_page_state(page, NR_SHMEM);
661         xa_unlock_irq(&mapping->i_pages);
662         put_page(page);
663         BUG_ON(error);
664 }
665
666 /*
667  * Remove swap entry from radix tree, free the swap and its page cache.
668  */
669 static int shmem_free_swap(struct address_space *mapping,
670                            pgoff_t index, void *radswap)
671 {
672         void *old;
673
674         xa_lock_irq(&mapping->i_pages);
675         old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
676         xa_unlock_irq(&mapping->i_pages);
677         if (old != radswap)
678                 return -ENOENT;
679         free_swap_and_cache(radix_to_swp_entry(radswap));
680         return 0;
681 }
682
683 /*
684  * Determine (in bytes) how many of the shmem object's pages mapped by the
685  * given offsets are swapped out.
686  *
687  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
688  * as long as the inode doesn't go away and racy results are not a problem.
689  */
690 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
691                                                 pgoff_t start, pgoff_t end)
692 {
693         struct radix_tree_iter iter;
694         void __rcu **slot;
695         struct page *page;
696         unsigned long swapped = 0;
697
698         rcu_read_lock();
699
700         radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
701                 if (iter.index >= end)
702                         break;
703
704                 page = radix_tree_deref_slot(slot);
705
706                 if (radix_tree_deref_retry(page)) {
707                         slot = radix_tree_iter_retry(&iter);
708                         continue;
709                 }
710
711                 if (radix_tree_exceptional_entry(page))
712                         swapped++;
713
714                 if (need_resched()) {
715                         slot = radix_tree_iter_resume(slot, &iter);
716                         cond_resched_rcu();
717                 }
718         }
719
720         rcu_read_unlock();
721
722         return swapped << PAGE_SHIFT;
723 }
724
725 /*
726  * Determine (in bytes) how many of the shmem object's pages mapped by the
727  * given vma is swapped out.
728  *
729  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
730  * as long as the inode doesn't go away and racy results are not a problem.
731  */
732 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
733 {
734         struct inode *inode = file_inode(vma->vm_file);
735         struct shmem_inode_info *info = SHMEM_I(inode);
736         struct address_space *mapping = inode->i_mapping;
737         unsigned long swapped;
738
739         /* Be careful as we don't hold info->lock */
740         swapped = READ_ONCE(info->swapped);
741
742         /*
743          * The easier cases are when the shmem object has nothing in swap, or
744          * the vma maps it whole. Then we can simply use the stats that we
745          * already track.
746          */
747         if (!swapped)
748                 return 0;
749
750         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
751                 return swapped << PAGE_SHIFT;
752
753         /* Here comes the more involved part */
754         return shmem_partial_swap_usage(mapping,
755                         linear_page_index(vma, vma->vm_start),
756                         linear_page_index(vma, vma->vm_end));
757 }
758
759 /*
760  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
761  */
762 void shmem_unlock_mapping(struct address_space *mapping)
763 {
764         struct pagevec pvec;
765         pgoff_t indices[PAGEVEC_SIZE];
766         pgoff_t index = 0;
767
768         pagevec_init(&pvec);
769         /*
770          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
771          */
772         while (!mapping_unevictable(mapping)) {
773                 /*
774                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
775                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
776                  */
777                 pvec.nr = find_get_entries(mapping, index,
778                                            PAGEVEC_SIZE, pvec.pages, indices);
779                 if (!pvec.nr)
780                         break;
781                 index = indices[pvec.nr - 1] + 1;
782                 pagevec_remove_exceptionals(&pvec);
783                 check_move_unevictable_pages(pvec.pages, pvec.nr);
784                 pagevec_release(&pvec);
785                 cond_resched();
786         }
787 }
788
789 /*
790  * Remove range of pages and swap entries from radix tree, and free them.
791  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
792  */
793 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
794                                                                  bool unfalloc)
795 {
796         struct address_space *mapping = inode->i_mapping;
797         struct shmem_inode_info *info = SHMEM_I(inode);
798         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
799         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
800         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
801         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
802         struct pagevec pvec;
803         pgoff_t indices[PAGEVEC_SIZE];
804         long nr_swaps_freed = 0;
805         pgoff_t index;
806         int i;
807
808         if (lend == -1)
809                 end = -1;       /* unsigned, so actually very big */
810
811         pagevec_init(&pvec);
812         index = start;
813         while (index < end) {
814                 pvec.nr = find_get_entries(mapping, index,
815                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
816                         pvec.pages, indices);
817                 if (!pvec.nr)
818                         break;
819                 for (i = 0; i < pagevec_count(&pvec); i++) {
820                         struct page *page = pvec.pages[i];
821
822                         index = indices[i];
823                         if (index >= end)
824                                 break;
825
826                         if (radix_tree_exceptional_entry(page)) {
827                                 if (unfalloc)
828                                         continue;
829                                 nr_swaps_freed += !shmem_free_swap(mapping,
830                                                                 index, page);
831                                 continue;
832                         }
833
834                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
835
836                         if (!trylock_page(page))
837                                 continue;
838
839                         if (PageTransTail(page)) {
840                                 /* Middle of THP: zero out the page */
841                                 clear_highpage(page);
842                                 unlock_page(page);
843                                 continue;
844                         } else if (PageTransHuge(page)) {
845                                 if (index == round_down(end, HPAGE_PMD_NR)) {
846                                         /*
847                                          * Range ends in the middle of THP:
848                                          * zero out the page
849                                          */
850                                         clear_highpage(page);
851                                         unlock_page(page);
852                                         continue;
853                                 }
854                                 index += HPAGE_PMD_NR - 1;
855                                 i += HPAGE_PMD_NR - 1;
856                         }
857
858                         if (!unfalloc || !PageUptodate(page)) {
859                                 VM_BUG_ON_PAGE(PageTail(page), page);
860                                 if (page_mapping(page) == mapping) {
861                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
862                                         truncate_inode_page(mapping, page);
863                                 }
864                         }
865                         unlock_page(page);
866                 }
867                 pagevec_remove_exceptionals(&pvec);
868                 pagevec_release(&pvec);
869                 cond_resched();
870                 index++;
871         }
872
873         if (partial_start) {
874                 struct page *page = NULL;
875                 shmem_getpage(inode, start - 1, &page, SGP_READ);
876                 if (page) {
877                         unsigned int top = PAGE_SIZE;
878                         if (start > end) {
879                                 top = partial_end;
880                                 partial_end = 0;
881                         }
882                         zero_user_segment(page, partial_start, top);
883                         set_page_dirty(page);
884                         unlock_page(page);
885                         put_page(page);
886                 }
887         }
888         if (partial_end) {
889                 struct page *page = NULL;
890                 shmem_getpage(inode, end, &page, SGP_READ);
891                 if (page) {
892                         zero_user_segment(page, 0, partial_end);
893                         set_page_dirty(page);
894                         unlock_page(page);
895                         put_page(page);
896                 }
897         }
898         if (start >= end)
899                 return;
900
901         index = start;
902         while (index < end) {
903                 cond_resched();
904
905                 pvec.nr = find_get_entries(mapping, index,
906                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
907                                 pvec.pages, indices);
908                 if (!pvec.nr) {
909                         /* If all gone or hole-punch or unfalloc, we're done */
910                         if (index == start || end != -1)
911                                 break;
912                         /* But if truncating, restart to make sure all gone */
913                         index = start;
914                         continue;
915                 }
916                 for (i = 0; i < pagevec_count(&pvec); i++) {
917                         struct page *page = pvec.pages[i];
918
919                         index = indices[i];
920                         if (index >= end)
921                                 break;
922
923                         if (radix_tree_exceptional_entry(page)) {
924                                 if (unfalloc)
925                                         continue;
926                                 if (shmem_free_swap(mapping, index, page)) {
927                                         /* Swap was replaced by page: retry */
928                                         index--;
929                                         break;
930                                 }
931                                 nr_swaps_freed++;
932                                 continue;
933                         }
934
935                         lock_page(page);
936
937                         if (PageTransTail(page)) {
938                                 /* Middle of THP: zero out the page */
939                                 clear_highpage(page);
940                                 unlock_page(page);
941                                 /*
942                                  * Partial thp truncate due 'start' in middle
943                                  * of THP: don't need to look on these pages
944                                  * again on !pvec.nr restart.
945                                  */
946                                 if (index != round_down(end, HPAGE_PMD_NR))
947                                         start++;
948                                 continue;
949                         } else if (PageTransHuge(page)) {
950                                 if (index == round_down(end, HPAGE_PMD_NR)) {
951                                         /*
952                                          * Range ends in the middle of THP:
953                                          * zero out the page
954                                          */
955                                         clear_highpage(page);
956                                         unlock_page(page);
957                                         continue;
958                                 }
959                                 index += HPAGE_PMD_NR - 1;
960                                 i += HPAGE_PMD_NR - 1;
961                         }
962
963                         if (!unfalloc || !PageUptodate(page)) {
964                                 VM_BUG_ON_PAGE(PageTail(page), page);
965                                 if (page_mapping(page) == mapping) {
966                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
967                                         truncate_inode_page(mapping, page);
968                                 } else {
969                                         /* Page was replaced by swap: retry */
970                                         unlock_page(page);
971                                         index--;
972                                         break;
973                                 }
974                         }
975                         unlock_page(page);
976                 }
977                 pagevec_remove_exceptionals(&pvec);
978                 pagevec_release(&pvec);
979                 index++;
980         }
981
982         spin_lock_irq(&info->lock);
983         info->swapped -= nr_swaps_freed;
984         shmem_recalc_inode(inode);
985         spin_unlock_irq(&info->lock);
986 }
987
988 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
989 {
990         shmem_undo_range(inode, lstart, lend, false);
991         inode->i_ctime = inode->i_mtime = current_time(inode);
992 }
993 EXPORT_SYMBOL_GPL(shmem_truncate_range);
994
995 static int shmem_getattr(const struct path *path, struct kstat *stat,
996                          u32 request_mask, unsigned int query_flags)
997 {
998         struct inode *inode = path->dentry->d_inode;
999         struct shmem_inode_info *info = SHMEM_I(inode);
1000         struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
1001
1002         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1003                 spin_lock_irq(&info->lock);
1004                 shmem_recalc_inode(inode);
1005                 spin_unlock_irq(&info->lock);
1006         }
1007         generic_fillattr(inode, stat);
1008
1009         if (is_huge_enabled(sb_info))
1010                 stat->blksize = HPAGE_PMD_SIZE;
1011
1012         return 0;
1013 }
1014
1015 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1016 {
1017         struct inode *inode = d_inode(dentry);
1018         struct shmem_inode_info *info = SHMEM_I(inode);
1019         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1020         int error;
1021
1022         error = setattr_prepare(dentry, attr);
1023         if (error)
1024                 return error;
1025
1026         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1027                 loff_t oldsize = inode->i_size;
1028                 loff_t newsize = attr->ia_size;
1029
1030                 /* protected by i_mutex */
1031                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1032                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1033                         return -EPERM;
1034
1035                 if (newsize != oldsize) {
1036                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1037                                         oldsize, newsize);
1038                         if (error)
1039                                 return error;
1040                         i_size_write(inode, newsize);
1041                         inode->i_ctime = inode->i_mtime = current_time(inode);
1042                 }
1043                 if (newsize <= oldsize) {
1044                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1045                         if (oldsize > holebegin)
1046                                 unmap_mapping_range(inode->i_mapping,
1047                                                         holebegin, 0, 1);
1048                         if (info->alloced)
1049                                 shmem_truncate_range(inode,
1050                                                         newsize, (loff_t)-1);
1051                         /* unmap again to remove racily COWed private pages */
1052                         if (oldsize > holebegin)
1053                                 unmap_mapping_range(inode->i_mapping,
1054                                                         holebegin, 0, 1);
1055
1056                         /*
1057                          * Part of the huge page can be beyond i_size: subject
1058                          * to shrink under memory pressure.
1059                          */
1060                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1061                                 spin_lock(&sbinfo->shrinklist_lock);
1062                                 /*
1063                                  * _careful to defend against unlocked access to
1064                                  * ->shrink_list in shmem_unused_huge_shrink()
1065                                  */
1066                                 if (list_empty_careful(&info->shrinklist)) {
1067                                         list_add_tail(&info->shrinklist,
1068                                                         &sbinfo->shrinklist);
1069                                         sbinfo->shrinklist_len++;
1070                                 }
1071                                 spin_unlock(&sbinfo->shrinklist_lock);
1072                         }
1073                 }
1074         }
1075
1076         setattr_copy(inode, attr);
1077         if (attr->ia_valid & ATTR_MODE)
1078                 error = posix_acl_chmod(inode, inode->i_mode);
1079         return error;
1080 }
1081
1082 static void shmem_evict_inode(struct inode *inode)
1083 {
1084         struct shmem_inode_info *info = SHMEM_I(inode);
1085         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1086
1087         if (inode->i_mapping->a_ops == &shmem_aops) {
1088                 shmem_unacct_size(info->flags, inode->i_size);
1089                 inode->i_size = 0;
1090                 shmem_truncate_range(inode, 0, (loff_t)-1);
1091                 if (!list_empty(&info->shrinklist)) {
1092                         spin_lock(&sbinfo->shrinklist_lock);
1093                         if (!list_empty(&info->shrinklist)) {
1094                                 list_del_init(&info->shrinklist);
1095                                 sbinfo->shrinklist_len--;
1096                         }
1097                         spin_unlock(&sbinfo->shrinklist_lock);
1098                 }
1099                 if (!list_empty(&info->swaplist)) {
1100                         mutex_lock(&shmem_swaplist_mutex);
1101                         list_del_init(&info->swaplist);
1102                         mutex_unlock(&shmem_swaplist_mutex);
1103                 }
1104         }
1105
1106         simple_xattrs_free(&info->xattrs);
1107         WARN_ON(inode->i_blocks);
1108         shmem_free_inode(inode->i_sb);
1109         clear_inode(inode);
1110 }
1111
1112 static unsigned long find_swap_entry(struct radix_tree_root *root, void *item)
1113 {
1114         struct radix_tree_iter iter;
1115         void __rcu **slot;
1116         unsigned long found = -1;
1117         unsigned int checked = 0;
1118
1119         rcu_read_lock();
1120         radix_tree_for_each_slot(slot, root, &iter, 0) {
1121                 void *entry = radix_tree_deref_slot(slot);
1122
1123                 if (radix_tree_deref_retry(entry)) {
1124                         slot = radix_tree_iter_retry(&iter);
1125                         continue;
1126                 }
1127                 if (entry == item) {
1128                         found = iter.index;
1129                         break;
1130                 }
1131                 checked++;
1132                 if ((checked % 4096) != 0)
1133                         continue;
1134                 slot = radix_tree_iter_resume(slot, &iter);
1135                 cond_resched_rcu();
1136         }
1137
1138         rcu_read_unlock();
1139         return found;
1140 }
1141
1142 /*
1143  * If swap found in inode, free it and move page from swapcache to filecache.
1144  */
1145 static int shmem_unuse_inode(struct shmem_inode_info *info,
1146                              swp_entry_t swap, struct page **pagep)
1147 {
1148         struct address_space *mapping = info->vfs_inode.i_mapping;
1149         void *radswap;
1150         pgoff_t index;
1151         gfp_t gfp;
1152         int error = 0;
1153
1154         radswap = swp_to_radix_entry(swap);
1155         index = find_swap_entry(&mapping->i_pages, radswap);
1156         if (index == -1)
1157                 return -EAGAIN; /* tell shmem_unuse we found nothing */
1158
1159         /*
1160          * Move _head_ to start search for next from here.
1161          * But be careful: shmem_evict_inode checks list_empty without taking
1162          * mutex, and there's an instant in list_move_tail when info->swaplist
1163          * would appear empty, if it were the only one on shmem_swaplist.
1164          */
1165         if (shmem_swaplist.next != &info->swaplist)
1166                 list_move_tail(&shmem_swaplist, &info->swaplist);
1167
1168         gfp = mapping_gfp_mask(mapping);
1169         if (shmem_should_replace_page(*pagep, gfp)) {
1170                 mutex_unlock(&shmem_swaplist_mutex);
1171                 error = shmem_replace_page(pagep, gfp, info, index);
1172                 mutex_lock(&shmem_swaplist_mutex);
1173                 /*
1174                  * We needed to drop mutex to make that restrictive page
1175                  * allocation, but the inode might have been freed while we
1176                  * dropped it: although a racing shmem_evict_inode() cannot
1177                  * complete without emptying the radix_tree, our page lock
1178                  * on this swapcache page is not enough to prevent that -
1179                  * free_swap_and_cache() of our swap entry will only
1180                  * trylock_page(), removing swap from radix_tree whatever.
1181                  *
1182                  * We must not proceed to shmem_add_to_page_cache() if the
1183                  * inode has been freed, but of course we cannot rely on
1184                  * inode or mapping or info to check that.  However, we can
1185                  * safely check if our swap entry is still in use (and here
1186                  * it can't have got reused for another page): if it's still
1187                  * in use, then the inode cannot have been freed yet, and we
1188                  * can safely proceed (if it's no longer in use, that tells
1189                  * nothing about the inode, but we don't need to unuse swap).
1190                  */
1191                 if (!page_swapcount(*pagep))
1192                         error = -ENOENT;
1193         }
1194
1195         /*
1196          * We rely on shmem_swaplist_mutex, not only to protect the swaplist,
1197          * but also to hold up shmem_evict_inode(): so inode cannot be freed
1198          * beneath us (pagelock doesn't help until the page is in pagecache).
1199          */
1200         if (!error)
1201                 error = shmem_add_to_page_cache(*pagep, mapping, index,
1202                                                 radswap);
1203         if (error != -ENOMEM) {
1204                 /*
1205                  * Truncation and eviction use free_swap_and_cache(), which
1206                  * only does trylock page: if we raced, best clean up here.
1207                  */
1208                 delete_from_swap_cache(*pagep);
1209                 set_page_dirty(*pagep);
1210                 if (!error) {
1211                         spin_lock_irq(&info->lock);
1212                         info->swapped--;
1213                         spin_unlock_irq(&info->lock);
1214                         swap_free(swap);
1215                 }
1216         }
1217         return error;
1218 }
1219
1220 /*
1221  * Search through swapped inodes to find and replace swap by page.
1222  */
1223 int shmem_unuse(swp_entry_t swap, struct page *page)
1224 {
1225         struct list_head *this, *next;
1226         struct shmem_inode_info *info;
1227         struct mem_cgroup *memcg;
1228         int error = 0;
1229
1230         /*
1231          * There's a faint possibility that swap page was replaced before
1232          * caller locked it: caller will come back later with the right page.
1233          */
1234         if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val))
1235                 goto out;
1236
1237         /*
1238          * Charge page using GFP_KERNEL while we can wait, before taking
1239          * the shmem_swaplist_mutex which might hold up shmem_writepage().
1240          * Charged back to the user (not to caller) when swap account is used.
1241          */
1242         error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg,
1243                         false);
1244         if (error)
1245                 goto out;
1246         /* No radix_tree_preload: swap entry keeps a place for page in tree */
1247         error = -EAGAIN;
1248
1249         mutex_lock(&shmem_swaplist_mutex);
1250         list_for_each_safe(this, next, &shmem_swaplist) {
1251                 info = list_entry(this, struct shmem_inode_info, swaplist);
1252                 if (info->swapped)
1253                         error = shmem_unuse_inode(info, swap, &page);
1254                 else
1255                         list_del_init(&info->swaplist);
1256                 cond_resched();
1257                 if (error != -EAGAIN)
1258                         break;
1259                 /* found nothing in this: move on to search the next */
1260         }
1261         mutex_unlock(&shmem_swaplist_mutex);
1262
1263         if (error) {
1264                 if (error != -ENOMEM)
1265                         error = 0;
1266                 mem_cgroup_cancel_charge(page, memcg, false);
1267         } else
1268                 mem_cgroup_commit_charge(page, memcg, true, false);
1269 out:
1270         unlock_page(page);
1271         put_page(page);
1272         return error;
1273 }
1274
1275 /*
1276  * Move the page from the page cache to the swap cache.
1277  */
1278 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1279 {
1280         struct shmem_inode_info *info;
1281         struct address_space *mapping;
1282         struct inode *inode;
1283         swp_entry_t swap;
1284         pgoff_t index;
1285
1286         VM_BUG_ON_PAGE(PageCompound(page), page);
1287         BUG_ON(!PageLocked(page));
1288         mapping = page->mapping;
1289         index = page->index;
1290         inode = mapping->host;
1291         info = SHMEM_I(inode);
1292         if (info->flags & VM_LOCKED)
1293                 goto redirty;
1294         if (!total_swap_pages)
1295                 goto redirty;
1296
1297         /*
1298          * Our capabilities prevent regular writeback or sync from ever calling
1299          * shmem_writepage; but a stacking filesystem might use ->writepage of
1300          * its underlying filesystem, in which case tmpfs should write out to
1301          * swap only in response to memory pressure, and not for the writeback
1302          * threads or sync.
1303          */
1304         if (!wbc->for_reclaim) {
1305                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1306                 goto redirty;
1307         }
1308
1309         /*
1310          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1311          * value into swapfile.c, the only way we can correctly account for a
1312          * fallocated page arriving here is now to initialize it and write it.
1313          *
1314          * That's okay for a page already fallocated earlier, but if we have
1315          * not yet completed the fallocation, then (a) we want to keep track
1316          * of this page in case we have to undo it, and (b) it may not be a
1317          * good idea to continue anyway, once we're pushing into swap.  So
1318          * reactivate the page, and let shmem_fallocate() quit when too many.
1319          */
1320         if (!PageUptodate(page)) {
1321                 if (inode->i_private) {
1322                         struct shmem_falloc *shmem_falloc;
1323                         spin_lock(&inode->i_lock);
1324                         shmem_falloc = inode->i_private;
1325                         if (shmem_falloc &&
1326                             !shmem_falloc->waitq &&
1327                             index >= shmem_falloc->start &&
1328                             index < shmem_falloc->next)
1329                                 shmem_falloc->nr_unswapped++;
1330                         else
1331                                 shmem_falloc = NULL;
1332                         spin_unlock(&inode->i_lock);
1333                         if (shmem_falloc)
1334                                 goto redirty;
1335                 }
1336                 clear_highpage(page);
1337                 flush_dcache_page(page);
1338                 SetPageUptodate(page);
1339         }
1340
1341         swap = get_swap_page(page);
1342         if (!swap.val)
1343                 goto redirty;
1344
1345         /*
1346          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1347          * if it's not already there.  Do it now before the page is
1348          * moved to swap cache, when its pagelock no longer protects
1349          * the inode from eviction.  But don't unlock the mutex until
1350          * we've incremented swapped, because shmem_unuse_inode() will
1351          * prune a !swapped inode from the swaplist under this mutex.
1352          */
1353         mutex_lock(&shmem_swaplist_mutex);
1354         if (list_empty(&info->swaplist))
1355                 list_add_tail(&info->swaplist, &shmem_swaplist);
1356
1357         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1358                 spin_lock_irq(&info->lock);
1359                 shmem_recalc_inode(inode);
1360                 info->swapped++;
1361                 spin_unlock_irq(&info->lock);
1362
1363                 swap_shmem_alloc(swap);
1364                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1365
1366                 mutex_unlock(&shmem_swaplist_mutex);
1367                 BUG_ON(page_mapped(page));
1368                 swap_writepage(page, wbc);
1369                 return 0;
1370         }
1371
1372         mutex_unlock(&shmem_swaplist_mutex);
1373         put_swap_page(page, swap);
1374 redirty:
1375         set_page_dirty(page);
1376         if (wbc->for_reclaim)
1377                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1378         unlock_page(page);
1379         return 0;
1380 }
1381
1382 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1383 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1384 {
1385         char buffer[64];
1386
1387         if (!mpol || mpol->mode == MPOL_DEFAULT)
1388                 return;         /* show nothing */
1389
1390         mpol_to_str(buffer, sizeof(buffer), mpol);
1391
1392         seq_printf(seq, ",mpol=%s", buffer);
1393 }
1394
1395 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1396 {
1397         struct mempolicy *mpol = NULL;
1398         if (sbinfo->mpol) {
1399                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1400                 mpol = sbinfo->mpol;
1401                 mpol_get(mpol);
1402                 spin_unlock(&sbinfo->stat_lock);
1403         }
1404         return mpol;
1405 }
1406 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1407 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1408 {
1409 }
1410 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1411 {
1412         return NULL;
1413 }
1414 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1415 #ifndef CONFIG_NUMA
1416 #define vm_policy vm_private_data
1417 #endif
1418
1419 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1420                 struct shmem_inode_info *info, pgoff_t index)
1421 {
1422         /* Create a pseudo vma that just contains the policy */
1423         memset(vma, 0, sizeof(*vma));
1424         /* Bias interleave by inode number to distribute better across nodes */
1425         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1426         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1427 }
1428
1429 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1430 {
1431         /* Drop reference taken by mpol_shared_policy_lookup() */
1432         mpol_cond_put(vma->vm_policy);
1433 }
1434
1435 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1436                         struct shmem_inode_info *info, pgoff_t index)
1437 {
1438         struct vm_area_struct pvma;
1439         struct page *page;
1440         struct vm_fault vmf;
1441
1442         shmem_pseudo_vma_init(&pvma, info, index);
1443         vmf.vma = &pvma;
1444         vmf.address = 0;
1445         page = swap_cluster_readahead(swap, gfp, &vmf);
1446         shmem_pseudo_vma_destroy(&pvma);
1447
1448         return page;
1449 }
1450
1451 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1452                 struct shmem_inode_info *info, pgoff_t index)
1453 {
1454         struct vm_area_struct pvma;
1455         struct inode *inode = &info->vfs_inode;
1456         struct address_space *mapping = inode->i_mapping;
1457         pgoff_t idx, hindex;
1458         void __rcu **results;
1459         struct page *page;
1460
1461         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1462                 return NULL;
1463
1464         hindex = round_down(index, HPAGE_PMD_NR);
1465         rcu_read_lock();
1466         if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1467                                 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1468                 rcu_read_unlock();
1469                 return NULL;
1470         }
1471         rcu_read_unlock();
1472
1473         shmem_pseudo_vma_init(&pvma, info, hindex);
1474         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1475                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1476         shmem_pseudo_vma_destroy(&pvma);
1477         if (page)
1478                 prep_transhuge_page(page);
1479         return page;
1480 }
1481
1482 static struct page *shmem_alloc_page(gfp_t gfp,
1483                         struct shmem_inode_info *info, pgoff_t index)
1484 {
1485         struct vm_area_struct pvma;
1486         struct page *page;
1487
1488         shmem_pseudo_vma_init(&pvma, info, index);
1489         page = alloc_page_vma(gfp, &pvma, 0);
1490         shmem_pseudo_vma_destroy(&pvma);
1491
1492         return page;
1493 }
1494
1495 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1496                 struct inode *inode,
1497                 pgoff_t index, bool huge)
1498 {
1499         struct shmem_inode_info *info = SHMEM_I(inode);
1500         struct page *page;
1501         int nr;
1502         int err = -ENOSPC;
1503
1504         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1505                 huge = false;
1506         nr = huge ? HPAGE_PMD_NR : 1;
1507
1508         if (!shmem_inode_acct_block(inode, nr))
1509                 goto failed;
1510
1511         if (huge)
1512                 page = shmem_alloc_hugepage(gfp, info, index);
1513         else
1514                 page = shmem_alloc_page(gfp, info, index);
1515         if (page) {
1516                 __SetPageLocked(page);
1517                 __SetPageSwapBacked(page);
1518                 return page;
1519         }
1520
1521         err = -ENOMEM;
1522         shmem_inode_unacct_blocks(inode, nr);
1523 failed:
1524         return ERR_PTR(err);
1525 }
1526
1527 /*
1528  * When a page is moved from swapcache to shmem filecache (either by the
1529  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1530  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1531  * ignorance of the mapping it belongs to.  If that mapping has special
1532  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1533  * we may need to copy to a suitable page before moving to filecache.
1534  *
1535  * In a future release, this may well be extended to respect cpuset and
1536  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1537  * but for now it is a simple matter of zone.
1538  */
1539 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1540 {
1541         return page_zonenum(page) > gfp_zone(gfp);
1542 }
1543
1544 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1545                                 struct shmem_inode_info *info, pgoff_t index)
1546 {
1547         struct page *oldpage, *newpage;
1548         struct address_space *swap_mapping;
1549         pgoff_t swap_index;
1550         int error;
1551
1552         oldpage = *pagep;
1553         swap_index = page_private(oldpage);
1554         swap_mapping = page_mapping(oldpage);
1555
1556         /*
1557          * We have arrived here because our zones are constrained, so don't
1558          * limit chance of success by further cpuset and node constraints.
1559          */
1560         gfp &= ~GFP_CONSTRAINT_MASK;
1561         newpage = shmem_alloc_page(gfp, info, index);
1562         if (!newpage)
1563                 return -ENOMEM;
1564
1565         get_page(newpage);
1566         copy_highpage(newpage, oldpage);
1567         flush_dcache_page(newpage);
1568
1569         __SetPageLocked(newpage);
1570         __SetPageSwapBacked(newpage);
1571         SetPageUptodate(newpage);
1572         set_page_private(newpage, swap_index);
1573         SetPageSwapCache(newpage);
1574
1575         /*
1576          * Our caller will very soon move newpage out of swapcache, but it's
1577          * a nice clean interface for us to replace oldpage by newpage there.
1578          */
1579         xa_lock_irq(&swap_mapping->i_pages);
1580         error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1581                                                                    newpage);
1582         if (!error) {
1583                 __inc_node_page_state(newpage, NR_FILE_PAGES);
1584                 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1585         }
1586         xa_unlock_irq(&swap_mapping->i_pages);
1587
1588         if (unlikely(error)) {
1589                 /*
1590                  * Is this possible?  I think not, now that our callers check
1591                  * both PageSwapCache and page_private after getting page lock;
1592                  * but be defensive.  Reverse old to newpage for clear and free.
1593                  */
1594                 oldpage = newpage;
1595         } else {
1596                 mem_cgroup_migrate(oldpage, newpage);
1597                 lru_cache_add_anon(newpage);
1598                 *pagep = newpage;
1599         }
1600
1601         ClearPageSwapCache(oldpage);
1602         set_page_private(oldpage, 0);
1603
1604         unlock_page(oldpage);
1605         put_page(oldpage);
1606         put_page(oldpage);
1607         return error;
1608 }
1609
1610 /*
1611  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1612  *
1613  * If we allocate a new one we do not mark it dirty. That's up to the
1614  * vm. If we swap it in we mark it dirty since we also free the swap
1615  * entry since a page cannot live in both the swap and page cache.
1616  *
1617  * fault_mm and fault_type are only supplied by shmem_fault:
1618  * otherwise they are NULL.
1619  */
1620 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1621         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1622         struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type)
1623 {
1624         struct address_space *mapping = inode->i_mapping;
1625         struct shmem_inode_info *info = SHMEM_I(inode);
1626         struct shmem_sb_info *sbinfo;
1627         struct mm_struct *charge_mm;
1628         struct mem_cgroup *memcg;
1629         struct page *page;
1630         swp_entry_t swap;
1631         enum sgp_type sgp_huge = sgp;
1632         pgoff_t hindex = index;
1633         int error;
1634         int once = 0;
1635         int alloced = 0;
1636
1637         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1638                 return -EFBIG;
1639         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1640                 sgp = SGP_CACHE;
1641 repeat:
1642         swap.val = 0;
1643         page = find_lock_entry(mapping, index);
1644         if (radix_tree_exceptional_entry(page)) {
1645                 swap = radix_to_swp_entry(page);
1646                 page = NULL;
1647         }
1648
1649         if (sgp <= SGP_CACHE &&
1650             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1651                 error = -EINVAL;
1652                 goto unlock;
1653         }
1654
1655         if (page && sgp == SGP_WRITE)
1656                 mark_page_accessed(page);
1657
1658         /* fallocated page? */
1659         if (page && !PageUptodate(page)) {
1660                 if (sgp != SGP_READ)
1661                         goto clear;
1662                 unlock_page(page);
1663                 put_page(page);
1664                 page = NULL;
1665         }
1666         if (page || (sgp == SGP_READ && !swap.val)) {
1667                 *pagep = page;
1668                 return 0;
1669         }
1670
1671         /*
1672          * Fast cache lookup did not find it:
1673          * bring it back from swap or allocate.
1674          */
1675         sbinfo = SHMEM_SB(inode->i_sb);
1676         charge_mm = vma ? vma->vm_mm : current->mm;
1677
1678         if (swap.val) {
1679                 /* Look it up and read it in.. */
1680                 page = lookup_swap_cache(swap, NULL, 0);
1681                 if (!page) {
1682                         /* Or update major stats only when swapin succeeds?? */
1683                         if (fault_type) {
1684                                 *fault_type |= VM_FAULT_MAJOR;
1685                                 count_vm_event(PGMAJFAULT);
1686                                 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1687                         }
1688                         /* Here we actually start the io */
1689                         page = shmem_swapin(swap, gfp, info, index);
1690                         if (!page) {
1691                                 error = -ENOMEM;
1692                                 goto failed;
1693                         }
1694                 }
1695
1696                 /* We have to do this with page locked to prevent races */
1697                 lock_page(page);
1698                 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1699                     !shmem_confirm_swap(mapping, index, swap)) {
1700                         error = -EEXIST;        /* try again */
1701                         goto unlock;
1702                 }
1703                 if (!PageUptodate(page)) {
1704                         error = -EIO;
1705                         goto failed;
1706                 }
1707                 wait_on_page_writeback(page);
1708
1709                 if (shmem_should_replace_page(page, gfp)) {
1710                         error = shmem_replace_page(&page, gfp, info, index);
1711                         if (error)
1712                                 goto failed;
1713                 }
1714
1715                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1716                                 false);
1717                 if (!error) {
1718                         error = shmem_add_to_page_cache(page, mapping, index,
1719                                                 swp_to_radix_entry(swap));
1720                         /*
1721                          * We already confirmed swap under page lock, and make
1722                          * no memory allocation here, so usually no possibility
1723                          * of error; but free_swap_and_cache() only trylocks a
1724                          * page, so it is just possible that the entry has been
1725                          * truncated or holepunched since swap was confirmed.
1726                          * shmem_undo_range() will have done some of the
1727                          * unaccounting, now delete_from_swap_cache() will do
1728                          * the rest.
1729                          * Reset swap.val? No, leave it so "failed" goes back to
1730                          * "repeat": reading a hole and writing should succeed.
1731                          */
1732                         if (error) {
1733                                 mem_cgroup_cancel_charge(page, memcg, false);
1734                                 delete_from_swap_cache(page);
1735                         }
1736                 }
1737                 if (error)
1738                         goto failed;
1739
1740                 mem_cgroup_commit_charge(page, memcg, true, false);
1741
1742                 spin_lock_irq(&info->lock);
1743                 info->swapped--;
1744                 shmem_recalc_inode(inode);
1745                 spin_unlock_irq(&info->lock);
1746
1747                 if (sgp == SGP_WRITE)
1748                         mark_page_accessed(page);
1749
1750                 delete_from_swap_cache(page);
1751                 set_page_dirty(page);
1752                 swap_free(swap);
1753
1754         } else {
1755                 if (vma && userfaultfd_missing(vma)) {
1756                         *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1757                         return 0;
1758                 }
1759
1760                 /* shmem_symlink() */
1761                 if (mapping->a_ops != &shmem_aops)
1762                         goto alloc_nohuge;
1763                 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1764                         goto alloc_nohuge;
1765                 if (shmem_huge == SHMEM_HUGE_FORCE)
1766                         goto alloc_huge;
1767                 switch (sbinfo->huge) {
1768                         loff_t i_size;
1769                         pgoff_t off;
1770                 case SHMEM_HUGE_NEVER:
1771                         goto alloc_nohuge;
1772                 case SHMEM_HUGE_WITHIN_SIZE:
1773                         off = round_up(index, HPAGE_PMD_NR);
1774                         i_size = round_up(i_size_read(inode), PAGE_SIZE);
1775                         if (i_size >= HPAGE_PMD_SIZE &&
1776                                         i_size >> PAGE_SHIFT >= off)
1777                                 goto alloc_huge;
1778                         /* fallthrough */
1779                 case SHMEM_HUGE_ADVISE:
1780                         if (sgp_huge == SGP_HUGE)
1781                                 goto alloc_huge;
1782                         /* TODO: implement fadvise() hints */
1783                         goto alloc_nohuge;
1784                 }
1785
1786 alloc_huge:
1787                 page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1788                 if (IS_ERR(page)) {
1789 alloc_nohuge:           page = shmem_alloc_and_acct_page(gfp, inode,
1790                                         index, false);
1791                 }
1792                 if (IS_ERR(page)) {
1793                         int retry = 5;
1794                         error = PTR_ERR(page);
1795                         page = NULL;
1796                         if (error != -ENOSPC)
1797                                 goto failed;
1798                         /*
1799                          * Try to reclaim some spece by splitting a huge page
1800                          * beyond i_size on the filesystem.
1801                          */
1802                         while (retry--) {
1803                                 int ret;
1804                                 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1805                                 if (ret == SHRINK_STOP)
1806                                         break;
1807                                 if (ret)
1808                                         goto alloc_nohuge;
1809                         }
1810                         goto failed;
1811                 }
1812
1813                 if (PageTransHuge(page))
1814                         hindex = round_down(index, HPAGE_PMD_NR);
1815                 else
1816                         hindex = index;
1817
1818                 if (sgp == SGP_WRITE)
1819                         __SetPageReferenced(page);
1820
1821                 error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg,
1822                                 PageTransHuge(page));
1823                 if (error)
1824                         goto unacct;
1825                 error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK,
1826                                 compound_order(page));
1827                 if (!error) {
1828                         error = shmem_add_to_page_cache(page, mapping, hindex,
1829                                                         NULL);
1830                         radix_tree_preload_end();
1831                 }
1832                 if (error) {
1833                         mem_cgroup_cancel_charge(page, memcg,
1834                                         PageTransHuge(page));
1835                         goto unacct;
1836                 }
1837                 mem_cgroup_commit_charge(page, memcg, false,
1838                                 PageTransHuge(page));
1839                 lru_cache_add_anon(page);
1840
1841                 spin_lock_irq(&info->lock);
1842                 info->alloced += 1 << compound_order(page);
1843                 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1844                 shmem_recalc_inode(inode);
1845                 spin_unlock_irq(&info->lock);
1846                 alloced = true;
1847
1848                 if (PageTransHuge(page) &&
1849                                 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1850                                 hindex + HPAGE_PMD_NR - 1) {
1851                         /*
1852                          * Part of the huge page is beyond i_size: subject
1853                          * to shrink under memory pressure.
1854                          */
1855                         spin_lock(&sbinfo->shrinklist_lock);
1856                         /*
1857                          * _careful to defend against unlocked access to
1858                          * ->shrink_list in shmem_unused_huge_shrink()
1859                          */
1860                         if (list_empty_careful(&info->shrinklist)) {
1861                                 list_add_tail(&info->shrinklist,
1862                                                 &sbinfo->shrinklist);
1863                                 sbinfo->shrinklist_len++;
1864                         }
1865                         spin_unlock(&sbinfo->shrinklist_lock);
1866                 }
1867
1868                 /*
1869                  * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1870                  */
1871                 if (sgp == SGP_FALLOC)
1872                         sgp = SGP_WRITE;
1873 clear:
1874                 /*
1875                  * Let SGP_WRITE caller clear ends if write does not fill page;
1876                  * but SGP_FALLOC on a page fallocated earlier must initialize
1877                  * it now, lest undo on failure cancel our earlier guarantee.
1878                  */
1879                 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1880                         struct page *head = compound_head(page);
1881                         int i;
1882
1883                         for (i = 0; i < (1 << compound_order(head)); i++) {
1884                                 clear_highpage(head + i);
1885                                 flush_dcache_page(head + i);
1886                         }
1887                         SetPageUptodate(head);
1888                 }
1889         }
1890
1891         /* Perhaps the file has been truncated since we checked */
1892         if (sgp <= SGP_CACHE &&
1893             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1894                 if (alloced) {
1895                         ClearPageDirty(page);
1896                         delete_from_page_cache(page);
1897                         spin_lock_irq(&info->lock);
1898                         shmem_recalc_inode(inode);
1899                         spin_unlock_irq(&info->lock);
1900                 }
1901                 error = -EINVAL;
1902                 goto unlock;
1903         }
1904         *pagep = page + index - hindex;
1905         return 0;
1906
1907         /*
1908          * Error recovery.
1909          */
1910 unacct:
1911         shmem_inode_unacct_blocks(inode, 1 << compound_order(page));
1912
1913         if (PageTransHuge(page)) {
1914                 unlock_page(page);
1915                 put_page(page);
1916                 goto alloc_nohuge;
1917         }
1918 failed:
1919         if (swap.val && !shmem_confirm_swap(mapping, index, swap))
1920                 error = -EEXIST;
1921 unlock:
1922         if (page) {
1923                 unlock_page(page);
1924                 put_page(page);
1925         }
1926         if (error == -ENOSPC && !once++) {
1927                 spin_lock_irq(&info->lock);
1928                 shmem_recalc_inode(inode);
1929                 spin_unlock_irq(&info->lock);
1930                 goto repeat;
1931         }
1932         if (error == -EEXIST)   /* from above or from radix_tree_insert */
1933                 goto repeat;
1934         return error;
1935 }
1936
1937 /*
1938  * This is like autoremove_wake_function, but it removes the wait queue
1939  * entry unconditionally - even if something else had already woken the
1940  * target.
1941  */
1942 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1943 {
1944         int ret = default_wake_function(wait, mode, sync, key);
1945         list_del_init(&wait->entry);
1946         return ret;
1947 }
1948
1949 static vm_fault_t shmem_fault(struct vm_fault *vmf)
1950 {
1951         struct vm_area_struct *vma = vmf->vma;
1952         struct inode *inode = file_inode(vma->vm_file);
1953         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1954         enum sgp_type sgp;
1955         int err;
1956         vm_fault_t ret = VM_FAULT_LOCKED;
1957
1958         /*
1959          * Trinity finds that probing a hole which tmpfs is punching can
1960          * prevent the hole-punch from ever completing: which in turn
1961          * locks writers out with its hold on i_mutex.  So refrain from
1962          * faulting pages into the hole while it's being punched.  Although
1963          * shmem_undo_range() does remove the additions, it may be unable to
1964          * keep up, as each new page needs its own unmap_mapping_range() call,
1965          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1966          *
1967          * It does not matter if we sometimes reach this check just before the
1968          * hole-punch begins, so that one fault then races with the punch:
1969          * we just need to make racing faults a rare case.
1970          *
1971          * The implementation below would be much simpler if we just used a
1972          * standard mutex or completion: but we cannot take i_mutex in fault,
1973          * and bloating every shmem inode for this unlikely case would be sad.
1974          */
1975         if (unlikely(inode->i_private)) {
1976                 struct shmem_falloc *shmem_falloc;
1977
1978                 spin_lock(&inode->i_lock);
1979                 shmem_falloc = inode->i_private;
1980                 if (shmem_falloc &&
1981                     shmem_falloc->waitq &&
1982                     vmf->pgoff >= shmem_falloc->start &&
1983                     vmf->pgoff < shmem_falloc->next) {
1984                         wait_queue_head_t *shmem_falloc_waitq;
1985                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
1986
1987                         ret = VM_FAULT_NOPAGE;
1988                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
1989                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
1990                                 /* It's polite to up mmap_sem if we can */
1991                                 up_read(&vma->vm_mm->mmap_sem);
1992                                 ret = VM_FAULT_RETRY;
1993                         }
1994
1995                         shmem_falloc_waitq = shmem_falloc->waitq;
1996                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
1997                                         TASK_UNINTERRUPTIBLE);
1998                         spin_unlock(&inode->i_lock);
1999                         schedule();
2000
2001                         /*
2002                          * shmem_falloc_waitq points into the shmem_fallocate()
2003                          * stack of the hole-punching task: shmem_falloc_waitq
2004                          * is usually invalid by the time we reach here, but
2005                          * finish_wait() does not dereference it in that case;
2006                          * though i_lock needed lest racing with wake_up_all().
2007                          */
2008                         spin_lock(&inode->i_lock);
2009                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2010                         spin_unlock(&inode->i_lock);
2011                         return ret;
2012                 }
2013                 spin_unlock(&inode->i_lock);
2014         }
2015
2016         sgp = SGP_CACHE;
2017
2018         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2019             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2020                 sgp = SGP_NOHUGE;
2021         else if (vma->vm_flags & VM_HUGEPAGE)
2022                 sgp = SGP_HUGE;
2023
2024         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2025                                   gfp, vma, vmf, &ret);
2026         if (err)
2027                 return vmf_error(err);
2028         return ret;
2029 }
2030
2031 unsigned long shmem_get_unmapped_area(struct file *file,
2032                                       unsigned long uaddr, unsigned long len,
2033                                       unsigned long pgoff, unsigned long flags)
2034 {
2035         unsigned long (*get_area)(struct file *,
2036                 unsigned long, unsigned long, unsigned long, unsigned long);
2037         unsigned long addr;
2038         unsigned long offset;
2039         unsigned long inflated_len;
2040         unsigned long inflated_addr;
2041         unsigned long inflated_offset;
2042
2043         if (len > TASK_SIZE)
2044                 return -ENOMEM;
2045
2046         get_area = current->mm->get_unmapped_area;
2047         addr = get_area(file, uaddr, len, pgoff, flags);
2048
2049         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2050                 return addr;
2051         if (IS_ERR_VALUE(addr))
2052                 return addr;
2053         if (addr & ~PAGE_MASK)
2054                 return addr;
2055         if (addr > TASK_SIZE - len)
2056                 return addr;
2057
2058         if (shmem_huge == SHMEM_HUGE_DENY)
2059                 return addr;
2060         if (len < HPAGE_PMD_SIZE)
2061                 return addr;
2062         if (flags & MAP_FIXED)
2063                 return addr;
2064         /*
2065          * Our priority is to support MAP_SHARED mapped hugely;
2066          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2067          * But if caller specified an address hint, respect that as before.
2068          */
2069         if (uaddr)
2070                 return addr;
2071
2072         if (shmem_huge != SHMEM_HUGE_FORCE) {
2073                 struct super_block *sb;
2074
2075                 if (file) {
2076                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2077                         sb = file_inode(file)->i_sb;
2078                 } else {
2079                         /*
2080                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2081                          * for "/dev/zero", to create a shared anonymous object.
2082                          */
2083                         if (IS_ERR(shm_mnt))
2084                                 return addr;
2085                         sb = shm_mnt->mnt_sb;
2086                 }
2087                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2088                         return addr;
2089         }
2090
2091         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2092         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2093                 return addr;
2094         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2095                 return addr;
2096
2097         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2098         if (inflated_len > TASK_SIZE)
2099                 return addr;
2100         if (inflated_len < len)
2101                 return addr;
2102
2103         inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2104         if (IS_ERR_VALUE(inflated_addr))
2105                 return addr;
2106         if (inflated_addr & ~PAGE_MASK)
2107                 return addr;
2108
2109         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2110         inflated_addr += offset - inflated_offset;
2111         if (inflated_offset > offset)
2112                 inflated_addr += HPAGE_PMD_SIZE;
2113
2114         if (inflated_addr > TASK_SIZE - len)
2115                 return addr;
2116         return inflated_addr;
2117 }
2118
2119 #ifdef CONFIG_NUMA
2120 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2121 {
2122         struct inode *inode = file_inode(vma->vm_file);
2123         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2124 }
2125
2126 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2127                                           unsigned long addr)
2128 {
2129         struct inode *inode = file_inode(vma->vm_file);
2130         pgoff_t index;
2131
2132         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2133         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2134 }
2135 #endif
2136
2137 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2138 {
2139         struct inode *inode = file_inode(file);
2140         struct shmem_inode_info *info = SHMEM_I(inode);
2141         int retval = -ENOMEM;
2142
2143         spin_lock_irq(&info->lock);
2144         if (lock && !(info->flags & VM_LOCKED)) {
2145                 if (!user_shm_lock(inode->i_size, user))
2146                         goto out_nomem;
2147                 info->flags |= VM_LOCKED;
2148                 mapping_set_unevictable(file->f_mapping);
2149         }
2150         if (!lock && (info->flags & VM_LOCKED) && user) {
2151                 user_shm_unlock(inode->i_size, user);
2152                 info->flags &= ~VM_LOCKED;
2153                 mapping_clear_unevictable(file->f_mapping);
2154         }
2155         retval = 0;
2156
2157 out_nomem:
2158         spin_unlock_irq(&info->lock);
2159         return retval;
2160 }
2161
2162 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2163 {
2164         file_accessed(file);
2165         vma->vm_ops = &shmem_vm_ops;
2166         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2167                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2168                         (vma->vm_end & HPAGE_PMD_MASK)) {
2169                 khugepaged_enter(vma, vma->vm_flags);
2170         }
2171         return 0;
2172 }
2173
2174 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2175                                      umode_t mode, dev_t dev, unsigned long flags)
2176 {
2177         struct inode *inode;
2178         struct shmem_inode_info *info;
2179         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2180
2181         if (shmem_reserve_inode(sb))
2182                 return NULL;
2183
2184         inode = new_inode(sb);
2185         if (inode) {
2186                 inode->i_ino = get_next_ino();
2187                 inode_init_owner(inode, dir, mode);
2188                 inode->i_blocks = 0;
2189                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2190                 inode->i_generation = get_seconds();
2191                 info = SHMEM_I(inode);
2192                 memset(info, 0, (char *)inode - (char *)info);
2193                 spin_lock_init(&info->lock);
2194                 info->seals = F_SEAL_SEAL;
2195                 info->flags = flags & VM_NORESERVE;
2196                 INIT_LIST_HEAD(&info->shrinklist);
2197                 INIT_LIST_HEAD(&info->swaplist);
2198                 simple_xattrs_init(&info->xattrs);
2199                 cache_no_acl(inode);
2200
2201                 switch (mode & S_IFMT) {
2202                 default:
2203                         inode->i_op = &shmem_special_inode_operations;
2204                         init_special_inode(inode, mode, dev);
2205                         break;
2206                 case S_IFREG:
2207                         inode->i_mapping->a_ops = &shmem_aops;
2208                         inode->i_op = &shmem_inode_operations;
2209                         inode->i_fop = &shmem_file_operations;
2210                         mpol_shared_policy_init(&info->policy,
2211                                                  shmem_get_sbmpol(sbinfo));
2212                         break;
2213                 case S_IFDIR:
2214                         inc_nlink(inode);
2215                         /* Some things misbehave if size == 0 on a directory */
2216                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2217                         inode->i_op = &shmem_dir_inode_operations;
2218                         inode->i_fop = &simple_dir_operations;
2219                         break;
2220                 case S_IFLNK:
2221                         /*
2222                          * Must not load anything in the rbtree,
2223                          * mpol_free_shared_policy will not be called.
2224                          */
2225                         mpol_shared_policy_init(&info->policy, NULL);
2226                         break;
2227                 }
2228         } else
2229                 shmem_free_inode(sb);
2230         return inode;
2231 }
2232
2233 bool shmem_mapping(struct address_space *mapping)
2234 {
2235         return mapping->a_ops == &shmem_aops;
2236 }
2237
2238 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2239                                   pmd_t *dst_pmd,
2240                                   struct vm_area_struct *dst_vma,
2241                                   unsigned long dst_addr,
2242                                   unsigned long src_addr,
2243                                   bool zeropage,
2244                                   struct page **pagep)
2245 {
2246         struct inode *inode = file_inode(dst_vma->vm_file);
2247         struct shmem_inode_info *info = SHMEM_I(inode);
2248         struct address_space *mapping = inode->i_mapping;
2249         gfp_t gfp = mapping_gfp_mask(mapping);
2250         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2251         struct mem_cgroup *memcg;
2252         spinlock_t *ptl;
2253         void *page_kaddr;
2254         struct page *page;
2255         pte_t _dst_pte, *dst_pte;
2256         int ret;
2257
2258         ret = -ENOMEM;
2259         if (!shmem_inode_acct_block(inode, 1))
2260                 goto out;
2261
2262         if (!*pagep) {
2263                 page = shmem_alloc_page(gfp, info, pgoff);
2264                 if (!page)
2265                         goto out_unacct_blocks;
2266
2267                 if (!zeropage) {        /* mcopy_atomic */
2268                         page_kaddr = kmap_atomic(page);
2269                         ret = copy_from_user(page_kaddr,
2270                                              (const void __user *)src_addr,
2271                                              PAGE_SIZE);
2272                         kunmap_atomic(page_kaddr);
2273
2274                         /* fallback to copy_from_user outside mmap_sem */
2275                         if (unlikely(ret)) {
2276                                 *pagep = page;
2277                                 shmem_inode_unacct_blocks(inode, 1);
2278                                 /* don't free the page */
2279                                 return -EFAULT;
2280                         }
2281                 } else {                /* mfill_zeropage_atomic */
2282                         clear_highpage(page);
2283                 }
2284         } else {
2285                 page = *pagep;
2286                 *pagep = NULL;
2287         }
2288
2289         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2290         __SetPageLocked(page);
2291         __SetPageSwapBacked(page);
2292         __SetPageUptodate(page);
2293
2294         ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false);
2295         if (ret)
2296                 goto out_release;
2297
2298         ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK);
2299         if (!ret) {
2300                 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL);
2301                 radix_tree_preload_end();
2302         }
2303         if (ret)
2304                 goto out_release_uncharge;
2305
2306         mem_cgroup_commit_charge(page, memcg, false, false);
2307
2308         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2309         if (dst_vma->vm_flags & VM_WRITE)
2310                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2311
2312         ret = -EEXIST;
2313         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2314         if (!pte_none(*dst_pte))
2315                 goto out_release_uncharge_unlock;
2316
2317         lru_cache_add_anon(page);
2318
2319         spin_lock(&info->lock);
2320         info->alloced++;
2321         inode->i_blocks += BLOCKS_PER_PAGE;
2322         shmem_recalc_inode(inode);
2323         spin_unlock(&info->lock);
2324
2325         inc_mm_counter(dst_mm, mm_counter_file(page));
2326         page_add_file_rmap(page, false);
2327         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2328
2329         /* No need to invalidate - it was non-present before */
2330         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2331         unlock_page(page);
2332         pte_unmap_unlock(dst_pte, ptl);
2333         ret = 0;
2334 out:
2335         return ret;
2336 out_release_uncharge_unlock:
2337         pte_unmap_unlock(dst_pte, ptl);
2338 out_release_uncharge:
2339         mem_cgroup_cancel_charge(page, memcg, false);
2340 out_release:
2341         unlock_page(page);
2342         put_page(page);
2343 out_unacct_blocks:
2344         shmem_inode_unacct_blocks(inode, 1);
2345         goto out;
2346 }
2347
2348 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2349                            pmd_t *dst_pmd,
2350                            struct vm_area_struct *dst_vma,
2351                            unsigned long dst_addr,
2352                            unsigned long src_addr,
2353                            struct page **pagep)
2354 {
2355         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2356                                       dst_addr, src_addr, false, pagep);
2357 }
2358
2359 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2360                              pmd_t *dst_pmd,
2361                              struct vm_area_struct *dst_vma,
2362                              unsigned long dst_addr)
2363 {
2364         struct page *page = NULL;
2365
2366         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2367                                       dst_addr, 0, true, &page);
2368 }
2369
2370 #ifdef CONFIG_TMPFS
2371 static const struct inode_operations shmem_symlink_inode_operations;
2372 static const struct inode_operations shmem_short_symlink_operations;
2373
2374 #ifdef CONFIG_TMPFS_XATTR
2375 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2376 #else
2377 #define shmem_initxattrs NULL
2378 #endif
2379
2380 static int
2381 shmem_write_begin(struct file *file, struct address_space *mapping,
2382                         loff_t pos, unsigned len, unsigned flags,
2383                         struct page **pagep, void **fsdata)
2384 {
2385         struct inode *inode = mapping->host;
2386         struct shmem_inode_info *info = SHMEM_I(inode);
2387         pgoff_t index = pos >> PAGE_SHIFT;
2388
2389         /* i_mutex is held by caller */
2390         if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) {
2391                 if (info->seals & F_SEAL_WRITE)
2392                         return -EPERM;
2393                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2394                         return -EPERM;
2395         }
2396
2397         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2398 }
2399
2400 static int
2401 shmem_write_end(struct file *file, struct address_space *mapping,
2402                         loff_t pos, unsigned len, unsigned copied,
2403                         struct page *page, void *fsdata)
2404 {
2405         struct inode *inode = mapping->host;
2406
2407         if (pos + copied > inode->i_size)
2408                 i_size_write(inode, pos + copied);
2409
2410         if (!PageUptodate(page)) {
2411                 struct page *head = compound_head(page);
2412                 if (PageTransCompound(page)) {
2413                         int i;
2414
2415                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2416                                 if (head + i == page)
2417                                         continue;
2418                                 clear_highpage(head + i);
2419                                 flush_dcache_page(head + i);
2420                         }
2421                 }
2422                 if (copied < PAGE_SIZE) {
2423                         unsigned from = pos & (PAGE_SIZE - 1);
2424                         zero_user_segments(page, 0, from,
2425                                         from + copied, PAGE_SIZE);
2426                 }
2427                 SetPageUptodate(head);
2428         }
2429         set_page_dirty(page);
2430         unlock_page(page);
2431         put_page(page);
2432
2433         return copied;
2434 }
2435
2436 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2437 {
2438         struct file *file = iocb->ki_filp;
2439         struct inode *inode = file_inode(file);
2440         struct address_space *mapping = inode->i_mapping;
2441         pgoff_t index;
2442         unsigned long offset;
2443         enum sgp_type sgp = SGP_READ;
2444         int error = 0;
2445         ssize_t retval = 0;
2446         loff_t *ppos = &iocb->ki_pos;
2447
2448         /*
2449          * Might this read be for a stacking filesystem?  Then when reading
2450          * holes of a sparse file, we actually need to allocate those pages,
2451          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2452          */
2453         if (!iter_is_iovec(to))
2454                 sgp = SGP_CACHE;
2455
2456         index = *ppos >> PAGE_SHIFT;
2457         offset = *ppos & ~PAGE_MASK;
2458
2459         for (;;) {
2460                 struct page *page = NULL;
2461                 pgoff_t end_index;
2462                 unsigned long nr, ret;
2463                 loff_t i_size = i_size_read(inode);
2464
2465                 end_index = i_size >> PAGE_SHIFT;
2466                 if (index > end_index)
2467                         break;
2468                 if (index == end_index) {
2469                         nr = i_size & ~PAGE_MASK;
2470                         if (nr <= offset)
2471                                 break;
2472                 }
2473
2474                 error = shmem_getpage(inode, index, &page, sgp);
2475                 if (error) {
2476                         if (error == -EINVAL)
2477                                 error = 0;
2478                         break;
2479                 }
2480                 if (page) {
2481                         if (sgp == SGP_CACHE)
2482                                 set_page_dirty(page);
2483                         unlock_page(page);
2484                 }
2485
2486                 /*
2487                  * We must evaluate after, since reads (unlike writes)
2488                  * are called without i_mutex protection against truncate
2489                  */
2490                 nr = PAGE_SIZE;
2491                 i_size = i_size_read(inode);
2492                 end_index = i_size >> PAGE_SHIFT;
2493                 if (index == end_index) {
2494                         nr = i_size & ~PAGE_MASK;
2495                         if (nr <= offset) {
2496                                 if (page)
2497                                         put_page(page);
2498                                 break;
2499                         }
2500                 }
2501                 nr -= offset;
2502
2503                 if (page) {
2504                         /*
2505                          * If users can be writing to this page using arbitrary
2506                          * virtual addresses, take care about potential aliasing
2507                          * before reading the page on the kernel side.
2508                          */
2509                         if (mapping_writably_mapped(mapping))
2510                                 flush_dcache_page(page);
2511                         /*
2512                          * Mark the page accessed if we read the beginning.
2513                          */
2514                         if (!offset)
2515                                 mark_page_accessed(page);
2516                 } else {
2517                         page = ZERO_PAGE(0);
2518                         get_page(page);
2519                 }
2520
2521                 /*
2522                  * Ok, we have the page, and it's up-to-date, so
2523                  * now we can copy it to user space...
2524                  */
2525                 ret = copy_page_to_iter(page, offset, nr, to);
2526                 retval += ret;
2527                 offset += ret;
2528                 index += offset >> PAGE_SHIFT;
2529                 offset &= ~PAGE_MASK;
2530
2531                 put_page(page);
2532                 if (!iov_iter_count(to))
2533                         break;
2534                 if (ret < nr) {
2535                         error = -EFAULT;
2536                         break;
2537                 }
2538                 cond_resched();
2539         }
2540
2541         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2542         file_accessed(file);
2543         return retval ? retval : error;
2544 }
2545
2546 /*
2547  * llseek SEEK_DATA or SEEK_HOLE through the radix_tree.
2548  */
2549 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2550                                     pgoff_t index, pgoff_t end, int whence)
2551 {
2552         struct page *page;
2553         struct pagevec pvec;
2554         pgoff_t indices[PAGEVEC_SIZE];
2555         bool done = false;
2556         int i;
2557
2558         pagevec_init(&pvec);
2559         pvec.nr = 1;            /* start small: we may be there already */
2560         while (!done) {
2561                 pvec.nr = find_get_entries(mapping, index,
2562                                         pvec.nr, pvec.pages, indices);
2563                 if (!pvec.nr) {
2564                         if (whence == SEEK_DATA)
2565                                 index = end;
2566                         break;
2567                 }
2568                 for (i = 0; i < pvec.nr; i++, index++) {
2569                         if (index < indices[i]) {
2570                                 if (whence == SEEK_HOLE) {
2571                                         done = true;
2572                                         break;
2573                                 }
2574                                 index = indices[i];
2575                         }
2576                         page = pvec.pages[i];
2577                         if (page && !radix_tree_exceptional_entry(page)) {
2578                                 if (!PageUptodate(page))
2579                                         page = NULL;
2580                         }
2581                         if (index >= end ||
2582                             (page && whence == SEEK_DATA) ||
2583                             (!page && whence == SEEK_HOLE)) {
2584                                 done = true;
2585                                 break;
2586                         }
2587                 }
2588                 pagevec_remove_exceptionals(&pvec);
2589                 pagevec_release(&pvec);
2590                 pvec.nr = PAGEVEC_SIZE;
2591                 cond_resched();
2592         }
2593         return index;
2594 }
2595
2596 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2597 {
2598         struct address_space *mapping = file->f_mapping;
2599         struct inode *inode = mapping->host;
2600         pgoff_t start, end;
2601         loff_t new_offset;
2602
2603         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2604                 return generic_file_llseek_size(file, offset, whence,
2605                                         MAX_LFS_FILESIZE, i_size_read(inode));
2606         inode_lock(inode);
2607         /* We're holding i_mutex so we can access i_size directly */
2608
2609         if (offset < 0)
2610                 offset = -EINVAL;
2611         else if (offset >= inode->i_size)
2612                 offset = -ENXIO;
2613         else {
2614                 start = offset >> PAGE_SHIFT;
2615                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2616                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2617                 new_offset <<= PAGE_SHIFT;
2618                 if (new_offset > offset) {
2619                         if (new_offset < inode->i_size)
2620                                 offset = new_offset;
2621                         else if (whence == SEEK_DATA)
2622                                 offset = -ENXIO;
2623                         else
2624                                 offset = inode->i_size;
2625                 }
2626         }
2627
2628         if (offset >= 0)
2629                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2630         inode_unlock(inode);
2631         return offset;
2632 }
2633
2634 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2635                                                          loff_t len)
2636 {
2637         struct inode *inode = file_inode(file);
2638         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2639         struct shmem_inode_info *info = SHMEM_I(inode);
2640         struct shmem_falloc shmem_falloc;
2641         pgoff_t start, index, end;
2642         int error;
2643
2644         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2645                 return -EOPNOTSUPP;
2646
2647         inode_lock(inode);
2648
2649         if (mode & FALLOC_FL_PUNCH_HOLE) {
2650                 struct address_space *mapping = file->f_mapping;
2651                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2652                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2653                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2654
2655                 /* protected by i_mutex */
2656                 if (info->seals & F_SEAL_WRITE) {
2657                         error = -EPERM;
2658                         goto out;
2659                 }
2660
2661                 shmem_falloc.waitq = &shmem_falloc_waitq;
2662                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2663                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2664                 spin_lock(&inode->i_lock);
2665                 inode->i_private = &shmem_falloc;
2666                 spin_unlock(&inode->i_lock);
2667
2668                 if ((u64)unmap_end > (u64)unmap_start)
2669                         unmap_mapping_range(mapping, unmap_start,
2670                                             1 + unmap_end - unmap_start, 0);
2671                 shmem_truncate_range(inode, offset, offset + len - 1);
2672                 /* No need to unmap again: hole-punching leaves COWed pages */
2673
2674                 spin_lock(&inode->i_lock);
2675                 inode->i_private = NULL;
2676                 wake_up_all(&shmem_falloc_waitq);
2677                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2678                 spin_unlock(&inode->i_lock);
2679                 error = 0;
2680                 goto out;
2681         }
2682
2683         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2684         error = inode_newsize_ok(inode, offset + len);
2685         if (error)
2686                 goto out;
2687
2688         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2689                 error = -EPERM;
2690                 goto out;
2691         }
2692
2693         start = offset >> PAGE_SHIFT;
2694         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2695         /* Try to avoid a swapstorm if len is impossible to satisfy */
2696         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2697                 error = -ENOSPC;
2698                 goto out;
2699         }
2700
2701         shmem_falloc.waitq = NULL;
2702         shmem_falloc.start = start;
2703         shmem_falloc.next  = start;
2704         shmem_falloc.nr_falloced = 0;
2705         shmem_falloc.nr_unswapped = 0;
2706         spin_lock(&inode->i_lock);
2707         inode->i_private = &shmem_falloc;
2708         spin_unlock(&inode->i_lock);
2709
2710         for (index = start; index < end; index++) {
2711                 struct page *page;
2712
2713                 /*
2714                  * Good, the fallocate(2) manpage permits EINTR: we may have
2715                  * been interrupted because we are using up too much memory.
2716                  */
2717                 if (signal_pending(current))
2718                         error = -EINTR;
2719                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2720                         error = -ENOMEM;
2721                 else
2722                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2723                 if (error) {
2724                         /* Remove the !PageUptodate pages we added */
2725                         if (index > start) {
2726                                 shmem_undo_range(inode,
2727                                     (loff_t)start << PAGE_SHIFT,
2728                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2729                         }
2730                         goto undone;
2731                 }
2732
2733                 /*
2734                  * Inform shmem_writepage() how far we have reached.
2735                  * No need for lock or barrier: we have the page lock.
2736                  */
2737                 shmem_falloc.next++;
2738                 if (!PageUptodate(page))
2739                         shmem_falloc.nr_falloced++;
2740
2741                 /*
2742                  * If !PageUptodate, leave it that way so that freeable pages
2743                  * can be recognized if we need to rollback on error later.
2744                  * But set_page_dirty so that memory pressure will swap rather
2745                  * than free the pages we are allocating (and SGP_CACHE pages
2746                  * might still be clean: we now need to mark those dirty too).
2747                  */
2748                 set_page_dirty(page);
2749                 unlock_page(page);
2750                 put_page(page);
2751                 cond_resched();
2752         }
2753
2754         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2755                 i_size_write(inode, offset + len);
2756         inode->i_ctime = current_time(inode);
2757 undone:
2758         spin_lock(&inode->i_lock);
2759         inode->i_private = NULL;
2760         spin_unlock(&inode->i_lock);
2761 out:
2762         inode_unlock(inode);
2763         return error;
2764 }
2765
2766 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2767 {
2768         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2769
2770         buf->f_type = TMPFS_MAGIC;
2771         buf->f_bsize = PAGE_SIZE;
2772         buf->f_namelen = NAME_MAX;
2773         if (sbinfo->max_blocks) {
2774                 buf->f_blocks = sbinfo->max_blocks;
2775                 buf->f_bavail =
2776                 buf->f_bfree  = sbinfo->max_blocks -
2777                                 percpu_counter_sum(&sbinfo->used_blocks);
2778         }
2779         if (sbinfo->max_inodes) {
2780                 buf->f_files = sbinfo->max_inodes;
2781                 buf->f_ffree = sbinfo->free_inodes;
2782         }
2783         /* else leave those fields 0 like simple_statfs */
2784         return 0;
2785 }
2786
2787 /*
2788  * File creation. Allocate an inode, and we're done..
2789  */
2790 static int
2791 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2792 {
2793         struct inode *inode;
2794         int error = -ENOSPC;
2795
2796         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2797         if (inode) {
2798                 error = simple_acl_create(dir, inode);
2799                 if (error)
2800                         goto out_iput;
2801                 error = security_inode_init_security(inode, dir,
2802                                                      &dentry->d_name,
2803                                                      shmem_initxattrs, NULL);
2804                 if (error && error != -EOPNOTSUPP)
2805                         goto out_iput;
2806
2807                 error = 0;
2808                 dir->i_size += BOGO_DIRENT_SIZE;
2809                 dir->i_ctime = dir->i_mtime = current_time(dir);
2810                 d_instantiate(dentry, inode);
2811                 dget(dentry); /* Extra count - pin the dentry in core */
2812         }
2813         return error;
2814 out_iput:
2815         iput(inode);
2816         return error;
2817 }
2818
2819 static int
2820 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2821 {
2822         struct inode *inode;
2823         int error = -ENOSPC;
2824
2825         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2826         if (inode) {
2827                 error = security_inode_init_security(inode, dir,
2828                                                      NULL,
2829                                                      shmem_initxattrs, NULL);
2830                 if (error && error != -EOPNOTSUPP)
2831                         goto out_iput;
2832                 error = simple_acl_create(dir, inode);
2833                 if (error)
2834                         goto out_iput;
2835                 d_tmpfile(dentry, inode);
2836         }
2837         return error;
2838 out_iput:
2839         iput(inode);
2840         return error;
2841 }
2842
2843 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2844 {
2845         int error;
2846
2847         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2848                 return error;
2849         inc_nlink(dir);
2850         return 0;
2851 }
2852
2853 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2854                 bool excl)
2855 {
2856         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2857 }
2858
2859 /*
2860  * Link a file..
2861  */
2862 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2863 {
2864         struct inode *inode = d_inode(old_dentry);
2865         int ret;
2866
2867         /*
2868          * No ordinary (disk based) filesystem counts links as inodes;
2869          * but each new link needs a new dentry, pinning lowmem, and
2870          * tmpfs dentries cannot be pruned until they are unlinked.
2871          */
2872         ret = shmem_reserve_inode(inode->i_sb);
2873         if (ret)
2874                 goto out;
2875
2876         dir->i_size += BOGO_DIRENT_SIZE;
2877         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2878         inc_nlink(inode);
2879         ihold(inode);   /* New dentry reference */
2880         dget(dentry);           /* Extra pinning count for the created dentry */
2881         d_instantiate(dentry, inode);
2882 out:
2883         return ret;
2884 }
2885
2886 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2887 {
2888         struct inode *inode = d_inode(dentry);
2889
2890         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2891                 shmem_free_inode(inode->i_sb);
2892
2893         dir->i_size -= BOGO_DIRENT_SIZE;
2894         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2895         drop_nlink(inode);
2896         dput(dentry);   /* Undo the count from "create" - this does all the work */
2897         return 0;
2898 }
2899
2900 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2901 {
2902         if (!simple_empty(dentry))
2903                 return -ENOTEMPTY;
2904
2905         drop_nlink(d_inode(dentry));
2906         drop_nlink(dir);
2907         return shmem_unlink(dir, dentry);
2908 }
2909
2910 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2911 {
2912         bool old_is_dir = d_is_dir(old_dentry);
2913         bool new_is_dir = d_is_dir(new_dentry);
2914
2915         if (old_dir != new_dir && old_is_dir != new_is_dir) {
2916                 if (old_is_dir) {
2917                         drop_nlink(old_dir);
2918                         inc_nlink(new_dir);
2919                 } else {
2920                         drop_nlink(new_dir);
2921                         inc_nlink(old_dir);
2922                 }
2923         }
2924         old_dir->i_ctime = old_dir->i_mtime =
2925         new_dir->i_ctime = new_dir->i_mtime =
2926         d_inode(old_dentry)->i_ctime =
2927         d_inode(new_dentry)->i_ctime = current_time(old_dir);
2928
2929         return 0;
2930 }
2931
2932 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2933 {
2934         struct dentry *whiteout;
2935         int error;
2936
2937         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
2938         if (!whiteout)
2939                 return -ENOMEM;
2940
2941         error = shmem_mknod(old_dir, whiteout,
2942                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
2943         dput(whiteout);
2944         if (error)
2945                 return error;
2946
2947         /*
2948          * Cheat and hash the whiteout while the old dentry is still in
2949          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
2950          *
2951          * d_lookup() will consistently find one of them at this point,
2952          * not sure which one, but that isn't even important.
2953          */
2954         d_rehash(whiteout);
2955         return 0;
2956 }
2957
2958 /*
2959  * The VFS layer already does all the dentry stuff for rename,
2960  * we just have to decrement the usage count for the target if
2961  * it exists so that the VFS layer correctly free's it when it
2962  * gets overwritten.
2963  */
2964 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
2965 {
2966         struct inode *inode = d_inode(old_dentry);
2967         int they_are_dirs = S_ISDIR(inode->i_mode);
2968
2969         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
2970                 return -EINVAL;
2971
2972         if (flags & RENAME_EXCHANGE)
2973                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
2974
2975         if (!simple_empty(new_dentry))
2976                 return -ENOTEMPTY;
2977
2978         if (flags & RENAME_WHITEOUT) {
2979                 int error;
2980
2981                 error = shmem_whiteout(old_dir, old_dentry);
2982                 if (error)
2983                         return error;
2984         }
2985
2986         if (d_really_is_positive(new_dentry)) {
2987                 (void) shmem_unlink(new_dir, new_dentry);
2988                 if (they_are_dirs) {
2989                         drop_nlink(d_inode(new_dentry));
2990                         drop_nlink(old_dir);
2991                 }
2992         } else if (they_are_dirs) {
2993                 drop_nlink(old_dir);
2994                 inc_nlink(new_dir);
2995         }
2996
2997         old_dir->i_size -= BOGO_DIRENT_SIZE;
2998         new_dir->i_size += BOGO_DIRENT_SIZE;
2999         old_dir->i_ctime = old_dir->i_mtime =
3000         new_dir->i_ctime = new_dir->i_mtime =
3001         inode->i_ctime = current_time(old_dir);
3002         return 0;
3003 }
3004
3005 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3006 {
3007         int error;
3008         int len;
3009         struct inode *inode;
3010         struct page *page;
3011
3012         len = strlen(symname) + 1;
3013         if (len > PAGE_SIZE)
3014                 return -ENAMETOOLONG;
3015
3016         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3017                                 VM_NORESERVE);
3018         if (!inode)
3019                 return -ENOSPC;
3020
3021         error = security_inode_init_security(inode, dir, &dentry->d_name,
3022                                              shmem_initxattrs, NULL);
3023         if (error) {
3024                 if (error != -EOPNOTSUPP) {
3025                         iput(inode);
3026                         return error;
3027                 }
3028                 error = 0;
3029         }
3030
3031         inode->i_size = len-1;
3032         if (len <= SHORT_SYMLINK_LEN) {
3033                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3034                 if (!inode->i_link) {
3035                         iput(inode);
3036                         return -ENOMEM;
3037                 }
3038                 inode->i_op = &shmem_short_symlink_operations;
3039         } else {
3040                 inode_nohighmem(inode);
3041                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3042                 if (error) {
3043                         iput(inode);
3044                         return error;
3045                 }
3046                 inode->i_mapping->a_ops = &shmem_aops;
3047                 inode->i_op = &shmem_symlink_inode_operations;
3048                 memcpy(page_address(page), symname, len);
3049                 SetPageUptodate(page);
3050                 set_page_dirty(page);
3051                 unlock_page(page);
3052                 put_page(page);
3053         }
3054         dir->i_size += BOGO_DIRENT_SIZE;
3055         dir->i_ctime = dir->i_mtime = current_time(dir);
3056         d_instantiate(dentry, inode);
3057         dget(dentry);
3058         return 0;
3059 }
3060
3061 static void shmem_put_link(void *arg)
3062 {
3063         mark_page_accessed(arg);
3064         put_page(arg);
3065 }
3066
3067 static const char *shmem_get_link(struct dentry *dentry,
3068                                   struct inode *inode,
3069                                   struct delayed_call *done)
3070 {
3071         struct page *page = NULL;
3072         int error;
3073         if (!dentry) {
3074                 page = find_get_page(inode->i_mapping, 0);
3075                 if (!page)
3076                         return ERR_PTR(-ECHILD);
3077                 if (!PageUptodate(page)) {
3078                         put_page(page);
3079                         return ERR_PTR(-ECHILD);
3080                 }
3081         } else {
3082                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3083                 if (error)
3084                         return ERR_PTR(error);
3085                 unlock_page(page);
3086         }
3087         set_delayed_call(done, shmem_put_link, page);
3088         return page_address(page);
3089 }
3090
3091 #ifdef CONFIG_TMPFS_XATTR
3092 /*
3093  * Superblocks without xattr inode operations may get some security.* xattr
3094  * support from the LSM "for free". As soon as we have any other xattrs
3095  * like ACLs, we also need to implement the security.* handlers at
3096  * filesystem level, though.
3097  */
3098
3099 /*
3100  * Callback for security_inode_init_security() for acquiring xattrs.
3101  */
3102 static int shmem_initxattrs(struct inode *inode,
3103                             const struct xattr *xattr_array,
3104                             void *fs_info)
3105 {
3106         struct shmem_inode_info *info = SHMEM_I(inode);
3107         const struct xattr *xattr;
3108         struct simple_xattr *new_xattr;
3109         size_t len;
3110
3111         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3112                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3113                 if (!new_xattr)
3114                         return -ENOMEM;
3115
3116                 len = strlen(xattr->name) + 1;
3117                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3118                                           GFP_KERNEL);
3119                 if (!new_xattr->name) {
3120                         kfree(new_xattr);
3121                         return -ENOMEM;
3122                 }
3123
3124                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3125                        XATTR_SECURITY_PREFIX_LEN);
3126                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3127                        xattr->name, len);
3128
3129                 simple_xattr_list_add(&info->xattrs, new_xattr);
3130         }
3131
3132         return 0;
3133 }
3134
3135 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3136                                    struct dentry *unused, struct inode *inode,
3137                                    const char *name, void *buffer, size_t size)
3138 {
3139         struct shmem_inode_info *info = SHMEM_I(inode);
3140
3141         name = xattr_full_name(handler, name);
3142         return simple_xattr_get(&info->xattrs, name, buffer, size);
3143 }
3144
3145 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3146                                    struct dentry *unused, struct inode *inode,
3147                                    const char *name, const void *value,
3148                                    size_t size, int flags)
3149 {
3150         struct shmem_inode_info *info = SHMEM_I(inode);
3151
3152         name = xattr_full_name(handler, name);
3153         return simple_xattr_set(&info->xattrs, name, value, size, flags);
3154 }
3155
3156 static const struct xattr_handler shmem_security_xattr_handler = {
3157         .prefix = XATTR_SECURITY_PREFIX,
3158         .get = shmem_xattr_handler_get,
3159         .set = shmem_xattr_handler_set,
3160 };
3161
3162 static const struct xattr_handler shmem_trusted_xattr_handler = {
3163         .prefix = XATTR_TRUSTED_PREFIX,
3164         .get = shmem_xattr_handler_get,
3165         .set = shmem_xattr_handler_set,
3166 };
3167
3168 static const struct xattr_handler *shmem_xattr_handlers[] = {
3169 #ifdef CONFIG_TMPFS_POSIX_ACL
3170         &posix_acl_access_xattr_handler,
3171         &posix_acl_default_xattr_handler,
3172 #endif
3173         &shmem_security_xattr_handler,
3174         &shmem_trusted_xattr_handler,
3175         NULL
3176 };
3177
3178 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3179 {
3180         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3181         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3182 }
3183 #endif /* CONFIG_TMPFS_XATTR */
3184
3185 static const struct inode_operations shmem_short_symlink_operations = {
3186         .get_link       = simple_get_link,
3187 #ifdef CONFIG_TMPFS_XATTR
3188         .listxattr      = shmem_listxattr,
3189 #endif
3190 };
3191
3192 static const struct inode_operations shmem_symlink_inode_operations = {
3193         .get_link       = shmem_get_link,
3194 #ifdef CONFIG_TMPFS_XATTR
3195         .listxattr      = shmem_listxattr,
3196 #endif
3197 };
3198
3199 static struct dentry *shmem_get_parent(struct dentry *child)
3200 {
3201         return ERR_PTR(-ESTALE);
3202 }
3203
3204 static int shmem_match(struct inode *ino, void *vfh)
3205 {
3206         __u32 *fh = vfh;
3207         __u64 inum = fh[2];
3208         inum = (inum << 32) | fh[1];
3209         return ino->i_ino == inum && fh[0] == ino->i_generation;
3210 }
3211
3212 /* Find any alias of inode, but prefer a hashed alias */
3213 static struct dentry *shmem_find_alias(struct inode *inode)
3214 {
3215         struct dentry *alias = d_find_alias(inode);
3216
3217         return alias ?: d_find_any_alias(inode);
3218 }
3219
3220
3221 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3222                 struct fid *fid, int fh_len, int fh_type)
3223 {
3224         struct inode *inode;
3225         struct dentry *dentry = NULL;
3226         u64 inum;
3227
3228         if (fh_len < 3)
3229                 return NULL;
3230
3231         inum = fid->raw[2];
3232         inum = (inum << 32) | fid->raw[1];
3233
3234         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3235                         shmem_match, fid->raw);
3236         if (inode) {
3237                 dentry = shmem_find_alias(inode);
3238                 iput(inode);
3239         }
3240
3241         return dentry;
3242 }
3243
3244 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3245                                 struct inode *parent)
3246 {
3247         if (*len < 3) {
3248                 *len = 3;
3249                 return FILEID_INVALID;
3250         }
3251
3252         if (inode_unhashed(inode)) {
3253                 /* Unfortunately insert_inode_hash is not idempotent,
3254                  * so as we hash inodes here rather than at creation
3255                  * time, we need a lock to ensure we only try
3256                  * to do it once
3257                  */
3258                 static DEFINE_SPINLOCK(lock);
3259                 spin_lock(&lock);
3260                 if (inode_unhashed(inode))
3261                         __insert_inode_hash(inode,
3262                                             inode->i_ino + inode->i_generation);
3263                 spin_unlock(&lock);
3264         }
3265
3266         fh[0] = inode->i_generation;
3267         fh[1] = inode->i_ino;
3268         fh[2] = ((__u64)inode->i_ino) >> 32;
3269
3270         *len = 3;
3271         return 1;
3272 }
3273
3274 static const struct export_operations shmem_export_ops = {
3275         .get_parent     = shmem_get_parent,
3276         .encode_fh      = shmem_encode_fh,
3277         .fh_to_dentry   = shmem_fh_to_dentry,
3278 };
3279
3280 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
3281                                bool remount)
3282 {
3283         char *this_char, *value, *rest;
3284         struct mempolicy *mpol = NULL;
3285         uid_t uid;
3286         gid_t gid;
3287
3288         while (options != NULL) {
3289                 this_char = options;
3290                 for (;;) {
3291                         /*
3292                          * NUL-terminate this option: unfortunately,
3293                          * mount options form a comma-separated list,
3294                          * but mpol's nodelist may also contain commas.
3295                          */
3296                         options = strchr(options, ',');
3297                         if (options == NULL)
3298                                 break;
3299                         options++;
3300                         if (!isdigit(*options)) {
3301                                 options[-1] = '\0';
3302                                 break;
3303                         }
3304                 }
3305                 if (!*this_char)
3306                         continue;
3307                 if ((value = strchr(this_char,'=')) != NULL) {
3308                         *value++ = 0;
3309                 } else {
3310                         pr_err("tmpfs: No value for mount option '%s'\n",
3311                                this_char);
3312                         goto error;
3313                 }
3314
3315                 if (!strcmp(this_char,"size")) {
3316                         unsigned long long size;
3317                         size = memparse(value,&rest);
3318                         if (*rest == '%') {
3319                                 size <<= PAGE_SHIFT;
3320                                 size *= totalram_pages;
3321                                 do_div(size, 100);
3322                                 rest++;
3323                         }
3324                         if (*rest)
3325                                 goto bad_val;
3326                         sbinfo->max_blocks =
3327                                 DIV_ROUND_UP(size, PAGE_SIZE);
3328                 } else if (!strcmp(this_char,"nr_blocks")) {
3329                         sbinfo->max_blocks = memparse(value, &rest);
3330                         if (*rest)
3331                                 goto bad_val;
3332                 } else if (!strcmp(this_char,"nr_inodes")) {
3333                         sbinfo->max_inodes = memparse(value, &rest);
3334                         if (*rest)
3335                                 goto bad_val;
3336                 } else if (!strcmp(this_char,"mode")) {
3337                         if (remount)
3338                                 continue;
3339                         sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
3340                         if (*rest)
3341                                 goto bad_val;
3342                 } else if (!strcmp(this_char,"uid")) {
3343                         if (remount)
3344                                 continue;
3345                         uid = simple_strtoul(value, &rest, 0);
3346                         if (*rest)
3347                                 goto bad_val;
3348                         sbinfo->uid = make_kuid(current_user_ns(), uid);
3349                         if (!uid_valid(sbinfo->uid))
3350                                 goto bad_val;
3351                 } else if (!strcmp(this_char,"gid")) {
3352                         if (remount)
3353                                 continue;
3354                         gid = simple_strtoul(value, &rest, 0);
3355                         if (*rest)
3356                                 goto bad_val;
3357                         sbinfo->gid = make_kgid(current_user_ns(), gid);
3358                         if (!gid_valid(sbinfo->gid))
3359                                 goto bad_val;
3360 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3361                 } else if (!strcmp(this_char, "huge")) {
3362                         int huge;
3363                         huge = shmem_parse_huge(value);
3364                         if (huge < 0)
3365                                 goto bad_val;
3366                         if (!has_transparent_hugepage() &&
3367                                         huge != SHMEM_HUGE_NEVER)
3368                                 goto bad_val;
3369                         sbinfo->huge = huge;
3370 #endif
3371 #ifdef CONFIG_NUMA
3372                 } else if (!strcmp(this_char,"mpol")) {
3373                         mpol_put(mpol);
3374                         mpol = NULL;
3375                         if (mpol_parse_str(value, &mpol))
3376                                 goto bad_val;
3377 #endif
3378                 } else {
3379                         pr_err("tmpfs: Bad mount option %s\n", this_char);
3380                         goto error;
3381                 }
3382         }
3383         sbinfo->mpol = mpol;
3384         return 0;
3385
3386 bad_val:
3387         pr_err("tmpfs: Bad value '%s' for mount option '%s'\n",
3388                value, this_char);
3389 error:
3390         mpol_put(mpol);
3391         return 1;
3392
3393 }
3394
3395 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
3396 {
3397         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3398         struct shmem_sb_info config = *sbinfo;
3399         unsigned long inodes;
3400         int error = -EINVAL;
3401
3402         config.mpol = NULL;
3403         if (shmem_parse_options(data, &config, true))
3404                 return error;
3405
3406         spin_lock(&sbinfo->stat_lock);
3407         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3408         if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
3409                 goto out;
3410         if (config.max_inodes < inodes)
3411                 goto out;
3412         /*
3413          * Those tests disallow limited->unlimited while any are in use;
3414          * but we must separately disallow unlimited->limited, because
3415          * in that case we have no record of how much is already in use.
3416          */
3417         if (config.max_blocks && !sbinfo->max_blocks)
3418