Blame view
mm/shmem.c
112 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 8 |
/* * Resizable virtual memory filesystem for Linux. * * Copyright (C) 2000 Linus Torvalds. * 2000 Transmeta Corp. * 2000-2001 Christoph Rohland * 2000-2001 SAP AG * 2002 Red Hat Inc. |
6922c0c7a tmpfs: convert sh... |
9 10 |
* Copyright (C) 2002-2011 Hugh Dickins. * Copyright (C) 2011 Google Inc. |
0edd73b33 [PATCH] shmem: re... |
11 |
* Copyright (C) 2002-2005 VERITAS Software Corporation. |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 16 17 |
* Copyright (C) 2004 Andi Kleen, SuSE Labs * * Extended attribute support for tmpfs: * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> * |
853ac43ab shmem: unify regu... |
18 19 20 |
* tiny-shmem: * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> * |
1da177e4c Linux-2.6.12-rc2 |
21 22 |
* This file is released under the GPL. */ |
853ac43ab shmem: unify regu... |
23 24 25 26 |
#include <linux/fs.h> #include <linux/init.h> #include <linux/vfs.h> #include <linux/mount.h> |
250297edf mm/shmem.c: remov... |
27 |
#include <linux/ramfs.h> |
caefba174 shmem: respect MA... |
28 |
#include <linux/pagemap.h> |
853ac43ab shmem: unify regu... |
29 30 |
#include <linux/file.h> #include <linux/mm.h> |
174cd4b1e sched/headers: Pr... |
31 |
#include <linux/sched/signal.h> |
b95f1b31b mm: Map most file... |
32 |
#include <linux/export.h> |
853ac43ab shmem: unify regu... |
33 |
#include <linux/swap.h> |
e2e40f2c1 fs: move struct k... |
34 |
#include <linux/uio.h> |
f3f0e1d21 khugepaged: add s... |
35 |
#include <linux/khugepaged.h> |
749df87bd mm/shmem: add hug... |
36 |
#include <linux/hugetlb.h> |
853ac43ab shmem: unify regu... |
37 |
|
95cc09d66 userfaultfd: shme... |
38 |
#include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */ |
853ac43ab shmem: unify regu... |
39 40 41 |
static struct vfsmount *shm_mnt; #ifdef CONFIG_SHMEM |
1da177e4c Linux-2.6.12-rc2 |
42 43 44 45 46 |
/* * This virtual memory filesystem is heavily based on the ramfs. It * extends ramfs by the ability to use swap and honor resource limits * which makes it a completely usable filesystem. */ |
39f0247d3 [PATCH] Access Co... |
47 |
#include <linux/xattr.h> |
a56942551 knfsd: exportfs: ... |
48 |
#include <linux/exportfs.h> |
1c7c474c3 make generic_acl ... |
49 |
#include <linux/posix_acl.h> |
feda821e7 fs: remove generi... |
50 |
#include <linux/posix_acl_xattr.h> |
1da177e4c Linux-2.6.12-rc2 |
51 |
#include <linux/mman.h> |
1da177e4c Linux-2.6.12-rc2 |
52 53 54 55 |
#include <linux/string.h> #include <linux/slab.h> #include <linux/backing-dev.h> #include <linux/shmem_fs.h> |
1da177e4c Linux-2.6.12-rc2 |
56 |
#include <linux/writeback.h> |
1da177e4c Linux-2.6.12-rc2 |
57 |
#include <linux/blkdev.h> |
bda97eab0 tmpfs: copy trunc... |
58 |
#include <linux/pagevec.h> |
41ffe5d5c tmpfs: miscellane... |
59 |
#include <linux/percpu_counter.h> |
83e4fa9c1 tmpfs: support fa... |
60 |
#include <linux/falloc.h> |
708e3508c tmpfs: clone shme... |
61 |
#include <linux/splice.h> |
1da177e4c Linux-2.6.12-rc2 |
62 63 64 65 |
#include <linux/security.h> #include <linux/swapops.h> #include <linux/mempolicy.h> #include <linux/namei.h> |
b00dc3ad7 [PATCH] tmpfs: fi... |
66 |
#include <linux/ctype.h> |
304dbdb7a [PATCH] add migra... |
67 |
#include <linux/migrate.h> |
c1f60a5a4 [PATCH] reduce MA... |
68 |
#include <linux/highmem.h> |
680d794ba mount options: fi... |
69 |
#include <linux/seq_file.h> |
925629278 integrity: specia... |
70 |
#include <linux/magic.h> |
9183df25f shm: add memfd_cr... |
71 |
#include <linux/syscalls.h> |
40e041a2c shm: add sealing API |
72 |
#include <linux/fcntl.h> |
9183df25f shm: add memfd_cr... |
73 |
#include <uapi/linux/memfd.h> |
cfda05267 userfaultfd: shme... |
74 |
#include <linux/userfaultfd_k.h> |
4c27fe4c4 userfaultfd: shme... |
75 |
#include <linux/rmap.h> |
2b4db7961 tmpfs: generate r... |
76 |
#include <linux/uuid.h> |
304dbdb7a [PATCH] add migra... |
77 |
|
7c0f6ba68 Replace <asm/uacc... |
78 |
#include <linux/uaccess.h> |
1da177e4c Linux-2.6.12-rc2 |
79 |
#include <asm/pgtable.h> |
dd56b0464 mm: page_alloc: h... |
80 |
#include "internal.h" |
09cbfeaf1 mm, fs: get rid o... |
81 82 |
#define BLOCKS_PER_PAGE (PAGE_SIZE/512) #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) |
1da177e4c Linux-2.6.12-rc2 |
83 |
|
1da177e4c Linux-2.6.12-rc2 |
84 85 |
/* Pretend that each entry is of this size in directory's i_size */ #define BOGO_DIRENT_SIZE 20 |
69f07ec93 tmpfs: use kmemdu... |
86 87 |
/* Symlink up to this size is kmalloc'ed instead of using a swappable page */ #define SHORT_SYMLINK_LEN 128 |
1aac14003 tmpfs: quit when ... |
88 |
/* |
f00cdc6df shmem: fix faulti... |
89 90 91 |
* shmem_fallocate communicates with shmem_fault or shmem_writepage via * inode->i_private (with i_mutex making sure that it has only one user at * a time): we would prefer not to enlarge the shmem inode just for that. |
1aac14003 tmpfs: quit when ... |
92 93 |
*/ struct shmem_falloc { |
8e205f779 shmem: fix faulti... |
94 |
wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ |
1aac14003 tmpfs: quit when ... |
95 96 97 98 99 |
pgoff_t start; /* start of range currently being fallocated */ pgoff_t next; /* the next page offset to be fallocated */ pgoff_t nr_falloced; /* how many new pages have been fallocated */ pgoff_t nr_unswapped; /* how often writepage refused to swap out */ }; |
b76db7354 mount-options-fix... |
100 |
#ifdef CONFIG_TMPFS |
680d794ba mount options: fi... |
101 102 103 104 105 106 107 108 109 |
static unsigned long shmem_default_max_blocks(void) { return totalram_pages / 2; } static unsigned long shmem_default_max_inodes(void) { return min(totalram_pages - totalhigh_pages, totalram_pages / 2); } |
b76db7354 mount-options-fix... |
110 |
#endif |
680d794ba mount options: fi... |
111 |
|
bde05d1cc shmem: replace pa... |
112 113 114 |
static bool shmem_should_replace_page(struct page *page, gfp_t gfp); static int shmem_replace_page(struct page **pagep, gfp_t gfp, struct shmem_inode_info *info, pgoff_t index); |
68da9f055 tmpfs: pass gfp t... |
115 |
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, |
9e18eb293 tmpfs: mem_cgroup... |
116 |
struct page **pagep, enum sgp_type sgp, |
cfda05267 userfaultfd: shme... |
117 118 |
gfp_t gfp, struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type); |
68da9f055 tmpfs: pass gfp t... |
119 |
|
f3f0e1d21 khugepaged: add s... |
120 |
int shmem_getpage(struct inode *inode, pgoff_t index, |
9e18eb293 tmpfs: mem_cgroup... |
121 |
struct page **pagep, enum sgp_type sgp) |
68da9f055 tmpfs: pass gfp t... |
122 123 |
{ return shmem_getpage_gfp(inode, index, pagep, sgp, |
cfda05267 userfaultfd: shme... |
124 |
mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); |
68da9f055 tmpfs: pass gfp t... |
125 |
} |
1da177e4c Linux-2.6.12-rc2 |
126 |
|
1da177e4c Linux-2.6.12-rc2 |
127 128 129 130 131 132 133 134 135 136 137 138 139 |
static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) { return sb->s_fs_info; } /* * shmem_file_setup pre-accounts the whole fixed size of a VM object, * for shared memory and for shared anonymous (/dev/zero) mappings * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), * consistent with the pre-accounting of private mappings ... */ static inline int shmem_acct_size(unsigned long flags, loff_t size) { |
0b0a0806b shmem: fix shared... |
140 |
return (flags & VM_NORESERVE) ? |
191c54244 mm: collapse secu... |
141 |
0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); |
1da177e4c Linux-2.6.12-rc2 |
142 143 144 145 |
} static inline void shmem_unacct_size(unsigned long flags, loff_t size) { |
0b0a0806b shmem: fix shared... |
146 |
if (!(flags & VM_NORESERVE)) |
1da177e4c Linux-2.6.12-rc2 |
147 148 |
vm_unacct_memory(VM_ACCT(size)); } |
771425179 shmem: update mem... |
149 150 151 152 153 154 155 156 157 158 159 160 |
static inline int shmem_reacct_size(unsigned long flags, loff_t oldsize, loff_t newsize) { if (!(flags & VM_NORESERVE)) { if (VM_ACCT(newsize) > VM_ACCT(oldsize)) return security_vm_enough_memory_mm(current->mm, VM_ACCT(newsize) - VM_ACCT(oldsize)); else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); } return 0; } |
1da177e4c Linux-2.6.12-rc2 |
161 162 |
/* * ... whereas tmpfs objects are accounted incrementally as |
75edd345e tmpfs: preliminar... |
163 |
* pages are allocated, in order to allow large sparse files. |
1da177e4c Linux-2.6.12-rc2 |
164 165 166 |
* shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. */ |
800d8c63b shmem: add huge p... |
167 |
static inline int shmem_acct_block(unsigned long flags, long pages) |
1da177e4c Linux-2.6.12-rc2 |
168 |
{ |
800d8c63b shmem: add huge p... |
169 170 171 172 173 |
if (!(flags & VM_NORESERVE)) return 0; return security_vm_enough_memory_mm(current->mm, pages * VM_ACCT(PAGE_SIZE)); |
1da177e4c Linux-2.6.12-rc2 |
174 175 176 177 |
} static inline void shmem_unacct_blocks(unsigned long flags, long pages) { |
0b0a0806b shmem: fix shared... |
178 |
if (flags & VM_NORESERVE) |
09cbfeaf1 mm, fs: get rid o... |
179 |
vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); |
1da177e4c Linux-2.6.12-rc2 |
180 |
} |
0f0796945 shmem: introduce ... |
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 |
static inline bool shmem_inode_acct_block(struct inode *inode, long pages) { struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); if (shmem_acct_block(info->flags, pages)) return false; if (sbinfo->max_blocks) { if (percpu_counter_compare(&sbinfo->used_blocks, sbinfo->max_blocks - pages) > 0) goto unacct; percpu_counter_add(&sbinfo->used_blocks, pages); } return true; unacct: shmem_unacct_blocks(info->flags, pages); return false; } static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) { struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); if (sbinfo->max_blocks) percpu_counter_sub(&sbinfo->used_blocks, pages); shmem_unacct_blocks(info->flags, pages); } |
759b9775c [PATCH] shmem and... |
212 |
static const struct super_operations shmem_ops; |
f5e54d6e5 [PATCH] mark addr... |
213 |
static const struct address_space_operations shmem_aops; |
15ad7cdcf [PATCH] struct se... |
214 |
static const struct file_operations shmem_file_operations; |
92e1d5be9 [PATCH] mark stru... |
215 216 217 |
static const struct inode_operations shmem_inode_operations; static const struct inode_operations shmem_dir_inode_operations; static const struct inode_operations shmem_special_inode_operations; |
f0f37e2f7 const: mark struc... |
218 |
static const struct vm_operations_struct shmem_vm_ops; |
779750d20 shmem: split huge... |
219 |
static struct file_system_type shmem_fs_type; |
1da177e4c Linux-2.6.12-rc2 |
220 |
|
b0506e488 userfaultfd: shme... |
221 222 223 224 |
bool vma_is_shmem(struct vm_area_struct *vma) { return vma->vm_ops == &shmem_vm_ops; } |
1da177e4c Linux-2.6.12-rc2 |
225 |
static LIST_HEAD(shmem_swaplist); |
cb5f7b9a4 tmpfs: make shmem... |
226 |
static DEFINE_MUTEX(shmem_swaplist_mutex); |
1da177e4c Linux-2.6.12-rc2 |
227 |
|
5b04c6890 shmem: factor out... |
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 |
static int shmem_reserve_inode(struct super_block *sb) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); if (sbinfo->max_inodes) { spin_lock(&sbinfo->stat_lock); if (!sbinfo->free_inodes) { spin_unlock(&sbinfo->stat_lock); return -ENOSPC; } sbinfo->free_inodes--; spin_unlock(&sbinfo->stat_lock); } return 0; } static void shmem_free_inode(struct super_block *sb) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); if (sbinfo->max_inodes) { spin_lock(&sbinfo->stat_lock); sbinfo->free_inodes++; spin_unlock(&sbinfo->stat_lock); } } |
467118102 mm/shmem and tiny... |
252 |
/** |
41ffe5d5c tmpfs: miscellane... |
253 |
* shmem_recalc_inode - recalculate the block usage of an inode |
1da177e4c Linux-2.6.12-rc2 |
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 |
* @inode: inode to recalc * * We have to calculate the free blocks since the mm can drop * undirtied hole pages behind our back. * * But normally info->alloced == inode->i_mapping->nrpages + info->swapped * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) * * It has to be called with the spinlock held. */ static void shmem_recalc_inode(struct inode *inode) { struct shmem_inode_info *info = SHMEM_I(inode); long freed; freed = info->alloced - info->swapped - inode->i_mapping->nrpages; if (freed > 0) { info->alloced -= freed; |
54af60421 tmpfs: convert sh... |
272 |
inode->i_blocks -= freed * BLOCKS_PER_PAGE; |
0f0796945 shmem: introduce ... |
273 |
shmem_inode_unacct_blocks(inode, freed); |
1da177e4c Linux-2.6.12-rc2 |
274 275 |
} } |
800d8c63b shmem: add huge p... |
276 277 278 |
bool shmem_charge(struct inode *inode, long pages) { struct shmem_inode_info *info = SHMEM_I(inode); |
4595ef88d shmem: make shmem... |
279 |
unsigned long flags; |
800d8c63b shmem: add huge p... |
280 |
|
0f0796945 shmem: introduce ... |
281 |
if (!shmem_inode_acct_block(inode, pages)) |
800d8c63b shmem: add huge p... |
282 |
return false; |
b1cc94ab2 shmem: shmem_char... |
283 |
|
4595ef88d shmem: make shmem... |
284 |
spin_lock_irqsave(&info->lock, flags); |
800d8c63b shmem: add huge p... |
285 286 287 |
info->alloced += pages; inode->i_blocks += pages * BLOCKS_PER_PAGE; shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
288 |
spin_unlock_irqrestore(&info->lock, flags); |
800d8c63b shmem: add huge p... |
289 |
inode->i_mapping->nrpages += pages; |
800d8c63b shmem: add huge p... |
290 291 292 293 294 295 |
return true; } void shmem_uncharge(struct inode *inode, long pages) { struct shmem_inode_info *info = SHMEM_I(inode); |
4595ef88d shmem: make shmem... |
296 |
unsigned long flags; |
800d8c63b shmem: add huge p... |
297 |
|
4595ef88d shmem: make shmem... |
298 |
spin_lock_irqsave(&info->lock, flags); |
800d8c63b shmem: add huge p... |
299 300 301 |
info->alloced -= pages; inode->i_blocks -= pages * BLOCKS_PER_PAGE; shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
302 |
spin_unlock_irqrestore(&info->lock, flags); |
800d8c63b shmem: add huge p... |
303 |
|
0f0796945 shmem: introduce ... |
304 |
shmem_inode_unacct_blocks(inode, pages); |
800d8c63b shmem: add huge p... |
305 |
} |
7a5d0fbb2 tmpfs: convert sh... |
306 307 308 309 310 311 |
/* * Replace item expected in radix tree by a new item, while holding tree lock. */ static int shmem_radix_tree_replace(struct address_space *mapping, pgoff_t index, void *expected, void *replacement) { |
f7942430e lib: radix-tree: ... |
312 |
struct radix_tree_node *node; |
7a5d0fbb2 tmpfs: convert sh... |
313 |
void **pslot; |
6dbaf22ce mm: shmem: save o... |
314 |
void *item; |
7a5d0fbb2 tmpfs: convert sh... |
315 316 |
VM_BUG_ON(!expected); |
6dbaf22ce mm: shmem: save o... |
317 |
VM_BUG_ON(!replacement); |
f7942430e lib: radix-tree: ... |
318 319 |
item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); if (!item) |
6dbaf22ce mm: shmem: save o... |
320 |
return -ENOENT; |
7a5d0fbb2 tmpfs: convert sh... |
321 322 |
if (item != expected) return -ENOENT; |
4d693d086 lib: radix-tree: ... |
323 324 |
__radix_tree_replace(&mapping->page_tree, node, pslot, replacement, NULL, NULL); |
7a5d0fbb2 tmpfs: convert sh... |
325 326 327 328 |
return 0; } /* |
d18992286 shmem: fix negati... |
329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 |
* Sometimes, before we decide whether to proceed or to fail, we must check * that an entry was not already brought back from swap by a racing thread. * * Checking page is not enough: by the time a SwapCache page is locked, it * might be reused, and again be SwapCache, using the same swap as before. */ static bool shmem_confirm_swap(struct address_space *mapping, pgoff_t index, swp_entry_t swap) { void *item; rcu_read_lock(); item = radix_tree_lookup(&mapping->page_tree, index); rcu_read_unlock(); return item == swp_to_radix_entry(swap); } /* |
5a6e75f81 shmem: prepare hu... |
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 |
* Definitions for "huge tmpfs": tmpfs mounted with the huge= option * * SHMEM_HUGE_NEVER: * disables huge pages for the mount; * SHMEM_HUGE_ALWAYS: * enables huge pages for the mount; * SHMEM_HUGE_WITHIN_SIZE: * only allocate huge pages if the page will be fully within i_size, * also respect fadvise()/madvise() hints; * SHMEM_HUGE_ADVISE: * only allocate huge pages if requested with fadvise()/madvise(); */ #define SHMEM_HUGE_NEVER 0 #define SHMEM_HUGE_ALWAYS 1 #define SHMEM_HUGE_WITHIN_SIZE 2 #define SHMEM_HUGE_ADVISE 3 /* * Special values. * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: * * SHMEM_HUGE_DENY: * disables huge on shm_mnt and all mounts, for emergency use; * SHMEM_HUGE_FORCE: * enables huge on shm_mnt and all mounts, w/o needing option, for testing; * */ #define SHMEM_HUGE_DENY (-1) #define SHMEM_HUGE_FORCE (-2) |
e496cf3d7 thp: introduce CO... |
377 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE |
5a6e75f81 shmem: prepare hu... |
378 379 380 |
/* ifdef here to avoid bloating shmem.o when not necessary */ int shmem_huge __read_mostly; |
f1f5929cd shmem: fix compil... |
381 |
#if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) |
5a6e75f81 shmem: prepare hu... |
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 |
static int shmem_parse_huge(const char *str) { if (!strcmp(str, "never")) return SHMEM_HUGE_NEVER; if (!strcmp(str, "always")) return SHMEM_HUGE_ALWAYS; if (!strcmp(str, "within_size")) return SHMEM_HUGE_WITHIN_SIZE; if (!strcmp(str, "advise")) return SHMEM_HUGE_ADVISE; if (!strcmp(str, "deny")) return SHMEM_HUGE_DENY; if (!strcmp(str, "force")) return SHMEM_HUGE_FORCE; return -EINVAL; } static const char *shmem_format_huge(int huge) { switch (huge) { case SHMEM_HUGE_NEVER: return "never"; case SHMEM_HUGE_ALWAYS: return "always"; case SHMEM_HUGE_WITHIN_SIZE: return "within_size"; case SHMEM_HUGE_ADVISE: return "advise"; case SHMEM_HUGE_DENY: return "deny"; case SHMEM_HUGE_FORCE: return "force"; default: VM_BUG_ON(1); return "bad_val"; } } |
f1f5929cd shmem: fix compil... |
419 |
#endif |
5a6e75f81 shmem: prepare hu... |
420 |
|
779750d20 shmem: split huge... |
421 422 423 424 |
static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_split) { LIST_HEAD(list), *pos, *next; |
253fd0f02 shmem: fix sleepi... |
425 |
LIST_HEAD(to_remove); |
779750d20 shmem: split huge... |
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 |
struct inode *inode; struct shmem_inode_info *info; struct page *page; unsigned long batch = sc ? sc->nr_to_scan : 128; int removed = 0, split = 0; if (list_empty(&sbinfo->shrinklist)) return SHRINK_STOP; spin_lock(&sbinfo->shrinklist_lock); list_for_each_safe(pos, next, &sbinfo->shrinklist) { info = list_entry(pos, struct shmem_inode_info, shrinklist); /* pin the inode */ inode = igrab(&info->vfs_inode); /* inode is about to be evicted */ if (!inode) { list_del_init(&info->shrinklist); removed++; goto next; } /* Check if there's anything to gain */ if (round_up(inode->i_size, PAGE_SIZE) == round_up(inode->i_size, HPAGE_PMD_SIZE)) { |
253fd0f02 shmem: fix sleepi... |
452 |
list_move(&info->shrinklist, &to_remove); |
779750d20 shmem: split huge... |
453 |
removed++; |
779750d20 shmem: split huge... |
454 455 456 457 458 459 460 461 462 |
goto next; } list_move(&info->shrinklist, &list); next: if (!--batch) break; } spin_unlock(&sbinfo->shrinklist_lock); |
253fd0f02 shmem: fix sleepi... |
463 464 465 466 467 468 |
list_for_each_safe(pos, next, &to_remove) { info = list_entry(pos, struct shmem_inode_info, shrinklist); inode = &info->vfs_inode; list_del_init(&info->shrinklist); iput(inode); } |
779750d20 shmem: split huge... |
469 470 471 472 473 |
list_for_each_safe(pos, next, &list) { int ret; info = list_entry(pos, struct shmem_inode_info, shrinklist); inode = &info->vfs_inode; |
f5dad0409 mm/shmem: do not ... |
474 475 |
if (nr_to_split && split >= nr_to_split) goto leave; |
779750d20 shmem: split huge... |
476 |
|
f5dad0409 mm/shmem: do not ... |
477 |
page = find_get_page(inode->i_mapping, |
779750d20 shmem: split huge... |
478 479 480 |
(inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); if (!page) goto drop; |
f5dad0409 mm/shmem: do not ... |
481 |
/* No huge page at the end of the file: nothing to split */ |
779750d20 shmem: split huge... |
482 |
if (!PageTransHuge(page)) { |
779750d20 shmem: split huge... |
483 484 485 |
put_page(page); goto drop; } |
f5dad0409 mm/shmem: do not ... |
486 487 488 489 490 491 492 493 494 495 496 |
/* * Leave the inode on the list if we failed to lock * the page at this time. * * Waiting for the lock may lead to deadlock in the * reclaim path. */ if (!trylock_page(page)) { put_page(page); goto leave; } |
779750d20 shmem: split huge... |
497 498 499 |
ret = split_huge_page(page); unlock_page(page); put_page(page); |
f5dad0409 mm/shmem: do not ... |
500 501 502 |
/* If split failed leave the inode on the list */ if (ret) goto leave; |
779750d20 shmem: split huge... |
503 504 505 506 507 |
split++; drop: list_del_init(&info->shrinklist); removed++; |
f5dad0409 mm/shmem: do not ... |
508 |
leave: |
779750d20 shmem: split huge... |
509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 |
iput(inode); } spin_lock(&sbinfo->shrinklist_lock); list_splice_tail(&list, &sbinfo->shrinklist); sbinfo->shrinklist_len -= removed; spin_unlock(&sbinfo->shrinklist_lock); return split; } static long shmem_unused_huge_scan(struct super_block *sb, struct shrink_control *sc) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); if (!READ_ONCE(sbinfo->shrinklist_len)) return SHRINK_STOP; return shmem_unused_huge_shrink(sbinfo, sc, 0); } static long shmem_unused_huge_count(struct super_block *sb, struct shrink_control *sc) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); return READ_ONCE(sbinfo->shrinklist_len); } |
e496cf3d7 thp: introduce CO... |
537 |
#else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ |
5a6e75f81 shmem: prepare hu... |
538 539 |
#define shmem_huge SHMEM_HUGE_DENY |
779750d20 shmem: split huge... |
540 541 542 543 544 |
static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, struct shrink_control *sc, unsigned long nr_to_split) { return 0; } |
e496cf3d7 thp: introduce CO... |
545 |
#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ |
5a6e75f81 shmem: prepare hu... |
546 547 |
/* |
46f65ec15 tmpfs: convert sh... |
548 549 550 551 |
* Like add_to_page_cache_locked, but error if expected item has gone. */ static int shmem_add_to_page_cache(struct page *page, struct address_space *mapping, |
fed400a18 mm/shmem.c: remov... |
552 |
pgoff_t index, void *expected) |
46f65ec15 tmpfs: convert sh... |
553 |
{ |
800d8c63b shmem: add huge p... |
554 |
int error, nr = hpage_nr_pages(page); |
46f65ec15 tmpfs: convert sh... |
555 |
|
800d8c63b shmem: add huge p... |
556 557 |
VM_BUG_ON_PAGE(PageTail(page), page); VM_BUG_ON_PAGE(index != round_down(index, nr), page); |
309381fea mm: dump page whe... |
558 559 |
VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageSwapBacked(page), page); |
800d8c63b shmem: add huge p... |
560 |
VM_BUG_ON(expected && PageTransHuge(page)); |
46f65ec15 tmpfs: convert sh... |
561 |
|
800d8c63b shmem: add huge p... |
562 |
page_ref_add(page, nr); |
b065b4321 shmem: cleanup sh... |
563 564 565 566 |
page->mapping = mapping; page->index = index; spin_lock_irq(&mapping->tree_lock); |
800d8c63b shmem: add huge p... |
567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 |
if (PageTransHuge(page)) { void __rcu **results; pgoff_t idx; int i; error = 0; if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, index, 1) && idx < index + HPAGE_PMD_NR) { error = -EEXIST; } if (!error) { for (i = 0; i < HPAGE_PMD_NR; i++) { error = radix_tree_insert(&mapping->page_tree, index + i, page + i); VM_BUG_ON(error); } count_vm_event(THP_FILE_ALLOC); } } else if (!expected) { |
b065b4321 shmem: cleanup sh... |
588 |
error = radix_tree_insert(&mapping->page_tree, index, page); |
800d8c63b shmem: add huge p... |
589 |
} else { |
b065b4321 shmem: cleanup sh... |
590 591 |
error = shmem_radix_tree_replace(mapping, index, expected, page); |
800d8c63b shmem: add huge p... |
592 |
} |
46f65ec15 tmpfs: convert sh... |
593 |
if (!error) { |
800d8c63b shmem: add huge p... |
594 595 |
mapping->nrpages += nr; if (PageTransHuge(page)) |
11fb99898 mm: move most fil... |
596 597 598 |
__inc_node_page_state(page, NR_SHMEM_THPS); __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); |
b065b4321 shmem: cleanup sh... |
599 600 601 602 |
spin_unlock_irq(&mapping->tree_lock); } else { page->mapping = NULL; spin_unlock_irq(&mapping->tree_lock); |
800d8c63b shmem: add huge p... |
603 |
page_ref_sub(page, nr); |
46f65ec15 tmpfs: convert sh... |
604 |
} |
46f65ec15 tmpfs: convert sh... |
605 606 607 608 |
return error; } /* |
6922c0c7a tmpfs: convert sh... |
609 610 611 612 613 614 |
* Like delete_from_page_cache, but substitutes swap for page. */ static void shmem_delete_from_page_cache(struct page *page, void *radswap) { struct address_space *mapping = page->mapping; int error; |
800d8c63b shmem: add huge p... |
615 |
VM_BUG_ON_PAGE(PageCompound(page), page); |
6922c0c7a tmpfs: convert sh... |
616 617 618 619 |
spin_lock_irq(&mapping->tree_lock); error = shmem_radix_tree_replace(mapping, page->index, page, radswap); page->mapping = NULL; mapping->nrpages--; |
11fb99898 mm: move most fil... |
620 621 |
__dec_node_page_state(page, NR_FILE_PAGES); __dec_node_page_state(page, NR_SHMEM); |
6922c0c7a tmpfs: convert sh... |
622 |
spin_unlock_irq(&mapping->tree_lock); |
09cbfeaf1 mm, fs: get rid o... |
623 |
put_page(page); |
6922c0c7a tmpfs: convert sh... |
624 625 626 627 |
BUG_ON(error); } /* |
7a5d0fbb2 tmpfs: convert sh... |
628 629 630 631 632 |
* Remove swap entry from radix tree, free the swap and its page cache. */ static int shmem_free_swap(struct address_space *mapping, pgoff_t index, void *radswap) { |
6dbaf22ce mm: shmem: save o... |
633 |
void *old; |
7a5d0fbb2 tmpfs: convert sh... |
634 635 |
spin_lock_irq(&mapping->tree_lock); |
6dbaf22ce mm: shmem: save o... |
636 |
old = radix_tree_delete_item(&mapping->page_tree, index, radswap); |
7a5d0fbb2 tmpfs: convert sh... |
637 |
spin_unlock_irq(&mapping->tree_lock); |
6dbaf22ce mm: shmem: save o... |
638 639 640 641 |
if (old != radswap) return -ENOENT; free_swap_and_cache(radix_to_swp_entry(radswap)); return 0; |
7a5d0fbb2 tmpfs: convert sh... |
642 643 644 |
} /* |
6a15a3709 mm, proc: reduce ... |
645 |
* Determine (in bytes) how many of the shmem object's pages mapped by the |
48131e03c mm, proc: reduce ... |
646 |
* given offsets are swapped out. |
6a15a3709 mm, proc: reduce ... |
647 648 649 650 |
* * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, * as long as the inode doesn't go away and racy results are not a problem. */ |
48131e03c mm, proc: reduce ... |
651 652 |
unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end) |
6a15a3709 mm, proc: reduce ... |
653 |
{ |
6a15a3709 mm, proc: reduce ... |
654 655 656 |
struct radix_tree_iter iter; void **slot; struct page *page; |
48131e03c mm, proc: reduce ... |
657 |
unsigned long swapped = 0; |
6a15a3709 mm, proc: reduce ... |
658 659 |
rcu_read_lock(); |
6a15a3709 mm, proc: reduce ... |
660 661 662 663 664 |
radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { if (iter.index >= end) break; page = radix_tree_deref_slot(slot); |
2cf938aae mm: use radix_tre... |
665 666 667 668 |
if (radix_tree_deref_retry(page)) { slot = radix_tree_iter_retry(&iter); continue; } |
6a15a3709 mm, proc: reduce ... |
669 670 671 672 673 |
if (radix_tree_exceptional_entry(page)) swapped++; if (need_resched()) { |
148deab22 radix-tree: impro... |
674 |
slot = radix_tree_iter_resume(slot, &iter); |
6a15a3709 mm, proc: reduce ... |
675 |
cond_resched_rcu(); |
6a15a3709 mm, proc: reduce ... |
676 677 678 679 680 681 682 683 684 |
} } rcu_read_unlock(); return swapped << PAGE_SHIFT; } /* |
48131e03c mm, proc: reduce ... |
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 |
* Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. * * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, * as long as the inode doesn't go away and racy results are not a problem. */ unsigned long shmem_swap_usage(struct vm_area_struct *vma) { struct inode *inode = file_inode(vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); struct address_space *mapping = inode->i_mapping; unsigned long swapped; /* Be careful as we don't hold info->lock */ swapped = READ_ONCE(info->swapped); /* * The easier cases are when the shmem object has nothing in swap, or * the vma maps it whole. Then we can simply use the stats that we * already track. */ if (!swapped) return 0; if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; /* Here comes the more involved part */ return shmem_partial_swap_usage(mapping, linear_page_index(vma, vma->vm_start), linear_page_index(vma, vma->vm_end)); } /* |
245132643 SHM_UNLOCK: fix U... |
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 |
* SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. */ void shmem_unlock_mapping(struct address_space *mapping) { struct pagevec pvec; pgoff_t indices[PAGEVEC_SIZE]; pgoff_t index = 0; pagevec_init(&pvec, 0); /* * Minor point, but we might as well stop if someone else SHM_LOCKs it. */ while (!mapping_unevictable(mapping)) { /* * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it * has finished, if it hits a row of PAGEVEC_SIZE swap entries. */ |
0cd6144aa mm + fs: prepare ... |
736 737 |
pvec.nr = find_get_entries(mapping, index, PAGEVEC_SIZE, pvec.pages, indices); |
245132643 SHM_UNLOCK: fix U... |
738 739 740 |
if (!pvec.nr) break; index = indices[pvec.nr - 1] + 1; |
0cd6144aa mm + fs: prepare ... |
741 |
pagevec_remove_exceptionals(&pvec); |
245132643 SHM_UNLOCK: fix U... |
742 743 744 745 |
check_move_unevictable_pages(pvec.pages, pvec.nr); pagevec_release(&pvec); cond_resched(); } |
7a5d0fbb2 tmpfs: convert sh... |
746 747 748 749 |
} /* * Remove range of pages and swap entries from radix tree, and free them. |
1635f6a74 tmpfs: undo fallo... |
750 |
* If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. |
7a5d0fbb2 tmpfs: convert sh... |
751 |
*/ |
1635f6a74 tmpfs: undo fallo... |
752 753 |
static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, bool unfalloc) |
1da177e4c Linux-2.6.12-rc2 |
754 |
{ |
285b2c4fd tmpfs: demolish o... |
755 |
struct address_space *mapping = inode->i_mapping; |
1da177e4c Linux-2.6.12-rc2 |
756 |
struct shmem_inode_info *info = SHMEM_I(inode); |
09cbfeaf1 mm, fs: get rid o... |
757 758 759 760 |
pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; pgoff_t end = (lend + 1) >> PAGE_SHIFT; unsigned int partial_start = lstart & (PAGE_SIZE - 1); unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); |
bda97eab0 tmpfs: copy trunc... |
761 |
struct pagevec pvec; |
7a5d0fbb2 tmpfs: convert sh... |
762 763 |
pgoff_t indices[PAGEVEC_SIZE]; long nr_swaps_freed = 0; |
285b2c4fd tmpfs: demolish o... |
764 |
pgoff_t index; |
bda97eab0 tmpfs: copy trunc... |
765 |
int i; |
83e4fa9c1 tmpfs: support fa... |
766 767 |
if (lend == -1) end = -1; /* unsigned, so actually very big */ |
bda97eab0 tmpfs: copy trunc... |
768 769 770 |
pagevec_init(&pvec, 0); index = start; |
83e4fa9c1 tmpfs: support fa... |
771 |
while (index < end) { |
0cd6144aa mm + fs: prepare ... |
772 773 774 |
pvec.nr = find_get_entries(mapping, index, min(end - index, (pgoff_t)PAGEVEC_SIZE), pvec.pages, indices); |
7a5d0fbb2 tmpfs: convert sh... |
775 776 |
if (!pvec.nr) break; |
bda97eab0 tmpfs: copy trunc... |
777 778 |
for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; |
7a5d0fbb2 tmpfs: convert sh... |
779 |
index = indices[i]; |
83e4fa9c1 tmpfs: support fa... |
780 |
if (index >= end) |
bda97eab0 tmpfs: copy trunc... |
781 |
break; |
7a5d0fbb2 tmpfs: convert sh... |
782 |
if (radix_tree_exceptional_entry(page)) { |
1635f6a74 tmpfs: undo fallo... |
783 784 |
if (unfalloc) continue; |
7a5d0fbb2 tmpfs: convert sh... |
785 786 |
nr_swaps_freed += !shmem_free_swap(mapping, index, page); |
bda97eab0 tmpfs: copy trunc... |
787 |
continue; |
7a5d0fbb2 tmpfs: convert sh... |
788 |
} |
800d8c63b shmem: add huge p... |
789 |
VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); |
7a5d0fbb2 tmpfs: convert sh... |
790 |
if (!trylock_page(page)) |
bda97eab0 tmpfs: copy trunc... |
791 |
continue; |
800d8c63b shmem: add huge p... |
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 |
if (PageTransTail(page)) { /* Middle of THP: zero out the page */ clear_highpage(page); unlock_page(page); continue; } else if (PageTransHuge(page)) { if (index == round_down(end, HPAGE_PMD_NR)) { /* * Range ends in the middle of THP: * zero out the page */ clear_highpage(page); unlock_page(page); continue; } index += HPAGE_PMD_NR - 1; i += HPAGE_PMD_NR - 1; } |
1635f6a74 tmpfs: undo fallo... |
811 |
if (!unfalloc || !PageUptodate(page)) { |
800d8c63b shmem: add huge p... |
812 813 |
VM_BUG_ON_PAGE(PageTail(page), page); if (page_mapping(page) == mapping) { |
309381fea mm: dump page whe... |
814 |
VM_BUG_ON_PAGE(PageWriteback(page), page); |
1635f6a74 tmpfs: undo fallo... |
815 816 |
truncate_inode_page(mapping, page); } |
bda97eab0 tmpfs: copy trunc... |
817 |
} |
bda97eab0 tmpfs: copy trunc... |
818 819 |
unlock_page(page); } |
0cd6144aa mm + fs: prepare ... |
820 |
pagevec_remove_exceptionals(&pvec); |
245132643 SHM_UNLOCK: fix U... |
821 |
pagevec_release(&pvec); |
bda97eab0 tmpfs: copy trunc... |
822 823 824 |
cond_resched(); index++; } |
1da177e4c Linux-2.6.12-rc2 |
825 |
|
83e4fa9c1 tmpfs: support fa... |
826 |
if (partial_start) { |
bda97eab0 tmpfs: copy trunc... |
827 |
struct page *page = NULL; |
9e18eb293 tmpfs: mem_cgroup... |
828 |
shmem_getpage(inode, start - 1, &page, SGP_READ); |
bda97eab0 tmpfs: copy trunc... |
829 |
if (page) { |
09cbfeaf1 mm, fs: get rid o... |
830 |
unsigned int top = PAGE_SIZE; |
83e4fa9c1 tmpfs: support fa... |
831 832 833 834 835 836 837 |
if (start > end) { top = partial_end; partial_end = 0; } zero_user_segment(page, partial_start, top); set_page_dirty(page); unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
838 |
put_page(page); |
83e4fa9c1 tmpfs: support fa... |
839 840 841 842 |
} } if (partial_end) { struct page *page = NULL; |
9e18eb293 tmpfs: mem_cgroup... |
843 |
shmem_getpage(inode, end, &page, SGP_READ); |
83e4fa9c1 tmpfs: support fa... |
844 845 |
if (page) { zero_user_segment(page, 0, partial_end); |
bda97eab0 tmpfs: copy trunc... |
846 847 |
set_page_dirty(page); unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
848 |
put_page(page); |
bda97eab0 tmpfs: copy trunc... |
849 850 |
} } |
83e4fa9c1 tmpfs: support fa... |
851 852 |
if (start >= end) return; |
bda97eab0 tmpfs: copy trunc... |
853 854 |
index = start; |
b1a366500 shmem: fix splici... |
855 |
while (index < end) { |
bda97eab0 tmpfs: copy trunc... |
856 |
cond_resched(); |
0cd6144aa mm + fs: prepare ... |
857 858 |
pvec.nr = find_get_entries(mapping, index, |
83e4fa9c1 tmpfs: support fa... |
859 |
min(end - index, (pgoff_t)PAGEVEC_SIZE), |
0cd6144aa mm + fs: prepare ... |
860 |
pvec.pages, indices); |
7a5d0fbb2 tmpfs: convert sh... |
861 |
if (!pvec.nr) { |
b1a366500 shmem: fix splici... |
862 863 |
/* If all gone or hole-punch or unfalloc, we're done */ if (index == start || end != -1) |
bda97eab0 tmpfs: copy trunc... |
864 |
break; |
b1a366500 shmem: fix splici... |
865 |
/* But if truncating, restart to make sure all gone */ |
bda97eab0 tmpfs: copy trunc... |
866 867 868 |
index = start; continue; } |
bda97eab0 tmpfs: copy trunc... |
869 870 |
for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; |
7a5d0fbb2 tmpfs: convert sh... |
871 |
index = indices[i]; |
83e4fa9c1 tmpfs: support fa... |
872 |
if (index >= end) |
bda97eab0 tmpfs: copy trunc... |
873 |
break; |
7a5d0fbb2 tmpfs: convert sh... |
874 |
if (radix_tree_exceptional_entry(page)) { |
1635f6a74 tmpfs: undo fallo... |
875 876 |
if (unfalloc) continue; |
b1a366500 shmem: fix splici... |
877 878 879 880 881 882 |
if (shmem_free_swap(mapping, index, page)) { /* Swap was replaced by page: retry */ index--; break; } nr_swaps_freed++; |
7a5d0fbb2 tmpfs: convert sh... |
883 884 |
continue; } |
bda97eab0 tmpfs: copy trunc... |
885 |
lock_page(page); |
800d8c63b shmem: add huge p... |
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 |
if (PageTransTail(page)) { /* Middle of THP: zero out the page */ clear_highpage(page); unlock_page(page); /* * Partial thp truncate due 'start' in middle * of THP: don't need to look on these pages * again on !pvec.nr restart. */ if (index != round_down(end, HPAGE_PMD_NR)) start++; continue; } else if (PageTransHuge(page)) { if (index == round_down(end, HPAGE_PMD_NR)) { /* * Range ends in the middle of THP: * zero out the page */ clear_highpage(page); unlock_page(page); continue; } index += HPAGE_PMD_NR - 1; i += HPAGE_PMD_NR - 1; } |
1635f6a74 tmpfs: undo fallo... |
912 |
if (!unfalloc || !PageUptodate(page)) { |
800d8c63b shmem: add huge p... |
913 914 |
VM_BUG_ON_PAGE(PageTail(page), page); if (page_mapping(page) == mapping) { |
309381fea mm: dump page whe... |
915 |
VM_BUG_ON_PAGE(PageWriteback(page), page); |
1635f6a74 tmpfs: undo fallo... |
916 |
truncate_inode_page(mapping, page); |
b1a366500 shmem: fix splici... |
917 918 919 920 921 |
} else { /* Page was replaced by swap: retry */ unlock_page(page); index--; break; |
1635f6a74 tmpfs: undo fallo... |
922 |
} |
7a5d0fbb2 tmpfs: convert sh... |
923 |
} |
bda97eab0 tmpfs: copy trunc... |
924 925 |
unlock_page(page); } |
0cd6144aa mm + fs: prepare ... |
926 |
pagevec_remove_exceptionals(&pvec); |
245132643 SHM_UNLOCK: fix U... |
927 |
pagevec_release(&pvec); |
bda97eab0 tmpfs: copy trunc... |
928 929 |
index++; } |
94c1e62df tmpfs: take contr... |
930 |
|
4595ef88d shmem: make shmem... |
931 |
spin_lock_irq(&info->lock); |
7a5d0fbb2 tmpfs: convert sh... |
932 |
info->swapped -= nr_swaps_freed; |
1da177e4c Linux-2.6.12-rc2 |
933 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
934 |
spin_unlock_irq(&info->lock); |
1635f6a74 tmpfs: undo fallo... |
935 |
} |
1da177e4c Linux-2.6.12-rc2 |
936 |
|
1635f6a74 tmpfs: undo fallo... |
937 938 939 |
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) { shmem_undo_range(inode, lstart, lend, false); |
078cd8279 fs: Replace CURRE... |
940 |
inode->i_ctime = inode->i_mtime = current_time(inode); |
1da177e4c Linux-2.6.12-rc2 |
941 |
} |
94c1e62df tmpfs: take contr... |
942 |
EXPORT_SYMBOL_GPL(shmem_truncate_range); |
1da177e4c Linux-2.6.12-rc2 |
943 |
|
a528d35e8 statx: Add a syst... |
944 945 |
static int shmem_getattr(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) |
44a30220b shmem: recalculat... |
946 |
{ |
a528d35e8 statx: Add a syst... |
947 |
struct inode *inode = path->dentry->d_inode; |
44a30220b shmem: recalculat... |
948 |
struct shmem_inode_info *info = SHMEM_I(inode); |
d0424c429 tmpfs: avoid a li... |
949 |
if (info->alloced - info->swapped != inode->i_mapping->nrpages) { |
4595ef88d shmem: make shmem... |
950 |
spin_lock_irq(&info->lock); |
d0424c429 tmpfs: avoid a li... |
951 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
952 |
spin_unlock_irq(&info->lock); |
d0424c429 tmpfs: avoid a li... |
953 |
} |
44a30220b shmem: recalculat... |
954 |
generic_fillattr(inode, stat); |
44a30220b shmem: recalculat... |
955 956 |
return 0; } |
94c1e62df tmpfs: take contr... |
957 |
static int shmem_setattr(struct dentry *dentry, struct iattr *attr) |
1da177e4c Linux-2.6.12-rc2 |
958 |
{ |
75c3cfa85 VFS: assorted wei... |
959 |
struct inode *inode = d_inode(dentry); |
40e041a2c shm: add sealing API |
960 |
struct shmem_inode_info *info = SHMEM_I(inode); |
779750d20 shmem: split huge... |
961 |
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); |
1da177e4c Linux-2.6.12-rc2 |
962 |
int error; |
31051c85b fs: Give dentry t... |
963 |
error = setattr_prepare(dentry, attr); |
db78b877f always call inode... |
964 965 |
if (error) return error; |
94c1e62df tmpfs: take contr... |
966 967 968 |
if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { loff_t oldsize = inode->i_size; loff_t newsize = attr->ia_size; |
3889e6e76 tmpfs: convert to... |
969 |
|
40e041a2c shm: add sealing API |
970 971 972 973 |
/* protected by i_mutex */ if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || (newsize > oldsize && (info->seals & F_SEAL_GROW))) return -EPERM; |
94c1e62df tmpfs: take contr... |
974 |
if (newsize != oldsize) { |
771425179 shmem: update mem... |
975 976 977 978 |
error = shmem_reacct_size(SHMEM_I(inode)->flags, oldsize, newsize); if (error) return error; |
94c1e62df tmpfs: take contr... |
979 |
i_size_write(inode, newsize); |
078cd8279 fs: Replace CURRE... |
980 |
inode->i_ctime = inode->i_mtime = current_time(inode); |
94c1e62df tmpfs: take contr... |
981 |
} |
afa2db2fb tmpfs: truncate p... |
982 |
if (newsize <= oldsize) { |
94c1e62df tmpfs: take contr... |
983 |
loff_t holebegin = round_up(newsize, PAGE_SIZE); |
d0424c429 tmpfs: avoid a li... |
984 985 986 987 988 989 |
if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); if (info->alloced) shmem_truncate_range(inode, newsize, (loff_t)-1); |
94c1e62df tmpfs: take contr... |
990 |
/* unmap again to remove racily COWed private pages */ |
d0424c429 tmpfs: avoid a li... |
991 992 993 |
if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); |
779750d20 shmem: split huge... |
994 995 996 997 998 999 1000 |
/* * Part of the huge page can be beyond i_size: subject * to shrink under memory pressure. */ if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { spin_lock(&sbinfo->shrinklist_lock); |
d041353dc mm: fix list corr... |
1001 1002 1003 1004 1005 |
/* * _careful to defend against unlocked access to * ->shrink_list in shmem_unused_huge_shrink() */ if (list_empty_careful(&info->shrinklist)) { |
779750d20 shmem: split huge... |
1006 1007 1008 1009 1010 1011 |
list_add_tail(&info->shrinklist, &sbinfo->shrinklist); sbinfo->shrinklist_len++; } spin_unlock(&sbinfo->shrinklist_lock); } |
94c1e62df tmpfs: take contr... |
1012 |
} |
1da177e4c Linux-2.6.12-rc2 |
1013 |
} |
db78b877f always call inode... |
1014 |
setattr_copy(inode, attr); |
db78b877f always call inode... |
1015 |
if (attr->ia_valid & ATTR_MODE) |
feda821e7 fs: remove generi... |
1016 |
error = posix_acl_chmod(inode, inode->i_mode); |
1da177e4c Linux-2.6.12-rc2 |
1017 1018 |
return error; } |
1f895f75d switch shmem.c to... |
1019 |
static void shmem_evict_inode(struct inode *inode) |
1da177e4c Linux-2.6.12-rc2 |
1020 |
{ |
1da177e4c Linux-2.6.12-rc2 |
1021 |
struct shmem_inode_info *info = SHMEM_I(inode); |
779750d20 shmem: split huge... |
1022 |
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); |
1da177e4c Linux-2.6.12-rc2 |
1023 |
|
3889e6e76 tmpfs: convert to... |
1024 |
if (inode->i_mapping->a_ops == &shmem_aops) { |
1da177e4c Linux-2.6.12-rc2 |
1025 1026 |
shmem_unacct_size(info->flags, inode->i_size); inode->i_size = 0; |
3889e6e76 tmpfs: convert to... |
1027 |
shmem_truncate_range(inode, 0, (loff_t)-1); |
779750d20 shmem: split huge... |
1028 1029 1030 1031 1032 1033 1034 1035 |
if (!list_empty(&info->shrinklist)) { spin_lock(&sbinfo->shrinklist_lock); if (!list_empty(&info->shrinklist)) { list_del_init(&info->shrinklist); sbinfo->shrinklist_len--; } spin_unlock(&sbinfo->shrinklist_lock); } |
1da177e4c Linux-2.6.12-rc2 |
1036 |
if (!list_empty(&info->swaplist)) { |
cb5f7b9a4 tmpfs: make shmem... |
1037 |
mutex_lock(&shmem_swaplist_mutex); |
1da177e4c Linux-2.6.12-rc2 |
1038 |
list_del_init(&info->swaplist); |
cb5f7b9a4 tmpfs: make shmem... |
1039 |
mutex_unlock(&shmem_swaplist_mutex); |
1da177e4c Linux-2.6.12-rc2 |
1040 |
} |
3ed47db34 make sure that fr... |
1041 |
} |
b09e0fa4b tmpfs: implement ... |
1042 |
|
38f386574 xattr: extract si... |
1043 |
simple_xattrs_free(&info->xattrs); |
0f3c42f52 tmpfs: change fin... |
1044 |
WARN_ON(inode->i_blocks); |
5b04c6890 shmem: factor out... |
1045 |
shmem_free_inode(inode->i_sb); |
dbd5768f8 vfs: Rename end_w... |
1046 |
clear_inode(inode); |
1da177e4c Linux-2.6.12-rc2 |
1047 |
} |
478922e2b radix-tree: delet... |
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 |
static unsigned long find_swap_entry(struct radix_tree_root *root, void *item) { struct radix_tree_iter iter; void **slot; unsigned long found = -1; unsigned int checked = 0; rcu_read_lock(); radix_tree_for_each_slot(slot, root, &iter, 0) { if (*slot == item) { found = iter.index; break; } checked++; if ((checked % 4096) != 0) continue; slot = radix_tree_iter_resume(slot, &iter); cond_resched_rcu(); } rcu_read_unlock(); return found; } |
46f65ec15 tmpfs: convert sh... |
1071 1072 1073 |
/* * If swap found in inode, free it and move page from swapcache to filecache. */ |
41ffe5d5c tmpfs: miscellane... |
1074 |
static int shmem_unuse_inode(struct shmem_inode_info *info, |
bde05d1cc shmem: replace pa... |
1075 |
swp_entry_t swap, struct page **pagep) |
1da177e4c Linux-2.6.12-rc2 |
1076 |
{ |
285b2c4fd tmpfs: demolish o... |
1077 |
struct address_space *mapping = info->vfs_inode.i_mapping; |
46f65ec15 tmpfs: convert sh... |
1078 |
void *radswap; |
41ffe5d5c tmpfs: miscellane... |
1079 |
pgoff_t index; |
bde05d1cc shmem: replace pa... |
1080 1081 |
gfp_t gfp; int error = 0; |
1da177e4c Linux-2.6.12-rc2 |
1082 |
|
46f65ec15 tmpfs: convert sh... |
1083 |
radswap = swp_to_radix_entry(swap); |
478922e2b radix-tree: delet... |
1084 |
index = find_swap_entry(&mapping->page_tree, radswap); |
46f65ec15 tmpfs: convert sh... |
1085 |
if (index == -1) |
00501b531 mm: memcontrol: r... |
1086 |
return -EAGAIN; /* tell shmem_unuse we found nothing */ |
2e0e26c76 tmpfs: open a win... |
1087 |
|
1b1b32f2c tmpfs: fix shmem_... |
1088 1089 |
/* * Move _head_ to start search for next from here. |
1f895f75d switch shmem.c to... |
1090 |
* But be careful: shmem_evict_inode checks list_empty without taking |
1b1b32f2c tmpfs: fix shmem_... |
1091 |
* mutex, and there's an instant in list_move_tail when info->swaplist |
285b2c4fd tmpfs: demolish o... |
1092 |
* would appear empty, if it were the only one on shmem_swaplist. |
1b1b32f2c tmpfs: fix shmem_... |
1093 1094 1095 |
*/ if (shmem_swaplist.next != &info->swaplist) list_move_tail(&shmem_swaplist, &info->swaplist); |
2e0e26c76 tmpfs: open a win... |
1096 |
|
bde05d1cc shmem: replace pa... |
1097 1098 1099 1100 1101 1102 1103 |
gfp = mapping_gfp_mask(mapping); if (shmem_should_replace_page(*pagep, gfp)) { mutex_unlock(&shmem_swaplist_mutex); error = shmem_replace_page(pagep, gfp, info, index); mutex_lock(&shmem_swaplist_mutex); /* * We needed to drop mutex to make that restrictive page |
0142ef6cd shmem: replace_pa... |
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 |
* allocation, but the inode might have been freed while we * dropped it: although a racing shmem_evict_inode() cannot * complete without emptying the radix_tree, our page lock * on this swapcache page is not enough to prevent that - * free_swap_and_cache() of our swap entry will only * trylock_page(), removing swap from radix_tree whatever. * * We must not proceed to shmem_add_to_page_cache() if the * inode has been freed, but of course we cannot rely on * inode or mapping or info to check that. However, we can * safely check if our swap entry is still in use (and here * it can't have got reused for another page): if it's still * in use, then the inode cannot have been freed yet, and we * can safely proceed (if it's no longer in use, that tells * nothing about the inode, but we don't need to unuse swap). |
bde05d1cc shmem: replace pa... |
1119 1120 1121 1122 |
*/ if (!page_swapcount(*pagep)) error = -ENOENT; } |
d13d14430 memcg: handle swa... |
1123 |
/* |
778dd893a tmpfs: fix race b... |
1124 1125 1126 |
* We rely on shmem_swaplist_mutex, not only to protect the swaplist, * but also to hold up shmem_evict_inode(): so inode cannot be freed * beneath us (pagelock doesn't help until the page is in pagecache). |
d13d14430 memcg: handle swa... |
1127 |
*/ |
bde05d1cc shmem: replace pa... |
1128 1129 |
if (!error) error = shmem_add_to_page_cache(*pagep, mapping, index, |
fed400a18 mm/shmem.c: remov... |
1130 |
radswap); |
48f170fb7 tmpfs: simplify u... |
1131 |
if (error != -ENOMEM) { |
46f65ec15 tmpfs: convert sh... |
1132 1133 1134 1135 |
/* * Truncation and eviction use free_swap_and_cache(), which * only does trylock page: if we raced, best clean up here. */ |
bde05d1cc shmem: replace pa... |
1136 1137 |
delete_from_swap_cache(*pagep); set_page_dirty(*pagep); |
46f65ec15 tmpfs: convert sh... |
1138 |
if (!error) { |
4595ef88d shmem: make shmem... |
1139 |
spin_lock_irq(&info->lock); |
46f65ec15 tmpfs: convert sh... |
1140 |
info->swapped--; |
4595ef88d shmem: make shmem... |
1141 |
spin_unlock_irq(&info->lock); |
46f65ec15 tmpfs: convert sh... |
1142 1143 |
swap_free(swap); } |
1da177e4c Linux-2.6.12-rc2 |
1144 |
} |
2e0e26c76 tmpfs: open a win... |
1145 |
return error; |
1da177e4c Linux-2.6.12-rc2 |
1146 1147 1148 |
} /* |
46f65ec15 tmpfs: convert sh... |
1149 |
* Search through swapped inodes to find and replace swap by page. |
1da177e4c Linux-2.6.12-rc2 |
1150 |
*/ |
41ffe5d5c tmpfs: miscellane... |
1151 |
int shmem_unuse(swp_entry_t swap, struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
1152 |
{ |
41ffe5d5c tmpfs: miscellane... |
1153 |
struct list_head *this, *next; |
1da177e4c Linux-2.6.12-rc2 |
1154 |
struct shmem_inode_info *info; |
00501b531 mm: memcontrol: r... |
1155 |
struct mem_cgroup *memcg; |
bde05d1cc shmem: replace pa... |
1156 1157 1158 1159 |
int error = 0; /* * There's a faint possibility that swap page was replaced before |
0142ef6cd shmem: replace_pa... |
1160 |
* caller locked it: caller will come back later with the right page. |
bde05d1cc shmem: replace pa... |
1161 |
*/ |
0142ef6cd shmem: replace_pa... |
1162 |
if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) |
bde05d1cc shmem: replace pa... |
1163 |
goto out; |
778dd893a tmpfs: fix race b... |
1164 1165 1166 1167 1168 |
/* * Charge page using GFP_KERNEL while we can wait, before taking * the shmem_swaplist_mutex which might hold up shmem_writepage(). * Charged back to the user (not to caller) when swap account is used. |
778dd893a tmpfs: fix race b... |
1169 |
*/ |
f627c2f53 memcg: adjust to ... |
1170 1171 |
error = mem_cgroup_try_charge(page, current->mm, GFP_KERNEL, &memcg, false); |
778dd893a tmpfs: fix race b... |
1172 1173 |
if (error) goto out; |
46f65ec15 tmpfs: convert sh... |
1174 |
/* No radix_tree_preload: swap entry keeps a place for page in tree */ |
00501b531 mm: memcontrol: r... |
1175 |
error = -EAGAIN; |
1da177e4c Linux-2.6.12-rc2 |
1176 |
|
cb5f7b9a4 tmpfs: make shmem... |
1177 |
mutex_lock(&shmem_swaplist_mutex); |
41ffe5d5c tmpfs: miscellane... |
1178 1179 |
list_for_each_safe(this, next, &shmem_swaplist) { info = list_entry(this, struct shmem_inode_info, swaplist); |
285b2c4fd tmpfs: demolish o... |
1180 |
if (info->swapped) |
00501b531 mm: memcontrol: r... |
1181 |
error = shmem_unuse_inode(info, swap, &page); |
6922c0c7a tmpfs: convert sh... |
1182 1183 |
else list_del_init(&info->swaplist); |
cb5f7b9a4 tmpfs: make shmem... |
1184 |
cond_resched(); |
00501b531 mm: memcontrol: r... |
1185 |
if (error != -EAGAIN) |
778dd893a tmpfs: fix race b... |
1186 |
break; |
00501b531 mm: memcontrol: r... |
1187 |
/* found nothing in this: move on to search the next */ |
1da177e4c Linux-2.6.12-rc2 |
1188 |
} |
cb5f7b9a4 tmpfs: make shmem... |
1189 |
mutex_unlock(&shmem_swaplist_mutex); |
778dd893a tmpfs: fix race b... |
1190 |
|
00501b531 mm: memcontrol: r... |
1191 1192 1193 |
if (error) { if (error != -ENOMEM) error = 0; |
f627c2f53 memcg: adjust to ... |
1194 |
mem_cgroup_cancel_charge(page, memcg, false); |
00501b531 mm: memcontrol: r... |
1195 |
} else |
f627c2f53 memcg: adjust to ... |
1196 |
mem_cgroup_commit_charge(page, memcg, true, false); |
778dd893a tmpfs: fix race b... |
1197 |
out: |
aaa468653 swap_info: note S... |
1198 |
unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
1199 |
put_page(page); |
778dd893a tmpfs: fix race b... |
1200 |
return error; |
1da177e4c Linux-2.6.12-rc2 |
1201 1202 1203 1204 1205 1206 1207 1208 |
} /* * Move the page from the page cache to the swap cache. */ static int shmem_writepage(struct page *page, struct writeback_control *wbc) { struct shmem_inode_info *info; |
1da177e4c Linux-2.6.12-rc2 |
1209 |
struct address_space *mapping; |
1da177e4c Linux-2.6.12-rc2 |
1210 |
struct inode *inode; |
6922c0c7a tmpfs: convert sh... |
1211 1212 |
swp_entry_t swap; pgoff_t index; |
1da177e4c Linux-2.6.12-rc2 |
1213 |
|
800d8c63b shmem: add huge p... |
1214 |
VM_BUG_ON_PAGE(PageCompound(page), page); |
1da177e4c Linux-2.6.12-rc2 |
1215 |
BUG_ON(!PageLocked(page)); |
1da177e4c Linux-2.6.12-rc2 |
1216 1217 1218 1219 1220 1221 |
mapping = page->mapping; index = page->index; inode = mapping->host; info = SHMEM_I(inode); if (info->flags & VM_LOCKED) goto redirty; |
d9fe526a8 tmpfs: allow file... |
1222 |
if (!total_swap_pages) |
1da177e4c Linux-2.6.12-rc2 |
1223 |
goto redirty; |
d9fe526a8 tmpfs: allow file... |
1224 |
/* |
97b713ba3 fs: kill BDI_CAP_... |
1225 1226 1227 1228 1229 |
* Our capabilities prevent regular writeback or sync from ever calling * shmem_writepage; but a stacking filesystem might use ->writepage of * its underlying filesystem, in which case tmpfs should write out to * swap only in response to memory pressure, and not for the writeback * threads or sync. |
d9fe526a8 tmpfs: allow file... |
1230 |
*/ |
48f170fb7 tmpfs: simplify u... |
1231 1232 1233 1234 |
if (!wbc->for_reclaim) { WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ goto redirty; } |
1635f6a74 tmpfs: undo fallo... |
1235 1236 1237 1238 1239 |
/* * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC * value into swapfile.c, the only way we can correctly account for a * fallocated page arriving here is now to initialize it and write it. |
1aac14003 tmpfs: quit when ... |
1240 1241 1242 1243 1244 1245 |
* * That's okay for a page already fallocated earlier, but if we have * not yet completed the fallocation, then (a) we want to keep track * of this page in case we have to undo it, and (b) it may not be a * good idea to continue anyway, once we're pushing into swap. So * reactivate the page, and let shmem_fallocate() quit when too many. |
1635f6a74 tmpfs: undo fallo... |
1246 1247 |
*/ if (!PageUptodate(page)) { |
1aac14003 tmpfs: quit when ... |
1248 1249 1250 1251 1252 |
if (inode->i_private) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; if (shmem_falloc && |
8e205f779 shmem: fix faulti... |
1253 |
!shmem_falloc->waitq && |
1aac14003 tmpfs: quit when ... |
1254 1255 1256 1257 1258 1259 1260 1261 1262 |
index >= shmem_falloc->start && index < shmem_falloc->next) shmem_falloc->nr_unswapped++; else shmem_falloc = NULL; spin_unlock(&inode->i_lock); if (shmem_falloc) goto redirty; } |
1635f6a74 tmpfs: undo fallo... |
1263 1264 1265 1266 |
clear_highpage(page); flush_dcache_page(page); SetPageUptodate(page); } |
38d8b4e6b mm, THP, swap: de... |
1267 |
swap = get_swap_page(page); |
48f170fb7 tmpfs: simplify u... |
1268 1269 |
if (!swap.val) goto redirty; |
d9fe526a8 tmpfs: allow file... |
1270 |
|
37e843511 mm: memcontrol: c... |
1271 1272 |
if (mem_cgroup_try_charge_swap(page, swap)) goto free_swap; |
b1dea800a tmpfs: fix race b... |
1273 1274 |
/* * Add inode to shmem_unuse()'s list of swapped-out inodes, |
6922c0c7a tmpfs: convert sh... |
1275 1276 |
* if it's not already there. Do it now before the page is * moved to swap cache, when its pagelock no longer protects |
b1dea800a tmpfs: fix race b... |
1277 |
* the inode from eviction. But don't unlock the mutex until |
6922c0c7a tmpfs: convert sh... |
1278 1279 |
* we've incremented swapped, because shmem_unuse_inode() will * prune a !swapped inode from the swaplist under this mutex. |
b1dea800a tmpfs: fix race b... |
1280 |
*/ |
48f170fb7 tmpfs: simplify u... |
1281 1282 1283 |
mutex_lock(&shmem_swaplist_mutex); if (list_empty(&info->swaplist)) list_add_tail(&info->swaplist, &shmem_swaplist); |
b1dea800a tmpfs: fix race b... |
1284 |
|
48f170fb7 tmpfs: simplify u... |
1285 |
if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { |
4595ef88d shmem: make shmem... |
1286 |
spin_lock_irq(&info->lock); |
6922c0c7a tmpfs: convert sh... |
1287 |
shmem_recalc_inode(inode); |
267a4c76b tmpfs: fix shmem_... |
1288 |
info->swapped++; |
4595ef88d shmem: make shmem... |
1289 |
spin_unlock_irq(&info->lock); |
6922c0c7a tmpfs: convert sh... |
1290 |
|
267a4c76b tmpfs: fix shmem_... |
1291 1292 |
swap_shmem_alloc(swap); shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); |
6922c0c7a tmpfs: convert sh... |
1293 |
mutex_unlock(&shmem_swaplist_mutex); |
d9fe526a8 tmpfs: allow file... |
1294 |
BUG_ON(page_mapped(page)); |
9fab5619b shmem: writepage ... |
1295 |
swap_writepage(page, wbc); |
1da177e4c Linux-2.6.12-rc2 |
1296 1297 |
return 0; } |
6922c0c7a tmpfs: convert sh... |
1298 |
mutex_unlock(&shmem_swaplist_mutex); |
37e843511 mm: memcontrol: c... |
1299 |
free_swap: |
75f6d6d29 mm, THP, swap: un... |
1300 |
put_swap_page(page, swap); |
1da177e4c Linux-2.6.12-rc2 |
1301 1302 |
redirty: set_page_dirty(page); |
d9fe526a8 tmpfs: allow file... |
1303 1304 1305 1306 |
if (wbc->for_reclaim) return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ unlock_page(page); return 0; |
1da177e4c Linux-2.6.12-rc2 |
1307 |
} |
75edd345e tmpfs: preliminar... |
1308 |
#if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) |
71fe804b6 mempolicy: use st... |
1309 |
static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) |
680d794ba mount options: fi... |
1310 |
{ |
095f1fc4e mempolicy: rework... |
1311 |
char buffer[64]; |
680d794ba mount options: fi... |
1312 |
|
71fe804b6 mempolicy: use st... |
1313 |
if (!mpol || mpol->mode == MPOL_DEFAULT) |
095f1fc4e mempolicy: rework... |
1314 |
return; /* show nothing */ |
680d794ba mount options: fi... |
1315 |
|
a7a88b237 mempolicy: remove... |
1316 |
mpol_to_str(buffer, sizeof(buffer), mpol); |
095f1fc4e mempolicy: rework... |
1317 1318 |
seq_printf(seq, ",mpol=%s", buffer); |
680d794ba mount options: fi... |
1319 |
} |
71fe804b6 mempolicy: use st... |
1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 |
static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) { struct mempolicy *mpol = NULL; if (sbinfo->mpol) { spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ mpol = sbinfo->mpol; mpol_get(mpol); spin_unlock(&sbinfo->stat_lock); } return mpol; } |
75edd345e tmpfs: preliminar... |
1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 |
#else /* !CONFIG_NUMA || !CONFIG_TMPFS */ static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) { } static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) { return NULL; } #endif /* CONFIG_NUMA && CONFIG_TMPFS */ #ifndef CONFIG_NUMA #define vm_policy vm_private_data #endif |
680d794ba mount options: fi... |
1344 |
|
800d8c63b shmem: add huge p... |
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 |
static void shmem_pseudo_vma_init(struct vm_area_struct *vma, struct shmem_inode_info *info, pgoff_t index) { /* Create a pseudo vma that just contains the policy */ vma->vm_start = 0; /* Bias interleave by inode number to distribute better across nodes */ vma->vm_pgoff = index + info->vfs_inode.i_ino; vma->vm_ops = NULL; vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); } static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) { /* Drop reference taken by mpol_shared_policy_lookup() */ mpol_cond_put(vma->vm_policy); } |
41ffe5d5c tmpfs: miscellane... |
1361 1362 |
static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, struct shmem_inode_info *info, pgoff_t index) |
1da177e4c Linux-2.6.12-rc2 |
1363 |
{ |
1da177e4c Linux-2.6.12-rc2 |
1364 |
struct vm_area_struct pvma; |
18a2f371f tmpfs: fix shared... |
1365 |
struct page *page; |
52cd3b074 mempolicy: rework... |
1366 |
|
800d8c63b shmem: add huge p... |
1367 |
shmem_pseudo_vma_init(&pvma, info, index); |
18a2f371f tmpfs: fix shared... |
1368 |
page = swapin_readahead(swap, gfp, &pvma, 0); |
800d8c63b shmem: add huge p... |
1369 |
shmem_pseudo_vma_destroy(&pvma); |
18a2f371f tmpfs: fix shared... |
1370 |
|
800d8c63b shmem: add huge p... |
1371 1372 1373 1374 1375 1376 1377 1378 1379 |
return page; } static struct page *shmem_alloc_hugepage(gfp_t gfp, struct shmem_inode_info *info, pgoff_t index) { struct vm_area_struct pvma; struct inode *inode = &info->vfs_inode; struct address_space *mapping = inode->i_mapping; |
4620a06e4 shmem: Fix link e... |
1380 |
pgoff_t idx, hindex; |
800d8c63b shmem: add huge p... |
1381 1382 |
void __rcu **results; struct page *page; |
e496cf3d7 thp: introduce CO... |
1383 |
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) |
800d8c63b shmem: add huge p... |
1384 |
return NULL; |
4620a06e4 shmem: Fix link e... |
1385 |
hindex = round_down(index, HPAGE_PMD_NR); |
800d8c63b shmem: add huge p... |
1386 1387 1388 1389 1390 1391 1392 |
rcu_read_lock(); if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, hindex, 1) && idx < hindex + HPAGE_PMD_NR) { rcu_read_unlock(); return NULL; } rcu_read_unlock(); |
18a2f371f tmpfs: fix shared... |
1393 |
|
800d8c63b shmem: add huge p... |
1394 1395 1396 1397 1398 1399 |
shmem_pseudo_vma_init(&pvma, info, hindex); page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); shmem_pseudo_vma_destroy(&pvma); if (page) prep_transhuge_page(page); |
18a2f371f tmpfs: fix shared... |
1400 |
return page; |
1da177e4c Linux-2.6.12-rc2 |
1401 |
} |
02098feaa swapin needs gfp_... |
1402 |
static struct page *shmem_alloc_page(gfp_t gfp, |
41ffe5d5c tmpfs: miscellane... |
1403 |
struct shmem_inode_info *info, pgoff_t index) |
1da177e4c Linux-2.6.12-rc2 |
1404 1405 |
{ struct vm_area_struct pvma; |
18a2f371f tmpfs: fix shared... |
1406 |
struct page *page; |
1da177e4c Linux-2.6.12-rc2 |
1407 |
|
800d8c63b shmem: add huge p... |
1408 1409 1410 1411 1412 1413 1414 1415 |
shmem_pseudo_vma_init(&pvma, info, index); page = alloc_page_vma(gfp, &pvma, 0); shmem_pseudo_vma_destroy(&pvma); return page; } static struct page *shmem_alloc_and_acct_page(gfp_t gfp, |
0f0796945 shmem: introduce ... |
1416 |
struct inode *inode, |
800d8c63b shmem: add huge p... |
1417 1418 |
pgoff_t index, bool huge) { |
0f0796945 shmem: introduce ... |
1419 |
struct shmem_inode_info *info = SHMEM_I(inode); |
800d8c63b shmem: add huge p... |
1420 1421 1422 |
struct page *page; int nr; int err = -ENOSPC; |
52cd3b074 mempolicy: rework... |
1423 |
|
e496cf3d7 thp: introduce CO... |
1424 |
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) |
800d8c63b shmem: add huge p... |
1425 1426 |
huge = false; nr = huge ? HPAGE_PMD_NR : 1; |
0f0796945 shmem: introduce ... |
1427 |
if (!shmem_inode_acct_block(inode, nr)) |
800d8c63b shmem: add huge p... |
1428 |
goto failed; |
800d8c63b shmem: add huge p... |
1429 1430 1431 1432 1433 |
if (huge) page = shmem_alloc_hugepage(gfp, info, index); else page = shmem_alloc_page(gfp, info, index); |
75edd345e tmpfs: preliminar... |
1434 1435 1436 |
if (page) { __SetPageLocked(page); __SetPageSwapBacked(page); |
800d8c63b shmem: add huge p... |
1437 |
return page; |
75edd345e tmpfs: preliminar... |
1438 |
} |
18a2f371f tmpfs: fix shared... |
1439 |
|
800d8c63b shmem: add huge p... |
1440 |
err = -ENOMEM; |
0f0796945 shmem: introduce ... |
1441 |
shmem_inode_unacct_blocks(inode, nr); |
800d8c63b shmem: add huge p... |
1442 1443 |
failed: return ERR_PTR(err); |
1da177e4c Linux-2.6.12-rc2 |
1444 |
} |
71fe804b6 mempolicy: use st... |
1445 |
|
1da177e4c Linux-2.6.12-rc2 |
1446 |
/* |
bde05d1cc shmem: replace pa... |
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 |
* When a page is moved from swapcache to shmem filecache (either by the * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of * shmem_unuse_inode()), it may have been read in earlier from swap, in * ignorance of the mapping it belongs to. If that mapping has special * constraints (like the gma500 GEM driver, which requires RAM below 4GB), * we may need to copy to a suitable page before moving to filecache. * * In a future release, this may well be extended to respect cpuset and * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); * but for now it is a simple matter of zone. */ static bool shmem_should_replace_page(struct page *page, gfp_t gfp) { return page_zonenum(page) > gfp_zone(gfp); } static int shmem_replace_page(struct page **pagep, gfp_t gfp, struct shmem_inode_info *info, pgoff_t index) { struct page *oldpage, *newpage; struct address_space *swap_mapping; pgoff_t swap_index; int error; oldpage = *pagep; swap_index = page_private(oldpage); swap_mapping = page_mapping(oldpage); /* * We have arrived here because our zones are constrained, so don't * limit chance of success by further cpuset and node constraints. */ gfp &= ~GFP_CONSTRAINT_MASK; newpage = shmem_alloc_page(gfp, info, index); if (!newpage) return -ENOMEM; |
bde05d1cc shmem: replace pa... |
1483 |
|
09cbfeaf1 mm, fs: get rid o... |
1484 |
get_page(newpage); |
bde05d1cc shmem: replace pa... |
1485 |
copy_highpage(newpage, oldpage); |
0142ef6cd shmem: replace_pa... |
1486 |
flush_dcache_page(newpage); |
bde05d1cc shmem: replace pa... |
1487 |
|
9956edf37 shmem: fix pagefl... |
1488 1489 |
__SetPageLocked(newpage); __SetPageSwapBacked(newpage); |
bde05d1cc shmem: replace pa... |
1490 |
SetPageUptodate(newpage); |
bde05d1cc shmem: replace pa... |
1491 |
set_page_private(newpage, swap_index); |
bde05d1cc shmem: replace pa... |
1492 1493 1494 1495 1496 1497 1498 1499 1500 |
SetPageSwapCache(newpage); /* * Our caller will very soon move newpage out of swapcache, but it's * a nice clean interface for us to replace oldpage by newpage there. */ spin_lock_irq(&swap_mapping->tree_lock); error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, newpage); |
0142ef6cd shmem: replace_pa... |
1501 |
if (!error) { |
11fb99898 mm: move most fil... |
1502 1503 |
__inc_node_page_state(newpage, NR_FILE_PAGES); __dec_node_page_state(oldpage, NR_FILE_PAGES); |
0142ef6cd shmem: replace_pa... |
1504 |
} |
bde05d1cc shmem: replace pa... |
1505 |
spin_unlock_irq(&swap_mapping->tree_lock); |
bde05d1cc shmem: replace pa... |
1506 |
|
0142ef6cd shmem: replace_pa... |
1507 1508 1509 1510 1511 1512 1513 1514 |
if (unlikely(error)) { /* * Is this possible? I think not, now that our callers check * both PageSwapCache and page_private after getting page lock; * but be defensive. Reverse old to newpage for clear and free. */ oldpage = newpage; } else { |
6a93ca8fd mm: migrate: do n... |
1515 |
mem_cgroup_migrate(oldpage, newpage); |
0142ef6cd shmem: replace_pa... |
1516 1517 1518 |
lru_cache_add_anon(newpage); *pagep = newpage; } |
bde05d1cc shmem: replace pa... |
1519 1520 1521 1522 1523 |
ClearPageSwapCache(oldpage); set_page_private(oldpage, 0); unlock_page(oldpage); |
09cbfeaf1 mm, fs: get rid o... |
1524 1525 |
put_page(oldpage); put_page(oldpage); |
0142ef6cd shmem: replace_pa... |
1526 |
return error; |
bde05d1cc shmem: replace pa... |
1527 1528 1529 |
} /* |
68da9f055 tmpfs: pass gfp t... |
1530 |
* shmem_getpage_gfp - find page in cache, or get from swap, or allocate |
1da177e4c Linux-2.6.12-rc2 |
1531 1532 1533 |
* * If we allocate a new one we do not mark it dirty. That's up to the * vm. If we swap it in we mark it dirty since we also free the swap |
9e18eb293 tmpfs: mem_cgroup... |
1534 1535 1536 1537 |
* entry since a page cannot live in both the swap and page cache. * * fault_mm and fault_type are only supplied by shmem_fault: * otherwise they are NULL. |
1da177e4c Linux-2.6.12-rc2 |
1538 |
*/ |
41ffe5d5c tmpfs: miscellane... |
1539 |
static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, |
9e18eb293 tmpfs: mem_cgroup... |
1540 |
struct page **pagep, enum sgp_type sgp, gfp_t gfp, |
cfda05267 userfaultfd: shme... |
1541 |
struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type) |
1da177e4c Linux-2.6.12-rc2 |
1542 1543 |
{ struct address_space *mapping = inode->i_mapping; |
23f919d4a shmem: avoid mayb... |
1544 |
struct shmem_inode_info *info = SHMEM_I(inode); |
1da177e4c Linux-2.6.12-rc2 |
1545 |
struct shmem_sb_info *sbinfo; |
9e18eb293 tmpfs: mem_cgroup... |
1546 |
struct mm_struct *charge_mm; |
00501b531 mm: memcontrol: r... |
1547 |
struct mem_cgroup *memcg; |
27ab70062 tmpfs: simplify f... |
1548 |
struct page *page; |
1da177e4c Linux-2.6.12-rc2 |
1549 |
swp_entry_t swap; |
657e3038c shmem, thp: respe... |
1550 |
enum sgp_type sgp_huge = sgp; |
800d8c63b shmem: add huge p... |
1551 |
pgoff_t hindex = index; |
1da177e4c Linux-2.6.12-rc2 |
1552 |
int error; |
54af60421 tmpfs: convert sh... |
1553 |
int once = 0; |
1635f6a74 tmpfs: undo fallo... |
1554 |
int alloced = 0; |
1da177e4c Linux-2.6.12-rc2 |
1555 |
|
09cbfeaf1 mm, fs: get rid o... |
1556 |
if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) |
1da177e4c Linux-2.6.12-rc2 |
1557 |
return -EFBIG; |
657e3038c shmem, thp: respe... |
1558 1559 |
if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) sgp = SGP_CACHE; |
1da177e4c Linux-2.6.12-rc2 |
1560 |
repeat: |
54af60421 tmpfs: convert sh... |
1561 |
swap.val = 0; |
0cd6144aa mm + fs: prepare ... |
1562 |
page = find_lock_entry(mapping, index); |
54af60421 tmpfs: convert sh... |
1563 1564 1565 1566 |
if (radix_tree_exceptional_entry(page)) { swap = radix_to_swp_entry(page); page = NULL; } |
75edd345e tmpfs: preliminar... |
1567 |
if (sgp <= SGP_CACHE && |
09cbfeaf1 mm, fs: get rid o... |
1568 |
((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { |
54af60421 tmpfs: convert sh... |
1569 |
error = -EINVAL; |
267a4c76b tmpfs: fix shmem_... |
1570 |
goto unlock; |
54af60421 tmpfs: convert sh... |
1571 |
} |
66d2f4d28 shmem: fix init_p... |
1572 1573 |
if (page && sgp == SGP_WRITE) mark_page_accessed(page); |
1635f6a74 tmpfs: undo fallo... |
1574 1575 1576 1577 1578 |
/* fallocated page? */ if (page && !PageUptodate(page)) { if (sgp != SGP_READ) goto clear; unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
1579 |
put_page(page); |
1635f6a74 tmpfs: undo fallo... |
1580 1581 |
page = NULL; } |
54af60421 tmpfs: convert sh... |
1582 |
if (page || (sgp == SGP_READ && !swap.val)) { |
54af60421 tmpfs: convert sh... |
1583 1584 |
*pagep = page; return 0; |
27ab70062 tmpfs: simplify f... |
1585 1586 1587 |
} /* |
54af60421 tmpfs: convert sh... |
1588 1589 |
* Fast cache lookup did not find it: * bring it back from swap or allocate. |
27ab70062 tmpfs: simplify f... |
1590 |
*/ |
54af60421 tmpfs: convert sh... |
1591 |
sbinfo = SHMEM_SB(inode->i_sb); |
cfda05267 userfaultfd: shme... |
1592 |
charge_mm = vma ? vma->vm_mm : current->mm; |
1da177e4c Linux-2.6.12-rc2 |
1593 |
|
1da177e4c Linux-2.6.12-rc2 |
1594 1595 |
if (swap.val) { /* Look it up and read it in.. */ |
ec560175c mm, swap: VMA bas... |
1596 |
page = lookup_swap_cache(swap, NULL, 0); |
27ab70062 tmpfs: simplify f... |
1597 |
if (!page) { |
9e18eb293 tmpfs: mem_cgroup... |
1598 1599 |
/* Or update major stats only when swapin succeeds?? */ if (fault_type) { |
68da9f055 tmpfs: pass gfp t... |
1600 |
*fault_type |= VM_FAULT_MAJOR; |
9e18eb293 tmpfs: mem_cgroup... |
1601 |
count_vm_event(PGMAJFAULT); |
2262185c5 mm: per-cgroup me... |
1602 |
count_memcg_event_mm(charge_mm, PGMAJFAULT); |
9e18eb293 tmpfs: mem_cgroup... |
1603 1604 |
} /* Here we actually start the io */ |
41ffe5d5c tmpfs: miscellane... |
1605 |
page = shmem_swapin(swap, gfp, info, index); |
27ab70062 tmpfs: simplify f... |
1606 |
if (!page) { |
54af60421 tmpfs: convert sh... |
1607 1608 |
error = -ENOMEM; goto failed; |
1da177e4c Linux-2.6.12-rc2 |
1609 |
} |
1da177e4c Linux-2.6.12-rc2 |
1610 1611 1612 |
} /* We have to do this with page locked to prevent races */ |
54af60421 tmpfs: convert sh... |
1613 |
lock_page(page); |
0142ef6cd shmem: replace_pa... |
1614 |
if (!PageSwapCache(page) || page_private(page) != swap.val || |
d18992286 shmem: fix negati... |
1615 |
!shmem_confirm_swap(mapping, index, swap)) { |
bde05d1cc shmem: replace pa... |
1616 |
error = -EEXIST; /* try again */ |
d18992286 shmem: fix negati... |
1617 |
goto unlock; |
bde05d1cc shmem: replace pa... |
1618 |
} |
27ab70062 tmpfs: simplify f... |
1619 |
if (!PageUptodate(page)) { |
1da177e4c Linux-2.6.12-rc2 |
1620 |
error = -EIO; |
54af60421 tmpfs: convert sh... |
1621 |
goto failed; |
1da177e4c Linux-2.6.12-rc2 |
1622 |
} |
54af60421 tmpfs: convert sh... |
1623 |
wait_on_page_writeback(page); |
bde05d1cc shmem: replace pa... |
1624 1625 1626 1627 |
if (shmem_should_replace_page(page, gfp)) { error = shmem_replace_page(&page, gfp, info, index); if (error) goto failed; |
1da177e4c Linux-2.6.12-rc2 |
1628 |
} |
27ab70062 tmpfs: simplify f... |
1629 |
|
9e18eb293 tmpfs: mem_cgroup... |
1630 |
error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, |
f627c2f53 memcg: adjust to ... |
1631 |
false); |
d18992286 shmem: fix negati... |
1632 |
if (!error) { |
aa3b18955 tmpfs: convert me... |
1633 |
error = shmem_add_to_page_cache(page, mapping, index, |
fed400a18 mm/shmem.c: remov... |
1634 |
swp_to_radix_entry(swap)); |
215c02bc3 tmpfs: fix shmem_... |
1635 1636 1637 1638 1639 1640 1641 1642 |
/* * We already confirmed swap under page lock, and make * no memory allocation here, so usually no possibility * of error; but free_swap_and_cache() only trylocks a * page, so it is just possible that the entry has been * truncated or holepunched since swap was confirmed. * shmem_undo_range() will have done some of the * unaccounting, now delete_from_swap_cache() will do |
93aa7d952 swap: remove unus... |
1643 |
* the rest. |
215c02bc3 tmpfs: fix shmem_... |
1644 1645 1646 |
* Reset swap.val? No, leave it so "failed" goes back to * "repeat": reading a hole and writing should succeed. */ |
00501b531 mm: memcontrol: r... |
1647 |
if (error) { |
f627c2f53 memcg: adjust to ... |
1648 |
mem_cgroup_cancel_charge(page, memcg, false); |
215c02bc3 tmpfs: fix shmem_... |
1649 |
delete_from_swap_cache(page); |
00501b531 mm: memcontrol: r... |
1650 |
} |
d18992286 shmem: fix negati... |
1651 |
} |
54af60421 tmpfs: convert sh... |
1652 1653 |
if (error) goto failed; |
f627c2f53 memcg: adjust to ... |
1654 |
mem_cgroup_commit_charge(page, memcg, true, false); |
00501b531 mm: memcontrol: r... |
1655 |
|
4595ef88d shmem: make shmem... |
1656 |
spin_lock_irq(&info->lock); |
285b2c4fd tmpfs: demolish o... |
1657 |
info->swapped--; |
54af60421 tmpfs: convert sh... |
1658 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
1659 |
spin_unlock_irq(&info->lock); |
54af60421 tmpfs: convert sh... |
1660 |
|
66d2f4d28 shmem: fix init_p... |
1661 1662 |
if (sgp == SGP_WRITE) mark_page_accessed(page); |
54af60421 tmpfs: convert sh... |
1663 |
delete_from_swap_cache(page); |
27ab70062 tmpfs: simplify f... |
1664 1665 |
set_page_dirty(page); swap_free(swap); |
54af60421 tmpfs: convert sh... |
1666 |
} else { |
cfda05267 userfaultfd: shme... |
1667 1668 1669 1670 |
if (vma && userfaultfd_missing(vma)) { *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); return 0; } |
800d8c63b shmem: add huge p... |
1671 1672 1673 |
/* shmem_symlink() */ if (mapping->a_ops != &shmem_aops) goto alloc_nohuge; |
657e3038c shmem, thp: respe... |
1674 |
if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) |
800d8c63b shmem: add huge p... |
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 |
goto alloc_nohuge; if (shmem_huge == SHMEM_HUGE_FORCE) goto alloc_huge; switch (sbinfo->huge) { loff_t i_size; pgoff_t off; case SHMEM_HUGE_NEVER: goto alloc_nohuge; case SHMEM_HUGE_WITHIN_SIZE: off = round_up(index, HPAGE_PMD_NR); i_size = round_up(i_size_read(inode), PAGE_SIZE); if (i_size >= HPAGE_PMD_SIZE && i_size >> PAGE_SHIFT >= off) goto alloc_huge; /* fallthrough */ case SHMEM_HUGE_ADVISE: |
657e3038c shmem, thp: respe... |
1691 1692 1693 |
if (sgp_huge == SGP_HUGE) goto alloc_huge; /* TODO: implement fadvise() hints */ |
800d8c63b shmem: add huge p... |
1694 |
goto alloc_nohuge; |
54af60421 tmpfs: convert sh... |
1695 |
} |
1da177e4c Linux-2.6.12-rc2 |
1696 |
|
800d8c63b shmem: add huge p... |
1697 |
alloc_huge: |
0f0796945 shmem: introduce ... |
1698 |
page = shmem_alloc_and_acct_page(gfp, inode, index, true); |
800d8c63b shmem: add huge p... |
1699 |
if (IS_ERR(page)) { |
0f0796945 shmem: introduce ... |
1700 |
alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, inode, |
800d8c63b shmem: add huge p... |
1701 |
index, false); |
1da177e4c Linux-2.6.12-rc2 |
1702 |
} |
800d8c63b shmem: add huge p... |
1703 |
if (IS_ERR(page)) { |
779750d20 shmem: split huge... |
1704 |
int retry = 5; |
800d8c63b shmem: add huge p... |
1705 1706 |
error = PTR_ERR(page); page = NULL; |
779750d20 shmem: split huge... |
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 |
if (error != -ENOSPC) goto failed; /* * Try to reclaim some spece by splitting a huge page * beyond i_size on the filesystem. */ while (retry--) { int ret; ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); if (ret == SHRINK_STOP) break; if (ret) goto alloc_nohuge; } |
800d8c63b shmem: add huge p... |
1721 1722 1723 1724 1725 1726 1727 |
goto failed; } if (PageTransHuge(page)) hindex = round_down(index, HPAGE_PMD_NR); else hindex = index; |
66d2f4d28 shmem: fix init_p... |
1728 |
if (sgp == SGP_WRITE) |
eb39d618f mm: replace init_... |
1729 |
__SetPageReferenced(page); |
66d2f4d28 shmem: fix init_p... |
1730 |
|
9e18eb293 tmpfs: mem_cgroup... |
1731 |
error = mem_cgroup_try_charge(page, charge_mm, gfp, &memcg, |
800d8c63b shmem: add huge p... |
1732 |
PageTransHuge(page)); |
54af60421 tmpfs: convert sh... |
1733 |
if (error) |
800d8c63b shmem: add huge p... |
1734 1735 1736 |
goto unacct; error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, compound_order(page)); |
b065b4321 shmem: cleanup sh... |
1737 |
if (!error) { |
800d8c63b shmem: add huge p... |
1738 |
error = shmem_add_to_page_cache(page, mapping, hindex, |
fed400a18 mm/shmem.c: remov... |
1739 |
NULL); |
b065b4321 shmem: cleanup sh... |
1740 1741 1742 |
radix_tree_preload_end(); } if (error) { |
800d8c63b shmem: add huge p... |
1743 1744 1745 |
mem_cgroup_cancel_charge(page, memcg, PageTransHuge(page)); goto unacct; |
b065b4321 shmem: cleanup sh... |
1746 |
} |
800d8c63b shmem: add huge p... |
1747 1748 |
mem_cgroup_commit_charge(page, memcg, false, PageTransHuge(page)); |
54af60421 tmpfs: convert sh... |
1749 |
lru_cache_add_anon(page); |
4595ef88d shmem: make shmem... |
1750 |
spin_lock_irq(&info->lock); |
800d8c63b shmem: add huge p... |
1751 1752 |
info->alloced += 1 << compound_order(page); inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); |
54af60421 tmpfs: convert sh... |
1753 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
1754 |
spin_unlock_irq(&info->lock); |
1635f6a74 tmpfs: undo fallo... |
1755 |
alloced = true; |
54af60421 tmpfs: convert sh... |
1756 |
|
779750d20 shmem: split huge... |
1757 1758 1759 1760 1761 1762 1763 1764 |
if (PageTransHuge(page) && DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < hindex + HPAGE_PMD_NR - 1) { /* * Part of the huge page is beyond i_size: subject * to shrink under memory pressure. */ spin_lock(&sbinfo->shrinklist_lock); |
d041353dc mm: fix list corr... |
1765 1766 1767 1768 1769 |
/* * _careful to defend against unlocked access to * ->shrink_list in shmem_unused_huge_shrink() */ if (list_empty_careful(&info->shrinklist)) { |
779750d20 shmem: split huge... |
1770 1771 1772 1773 1774 1775 |
list_add_tail(&info->shrinklist, &sbinfo->shrinklist); sbinfo->shrinklist_len++; } spin_unlock(&sbinfo->shrinklist_lock); } |
ec9516fbc tmpfs: optimize c... |
1776 |
/* |
1635f6a74 tmpfs: undo fallo... |
1777 1778 1779 1780 1781 1782 1783 1784 1785 |
* Let SGP_FALLOC use the SGP_WRITE optimization on a new page. */ if (sgp == SGP_FALLOC) sgp = SGP_WRITE; clear: /* * Let SGP_WRITE caller clear ends if write does not fill page; * but SGP_FALLOC on a page fallocated earlier must initialize * it now, lest undo on failure cancel our earlier guarantee. |
ec9516fbc tmpfs: optimize c... |
1786 |
*/ |
800d8c63b shmem: add huge p... |
1787 1788 1789 1790 1791 1792 1793 1794 1795 |
if (sgp != SGP_WRITE && !PageUptodate(page)) { struct page *head = compound_head(page); int i; for (i = 0; i < (1 << compound_order(head)); i++) { clear_highpage(head + i); flush_dcache_page(head + i); } SetPageUptodate(head); |
ec9516fbc tmpfs: optimize c... |
1796 |
} |
1da177e4c Linux-2.6.12-rc2 |
1797 |
} |
bde05d1cc shmem: replace pa... |
1798 |
|
54af60421 tmpfs: convert sh... |
1799 |
/* Perhaps the file has been truncated since we checked */ |
75edd345e tmpfs: preliminar... |
1800 |
if (sgp <= SGP_CACHE && |
09cbfeaf1 mm, fs: get rid o... |
1801 |
((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { |
267a4c76b tmpfs: fix shmem_... |
1802 1803 1804 |
if (alloced) { ClearPageDirty(page); delete_from_page_cache(page); |
4595ef88d shmem: make shmem... |
1805 |
spin_lock_irq(&info->lock); |
267a4c76b tmpfs: fix shmem_... |
1806 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
1807 |
spin_unlock_irq(&info->lock); |
267a4c76b tmpfs: fix shmem_... |
1808 |
} |
54af60421 tmpfs: convert sh... |
1809 |
error = -EINVAL; |
267a4c76b tmpfs: fix shmem_... |
1810 |
goto unlock; |
e83c32e8f tmpfs: simplify p... |
1811 |
} |
800d8c63b shmem: add huge p... |
1812 |
*pagep = page + index - hindex; |
54af60421 tmpfs: convert sh... |
1813 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
1814 |
|
59a16ead5 tmpfs: fix spurio... |
1815 |
/* |
54af60421 tmpfs: convert sh... |
1816 |
* Error recovery. |
59a16ead5 tmpfs: fix spurio... |
1817 |
*/ |
54af60421 tmpfs: convert sh... |
1818 |
unacct: |
0f0796945 shmem: introduce ... |
1819 |
shmem_inode_unacct_blocks(inode, 1 << compound_order(page)); |
800d8c63b shmem: add huge p... |
1820 1821 1822 1823 1824 1825 |
if (PageTransHuge(page)) { unlock_page(page); put_page(page); goto alloc_nohuge; } |
54af60421 tmpfs: convert sh... |
1826 |
failed: |
267a4c76b tmpfs: fix shmem_... |
1827 |
if (swap.val && !shmem_confirm_swap(mapping, index, swap)) |
d18992286 shmem: fix negati... |
1828 1829 |
error = -EEXIST; unlock: |
27ab70062 tmpfs: simplify f... |
1830 |
if (page) { |
54af60421 tmpfs: convert sh... |
1831 |
unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
1832 |
put_page(page); |
54af60421 tmpfs: convert sh... |
1833 1834 |
} if (error == -ENOSPC && !once++) { |
4595ef88d shmem: make shmem... |
1835 |
spin_lock_irq(&info->lock); |
54af60421 tmpfs: convert sh... |
1836 |
shmem_recalc_inode(inode); |
4595ef88d shmem: make shmem... |
1837 |
spin_unlock_irq(&info->lock); |
27ab70062 tmpfs: simplify f... |
1838 |
goto repeat; |
ff36b8016 shmem: reduce pag... |
1839 |
} |
d18992286 shmem: fix negati... |
1840 |
if (error == -EEXIST) /* from above or from radix_tree_insert */ |
54af60421 tmpfs: convert sh... |
1841 1842 |
goto repeat; return error; |
1da177e4c Linux-2.6.12-rc2 |
1843 |
} |
10d20bd25 shmem: fix shm fa... |
1844 1845 1846 1847 1848 |
/* * This is like autoremove_wake_function, but it removes the wait queue * entry unconditionally - even if something else had already woken the * target. */ |
ac6424b98 sched/wait: Renam... |
1849 |
static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) |
10d20bd25 shmem: fix shm fa... |
1850 1851 |
{ int ret = default_wake_function(wait, mode, sync, key); |
2055da973 sched/wait: Disam... |
1852 |
list_del_init(&wait->entry); |
10d20bd25 shmem: fix shm fa... |
1853 1854 |
return ret; } |
11bac8000 mm, fs: reduce fa... |
1855 |
static int shmem_fault(struct vm_fault *vmf) |
1da177e4c Linux-2.6.12-rc2 |
1856 |
{ |
11bac8000 mm, fs: reduce fa... |
1857 |
struct vm_area_struct *vma = vmf->vma; |
496ad9aa8 new helper: file_... |
1858 |
struct inode *inode = file_inode(vma->vm_file); |
9e18eb293 tmpfs: mem_cgroup... |
1859 |
gfp_t gfp = mapping_gfp_mask(inode->i_mapping); |
657e3038c shmem, thp: respe... |
1860 |
enum sgp_type sgp; |
1da177e4c Linux-2.6.12-rc2 |
1861 |
int error; |
68da9f055 tmpfs: pass gfp t... |
1862 |
int ret = VM_FAULT_LOCKED; |
1da177e4c Linux-2.6.12-rc2 |
1863 |
|
f00cdc6df shmem: fix faulti... |
1864 1865 1866 1867 |
/* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: which in turn * locks writers out with its hold on i_mutex. So refrain from |
8e205f779 shmem: fix faulti... |
1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 |
* faulting pages into the hole while it's being punched. Although * shmem_undo_range() does remove the additions, it may be unable to * keep up, as each new page needs its own unmap_mapping_range() call, * and the i_mmap tree grows ever slower to scan if new vmas are added. * * It does not matter if we sometimes reach this check just before the * hole-punch begins, so that one fault then races with the punch: * we just need to make racing faults a rare case. * * The implementation below would be much simpler if we just used a * standard mutex or completion: but we cannot take i_mutex in fault, * and bloating every shmem inode for this unlikely case would be sad. |
f00cdc6df shmem: fix faulti... |
1880 1881 1882 1883 1884 1885 |
*/ if (unlikely(inode->i_private)) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); shmem_falloc = inode->i_private; |
8e205f779 shmem: fix faulti... |
1886 1887 1888 1889 1890 |
if (shmem_falloc && shmem_falloc->waitq && vmf->pgoff >= shmem_falloc->start && vmf->pgoff < shmem_falloc->next) { wait_queue_head_t *shmem_falloc_waitq; |
10d20bd25 shmem: fix shm fa... |
1891 |
DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); |
8e205f779 shmem: fix faulti... |
1892 1893 |
ret = VM_FAULT_NOPAGE; |
f00cdc6df shmem: fix faulti... |
1894 1895 |
if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { |
8e205f779 shmem: fix faulti... |
1896 |
/* It's polite to up mmap_sem if we can */ |
f00cdc6df shmem: fix faulti... |
1897 |
up_read(&vma->vm_mm->mmap_sem); |
8e205f779 shmem: fix faulti... |
1898 |
ret = VM_FAULT_RETRY; |
f00cdc6df shmem: fix faulti... |
1899 |
} |
8e205f779 shmem: fix faulti... |
1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 |
shmem_falloc_waitq = shmem_falloc->waitq; prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, TASK_UNINTERRUPTIBLE); spin_unlock(&inode->i_lock); schedule(); /* * shmem_falloc_waitq points into the shmem_fallocate() * stack of the hole-punching task: shmem_falloc_waitq * is usually invalid by the time we reach here, but * finish_wait() does not dereference it in that case; * though i_lock needed lest racing with wake_up_all(). */ spin_lock(&inode->i_lock); finish_wait(shmem_falloc_waitq, &shmem_fault_wait); spin_unlock(&inode->i_lock); return ret; |
f00cdc6df shmem: fix faulti... |
1918 |
} |
8e205f779 shmem: fix faulti... |
1919 |
spin_unlock(&inode->i_lock); |
f00cdc6df shmem: fix faulti... |
1920 |
} |
657e3038c shmem, thp: respe... |
1921 |
sgp = SGP_CACHE; |
186003323 mm: make PR_SET_T... |
1922 1923 1924 |
if ((vma->vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) |
657e3038c shmem, thp: respe... |
1925 |
sgp = SGP_NOHUGE; |
186003323 mm: make PR_SET_T... |
1926 1927 |
else if (vma->vm_flags & VM_HUGEPAGE) sgp = SGP_HUGE; |
657e3038c shmem, thp: respe... |
1928 1929 |
error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, |
cfda05267 userfaultfd: shme... |
1930 |
gfp, vma, vmf, &ret); |
d0217ac04 mm: fault feedbac... |
1931 1932 |
if (error) return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); |
68da9f055 tmpfs: pass gfp t... |
1933 |
return ret; |
1da177e4c Linux-2.6.12-rc2 |
1934 |
} |
c01d5b300 shmem: get_unmapp... |
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 |
unsigned long shmem_get_unmapped_area(struct file *file, unsigned long uaddr, unsigned long len, unsigned long pgoff, unsigned long flags) { unsigned long (*get_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); unsigned long addr; unsigned long offset; unsigned long inflated_len; unsigned long inflated_addr; unsigned long inflated_offset; if (len > TASK_SIZE) return -ENOMEM; get_area = current->mm->get_unmapped_area; addr = get_area(file, uaddr, len, pgoff, flags); |
e496cf3d7 thp: introduce CO... |
1952 |
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) |
c01d5b300 shmem: get_unmapp... |
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 |
return addr; if (IS_ERR_VALUE(addr)) return addr; if (addr & ~PAGE_MASK) return addr; if (addr > TASK_SIZE - len) return addr; if (shmem_huge == SHMEM_HUGE_DENY) return addr; if (len < HPAGE_PMD_SIZE) return addr; if (flags & MAP_FIXED) return addr; /* * Our priority is to support MAP_SHARED mapped hugely; * and support MAP_PRIVATE mapped hugely too, until it is COWed. * But if caller specified an address hint, respect that as before. */ if (uaddr) return addr; if (shmem_huge != SHMEM_HUGE_FORCE) { struct super_block *sb; if (file) { VM_BUG_ON(file->f_op != &shmem_file_operations); sb = file_inode(file)->i_sb; } else { /* * Called directly from mm/mmap.c, or drivers/char/mem.c * for "/dev/zero", to create a shared anonymous object. */ if (IS_ERR(shm_mnt)) return addr; sb = shm_mnt->mnt_sb; } |
3089bf614 shmem: fix tmpfs ... |
1990 |
if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) |
c01d5b300 shmem: get_unmapp... |
1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 |
return addr; } offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); if (offset && offset + len < 2 * HPAGE_PMD_SIZE) return addr; if ((addr & (HPAGE_PMD_SIZE-1)) == offset) return addr; inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; if (inflated_len > TASK_SIZE) return addr; if (inflated_len < len) return addr; inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); if (IS_ERR_VALUE(inflated_addr)) return addr; if (inflated_addr & ~PAGE_MASK) return addr; inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); inflated_addr += offset - inflated_offset; if (inflated_offset > offset) inflated_addr += HPAGE_PMD_SIZE; if (inflated_addr > TASK_SIZE - len) return addr; return inflated_addr; } |
1da177e4c Linux-2.6.12-rc2 |
2021 |
#ifdef CONFIG_NUMA |
41ffe5d5c tmpfs: miscellane... |
2022 |
static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) |
1da177e4c Linux-2.6.12-rc2 |
2023 |
{ |
496ad9aa8 new helper: file_... |
2024 |
struct inode *inode = file_inode(vma->vm_file); |
41ffe5d5c tmpfs: miscellane... |
2025 |
return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); |
1da177e4c Linux-2.6.12-rc2 |
2026 |
} |
d8dc74f21 mm/shmem.c: make ... |
2027 2028 |
static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, unsigned long addr) |
1da177e4c Linux-2.6.12-rc2 |
2029 |
{ |
496ad9aa8 new helper: file_... |
2030 |
struct inode *inode = file_inode(vma->vm_file); |
41ffe5d5c tmpfs: miscellane... |
2031 |
pgoff_t index; |
1da177e4c Linux-2.6.12-rc2 |
2032 |
|
41ffe5d5c tmpfs: miscellane... |
2033 2034 |
index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); |
1da177e4c Linux-2.6.12-rc2 |
2035 2036 2037 2038 2039 |
} #endif int shmem_lock(struct file *file, int lock, struct user_struct *user) { |
496ad9aa8 new helper: file_... |
2040 |
struct inode *inode = file_inode(file); |
1da177e4c Linux-2.6.12-rc2 |
2041 2042 |
struct shmem_inode_info *info = SHMEM_I(inode); int retval = -ENOMEM; |
4595ef88d shmem: make shmem... |
2043 |
spin_lock_irq(&info->lock); |
1da177e4c Linux-2.6.12-rc2 |
2044 2045 2046 2047 |
if (lock && !(info->flags & VM_LOCKED)) { if (!user_shm_lock(inode->i_size, user)) goto out_nomem; info->flags |= VM_LOCKED; |
89e004ea5 SHM_LOCKED pages ... |
2048 |
mapping_set_unevictable(file->f_mapping); |
1da177e4c Linux-2.6.12-rc2 |
2049 2050 2051 2052 |
} if (!lock && (info->flags & VM_LOCKED) && user) { user_shm_unlock(inode->i_size, user); info->flags &= ~VM_LOCKED; |
89e004ea5 SHM_LOCKED pages ... |
2053 |
mapping_clear_unevictable(file->f_mapping); |
1da177e4c Linux-2.6.12-rc2 |
2054 2055 |
} retval = 0; |
89e004ea5 SHM_LOCKED pages ... |
2056 |
|
1da177e4c Linux-2.6.12-rc2 |
2057 |
out_nomem: |
4595ef88d shmem: make shmem... |
2058 |
spin_unlock_irq(&info->lock); |
1da177e4c Linux-2.6.12-rc2 |
2059 2060 |
return retval; } |
9b83a6a85 [PATCH] mm/{,tiny... |
2061 |
static int shmem_mmap(struct file *file, struct vm_area_struct *vma) |
1da177e4c Linux-2.6.12-rc2 |
2062 2063 2064 |
{ file_accessed(file); vma->vm_ops = &shmem_vm_ops; |
e496cf3d7 thp: introduce CO... |
2065 |
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && |
f3f0e1d21 khugepaged: add s... |
2066 2067 2068 2069 |
((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < (vma->vm_end & HPAGE_PMD_MASK)) { khugepaged_enter(vma, vma->vm_flags); } |
1da177e4c Linux-2.6.12-rc2 |
2070 2071 |
return 0; } |
454abafe9 ramfs: replace in... |
2072 |
static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, |
09208d150 shmem, ramfs: pro... |
2073 |
umode_t mode, dev_t dev, unsigned long flags) |
1da177e4c Linux-2.6.12-rc2 |
2074 2075 2076 2077 |
{ struct inode *inode; struct shmem_inode_info *info; struct shmem_sb_info *sbinfo = SHMEM_SB(sb); |
5b04c6890 shmem: factor out... |
2078 2079 |
if (shmem_reserve_inode(sb)) return NULL; |
1da177e4c Linux-2.6.12-rc2 |
2080 2081 2082 |
inode = new_inode(sb); if (inode) { |
85fe4025c fs: do not assign... |
2083 |
inode->i_ino = get_next_ino(); |
454abafe9 ramfs: replace in... |
2084 |
inode_init_owner(inode, dir, mode); |
1da177e4c Linux-2.6.12-rc2 |
2085 |
inode->i_blocks = 0; |
078cd8279 fs: Replace CURRE... |
2086 |
inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); |
91828a405 [PATCH] knfsd: ad... |
2087 |
inode->i_generation = get_seconds(); |
1da177e4c Linux-2.6.12-rc2 |
2088 2089 2090 |
info = SHMEM_I(inode); memset(info, 0, (char *)inode - (char *)info); spin_lock_init(&info->lock); |
40e041a2c shm: add sealing API |
2091 |
info->seals = F_SEAL_SEAL; |
0b0a0806b shmem: fix shared... |
2092 |
info->flags = flags & VM_NORESERVE; |
779750d20 shmem: split huge... |
2093 |
INIT_LIST_HEAD(&info->shrinklist); |
1da177e4c Linux-2.6.12-rc2 |
2094 |
INIT_LIST_HEAD(&info->swaplist); |
38f386574 xattr: extract si... |
2095 |
simple_xattrs_init(&info->xattrs); |
72c04902d Get "no acls for ... |
2096 |
cache_no_acl(inode); |
1da177e4c Linux-2.6.12-rc2 |
2097 2098 2099 |
switch (mode & S_IFMT) { default: |
39f0247d3 [PATCH] Access Co... |
2100 |
inode->i_op = &shmem_special_inode_operations; |
1da177e4c Linux-2.6.12-rc2 |
2101 2102 2103 |
init_special_inode(inode, mode, dev); break; case S_IFREG: |
14fcc23fd tmpfs: fix kernel... |
2104 |
inode->i_mapping->a_ops = &shmem_aops; |
1da177e4c Linux-2.6.12-rc2 |
2105 2106 |
inode->i_op = &shmem_inode_operations; inode->i_fop = &shmem_file_operations; |
71fe804b6 mempolicy: use st... |
2107 2108 |
mpol_shared_policy_init(&info->policy, shmem_get_sbmpol(sbinfo)); |
1da177e4c Linux-2.6.12-rc2 |
2109 2110 |
break; case S_IFDIR: |
d8c76e6f4 [PATCH] r/o bind ... |
2111 |
inc_nlink(inode); |
1da177e4c Linux-2.6.12-rc2 |
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 |
/* Some things misbehave if size == 0 on a directory */ inode->i_size = 2 * BOGO_DIRENT_SIZE; inode->i_op = &shmem_dir_inode_operations; inode->i_fop = &simple_dir_operations; break; case S_IFLNK: /* * Must not load anything in the rbtree, * mpol_free_shared_policy will not be called. */ |
71fe804b6 mempolicy: use st... |
2122 |
mpol_shared_policy_init(&info->policy, NULL); |
1da177e4c Linux-2.6.12-rc2 |
2123 2124 |
break; } |
6447b34fc mm: shmem.c: Corr... |
2125 2126 |
lockdep_annotate_inode_mutex_key(inode); |
5b04c6890 shmem: factor out... |
2127 2128 |
} else shmem_free_inode(sb); |
1da177e4c Linux-2.6.12-rc2 |
2129 2130 |
return inode; } |
0cd6144aa mm + fs: prepare ... |
2131 2132 |
bool shmem_mapping(struct address_space *mapping) { |
f8005451d tmpfs: change shm... |
2133 |
return mapping->a_ops == &shmem_aops; |
0cd6144aa mm + fs: prepare ... |
2134 |
} |
8d1039634 userfaultfd: shme... |
2135 2136 2137 2138 2139 2140 2141 |
static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, bool zeropage, struct page **pagep) |
4c27fe4c4 userfaultfd: shme... |
2142 2143 2144 |
{ struct inode *inode = file_inode(dst_vma->vm_file); struct shmem_inode_info *info = SHMEM_I(inode); |
4c27fe4c4 userfaultfd: shme... |
2145 2146 2147 2148 2149 2150 2151 2152 2153 |
struct address_space *mapping = inode->i_mapping; gfp_t gfp = mapping_gfp_mask(mapping); pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); struct mem_cgroup *memcg; spinlock_t *ptl; void *page_kaddr; struct page *page; pte_t _dst_pte, *dst_pte; int ret; |
cb658a453 userfaultfd: shme... |
2154 |
ret = -ENOMEM; |
0f0796945 shmem: introduce ... |
2155 |
if (!shmem_inode_acct_block(inode, 1)) |
cb658a453 userfaultfd: shme... |
2156 |
goto out; |
4c27fe4c4 userfaultfd: shme... |
2157 |
|
cb658a453 userfaultfd: shme... |
2158 |
if (!*pagep) { |
4c27fe4c4 userfaultfd: shme... |
2159 2160 |
page = shmem_alloc_page(gfp, info, pgoff); if (!page) |
0f0796945 shmem: introduce ... |
2161 |
goto out_unacct_blocks; |
4c27fe4c4 userfaultfd: shme... |
2162 |
|
8d1039634 userfaultfd: shme... |
2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 |
if (!zeropage) { /* mcopy_atomic */ page_kaddr = kmap_atomic(page); ret = copy_from_user(page_kaddr, (const void __user *)src_addr, PAGE_SIZE); kunmap_atomic(page_kaddr); /* fallback to copy_from_user outside mmap_sem */ if (unlikely(ret)) { *pagep = page; shmem_inode_unacct_blocks(inode, 1); /* don't free the page */ return -EFAULT; } } else { /* mfill_zeropage_atomic */ clear_highpage(page); |
4c27fe4c4 userfaultfd: shme... |
2179 2180 2181 2182 2183 |
} } else { page = *pagep; *pagep = NULL; } |
9cc90c664 userfaultfd: shme... |
2184 2185 2186 |
VM_BUG_ON(PageLocked(page) || PageSwapBacked(page)); __SetPageLocked(page); __SetPageSwapBacked(page); |
a425d3584 userfaultfd: shme... |
2187 |
__SetPageUptodate(page); |
9cc90c664 userfaultfd: shme... |
2188 |
|
4c27fe4c4 userfaultfd: shme... |
2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 |
ret = mem_cgroup_try_charge(page, dst_mm, gfp, &memcg, false); if (ret) goto out_release; ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK); if (!ret) { ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL); radix_tree_preload_end(); } if (ret) goto out_release_uncharge; mem_cgroup_commit_charge(page, memcg, false, false); _dst_pte = mk_pte(page, dst_vma->vm_page_prot); if (dst_vma->vm_flags & VM_WRITE) _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte)); ret = -EEXIST; dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); if (!pte_none(*dst_pte)) goto out_release_uncharge_unlock; |
4c27fe4c4 userfaultfd: shme... |
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 |
lru_cache_add_anon(page); spin_lock(&info->lock); info->alloced++; inode->i_blocks += BLOCKS_PER_PAGE; shmem_recalc_inode(inode); spin_unlock(&info->lock); inc_mm_counter(dst_mm, mm_counter_file(page)); page_add_file_rmap(page, false); set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); /* No need to invalidate - it was non-present before */ update_mmu_cache(dst_vma, dst_addr, dst_pte); unlock_page(page); pte_unmap_unlock(dst_pte, ptl); ret = 0; out: return ret; out_release_uncharge_unlock: pte_unmap_unlock(dst_pte, ptl); out_release_uncharge: mem_cgroup_cancel_charge(page, memcg, false); out_release: |
9cc90c664 userfaultfd: shme... |
2235 |
unlock_page(page); |
4c27fe4c4 userfaultfd: shme... |
2236 |
put_page(page); |
4c27fe4c4 userfaultfd: shme... |
2237 |
out_unacct_blocks: |
0f0796945 shmem: introduce ... |
2238 |
shmem_inode_unacct_blocks(inode, 1); |
4c27fe4c4 userfaultfd: shme... |
2239 2240 |
goto out; } |
8d1039634 userfaultfd: shme... |
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 |
int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, struct page **pagep) { return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, src_addr, false, pagep); } int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr) { struct page *page = NULL; return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 0, true, &page); } |
1da177e4c Linux-2.6.12-rc2 |
2262 |
#ifdef CONFIG_TMPFS |
92e1d5be9 [PATCH] mark stru... |
2263 |
static const struct inode_operations shmem_symlink_inode_operations; |
69f07ec93 tmpfs: use kmemdu... |
2264 |
static const struct inode_operations shmem_short_symlink_operations; |
1da177e4c Linux-2.6.12-rc2 |
2265 |
|
6d9d88d07 tmpfs: security x... |
2266 2267 2268 2269 2270 |
#ifdef CONFIG_TMPFS_XATTR static int shmem_initxattrs(struct inode *, const struct xattr *, void *); #else #define shmem_initxattrs NULL #endif |
1da177e4c Linux-2.6.12-rc2 |
2271 |
static int |
800d15a53 implement simple ... |
2272 2273 2274 |
shmem_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) |
1da177e4c Linux-2.6.12-rc2 |
2275 |
{ |
800d15a53 implement simple ... |
2276 |
struct inode *inode = mapping->host; |
40e041a2c shm: add sealing API |
2277 |
struct shmem_inode_info *info = SHMEM_I(inode); |
09cbfeaf1 mm, fs: get rid o... |
2278 |
pgoff_t index = pos >> PAGE_SHIFT; |
40e041a2c shm: add sealing API |
2279 2280 |
/* i_mutex is held by caller */ |
3f472cc97 mm/shmem.c: fix u... |
2281 |
if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) { |
40e041a2c shm: add sealing API |
2282 2283 2284 2285 2286 |
if (info->seals & F_SEAL_WRITE) return -EPERM; if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) return -EPERM; } |
9e18eb293 tmpfs: mem_cgroup... |
2287 |
return shmem_getpage(inode, index, pagep, SGP_WRITE); |
800d15a53 implement simple ... |
2288 2289 2290 2291 2292 2293 2294 2295 |
} static int shmem_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { struct inode *inode = mapping->host; |
d3602444e shmem_getpage ret... |
2296 2297 |
if (pos + copied > inode->i_size) i_size_write(inode, pos + copied); |
ec9516fbc tmpfs: optimize c... |
2298 |
if (!PageUptodate(page)) { |
800d8c63b shmem: add huge p... |
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 |
struct page *head = compound_head(page); if (PageTransCompound(page)) { int i; for (i = 0; i < HPAGE_PMD_NR; i++) { if (head + i == page) continue; clear_highpage(head + i); flush_dcache_page(head + i); } } |
09cbfeaf1 mm, fs: get rid o... |
2310 2311 |
if (copied < PAGE_SIZE) { unsigned from = pos & (PAGE_SIZE - 1); |
ec9516fbc tmpfs: optimize c... |
2312 |
zero_user_segments(page, 0, from, |
09cbfeaf1 mm, fs: get rid o... |
2313 |
from + copied, PAGE_SIZE); |
ec9516fbc tmpfs: optimize c... |
2314 |
} |
800d8c63b shmem: add huge p... |
2315 |
SetPageUptodate(head); |
ec9516fbc tmpfs: optimize c... |
2316 |
} |
800d15a53 implement simple ... |
2317 |
set_page_dirty(page); |
6746aff74 HWPOISON: shmem: ... |
2318 |
unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
2319 |
put_page(page); |
800d15a53 implement simple ... |
2320 |
|
800d15a53 implement simple ... |
2321 |
return copied; |
1da177e4c Linux-2.6.12-rc2 |
2322 |
} |
2ba5bbed0 shmem: switch to ... |
2323 |
static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) |
1da177e4c Linux-2.6.12-rc2 |
2324 |
{ |
6e58e79db introduce copy_pa... |
2325 2326 |
struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); |
1da177e4c Linux-2.6.12-rc2 |
2327 |
struct address_space *mapping = inode->i_mapping; |
41ffe5d5c tmpfs: miscellane... |
2328 2329 |
pgoff_t index; unsigned long offset; |
a0ee5ec52 tmpfs: allocate o... |
2330 |
enum sgp_type sgp = SGP_READ; |
f7c1d0742 mm: Initialize er... |
2331 |
int error = 0; |
cb66a7a1f kill generic_segm... |
2332 |
ssize_t retval = 0; |
6e58e79db introduce copy_pa... |
2333 |
loff_t *ppos = &iocb->ki_pos; |
a0ee5ec52 tmpfs: allocate o... |
2334 2335 2336 2337 2338 2339 |
/* * Might this read be for a stacking filesystem? Then when reading * holes of a sparse file, we actually need to allocate those pages, * and even mark them dirty, so it cannot exceed the max_blocks limit. */ |
777eda2c5 new helper: iter_... |
2340 |
if (!iter_is_iovec(to)) |
75edd345e tmpfs: preliminar... |
2341 |
sgp = SGP_CACHE; |
1da177e4c Linux-2.6.12-rc2 |
2342 |
|
09cbfeaf1 mm, fs: get rid o... |
2343 2344 |
index = *ppos >> PAGE_SHIFT; offset = *ppos & ~PAGE_MASK; |
1da177e4c Linux-2.6.12-rc2 |
2345 2346 2347 |
for (;;) { struct page *page = NULL; |
41ffe5d5c tmpfs: miscellane... |
2348 2349 |
pgoff_t end_index; unsigned long nr, ret; |
1da177e4c Linux-2.6.12-rc2 |
2350 |
loff_t i_size = i_size_read(inode); |
09cbfeaf1 mm, fs: get rid o... |
2351 |
end_index = i_size >> PAGE_SHIFT; |
1da177e4c Linux-2.6.12-rc2 |
2352 2353 2354 |
if (index > end_index) break; if (index == end_index) { |
09cbfeaf1 mm, fs: get rid o... |
2355 |
nr = i_size & ~PAGE_MASK; |
1da177e4c Linux-2.6.12-rc2 |
2356 2357 2358 |
if (nr <= offset) break; } |
9e18eb293 tmpfs: mem_cgroup... |
2359 |
error = shmem_getpage(inode, index, &page, sgp); |
6e58e79db introduce copy_pa... |
2360 2361 2362 |
if (error) { if (error == -EINVAL) error = 0; |
1da177e4c Linux-2.6.12-rc2 |
2363 2364 |
break; } |
75edd345e tmpfs: preliminar... |
2365 2366 2367 |
if (page) { if (sgp == SGP_CACHE) set_page_dirty(page); |
d3602444e shmem_getpage ret... |
2368 |
unlock_page(page); |
75edd345e tmpfs: preliminar... |
2369 |
} |
1da177e4c Linux-2.6.12-rc2 |
2370 2371 2372 |
/* * We must evaluate after, since reads (unlike writes) |
1b1dcc1b5 [PATCH] mutex sub... |
2373 |
* are called without i_mutex protection against truncate |
1da177e4c Linux-2.6.12-rc2 |
2374 |
*/ |
09cbfeaf1 mm, fs: get rid o... |
2375 |
nr = PAGE_SIZE; |
1da177e4c Linux-2.6.12-rc2 |
2376 |
i_size = i_size_read(inode); |
09cbfeaf1 mm, fs: get rid o... |
2377 |
end_index = i_size >> PAGE_SHIFT; |
1da177e4c Linux-2.6.12-rc2 |
2378 |
if (index == end_index) { |
09cbfeaf1 mm, fs: get rid o... |
2379 |
nr = i_size & ~PAGE_MASK; |
1da177e4c Linux-2.6.12-rc2 |
2380 2381 |
if (nr <= offset) { if (page) |
09cbfeaf1 mm, fs: get rid o... |
2382 |
put_page(page); |
1da177e4c Linux-2.6.12-rc2 |
2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 |
break; } } nr -= offset; if (page) { /* * If users can be writing to this page using arbitrary * virtual addresses, take care about potential aliasing * before reading the page on the kernel side. */ if (mapping_writably_mapped(mapping)) flush_dcache_page(page); /* * Mark the page accessed if we read the beginning. */ if (!offset) mark_page_accessed(page); |
b5810039a [PATCH] core remo... |
2401 |
} else { |
1da177e4c Linux-2.6.12-rc2 |
2402 |
page = ZERO_PAGE(0); |
09cbfeaf1 mm, fs: get rid o... |
2403 |
get_page(page); |
b5810039a [PATCH] core remo... |
2404 |
} |
1da177e4c Linux-2.6.12-rc2 |
2405 2406 2407 2408 |
/* * Ok, we have the page, and it's up-to-date, so * now we can copy it to user space... |
1da177e4c Linux-2.6.12-rc2 |
2409 |
*/ |
2ba5bbed0 shmem: switch to ... |
2410 |
ret = copy_page_to_iter(page, offset, nr, to); |
6e58e79db introduce copy_pa... |
2411 |
retval += ret; |
1da177e4c Linux-2.6.12-rc2 |
2412 |
offset += ret; |
09cbfeaf1 mm, fs: get rid o... |
2413 2414 |
index += offset >> PAGE_SHIFT; offset &= ~PAGE_MASK; |
1da177e4c Linux-2.6.12-rc2 |
2415 |
|
09cbfeaf1 mm, fs: get rid o... |
2416 |
put_page(page); |
2ba5bbed0 shmem: switch to ... |
2417 |
if (!iov_iter_count(to)) |
1da177e4c Linux-2.6.12-rc2 |
2418 |
break; |
6e58e79db introduce copy_pa... |
2419 2420 2421 2422 |
if (ret < nr) { error = -EFAULT; break; } |
1da177e4c Linux-2.6.12-rc2 |
2423 2424 |
cond_resched(); } |
09cbfeaf1 mm, fs: get rid o... |
2425 |
*ppos = ((loff_t) index << PAGE_SHIFT) + offset; |
6e58e79db introduce copy_pa... |
2426 2427 |
file_accessed(file); return retval ? retval : error; |
1da177e4c Linux-2.6.12-rc2 |
2428 |
} |
220f2ac91 tmpfs: support SE... |
2429 2430 2431 2432 |
/* * llseek SEEK_DATA or SEEK_HOLE through the radix_tree. */ static pgoff_t shmem_seek_hole_data(struct address_space *mapping, |
965c8e59c lseek: the "whenc... |
2433 |
pgoff_t index, pgoff_t end, int whence) |
220f2ac91 tmpfs: support SE... |
2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 |
{ struct page *page; struct pagevec pvec; pgoff_t indices[PAGEVEC_SIZE]; bool done = false; int i; pagevec_init(&pvec, 0); pvec.nr = 1; /* start small: we may be there already */ while (!done) { |
0cd6144aa mm + fs: prepare ... |
2444 |
pvec.nr = find_get_entries(mapping, index, |
220f2ac91 tmpfs: support SE... |
2445 2446 |
pvec.nr, pvec.pages, indices); if (!pvec.nr) { |
965c8e59c lseek: the "whenc... |
2447 |
if (whence == SEEK_DATA) |
220f2ac91 tmpfs: support SE... |
2448 2449 2450 2451 2452 |
index = end; break; } for (i = 0; i < pvec.nr; i++, index++) { if (index < indices[i]) { |
965c8e59c lseek: the "whenc... |
2453 |
if (whence == SEEK_HOLE) { |
220f2ac91 tmpfs: support SE... |
2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 |
done = true; break; } index = indices[i]; } page = pvec.pages[i]; if (page && !radix_tree_exceptional_entry(page)) { if (!PageUptodate(page)) page = NULL; } if (index >= end || |
965c8e59c lseek: the "whenc... |
2465 2466 |
(page && whence == SEEK_DATA) || (!page && whence == SEEK_HOLE)) { |
220f2ac91 tmpfs: support SE... |
2467 2468 2469 2470 |
done = true; break; } } |
0cd6144aa mm + fs: prepare ... |
2471 |
pagevec_remove_exceptionals(&pvec); |
220f2ac91 tmpfs: support SE... |
2472 2473 2474 2475 2476 2477 |
pagevec_release(&pvec); pvec.nr = PAGEVEC_SIZE; cond_resched(); } return index; } |
965c8e59c lseek: the "whenc... |
2478 |
static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) |
220f2ac91 tmpfs: support SE... |
2479 2480 2481 2482 2483 |
{ struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; pgoff_t start, end; loff_t new_offset; |
965c8e59c lseek: the "whenc... |
2484 2485 |
if (whence != SEEK_DATA && whence != SEEK_HOLE) return generic_file_llseek_size(file, offset, whence, |
220f2ac91 tmpfs: support SE... |
2486 |
MAX_LFS_FILESIZE, i_size_read(inode)); |
5955102c9 wrappers for ->i_... |
2487 |
inode_lock(inode); |
220f2ac91 tmpfs: support SE... |
2488 2489 2490 2491 2492 2493 2494 |
/* We're holding i_mutex so we can access i_size directly */ if (offset < 0) offset = -EINVAL; else if (offset >= inode->i_size) offset = -ENXIO; else { |
09cbfeaf1 mm, fs: get rid o... |
2495 2496 |
start = offset >> PAGE_SHIFT; end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; |
965c8e59c lseek: the "whenc... |
2497 |
new_offset = shmem_seek_hole_data(mapping, start, end, whence); |
09cbfeaf1 mm, fs: get rid o... |
2498 |
new_offset <<= PAGE_SHIFT; |
220f2ac91 tmpfs: support SE... |
2499 2500 2501 |
if (new_offset > offset) { if (new_offset < inode->i_size) offset = new_offset; |
965c8e59c lseek: the "whenc... |
2502 |
else if (whence == SEEK_DATA) |
220f2ac91 tmpfs: support SE... |
2503 2504 2505 2506 2507 |
offset = -ENXIO; else offset = inode->i_size; } } |
387aae6fd tmpfs: fix SEEK_D... |
2508 2509 |
if (offset >= 0) offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); |
5955102c9 wrappers for ->i_... |
2510 |
inode_unlock(inode); |
220f2ac91 tmpfs: support SE... |
2511 2512 |
return offset; } |
05f65b5c7 shm: wait for pin... |
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 |
/* * We need a tag: a new tag would expand every radix_tree_node by 8 bytes, * so reuse a tag which we firmly believe is never set or cleared on shmem. */ #define SHMEM_TAG_PINNED PAGECACHE_TAG_TOWRITE #define LAST_SCAN 4 /* about 150ms max */ static void shmem_tag_pins(struct address_space *mapping) { struct radix_tree_iter iter; void **slot; pgoff_t start; struct page *page; lru_add_drain(); start = 0; rcu_read_lock(); |
05f65b5c7 shm: wait for pin... |
2530 2531 2532 |
radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { page = radix_tree_deref_slot(slot); if (!page || radix_tree_exception(page)) { |
2cf938aae mm: use radix_tre... |
2533 2534 2535 2536 |
if (radix_tree_deref_retry(page)) { slot = radix_tree_iter_retry(&iter); continue; } |
05f65b5c7 shm: wait for pin... |
2537 2538 2539 2540 2541 2542 2543 2544 |
} else if (page_count(page) - page_mapcount(page) > 1) { spin_lock_irq(&mapping->tree_lock); radix_tree_tag_set(&mapping->page_tree, iter.index, SHMEM_TAG_PINNED); spin_unlock_irq(&mapping->tree_lock); } if (need_resched()) { |
148deab22 radix-tree: impro... |
2545 |
slot = radix_tree_iter_resume(slot, &iter); |
05f65b5c7 shm: wait for pin... |
2546 |
cond_resched_rcu(); |
05f65b5c7 shm: wait for pin... |
2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 |
} } rcu_read_unlock(); } /* * Setting SEAL_WRITE requires us to verify there's no pending writer. However, * via get_user_pages(), drivers might have some pending I/O without any active * user-space mappings (eg., direct-IO, AIO). Therefore, we look at all pages * and see whether it has an elevated ref-count. If so, we tag them and wait for * them to be dropped. * The caller must guarantee that no new user will acquire writable references * to those pages to avoid races. */ |
40e041a2c shm: add sealing API |
2561 2562 |
static int shmem_wait_for_pins(struct address_space *mapping) { |
05f65b5c7 shm: wait for pin... |
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 |
struct radix_tree_iter iter; void **slot; pgoff_t start; struct page *page; int error, scan; shmem_tag_pins(mapping); error = 0; for (scan = 0; scan <= LAST_SCAN; scan++) { if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) break; if (!scan) lru_add_drain_all(); else if (schedule_timeout_killable((HZ << scan) / 200)) scan = LAST_SCAN; start = 0; rcu_read_lock(); |
05f65b5c7 shm: wait for pin... |
2583 2584 2585 2586 2587 |
radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start, SHMEM_TAG_PINNED) { page = radix_tree_deref_slot(slot); if (radix_tree_exception(page)) { |
2cf938aae mm: use radix_tre... |
2588 2589 2590 2591 |
if (radix_tree_deref_retry(page)) { slot = radix_tree_iter_retry(&iter); continue; } |
05f65b5c7 shm: wait for pin... |
2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 |
page = NULL; } if (page && page_count(page) - page_mapcount(page) != 1) { if (scan < LAST_SCAN) goto continue_resched; /* * On the last scan, we clean up all those tags * we inserted; but make a note that we still * found pages pinned. */ error = -EBUSY; } spin_lock_irq(&mapping->tree_lock); radix_tree_tag_clear(&mapping->page_tree, iter.index, SHMEM_TAG_PINNED); spin_unlock_irq(&mapping->tree_lock); continue_resched: if (need_resched()) { |
148deab22 radix-tree: impro... |
2615 |
slot = radix_tree_iter_resume(slot, &iter); |
05f65b5c7 shm: wait for pin... |
2616 |
cond_resched_rcu(); |
05f65b5c7 shm: wait for pin... |
2617 2618 2619 2620 2621 2622 |
} } rcu_read_unlock(); } return error; |
40e041a2c shm: add sealing API |
2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 |
} #define F_ALL_SEALS (F_SEAL_SEAL | \ F_SEAL_SHRINK | \ F_SEAL_GROW | \ F_SEAL_WRITE) int shmem_add_seals(struct file *file, unsigned int seals) { struct inode *inode = file_inode(file); struct shmem_inode_info *info = SHMEM_I(inode); int error; /* * SEALING * Sealing allows multiple parties to share a shmem-file but restrict * access to a specific subset of file operations. Seals can only be * added, but never removed. This way, mutually untrusted parties can * share common memory regions with a well-defined policy. A malicious * peer can thus never perform unwanted operations on a shared object. * * Seals are only supported on special shmem-files and always affect * the whole underlying inode. Once a seal is set, it may prevent some * kinds of access to the file. Currently, the following seals are * defined: * SEAL_SEAL: Prevent further seals from being set on this file * SEAL_SHRINK: Prevent the file from shrinking * SEAL_GROW: Prevent the file from growing * SEAL_WRITE: Prevent write access to the file * * As we don't require any trust relationship between two parties, we * must prevent seals from being removed. Therefore, sealing a file * only adds a given set of seals to the file, it never touches * existing seals. Furthermore, the "setting seals"-operation can be * sealed itself, which basically prevents any further seal from being * added. * * Semantics of sealing are only defined on volatile files. Only * anonymous shmem files support sealing. More importantly, seals are * never written to disk. Therefore, there's no plan to support it on * other file types. */ if (file->f_op != &shmem_file_operations) return -EINVAL; if (!(file->f_mode & FMODE_WRITE)) return -EPERM; if (seals & ~(unsigned int)F_ALL_SEALS) return -EINVAL; |
5955102c9 wrappers for ->i_... |
2672 |
inode_lock(inode); |
40e041a2c shm: add sealing API |
2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 |
if (info->seals & F_SEAL_SEAL) { error = -EPERM; goto unlock; } if ((seals & F_SEAL_WRITE) && !(info->seals & F_SEAL_WRITE)) { error = mapping_deny_writable(file->f_mapping); if (error) goto unlock; error = shmem_wait_for_pins(file->f_mapping); if (error) { mapping_allow_writable(file->f_mapping); goto unlock; } } info->seals |= seals; error = 0; unlock: |
5955102c9 wrappers for ->i_... |
2695 |
inode_unlock(inode); |
40e041a2c shm: add sealing API |
2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 |
return error; } EXPORT_SYMBOL_GPL(shmem_add_seals); int shmem_get_seals(struct file *file) { if (file->f_op != &shmem_file_operations) return -EINVAL; return SHMEM_I(file_inode(file))->seals; } EXPORT_SYMBOL_GPL(shmem_get_seals); long shmem_fcntl(struct file *file, unsigned int cmd, unsigned long arg) { long error; switch (cmd) { case F_ADD_SEALS: /* disallow upper 32bit */ if (arg > UINT_MAX) return -EINVAL; error = shmem_add_seals(file, arg); break; case F_GET_SEALS: error = shmem_get_seals(file); break; default: error = -EINVAL; break; } return error; } |
83e4fa9c1 tmpfs: support fa... |
2731 2732 2733 |
static long shmem_fallocate(struct file *file, int mode, loff_t offset, loff_t len) { |
496ad9aa8 new helper: file_... |
2734 |
struct inode *inode = file_inode(file); |
e2d12e22c tmpfs: support fa... |
2735 |
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); |
40e041a2c shm: add sealing API |
2736 |
struct shmem_inode_info *info = SHMEM_I(inode); |
1aac14003 tmpfs: quit when ... |
2737 |
struct shmem_falloc shmem_falloc; |
e2d12e22c tmpfs: support fa... |
2738 2739 |
pgoff_t start, index, end; int error; |
83e4fa9c1 tmpfs: support fa... |
2740 |
|
13ace4d0d tmpfs: ZERO_RANGE... |
2741 2742 |
if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) return -EOPNOTSUPP; |
5955102c9 wrappers for ->i_... |
2743 |
inode_lock(inode); |
83e4fa9c1 tmpfs: support fa... |
2744 2745 2746 2747 2748 |
if (mode & FALLOC_FL_PUNCH_HOLE) { struct address_space *mapping = file->f_mapping; loff_t unmap_start = round_up(offset, PAGE_SIZE); loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; |
8e205f779 shmem: fix faulti... |
2749 |
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); |
83e4fa9c1 tmpfs: support fa... |
2750 |
|
40e041a2c shm: add sealing API |
2751 2752 2753 2754 2755 |
/* protected by i_mutex */ if (info->seals & F_SEAL_WRITE) { error = -EPERM; goto out; } |
8e205f779 shmem: fix faulti... |
2756 |
shmem_falloc.waitq = &shmem_falloc_waitq; |
f00cdc6df shmem: fix faulti... |
2757 2758 2759 2760 2761 |
shmem_falloc.start = unmap_start >> PAGE_SHIFT; shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; spin_lock(&inode->i_lock); inode->i_private = &shmem_falloc; spin_unlock(&inode->i_lock); |
83e4fa9c1 tmpfs: support fa... |
2762 2763 2764 2765 2766 |
if ((u64)unmap_end > (u64)unmap_start) unmap_mapping_range(mapping, unmap_start, 1 + unmap_end - unmap_start, 0); shmem_truncate_range(inode, offset, offset + len - 1); /* No need to unmap again: hole-punching leaves COWed pages */ |
8e205f779 shmem: fix faulti... |
2767 2768 2769 2770 |
spin_lock(&inode->i_lock); inode->i_private = NULL; wake_up_all(&shmem_falloc_waitq); |
2055da973 sched/wait: Disam... |
2771 |
WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); |
8e205f779 shmem: fix faulti... |
2772 |
spin_unlock(&inode->i_lock); |
83e4fa9c1 tmpfs: support fa... |
2773 |
error = 0; |
8e205f779 shmem: fix faulti... |
2774 |
goto out; |
e2d12e22c tmpfs: support fa... |
2775 2776 2777 2778 2779 2780 |
} /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ error = inode_newsize_ok(inode, offset + len); if (error) goto out; |
40e041a2c shm: add sealing API |
2781 2782 2783 2784 |
if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { error = -EPERM; goto out; } |
09cbfeaf1 mm, fs: get rid o... |
2785 2786 |
start = offset >> PAGE_SHIFT; end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
e2d12e22c tmpfs: support fa... |
2787 2788 2789 2790 |
/* Try to avoid a swapstorm if len is impossible to satisfy */ if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { error = -ENOSPC; goto out; |
83e4fa9c1 tmpfs: support fa... |
2791 |
} |
8e205f779 shmem: fix faulti... |
2792 |
shmem_falloc.waitq = NULL; |
1aac14003 tmpfs: quit when ... |
2793 2794 2795 2796 2797 2798 2799 |
shmem_falloc.start = start; shmem_falloc.next = start; shmem_falloc.nr_falloced = 0; shmem_falloc.nr_unswapped = 0; spin_lock(&inode->i_lock); inode->i_private = &shmem_falloc; spin_unlock(&inode->i_lock); |
e2d12e22c tmpfs: support fa... |
2800 2801 2802 2803 2804 2805 2806 2807 2808 |
for (index = start; index < end; index++) { struct page *page; /* * Good, the fallocate(2) manpage permits EINTR: we may have * been interrupted because we are using up too much memory. */ if (signal_pending(current)) error = -EINTR; |
1aac14003 tmpfs: quit when ... |
2809 2810 |
else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) error = -ENOMEM; |
e2d12e22c tmpfs: support fa... |
2811 |
else |
9e18eb293 tmpfs: mem_cgroup... |
2812 |
error = shmem_getpage(inode, index, &page, SGP_FALLOC); |
e2d12e22c tmpfs: support fa... |
2813 |
if (error) { |
1635f6a74 tmpfs: undo fallo... |
2814 |
/* Remove the !PageUptodate pages we added */ |
7f5565670 tmpfs: fix regres... |
2815 2816 2817 2818 2819 |
if (index > start) { shmem_undo_range(inode, (loff_t)start << PAGE_SHIFT, ((loff_t)index << PAGE_SHIFT) - 1, true); } |
1aac14003 tmpfs: quit when ... |
2820 |
goto undone; |
e2d12e22c tmpfs: support fa... |
2821 |
} |
e2d12e22c tmpfs: support fa... |
2822 |
/* |
1aac14003 tmpfs: quit when ... |
2823 2824 2825 2826 2827 2828 2829 2830 |
* Inform shmem_writepage() how far we have reached. * No need for lock or barrier: we have the page lock. */ shmem_falloc.next++; if (!PageUptodate(page)) shmem_falloc.nr_falloced++; /* |
1635f6a74 tmpfs: undo fallo... |
2831 2832 2833 |
* If !PageUptodate, leave it that way so that freeable pages * can be recognized if we need to rollback on error later. * But set_page_dirty so that memory pressure will swap rather |
e2d12e22c tmpfs: support fa... |
2834 2835 2836 2837 2838 |
* than free the pages we are allocating (and SGP_CACHE pages * might still be clean: we now need to mark those dirty too). */ set_page_dirty(page); unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
2839 |
put_page(page); |
e2d12e22c tmpfs: support fa... |
2840 2841 2842 2843 2844 |
cond_resched(); } if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) i_size_write(inode, offset + len); |
078cd8279 fs: Replace CURRE... |
2845 |
inode->i_ctime = current_time(inode); |
1aac14003 tmpfs: quit when ... |
2846 2847 2848 2849 |
undone: spin_lock(&inode->i_lock); inode->i_private = NULL; spin_unlock(&inode->i_lock); |
e2d12e22c tmpfs: support fa... |
2850 |
out: |
5955102c9 wrappers for ->i_... |
2851 |
inode_unlock(inode); |
83e4fa9c1 tmpfs: support fa... |
2852 2853 |
return error; } |
726c33422 [PATCH] VFS: Perm... |
2854 |
static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) |
1da177e4c Linux-2.6.12-rc2 |
2855 |
{ |
726c33422 [PATCH] VFS: Perm... |
2856 |
struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); |
1da177e4c Linux-2.6.12-rc2 |
2857 2858 |
buf->f_type = TMPFS_MAGIC; |
09cbfeaf1 mm, fs: get rid o... |
2859 |
buf->f_bsize = PAGE_SIZE; |
1da177e4c Linux-2.6.12-rc2 |
2860 |
buf->f_namelen = NAME_MAX; |
0edd73b33 [PATCH] shmem: re... |
2861 |
if (sbinfo->max_blocks) { |
1da177e4c Linux-2.6.12-rc2 |
2862 |
buf->f_blocks = sbinfo->max_blocks; |
41ffe5d5c tmpfs: miscellane... |
2863 2864 2865 |
buf->f_bavail = buf->f_bfree = sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks); |
0edd73b33 [PATCH] shmem: re... |
2866 2867 |
} if (sbinfo->max_inodes) { |
1da177e4c Linux-2.6.12-rc2 |
2868 2869 |
buf->f_files = sbinfo->max_inodes; buf->f_ffree = sbinfo->free_inodes; |
1da177e4c Linux-2.6.12-rc2 |
2870 2871 2872 2873 2874 2875 2876 2877 2878 |
} /* else leave those fields 0 like simple_statfs */ return 0; } /* * File creation. Allocate an inode, and we're done.. */ static int |
1a67aafb5 switch ->mknod() ... |
2879 |
shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) |
1da177e4c Linux-2.6.12-rc2 |
2880 |
{ |
0b0a0806b shmem: fix shared... |
2881 |
struct inode *inode; |
1da177e4c Linux-2.6.12-rc2 |
2882 |
int error = -ENOSPC; |
454abafe9 ramfs: replace in... |
2883 |
inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); |
1da177e4c Linux-2.6.12-rc2 |
2884 |
if (inode) { |
feda821e7 fs: remove generi... |
2885 2886 2887 |
error = simple_acl_create(dir, inode); if (error) goto out_iput; |
2a7dba391 fs/vfs/security: ... |
2888 |
error = security_inode_init_security(inode, dir, |
9d8f13ba3 security: new sec... |
2889 |
&dentry->d_name, |
6d9d88d07 tmpfs: security x... |
2890 |
shmem_initxattrs, NULL); |
feda821e7 fs: remove generi... |
2891 2892 |
if (error && error != -EOPNOTSUPP) goto out_iput; |
37ec43cdc evm: calculate HM... |
2893 |
|
718deb6b6 Fix breakage in s... |
2894 |
error = 0; |
1da177e4c Linux-2.6.12-rc2 |
2895 |
dir->i_size += BOGO_DIRENT_SIZE; |
078cd8279 fs: Replace CURRE... |
2896 |
dir->i_ctime = dir->i_mtime = current_time(dir); |
1da177e4c Linux-2.6.12-rc2 |
2897 2898 |
d_instantiate(dentry, inode); dget(dentry); /* Extra count - pin the dentry in core */ |
1da177e4c Linux-2.6.12-rc2 |
2899 2900 |
} return error; |
feda821e7 fs: remove generi... |
2901 2902 2903 |
out_iput: iput(inode); return error; |
1da177e4c Linux-2.6.12-rc2 |
2904 |
} |
60545d0d4 [O_TMPFILE] it's ... |
2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 |
static int shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) { struct inode *inode; int error = -ENOSPC; inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); if (inode) { error = security_inode_init_security(inode, dir, NULL, shmem_initxattrs, NULL); |
feda821e7 fs: remove generi... |
2916 2917 2918 2919 2920 |
if (error && error != -EOPNOTSUPP) goto out_iput; error = simple_acl_create(dir, inode); if (error) goto out_iput; |
60545d0d4 [O_TMPFILE] it's ... |
2921 2922 2923 |
d_tmpfile(dentry, inode); } return error; |
feda821e7 fs: remove generi... |
2924 2925 2926 |
out_iput: iput(inode); return error; |
60545d0d4 [O_TMPFILE] it's ... |
2927 |
} |
18bb1db3e switch vfs_mkdir(... |
2928 |
static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) |
1da177e4c Linux-2.6.12-rc2 |
2929 2930 2931 2932 2933 |
{ int error; if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) return error; |
d8c76e6f4 [PATCH] r/o bind ... |
2934 |
inc_nlink(dir); |
1da177e4c Linux-2.6.12-rc2 |
2935 2936 |
return 0; } |
4acdaf27e switch ->create()... |
2937 |
static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, |
ebfc3b49a don't pass nameid... |
2938 |
bool excl) |
1da177e4c Linux-2.6.12-rc2 |
2939 2940 2941 2942 2943 2944 2945 2946 2947 |
{ return shmem_mknod(dir, dentry, mode | S_IFREG, 0); } /* * Link a file.. */ static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) { |
75c3cfa85 VFS: assorted wei... |
2948 |
struct inode *inode = d_inode(old_dentry); |
5b04c6890 shmem: factor out... |
2949 |
int ret; |
1da177e4c Linux-2.6.12-rc2 |
2950 2951 2952 2953 2954 2955 |
/* * No ordinary (disk based) filesystem counts links as inodes; * but each new link needs a new dentry, pinning lowmem, and * tmpfs dentries cannot be pruned until they are unlinked. */ |
5b04c6890 shmem: factor out... |
2956 2957 2958 |
ret = shmem_reserve_inode(inode->i_sb); if (ret) goto out; |
1da177e4c Linux-2.6.12-rc2 |
2959 2960 |
dir->i_size += BOGO_DIRENT_SIZE; |
078cd8279 fs: Replace CURRE... |
2961 |
inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); |
d8c76e6f4 [PATCH] r/o bind ... |
2962 |
inc_nlink(inode); |
7de9c6ee3 new helper: ihold() |
2963 |
ihold(inode); /* New dentry reference */ |
1da177e4c Linux-2.6.12-rc2 |
2964 2965 |
dget(dentry); /* Extra pinning count for the created dentry */ d_instantiate(dentry, inode); |
5b04c6890 shmem: factor out... |
2966 2967 |
out: return ret; |
1da177e4c Linux-2.6.12-rc2 |
2968 2969 2970 2971 |
} static int shmem_unlink(struct inode *dir, struct dentry *dentry) { |
75c3cfa85 VFS: assorted wei... |
2972 |
struct inode *inode = d_inode(dentry); |
1da177e4c Linux-2.6.12-rc2 |
2973 |
|
5b04c6890 shmem: factor out... |
2974 2975 |
if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) shmem_free_inode(inode->i_sb); |
1da177e4c Linux-2.6.12-rc2 |
2976 2977 |
dir->i_size -= BOGO_DIRENT_SIZE; |
078cd8279 fs: Replace CURRE... |
2978 |
inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); |
9a53c3a78 [PATCH] r/o bind ... |
2979 |
drop_nlink(inode); |
1da177e4c Linux-2.6.12-rc2 |
2980 2981 2982 2983 2984 2985 2986 2987 |
dput(dentry); /* Undo the count from "create" - this does all the work */ return 0; } static int shmem_rmdir(struct inode *dir, struct dentry *dentry) { if (!simple_empty(dentry)) return -ENOTEMPTY; |
75c3cfa85 VFS: assorted wei... |
2988 |
drop_nlink(d_inode(dentry)); |
9a53c3a78 [PATCH] r/o bind ... |
2989 |
drop_nlink(dir); |
1da177e4c Linux-2.6.12-rc2 |
2990 2991 |
return shmem_unlink(dir, dentry); } |
37456771c shmem: support RE... |
2992 2993 |
static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) { |
e36cb0b89 VFS: (Scripted) C... |
2994 2995 |
bool old_is_dir = d_is_dir(old_dentry); bool new_is_dir = d_is_dir(new_dentry); |
37456771c shmem: support RE... |
2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 |
if (old_dir != new_dir && old_is_dir != new_is_dir) { if (old_is_dir) { drop_nlink(old_dir); inc_nlink(new_dir); } else { drop_nlink(new_dir); inc_nlink(old_dir); } } old_dir->i_ctime = old_dir->i_mtime = new_dir->i_ctime = new_dir->i_mtime = |
75c3cfa85 VFS: assorted wei... |
3008 |
d_inode(old_dentry)->i_ctime = |
078cd8279 fs: Replace CURRE... |
3009 |
d_inode(new_dentry)->i_ctime = current_time(old_dir); |
37456771c shmem: support RE... |
3010 3011 3012 |
return 0; } |
46fdb794e shmem: support RE... |
3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 |
static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) { struct dentry *whiteout; int error; whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); if (!whiteout) return -ENOMEM; error = shmem_mknod(old_dir, whiteout, S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); dput(whiteout); if (error) return error; /* * Cheat and hash the whiteout while the old dentry is still in * place, instead of playing games with FS_RENAME_DOES_D_MOVE. * * d_lookup() will consistently find one of them at this point, * not sure which one, but that isn't even important. */ d_rehash(whiteout); return 0; } |
1da177e4c Linux-2.6.12-rc2 |
3038 3039 3040 3041 3042 3043 |
/* * The VFS layer already does all the dentry stuff for rename, * we just have to decrement the usage count for the target if * it exists so that the VFS layer correctly free's it when it * gets overwritten. */ |
3b69ff51d shmem: support RE... |
3044 |
static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) |
1da177e4c Linux-2.6.12-rc2 |
3045 |
{ |
75c3cfa85 VFS: assorted wei... |
3046 |
struct inode *inode = d_inode(old_dentry); |
1da177e4c Linux-2.6.12-rc2 |
3047 |
int they_are_dirs = S_ISDIR(inode->i_mode); |
46fdb794e shmem: support RE... |
3048 |
if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) |
3b69ff51d shmem: support RE... |
3049 |
return -EINVAL; |
37456771c shmem: support RE... |
3050 3051 |
if (flags & RENAME_EXCHANGE) return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); |
1da177e4c Linux-2.6.12-rc2 |
3052 3053 |
if (!simple_empty(new_dentry)) return -ENOTEMPTY; |
46fdb794e shmem: support RE... |
3054 3055 3056 3057 3058 3059 3060 |
if (flags & RENAME_WHITEOUT) { int error; error = shmem_whiteout(old_dir, old_dentry); if (error) return error; } |
75c3cfa85 VFS: assorted wei... |
3061 |
if (d_really_is_positive(new_dentry)) { |
1da177e4c Linux-2.6.12-rc2 |
3062 |
(void) shmem_unlink(new_dir, new_dentry); |
b928095b0 shmem: fix nlink ... |
3063 |
if (they_are_dirs) { |
75c3cfa85 VFS: assorted wei... |
3064 |
drop_nlink(d_inode(new_dentry)); |
9a53c3a78 [PATCH] r/o bind ... |
3065 |
drop_nlink(old_dir); |
b928095b0 shmem: fix nlink ... |
3066 |
} |
1da177e4c Linux-2.6.12-rc2 |
3067 |
} else if (they_are_dirs) { |
9a53c3a78 [PATCH] r/o bind ... |
3068 |
drop_nlink(old_dir); |
d8c76e6f4 [PATCH] r/o bind ... |
3069 |
inc_nlink(new_dir); |
1da177e4c Linux-2.6.12-rc2 |
3070 3071 3072 3073 3074 3075 |
} old_dir->i_size -= BOGO_DIRENT_SIZE; new_dir->i_size += BOGO_DIRENT_SIZE; old_dir->i_ctime = old_dir->i_mtime = new_dir->i_ctime = new_dir->i_mtime = |
078cd8279 fs: Replace CURRE... |
3076 |
inode->i_ctime = current_time(old_dir); |
1da177e4c Linux-2.6.12-rc2 |
3077 3078 3079 3080 3081 3082 3083 3084 |
return 0; } static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) { int error; int len; struct inode *inode; |
9276aad6c tmpfs: remove_shm... |
3085 |
struct page *page; |
1da177e4c Linux-2.6.12-rc2 |
3086 3087 3088 |
struct shmem_inode_info *info; len = strlen(symname) + 1; |
09cbfeaf1 mm, fs: get rid o... |
3089 |
if (len > PAGE_SIZE) |
1da177e4c Linux-2.6.12-rc2 |
3090 |
return -ENAMETOOLONG; |
454abafe9 ramfs: replace in... |
3091 |
inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0, VM_NORESERVE); |
1da177e4c Linux-2.6.12-rc2 |
3092 3093 |
if (!inode) return -ENOSPC; |
9d8f13ba3 security: new sec... |
3094 |
error = security_inode_init_security(inode, dir, &dentry->d_name, |
6d9d88d07 tmpfs: security x... |
3095 |
shmem_initxattrs, NULL); |
570bc1c2e [PATCH] tmpfs: En... |
3096 3097 3098 3099 3100 3101 3102 |
if (error) { if (error != -EOPNOTSUPP) { iput(inode); return error; } error = 0; } |
1da177e4c Linux-2.6.12-rc2 |
3103 3104 |
info = SHMEM_I(inode); inode->i_size = len-1; |
69f07ec93 tmpfs: use kmemdu... |
3105 |
if (len <= SHORT_SYMLINK_LEN) { |
3ed47db34 make sure that fr... |
3106 3107 |
inode->i_link = kmemdup(symname, len, GFP_KERNEL); if (!inode->i_link) { |
69f07ec93 tmpfs: use kmemdu... |
3108 3109 3110 3111 |
iput(inode); return -ENOMEM; } inode->i_op = &shmem_short_symlink_operations; |
1da177e4c Linux-2.6.12-rc2 |
3112 |
} else { |
e8ecde25f Make sure that hi... |
3113 |
inode_nohighmem(inode); |
9e18eb293 tmpfs: mem_cgroup... |
3114 |
error = shmem_getpage(inode, 0, &page, SGP_WRITE); |
1da177e4c Linux-2.6.12-rc2 |
3115 3116 3117 3118 |
if (error) { iput(inode); return error; } |
14fcc23fd tmpfs: fix kernel... |
3119 |
inode->i_mapping->a_ops = &shmem_aops; |
1da177e4c Linux-2.6.12-rc2 |
3120 |
inode->i_op = &shmem_symlink_inode_operations; |
21fc61c73 don't put symlink... |
3121 |
memcpy(page_address(page), symname, len); |
ec9516fbc tmpfs: optimize c... |
3122 |
SetPageUptodate(page); |
1da177e4c Linux-2.6.12-rc2 |
3123 |
set_page_dirty(page); |
6746aff74 HWPOISON: shmem: ... |
3124 |
unlock_page(page); |
09cbfeaf1 mm, fs: get rid o... |
3125 |
put_page(page); |
1da177e4c Linux-2.6.12-rc2 |
3126 |
} |
1da177e4c Linux-2.6.12-rc2 |
3127 |
dir->i_size += BOGO_DIRENT_SIZE; |
078cd8279 fs: Replace CURRE... |
3128 |
dir->i_ctime = dir->i_mtime = current_time(dir); |
1da177e4c Linux-2.6.12-rc2 |
3129 3130 3131 3132 |
d_instantiate(dentry, inode); dget(dentry); return 0; } |
fceef393a switch ->get_link... |
3133 |
static void shmem_put_link(void *arg) |
1da177e4c Linux-2.6.12-rc2 |
3134 |
{ |
fceef393a switch ->get_link... |
3135 3136 |
mark_page_accessed(arg); put_page(arg); |
1da177e4c Linux-2.6.12-rc2 |
3137 |
} |
6b2553918 replace ->follow_... |
3138 |
static const char *shmem_get_link(struct dentry *dentry, |
fceef393a switch ->get_link... |
3139 3140 |
struct inode *inode, struct delayed_call *done) |
1da177e4c Linux-2.6.12-rc2 |
3141 |
{ |
1da177e4c Linux-2.6.12-rc2 |
3142 |
struct page *page = NULL; |
6b2553918 replace ->follow_... |
3143 |
int error; |
6a6c99049 teach shmem_get_l... |
3144 3145 3146 3147 3148 3149 3150 3151 3152 |
if (!dentry) { page = find_get_page(inode->i_mapping, 0); if (!page) return ERR_PTR(-ECHILD); if (!PageUptodate(page)) { put_page(page); return ERR_PTR(-ECHILD); } } else { |
9e18eb293 tmpfs: mem_cgroup... |
3153 |
error = shmem_getpage(inode, 0, &page, SGP_READ); |
6a6c99049 teach shmem_get_l... |
3154 3155 3156 3157 |
if (error) return ERR_PTR(error); unlock_page(page); } |
fceef393a switch ->get_link... |
3158 |
set_delayed_call(done, shmem_put_link, page); |
21fc61c73 don't put symlink... |
3159 |
return page_address(page); |
1da177e4c Linux-2.6.12-rc2 |
3160 |
} |
b09e0fa4b tmpfs: implement ... |
3161 |
#ifdef CONFIG_TMPFS_XATTR |
467118102 mm/shmem and tiny... |
3162 |
/* |
b09e0fa4b tmpfs: implement ... |
3163 3164 |
* Superblocks without xattr inode operations may get some security.* xattr * support from the LSM "for free". As soon as we have any other xattrs |
39f0247d3 [PATCH] Access Co... |
3165 3166 3167 |
* like ACLs, we also need to implement the security.* handlers at * filesystem level, though. */ |
6d9d88d07 tmpfs: security x... |
3168 |
/* |
6d9d88d07 tmpfs: security x... |
3169 3170 3171 3172 3173 3174 3175 3176 |
* Callback for security_inode_init_security() for acquiring xattrs. */ static int shmem_initxattrs(struct inode *inode, const struct xattr *xattr_array, void *fs_info) { struct shmem_inode_info *info = SHMEM_I(inode); const struct xattr *xattr; |
38f386574 xattr: extract si... |
3177 |
struct simple_xattr *new_xattr; |
6d9d88d07 tmpfs: security x... |
3178 3179 3180 |
size_t len; for (xattr = xattr_array; xattr->name != NULL; xattr++) { |
38f386574 xattr: extract si... |
3181 |
new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); |
6d9d88d07 tmpfs: security x... |
3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 |
if (!new_xattr) return -ENOMEM; len = strlen(xattr->name) + 1; new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, GFP_KERNEL); if (!new_xattr->name) { kfree(new_xattr); return -ENOMEM; } memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, xattr->name, len); |
38f386574 xattr: extract si... |
3197 |
simple_xattr_list_add(&info->xattrs, new_xattr); |
6d9d88d07 tmpfs: security x... |
3198 3199 3200 3201 |
} return 0; } |
aa7c5241c tmpfs: Use xattr ... |
3202 |
static int shmem_xattr_handler_get(const struct xattr_handler *handler, |
b296821a7 xattr_handler: pa... |
3203 3204 |
struct dentry *unused, struct inode *inode, const char *name, void *buffer, size_t size) |
b09e0fa4b tmpfs: implement ... |
3205 |
{ |
b296821a7 xattr_handler: pa... |
3206 |
struct shmem_inode_info *info = SHMEM_I(inode); |
b09e0fa4b tmpfs: implement ... |
3207 |
|
aa7c5241c tmpfs: Use xattr ... |
3208 |
name = xattr_full_name(handler, name); |
38f386574 xattr: extract si... |
3209 |
return simple_xattr_get(&info->xattrs, name, buffer, size); |
b09e0fa4b tmpfs: implement ... |
3210 |
} |
aa7c5241c tmpfs: Use xattr ... |
3211 |
static int shmem_xattr_handler_set(const struct xattr_handler *handler, |
593012268 switch xattr_hand... |
3212 3213 3214 |
struct dentry *unused, struct inode *inode, const char *name, const void *value, size_t size, int flags) |
b09e0fa4b tmpfs: implement ... |
3215 |
{ |
593012268 switch xattr_hand... |
3216 |
struct shmem_inode_info *info = SHMEM_I(inode); |
b09e0fa4b tmpfs: implement ... |
3217 |
|
aa7c5241c tmpfs: Use xattr ... |
3218 |
name = xattr_full_name(handler, name); |
38f386574 xattr: extract si... |
3219 |
return simple_xattr_set(&info->xattrs, name, value, size, flags); |
b09e0fa4b tmpfs: implement ... |
3220 |
} |
aa7c5241c tmpfs: Use xattr ... |
3221 3222 3223 3224 3225 |
static const struct xattr_handler shmem_security_xattr_handler = { .prefix = XATTR_SECURITY_PREFIX, .get = shmem_xattr_handler_get, .set = shmem_xattr_handler_set, }; |
b09e0fa4b tmpfs: implement ... |
3226 |
|
aa7c5241c tmpfs: Use xattr ... |
3227 3228 3229 3230 3231 |
static const struct xattr_handler shmem_trusted_xattr_handler = { .prefix = XATTR_TRUSTED_PREFIX, .get = shmem_xattr_handler_get, .set = shmem_xattr_handler_set, }; |
b09e0fa4b tmpfs: implement ... |
3232 |
|
aa7c5241c tmpfs: Use xattr ... |
3233 3234 3235 3236 3237 3238 3239 3240 3241 |
static const struct xattr_handler *shmem_xattr_handlers[] = { #ifdef CONFIG_TMPFS_POSIX_ACL &posix_acl_access_xattr_handler, &posix_acl_default_xattr_handler, #endif &shmem_security_xattr_handler, &shmem_trusted_xattr_handler, NULL }; |
b09e0fa4b tmpfs: implement ... |
3242 3243 3244 |
static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) { |
75c3cfa85 VFS: assorted wei... |
3245 |
struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); |
786534b92 tmpfs: listxattr ... |
3246 |
return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); |
b09e0fa4b tmpfs: implement ... |
3247 3248 |
} #endif /* CONFIG_TMPFS_XATTR */ |
69f07ec93 tmpfs: use kmemdu... |
3249 |
static const struct inode_operations shmem_short_symlink_operations = { |
6b2553918 replace ->follow_... |
3250 |
.get_link = simple_get_link, |
b09e0fa4b tmpfs: implement ... |
3251 |
#ifdef CONFIG_TMPFS_XATTR |
b09e0fa4b tmpfs: implement ... |
3252 |
.listxattr = shmem_listxattr, |
b09e0fa4b tmpfs: implement ... |
3253 3254 3255 3256 |
#endif }; static const struct inode_operations shmem_symlink_inode_operations = { |
6b2553918 replace ->follow_... |
3257 |
.get_link = shmem_get_link, |
b09e0fa4b tmpfs: implement ... |
3258 |
#ifdef CONFIG_TMPFS_XATTR |
b09e0fa4b tmpfs: implement ... |
3259 |
.listxattr = shmem_listxattr, |
39f0247d3 [PATCH] Access Co... |
3260 |
#endif |
b09e0fa4b tmpfs: implement ... |
3261 |
}; |
39f0247d3 [PATCH] Access Co... |
3262 |
|
91828a405 [PATCH] knfsd: ad... |
3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 |
static struct dentry *shmem_get_parent(struct dentry *child) { return ERR_PTR(-ESTALE); } static int shmem_match(struct inode *ino, void *vfh) { __u32 *fh = vfh; __u64 inum = fh[2]; inum = (inum << 32) | fh[1]; return ino->i_ino == inum && fh[0] == ino->i_generation; } |
480b116c9 shmem: new export... |
3275 3276 |
static struct dentry *shmem_fh_to_dentry(struct super_block *sb, struct fid *fid, int fh_len, int fh_type) |
91828a405 [PATCH] knfsd: ad... |
3277 |
{ |
91828a405 [PATCH] knfsd: ad... |
3278 |
struct inode *inode; |
480b116c9 shmem: new export... |
3279 |
struct dentry *dentry = NULL; |
35c2a7f49 tmpfs,ceph,gfs2,i... |
3280 |
u64 inum; |
480b116c9 shmem: new export... |
3281 3282 3283 |
if (fh_len < 3) return NULL; |
91828a405 [PATCH] knfsd: ad... |
3284 |
|
35c2a7f49 tmpfs,ceph,gfs2,i... |
3285 3286 |
inum = fid->raw[2]; inum = (inum << 32) | fid->raw[1]; |
480b116c9 shmem: new export... |
3287 3288 |
inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), shmem_match, fid->raw); |
91828a405 [PATCH] knfsd: ad... |
3289 |
if (inode) { |
480b116c9 shmem: new export... |
3290 |
dentry = d_find_alias(inode); |
91828a405 [PATCH] knfsd: ad... |
3291 3292 |
iput(inode); } |
480b116c9 shmem: new export... |
3293 |
return dentry; |
91828a405 [PATCH] knfsd: ad... |
3294 |
} |
b0b0382bb ->encode_fh() API... |
3295 3296 |
static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, struct inode *parent) |
91828a405 [PATCH] knfsd: ad... |
3297 |
{ |
5fe0c2378 exportfs: Return ... |
3298 3299 |
if (*len < 3) { *len = 3; |
94e07a759 fs: encode_fh: re... |
3300 |
return FILEID_INVALID; |
5fe0c2378 exportfs: Return ... |
3301 |
} |
91828a405 [PATCH] knfsd: ad... |
3302 |
|
1d3382cbf new helper: inode... |
3303 |
if (inode_unhashed(inode)) { |
91828a405 [PATCH] knfsd: ad... |
3304 3305 3306 3307 3308 3309 3310 |
/* Unfortunately insert_inode_hash is not idempotent, * so as we hash inodes here rather than at creation * time, we need a lock to ensure we only try * to do it once */ static DEFINE_SPINLOCK(lock); spin_lock(&lock); |
1d3382cbf new helper: inode... |
3311 |
if (inode_unhashed(inode)) |
91828a405 [PATCH] knfsd: ad... |
3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 |
__insert_inode_hash(inode, inode->i_ino + inode->i_generation); spin_unlock(&lock); } fh[0] = inode->i_generation; fh[1] = inode->i_ino; fh[2] = ((__u64)inode->i_ino) >> 32; *len = 3; return 1; } |
396551644 exportfs: make st... |
3324 |
static const struct export_operations shmem_export_ops = { |
91828a405 [PATCH] knfsd: ad... |
3325 |
.get_parent = shmem_get_parent, |
91828a405 [PATCH] knfsd: ad... |
3326 |
.encode_fh = shmem_encode_fh, |
480b116c9 shmem: new export... |
3327 |
.fh_to_dentry = shmem_fh_to_dentry, |
91828a405 [PATCH] knfsd: ad... |
3328 |
}; |
680d794ba mount options: fi... |
3329 3330 |
static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, bool remount) |
1da177e4c Linux-2.6.12-rc2 |
3331 3332 |
{ char *this_char, *value, *rest; |
49cd0a5c2 tmpfs: fix mempol... |
3333 |
struct mempolicy *mpol = NULL; |
8751e0395 userns: Convert t... |
3334 3335 |
uid_t uid; gid_t gid; |
1da177e4c Linux-2.6.12-rc2 |
3336 |
|
b00dc3ad7 [PATCH] tmpfs: fi... |
3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 |
while (options != NULL) { this_char = options; for (;;) { /* * NUL-terminate this option: unfortunately, * mount options form a comma-separated list, * but mpol's nodelist may also contain commas. */ options = strchr(options, ','); if (options == NULL) break; options++; if (!isdigit(*options)) { options[-1] = '\0'; break; } } |
1da177e4c Linux-2.6.12-rc2 |
3354 3355 3356 3357 3358 |
if (!*this_char) continue; if ((value = strchr(this_char,'=')) != NULL) { *value++ = 0; } else { |
1170532bb mm: convert print... |
3359 3360 3361 |
pr_err("tmpfs: No value for mount option '%s' ", this_char); |
49cd0a5c2 tmpfs: fix mempol... |
3362 |
goto error; |
1da177e4c Linux-2.6.12-rc2 |
3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 |
} if (!strcmp(this_char,"size")) { unsigned long long size; size = memparse(value,&rest); if (*rest == '%') { size <<= PAGE_SHIFT; size *= totalram_pages; do_div(size, 100); rest++; } if (*rest) goto bad_val; |
680d794ba mount options: fi... |
3376 |
sbinfo->max_blocks = |
09cbfeaf1 mm, fs: get rid o... |
3377 |
DIV_ROUND_UP(size, PAGE_SIZE); |
1da177e4c Linux-2.6.12-rc2 |
3378 |
} else if (!strcmp(this_char,"nr_blocks")) { |
680d794ba mount options: fi... |
3379 |
sbinfo->max_blocks = memparse(value, &rest); |
1da177e4c Linux-2.6.12-rc2 |
3380 3381 3382 |
if (*rest) goto bad_val; } else if (!strcmp(this_char,"nr_inodes")) { |
680d794ba mount options: fi... |
3383 |
sbinfo->max_inodes = memparse(value, &rest); |
1da177e4c Linux-2.6.12-rc2 |
3384 3385 3386 |
if (*rest) goto bad_val; } else if (!strcmp(this_char,"mode")) { |
680d794ba mount options: fi... |
3387 |
if (remount) |
1da177e4c Linux-2.6.12-rc2 |
3388 |
continue; |
680d794ba mount options: fi... |
3389 |
sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; |
1da177e4c Linux-2.6.12-rc2 |
3390 3391 3392 |
if (*rest) goto bad_val; } else if (!strcmp(this_char,"uid")) { |
680d794ba mount options: fi... |
3393 |
if (remount) |
1da177e4c Linux-2.6.12-rc2 |
3394 |
continue; |
8751e0395 userns: Convert t... |
3395 |
uid = simple_strtoul(value, &rest, 0); |
1da177e4c Linux-2.6.12-rc2 |
3396 3397 |
if (*rest) goto bad_val; |
8751e0395 userns: Convert t... |
3398 3399 3400 |
sbinfo->uid = make_kuid(current_user_ns(), uid); if (!uid_valid(sbinfo->uid)) goto bad_val; |
1da177e4c Linux-2.6.12-rc2 |
3401 |
} else if (!strcmp(this_char,"gid")) { |
680d794ba mount options: fi... |
3402 |
if (remount) |
1da177e4c Linux-2.6.12-rc2 |
3403 |
continue; |
8751e0395 userns: Convert t... |
3404 |
gid = simple_strtoul(value, &rest, 0); |
1da177e4c Linux-2.6.12-rc2 |
3405 3406 |
if (*rest) goto bad_val; |
8751e0395 userns: Convert t... |
3407 3408 3409 |
sbinfo->gid = make_kgid(current_user_ns(), gid); if (!gid_valid(sbinfo->gid)) goto bad_val; |
e496cf3d7 thp: introduce CO... |
3410 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE |
5a6e75f81 shmem: prepare hu... |
3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 |
} else if (!strcmp(this_char, "huge")) { int huge; huge = shmem_parse_huge(value); if (huge < 0) goto bad_val; if (!has_transparent_hugepage() && huge != SHMEM_HUGE_NEVER) goto bad_val; sbinfo->huge = huge; #endif #ifdef CONFIG_NUMA |
7339ff830 [PATCH] Add tmpfs... |
3422 |
} else if (!strcmp(this_char,"mpol")) { |
49cd0a5c2 tmpfs: fix mempol... |
3423 3424 3425 |
mpol_put(mpol); mpol = NULL; if (mpol_parse_str(value, &mpol)) |
7339ff830 [PATCH] Add tmpfs... |
3426 |
goto bad_val; |
5a6e75f81 shmem: prepare hu... |
3427 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
3428 |
} else { |
1170532bb mm: convert print... |
3429 3430 |
pr_err("tmpfs: Bad mount option %s ", this_char); |
49cd0a5c2 tmpfs: fix mempol... |
3431 |
goto error; |
1da177e4c Linux-2.6.12-rc2 |
3432 3433 |
} } |
49cd0a5c2 tmpfs: fix mempol... |
3434 |
sbinfo->mpol = mpol; |
1da177e4c Linux-2.6.12-rc2 |
3435 3436 3437 |
return 0; bad_val: |
1170532bb mm: convert print... |
3438 3439 |
pr_err("tmpfs: Bad value '%s' for mount option '%s' ", |
1da177e4c Linux-2.6.12-rc2 |
3440 |
value, this_char); |
49cd0a5c2 tmpfs: fix mempol... |
3441 3442 |
error: mpol_put(mpol); |
1da177e4c Linux-2.6.12-rc2 |
3443 3444 3445 3446 3447 3448 3449 |
return 1; } static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); |
680d794ba mount options: fi... |
3450 |
struct shmem_sb_info config = *sbinfo; |
0edd73b33 [PATCH] shmem: re... |
3451 3452 |
unsigned long inodes; int error = -EINVAL; |
5f00110f7 tmpfs: fix use-af... |
3453 |
config.mpol = NULL; |
680d794ba mount options: fi... |
3454 |
if (shmem_parse_options(data, &config, true)) |
0edd73b33 [PATCH] shmem: re... |
3455 |
return error; |
1da177e4c Linux-2.6.12-rc2 |
3456 |
|
0edd73b33 [PATCH] shmem: re... |
3457 |
spin_lock(&sbinfo->stat_lock); |
0edd73b33 [PATCH] shmem: re... |
3458 |
inodes = sbinfo->max_inodes - sbinfo->free_inodes; |
7e496299d tmpfs: make tmpfs... |
3459 |
if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) |
0edd73b33 [PATCH] shmem: re... |
3460 |
goto out; |
680d794ba mount options: fi... |
3461 |
if (config.max_inodes < inodes) |
0edd73b33 [PATCH] shmem: re... |
3462 3463 |
goto out; /* |
54af60421 tmpfs: convert sh... |
3464 |
* Those tests disallow limited->unlimited while any are in use; |
0edd73b33 [PATCH] shmem: re... |
3465 3466 3467 |
* but we must separately disallow unlimited->limited, because * in that case we have no record of how much is already in use. */ |
680d794ba mount options: fi... |
3468 |
if (config.max_blocks && !sbinfo->max_blocks) |
0edd73b33 [PATCH] shmem: re... |
3469 |
goto out; |
680d794ba mount options: fi... |
3470 |
if (config.max_inodes && !sbinfo->max_inodes) |
0edd73b33 [PATCH] shmem: re... |
3471 3472 3473 |
goto out; error = 0; |
5a6e75f81 shmem: prepare hu... |
3474 |
sbinfo->huge = config.huge; |
680d794ba mount options: fi... |
3475 |
sbinfo->max_blocks = config.max_blocks; |
680d794ba mount options: fi... |
3476 3477 |
sbinfo->max_inodes = config.max_inodes; sbinfo->free_inodes = config.max_inodes - inodes; |
71fe804b6 mempolicy: use st... |
3478 |
|
5f00110f7 tmpfs: fix use-af... |
3479 3480 3481 3482 3483 3484 3485 |
/* * Preserve previous mempolicy unless mpol remount option was specified. */ if (config.mpol) { mpol_put(sbinfo->mpol); sbinfo->mpol = config.mpol; /* transfers initial ref */ } |
0edd73b33 [PATCH] shmem: re... |
3486 3487 3488 |
out: spin_unlock(&sbinfo->stat_lock); return error; |
1da177e4c Linux-2.6.12-rc2 |
3489 |
} |
680d794ba mount options: fi... |
3490 |
|
34c80b1d9 vfs: switch ->sho... |
3491 |
static int shmem_show_options(struct seq_file *seq, struct dentry *root) |
680d794ba mount options: fi... |
3492 |
{ |
34c80b1d9 vfs: switch ->sho... |
3493 |
struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); |
680d794ba mount options: fi... |
3494 3495 3496 |
if (sbinfo->max_blocks != shmem_default_max_blocks()) seq_printf(seq, ",size=%luk", |
09cbfeaf1 mm, fs: get rid o... |
3497 |
sbinfo->max_blocks << (PAGE_SHIFT - 10)); |
680d794ba mount options: fi... |
3498 3499 3500 |
if (sbinfo->max_inodes != shmem_default_max_inodes()) seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); if (sbinfo->mode != (S_IRWXUGO | S_ISVTX)) |
09208d150 shmem, ramfs: pro... |
3501 |
seq_printf(seq, ",mode=%03ho", sbinfo->mode); |
8751e0395 userns: Convert t... |
3502 3503 3504 3505 3506 3507 |
if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) seq_printf(seq, ",uid=%u", from_kuid_munged(&init_user_ns, sbinfo->uid)); if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, sbinfo->gid)); |
e496cf3d7 thp: introduce CO... |
3508 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE |
5a6e75f81 shmem: prepare hu... |
3509 3510 3511 3512 |
/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ if (sbinfo->huge) seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); #endif |
71fe804b6 mempolicy: use st... |
3513 |
shmem_show_mpol(seq, sbinfo->mpol); |
680d794ba mount options: fi... |
3514 3515 |
return 0; } |
9183df25f shm: add memfd_cr... |
3516 3517 3518 3519 |
#define MFD_NAME_PREFIX "memfd:" #define MFD_NAME_PREFIX_LEN (sizeof(MFD_NAME_PREFIX) - 1) #define MFD_NAME_MAX_LEN (NAME_MAX - MFD_NAME_PREFIX_LEN) |
749df87bd mm/shmem: add hug... |
3520 |
#define MFD_ALL_FLAGS (MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB) |
9183df25f shm: add memfd_cr... |
3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 |
SYSCALL_DEFINE2(memfd_create, const char __user *, uname, unsigned int, flags) { struct shmem_inode_info *info; struct file *file; int fd, error; char *name; long len; |
749df87bd mm/shmem: add hug... |
3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 |
if (!(flags & MFD_HUGETLB)) { if (flags & ~(unsigned int)MFD_ALL_FLAGS) return -EINVAL; } else { /* Sealing not supported in hugetlbfs (MFD_HUGETLB) */ if (flags & MFD_ALLOW_SEALING) return -EINVAL; /* Allow huge page size encoding in flags. */ if (flags & ~(unsigned int)(MFD_ALL_FLAGS | (MFD_HUGE_MASK << MFD_HUGE_SHIFT))) return -EINVAL; } |
9183df25f shm: add memfd_cr... |
3543 3544 3545 3546 3547 3548 3549 |
/* length includes terminating zero */ len = strnlen_user(uname, MFD_NAME_MAX_LEN + 1); if (len <= 0) return -EFAULT; if (len > MFD_NAME_MAX_LEN + 1) return -EINVAL; |
0ee931c4e mm: treewide: rem... |
3550 |
name = kmalloc(len + MFD_NAME_PREFIX_LEN, GFP_KERNEL); |
9183df25f shm: add memfd_cr... |
3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 |
if (!name) return -ENOMEM; strcpy(name, MFD_NAME_PREFIX); if (copy_from_user(&name[MFD_NAME_PREFIX_LEN], uname, len)) { error = -EFAULT; goto err_name; } /* terminating-zero may have changed after strnlen_user() returned */ if (name[len + MFD_NAME_PREFIX_LEN - 1]) { error = -EFAULT; goto err_name; } fd = get_unused_fd_flags((flags & MFD_CLOEXEC) ? O_CLOEXEC : 0); if (fd < 0) { error = fd; goto err_name; } |
749df87bd mm/shmem: add hug... |
3571 3572 3573 3574 3575 3576 3577 3578 3579 |
if (flags & MFD_HUGETLB) { struct user_struct *user = NULL; file = hugetlb_file_setup(name, 0, VM_NORESERVE, &user, HUGETLB_ANONHUGE_INODE, (flags >> MFD_HUGE_SHIFT) & MFD_HUGE_MASK); } else file = shmem_file_setup(name, 0, VM_NORESERVE); |
9183df25f shm: add memfd_cr... |
3580 3581 3582 3583 |
if (IS_ERR(file)) { error = PTR_ERR(file); goto err_fd; } |
9183df25f shm: add memfd_cr... |
3584 3585 |
file->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; file->f_flags |= O_RDWR | O_LARGEFILE; |
749df87bd mm/shmem: add hug... |
3586 3587 3588 3589 3590 3591 3592 |
if (flags & MFD_ALLOW_SEALING) { /* * flags check at beginning of function ensures * this is not a hugetlbfs (MFD_HUGETLB) file. */ info = SHMEM_I(file_inode(file)); |
9183df25f shm: add memfd_cr... |
3593 |
info->seals &= ~F_SEAL_SEAL; |
749df87bd mm/shmem: add hug... |
3594 |
} |
9183df25f shm: add memfd_cr... |
3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 |
fd_install(fd, file); kfree(name); return fd; err_fd: put_unused_fd(fd); err_name: kfree(name); return error; } |
680d794ba mount options: fi... |
3606 |
#endif /* CONFIG_TMPFS */ |
1da177e4c Linux-2.6.12-rc2 |
3607 3608 3609 |
static void shmem_put_super(struct super_block *sb) { |
602586a83 shmem: put_super ... |
3610 3611 3612 |
struct shmem_sb_info *sbinfo = SHMEM_SB(sb); percpu_counter_destroy(&sbinfo->used_blocks); |
49cd0a5c2 tmpfs: fix mempol... |
3613 |
mpol_put(sbinfo->mpol); |
602586a83 shmem: put_super ... |
3614 |
kfree(sbinfo); |
1da177e4c Linux-2.6.12-rc2 |
3615 3616 |
sb->s_fs_info = NULL; } |
2b2af54a5 Driver Core: devt... |
3617 |
int shmem_fill_super(struct super_block *sb, void *data, int silent) |
1da177e4c Linux-2.6.12-rc2 |
3618 3619 |
{ struct inode *inode; |
0edd73b33 [PATCH] shmem: re... |
3620 |
struct shmem_sb_info *sbinfo; |
680d794ba mount options: fi... |
3621 3622 3623 |
int err = -ENOMEM; /* Round up to L1_CACHE_BYTES to resist false sharing */ |
425fbf047 shmem: initialize... |
3624 |
sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), |
680d794ba mount options: fi... |
3625 3626 3627 |
L1_CACHE_BYTES), GFP_KERNEL); if (!sbinfo) return -ENOMEM; |
680d794ba mount options: fi... |
3628 |
sbinfo->mode = S_IRWXUGO | S_ISVTX; |
76aac0e9a CRED: Wrap task c... |
3629 3630 |
sbinfo->uid = current_fsuid(); sbinfo->gid = current_fsgid(); |
680d794ba mount options: fi... |
3631 |
sb->s_fs_info = sbinfo; |
1da177e4c Linux-2.6.12-rc2 |
3632 |
|
0edd73b33 [PATCH] shmem: re... |
3633 |
#ifdef CONFIG_TMPFS |
1da177e4c Linux-2.6.12-rc2 |
3634 3635 3636 3637 3638 |
/* * Per default we only allow half of the physical ram per * tmpfs instance, limiting inodes to one per page of lowmem; * but the internal instance is left unlimited. */ |
ca4e05195 shm_mnt is as lon... |
3639 |
if (!(sb->s_flags & MS_KERNMOUNT)) { |
680d794ba mount options: fi... |
3640 3641 3642 3643 3644 3645 |
sbinfo->max_blocks = shmem_default_max_blocks(); sbinfo->max_inodes = shmem_default_max_inodes(); if (shmem_parse_options(data, sbinfo, false)) { err = -EINVAL; goto failed; } |
ca4e05195 shm_mnt is as lon... |
3646 3647 |
} else { sb->s_flags |= MS_NOUSER; |
1da177e4c Linux-2.6.12-rc2 |
3648 |
} |
91828a405 [PATCH] knfsd: ad... |
3649 |
sb->s_export_op = &shmem_export_ops; |
2f6e38f3c tmpfs: enable NOS... |
3650 |
sb->s_flags |= MS_NOSEC; |
1da177e4c Linux-2.6.12-rc2 |
3651 3652 3653 |
#else sb->s_flags |= MS_NOUSER; #endif |
0edd73b33 [PATCH] shmem: re... |
3654 |
spin_lock_init(&sbinfo->stat_lock); |
908c7f194 percpu_counter: a... |
3655 |
if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) |
602586a83 shmem: put_super ... |
3656 |
goto failed; |
680d794ba mount options: fi... |
3657 |
sbinfo->free_inodes = sbinfo->max_inodes; |
779750d20 shmem: split huge... |
3658 3659 |
spin_lock_init(&sbinfo->shrinklist_lock); INIT_LIST_HEAD(&sbinfo->shrinklist); |
0edd73b33 [PATCH] shmem: re... |
3660 |
|
285b2c4fd tmpfs: demolish o... |
3661 |
sb->s_maxbytes = MAX_LFS_FILESIZE; |
09cbfeaf1 mm, fs: get rid o... |
3662 3663 |
sb->s_blocksize = PAGE_SIZE; sb->s_blocksize_bits = PAGE_SHIFT; |
1da177e4c Linux-2.6.12-rc2 |
3664 3665 |
sb->s_magic = TMPFS_MAGIC; sb->s_op = &shmem_ops; |
cfd95a9cf [PATCH] tmpfs: ti... |
3666 |
sb->s_time_gran = 1; |
b09e0fa4b tmpfs: implement ... |
3667 |
#ifdef CONFIG_TMPFS_XATTR |
39f0247d3 [PATCH] Access Co... |
3668 |
sb->s_xattr = shmem_xattr_handlers; |
b09e0fa4b tmpfs: implement ... |
3669 3670 |
#endif #ifdef CONFIG_TMPFS_POSIX_ACL |
39f0247d3 [PATCH] Access Co... |
3671 3672 |
sb->s_flags |= MS_POSIXACL; #endif |
2b4db7961 tmpfs: generate r... |
3673 |
uuid_gen(&sb->s_uuid); |
0edd73b33 [PATCH] shmem: re... |
3674 |
|
454abafe9 ramfs: replace in... |
3675 |
inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); |
1da177e4c Linux-2.6.12-rc2 |
3676 3677 |
if (!inode) goto failed; |
680d794ba mount options: fi... |
3678 3679 |
inode->i_uid = sbinfo->uid; inode->i_gid = sbinfo->gid; |
318ceed08 tidy up after d_m... |
3680 3681 |
sb->s_root = d_make_root(inode); if (!sb->s_root) |
48fde701a switch open-coded... |
3682 |
goto failed; |
1da177e4c Linux-2.6.12-rc2 |
3683 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
3684 3685 3686 3687 |
failed: shmem_put_super(sb); return err; } |
fcc234f88 [PATCH] mm: kill ... |
3688 |
static struct kmem_cache *shmem_inode_cachep; |
1da177e4c Linux-2.6.12-rc2 |
3689 3690 3691 |
static struct inode *shmem_alloc_inode(struct super_block *sb) { |
41ffe5d5c tmpfs: miscellane... |
3692 3693 3694 |
struct shmem_inode_info *info; info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); if (!info) |
1da177e4c Linux-2.6.12-rc2 |
3695 |
return NULL; |
41ffe5d5c tmpfs: miscellane... |
3696 |
return &info->vfs_inode; |
1da177e4c Linux-2.6.12-rc2 |
3697 |
} |
41ffe5d5c tmpfs: miscellane... |
3698 |
static void shmem_destroy_callback(struct rcu_head *head) |
fa0d7e3de fs: icache RCU fr... |
3699 3700 |
{ struct inode *inode = container_of(head, struct inode, i_rcu); |
84e710da2 parallel lookups ... |
3701 3702 |
if (S_ISLNK(inode->i_mode)) kfree(inode->i_link); |
fa0d7e3de fs: icache RCU fr... |
3703 3704 |
kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); } |
1da177e4c Linux-2.6.12-rc2 |
3705 3706 |
static void shmem_destroy_inode(struct inode *inode) { |
09208d150 shmem, ramfs: pro... |
3707 |
if (S_ISREG(inode->i_mode)) |
1da177e4c Linux-2.6.12-rc2 |
3708 |
mpol_free_shared_policy(&SHMEM_I(inode)->policy); |
41ffe5d5c tmpfs: miscellane... |
3709 |
call_rcu(&inode->i_rcu, shmem_destroy_callback); |
1da177e4c Linux-2.6.12-rc2 |
3710 |
} |
41ffe5d5c tmpfs: miscellane... |
3711 |
static void shmem_init_inode(void *foo) |
1da177e4c Linux-2.6.12-rc2 |
3712 |
{ |
41ffe5d5c tmpfs: miscellane... |
3713 3714 |
struct shmem_inode_info *info = foo; inode_init_once(&info->vfs_inode); |
1da177e4c Linux-2.6.12-rc2 |
3715 |
} |
41ffe5d5c tmpfs: miscellane... |
3716 |
static int shmem_init_inodecache(void) |
1da177e4c Linux-2.6.12-rc2 |
3717 3718 3719 |
{ shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", sizeof(struct shmem_inode_info), |
5d097056c kmemcg: account c... |
3720 |
0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); |
1da177e4c Linux-2.6.12-rc2 |
3721 3722 |
return 0; } |
41ffe5d5c tmpfs: miscellane... |
3723 |
static void shmem_destroy_inodecache(void) |
1da177e4c Linux-2.6.12-rc2 |
3724 |
{ |
1a1d92c10 [PATCH] Really ig... |
3725 |
kmem_cache_destroy(shmem_inode_cachep); |
1da177e4c Linux-2.6.12-rc2 |
3726 |
} |
f5e54d6e5 [PATCH] mark addr... |
3727 |
static const struct address_space_operations shmem_aops = { |
1da177e4c Linux-2.6.12-rc2 |
3728 |
.writepage = shmem_writepage, |
767193253 [PATCH] simplify ... |
3729 |
.set_page_dirty = __set_page_dirty_no_writeback, |
1da177e4c Linux-2.6.12-rc2 |
3730 |
#ifdef CONFIG_TMPFS |
800d15a53 implement simple ... |
3731 3732 |
.write_begin = shmem_write_begin, .write_end = shmem_write_end, |
1da177e4c Linux-2.6.12-rc2 |
3733 |
#endif |
1c93923cc include/linux/mig... |
3734 |
#ifdef CONFIG_MIGRATION |
304dbdb7a [PATCH] add migra... |
3735 |
.migratepage = migrate_page, |
1c93923cc include/linux/mig... |
3736 |
#endif |
aa261f549 HWPOISON: Enable ... |
3737 |
.error_remove_page = generic_error_remove_page, |
1da177e4c Linux-2.6.12-rc2 |
3738 |
}; |
15ad7cdcf [PATCH] struct se... |
3739 |
static const struct file_operations shmem_file_operations = { |
1da177e4c Linux-2.6.12-rc2 |
3740 |
.mmap = shmem_mmap, |
c01d5b300 shmem: get_unmapp... |
3741 |
.get_unmapped_area = shmem_get_unmapped_area, |
1da177e4c Linux-2.6.12-rc2 |
3742 |
#ifdef CONFIG_TMPFS |
220f2ac91 tmpfs: support SE... |
3743 |
.llseek = shmem_file_llseek, |
2ba5bbed0 shmem: switch to ... |
3744 |
.read_iter = shmem_file_read_iter, |
8174202b3 write_iter varian... |
3745 |
.write_iter = generic_file_write_iter, |
1b061d924 rename the generi... |
3746 |
.fsync = noop_fsync, |
82c156f85 switch generic_fi... |
3747 |
.splice_read = generic_file_splice_read, |
f6cb85d00 shmem: switch to ... |
3748 |
.splice_write = iter_file_splice_write, |
83e4fa9c1 tmpfs: support fa... |
3749 |
.fallocate = shmem_fallocate, |
1da177e4c Linux-2.6.12-rc2 |
3750 3751 |
#endif }; |
92e1d5be9 [PATCH] mark stru... |
3752 |
static const struct inode_operations shmem_inode_operations = { |
44a30220b shmem: recalculat... |
3753 |
.getattr = shmem_getattr, |
94c1e62df tmpfs: take contr... |
3754 |
.setattr = shmem_setattr, |
b09e0fa4b tmpfs: implement ... |
3755 |
#ifdef CONFIG_TMPFS_XATTR |
b09e0fa4b tmpfs: implement ... |
3756 |
.listxattr = shmem_listxattr, |
feda821e7 fs: remove generi... |
3757 |
.set_acl = simple_set_acl, |
b09e0fa4b tmpfs: implement ... |
3758 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
3759 |
}; |
92e1d5be9 [PATCH] mark stru... |
3760 |
static const struct inode_operations shmem_dir_inode_operations = { |
1da177e4c Linux-2.6.12-rc2 |
3761 3762 3763 3764 3765 3766 3767 3768 3769 |
#ifdef CONFIG_TMPFS .create = shmem_create, .lookup = simple_lookup, .link = shmem_link, .unlink = shmem_unlink, .symlink = shmem_symlink, .mkdir = shmem_mkdir, .rmdir = shmem_rmdir, .mknod = shmem_mknod, |
2773bf00a fs: rename "renam... |
3770 |
.rename = shmem_rename2, |
60545d0d4 [O_TMPFILE] it's ... |
3771 |
.tmpfile = shmem_tmpfile, |
1da177e4c Linux-2.6.12-rc2 |
3772 |
#endif |
b09e0fa4b tmpfs: implement ... |
3773 |
#ifdef CONFIG_TMPFS_XATTR |
b09e0fa4b tmpfs: implement ... |
3774 |
.listxattr = shmem_listxattr, |
b09e0fa4b tmpfs: implement ... |
3775 |
#endif |
39f0247d3 [PATCH] Access Co... |
3776 |
#ifdef CONFIG_TMPFS_POSIX_ACL |
94c1e62df tmpfs: take contr... |
3777 |
.setattr = shmem_setattr, |
feda821e7 fs: remove generi... |
3778 |
.set_acl = simple_set_acl, |
39f0247d3 [PATCH] Access Co... |
3779 3780 |
#endif }; |
92e1d5be9 [PATCH] mark stru... |
3781 |
static const struct inode_operations shmem_special_inode_operations = { |
b09e0fa4b tmpfs: implement ... |
3782 |
#ifdef CONFIG_TMPFS_XATTR |
b09e0fa4b tmpfs: implement ... |
3783 |
.listxattr = shmem_listxattr, |
b09e0fa4b tmpfs: implement ... |
3784 |
#endif |
39f0247d3 [PATCH] Access Co... |
3785 |
#ifdef CONFIG_TMPFS_POSIX_ACL |
94c1e62df tmpfs: take contr... |
3786 |
.setattr = shmem_setattr, |
feda821e7 fs: remove generi... |
3787 |
.set_acl = simple_set_acl, |
39f0247d3 [PATCH] Access Co... |
3788 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
3789 |
}; |
759b9775c [PATCH] shmem and... |
3790 |
static const struct super_operations shmem_ops = { |
1da177e4c Linux-2.6.12-rc2 |
3791 3792 3793 3794 3795 |
.alloc_inode = shmem_alloc_inode, .destroy_inode = shmem_destroy_inode, #ifdef CONFIG_TMPFS .statfs = shmem_statfs, .remount_fs = shmem_remount_fs, |
680d794ba mount options: fi... |
3796 |
.show_options = shmem_show_options, |
1da177e4c Linux-2.6.12-rc2 |
3797 |
#endif |
1f895f75d switch shmem.c to... |
3798 |
.evict_inode = shmem_evict_inode, |
1da177e4c Linux-2.6.12-rc2 |
3799 3800 |
.drop_inode = generic_delete_inode, .put_super = shmem_put_super, |
779750d20 shmem: split huge... |
3801 3802 3803 3804 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE .nr_cached_objects = shmem_unused_huge_count, .free_cached_objects = shmem_unused_huge_scan, #endif |
1da177e4c Linux-2.6.12-rc2 |
3805 |
}; |
f0f37e2f7 const: mark struc... |
3806 |
static const struct vm_operations_struct shmem_vm_ops = { |
54cb8821d mm: merge populat... |
3807 |
.fault = shmem_fault, |
d7c175517 mm: implement ->m... |
3808 |
.map_pages = filemap_map_pages, |
1da177e4c Linux-2.6.12-rc2 |
3809 3810 3811 3812 3813 |
#ifdef CONFIG_NUMA .set_policy = shmem_set_policy, .get_policy = shmem_get_policy, #endif }; |
3c26ff6e4 convert get_sb_no... |
3814 3815 |
static struct dentry *shmem_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data) |
1da177e4c Linux-2.6.12-rc2 |
3816 |
{ |
3c26ff6e4 convert get_sb_no... |
3817 |
return mount_nodev(fs_type, flags, data, shmem_fill_super); |
1da177e4c Linux-2.6.12-rc2 |
3818 |
} |
41ffe5d5c tmpfs: miscellane... |
3819 |
static struct file_system_type shmem_fs_type = { |
1da177e4c Linux-2.6.12-rc2 |
3820 3821 |
.owner = THIS_MODULE, .name = "tmpfs", |
3c26ff6e4 convert get_sb_no... |
3822 |
.mount = shmem_mount, |
1da177e4c Linux-2.6.12-rc2 |
3823 |
.kill_sb = kill_litter_super, |
2b8576cb0 userns: Allow the... |
3824 |
.fs_flags = FS_USERNS_MOUNT, |
1da177e4c Linux-2.6.12-rc2 |
3825 |
}; |
1da177e4c Linux-2.6.12-rc2 |
3826 |
|
41ffe5d5c tmpfs: miscellane... |
3827 |
int __init shmem_init(void) |
1da177e4c Linux-2.6.12-rc2 |
3828 3829 |
{ int error; |
16203a7a9 initmpfs: make ro... |
3830 3831 3832 |
/* If rootfs called this, don't re-init */ if (shmem_inode_cachep) return 0; |
41ffe5d5c tmpfs: miscellane... |
3833 |
error = shmem_init_inodecache(); |
1da177e4c Linux-2.6.12-rc2 |
3834 3835 |
if (error) goto out3; |
41ffe5d5c tmpfs: miscellane... |
3836 |
error = register_filesystem(&shmem_fs_type); |
1da177e4c Linux-2.6.12-rc2 |
3837 |
if (error) { |
1170532bb mm: convert print... |
3838 3839 |
pr_err("Could not register tmpfs "); |
1da177e4c Linux-2.6.12-rc2 |
3840 3841 |
goto out2; } |
95dc112a5 [PATCH] devfs: Re... |
3842 |
|
ca4e05195 shm_mnt is as lon... |
3843 |
shm_mnt = kern_mount(&shmem_fs_type); |
1da177e4c Linux-2.6.12-rc2 |
3844 3845 |
if (IS_ERR(shm_mnt)) { error = PTR_ERR(shm_mnt); |
1170532bb mm: convert print... |
3846 3847 |
pr_err("Could not kern_mount tmpfs "); |
1da177e4c Linux-2.6.12-rc2 |
3848 3849 |
goto out1; } |
5a6e75f81 shmem: prepare hu... |
3850 |
|
e496cf3d7 thp: introduce CO... |
3851 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE |
435c0b87d mm, shmem: fix ha... |
3852 |
if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) |
5a6e75f81 shmem: prepare hu... |
3853 3854 3855 3856 |
SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; else shmem_huge = 0; /* just in case it was patched */ #endif |
1da177e4c Linux-2.6.12-rc2 |
3857 3858 3859 |
return 0; out1: |
41ffe5d5c tmpfs: miscellane... |
3860 |
unregister_filesystem(&shmem_fs_type); |
1da177e4c Linux-2.6.12-rc2 |
3861 |
out2: |
41ffe5d5c tmpfs: miscellane... |
3862 |
shmem_destroy_inodecache(); |
1da177e4c Linux-2.6.12-rc2 |
3863 3864 3865 3866 |
out3: shm_mnt = ERR_PTR(error); return error; } |
853ac43ab shmem: unify regu... |
3867 |
|
e496cf3d7 thp: introduce CO... |
3868 |
#if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) |
5a6e75f81 shmem: prepare hu... |
3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 |
static ssize_t shmem_enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { int values[] = { SHMEM_HUGE_ALWAYS, SHMEM_HUGE_WITHIN_SIZE, SHMEM_HUGE_ADVISE, SHMEM_HUGE_NEVER, SHMEM_HUGE_DENY, SHMEM_HUGE_FORCE, }; int i, count; for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; count += sprintf(buf + count, fmt, shmem_format_huge(values[i])); } buf[count - 1] = ' '; return count; } static ssize_t shmem_enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { char tmp[16]; int huge; if (count + 1 > sizeof(tmp)) return -EINVAL; memcpy(tmp, buf, count); tmp[count] = '\0'; if (count && tmp[count - 1] == ' ') tmp[count - 1] = '\0'; huge = shmem_parse_huge(tmp); if (huge == -EINVAL) return -EINVAL; if (!has_transparent_hugepage() && huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) return -EINVAL; shmem_huge = huge; |
435c0b87d mm, shmem: fix ha... |
3915 |
if (shmem_huge > SHMEM_HUGE_DENY) |
5a6e75f81 shmem: prepare hu... |
3916 3917 3918 3919 3920 3921 |
SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; return count; } struct kobj_attribute shmem_enabled_attr = __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); |
3b33719c9 thp: move shmem_h... |
3922 |
#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ |
f3f0e1d21 khugepaged: add s... |
3923 |
|
3b33719c9 thp: move shmem_h... |
3924 |
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE |
f3f0e1d21 khugepaged: add s... |
3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 |
bool shmem_huge_enabled(struct vm_area_struct *vma) { struct inode *inode = file_inode(vma->vm_file); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); loff_t i_size; pgoff_t off; if (shmem_huge == SHMEM_HUGE_FORCE) return true; if (shmem_huge == SHMEM_HUGE_DENY) return false; switch (sbinfo->huge) { case SHMEM_HUGE_NEVER: return false; case SHMEM_HUGE_ALWAYS: return true; case SHMEM_HUGE_WITHIN_SIZE: off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); i_size = round_up(i_size_read(inode), PAGE_SIZE); if (i_size >= HPAGE_PMD_SIZE && i_size >> PAGE_SHIFT >= off) return true; case SHMEM_HUGE_ADVISE: /* TODO: implement fadvise() hints */ return (vma->vm_flags & VM_HUGEPAGE); default: VM_BUG_ON(1); return false; } } |
3b33719c9 thp: move shmem_h... |
3955 |
#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ |
5a6e75f81 shmem: prepare hu... |
3956 |
|
853ac43ab shmem: unify regu... |
3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 |
#else /* !CONFIG_SHMEM */ /* * tiny-shmem: simple shmemfs and tmpfs using ramfs code * * This is intended for small system where the benefits of the full * shmem code (swap-backed and resource-limited) are outweighed by * their complexity. On systems without swap this code should be * effectively equivalent, but much lighter weight. */ |
41ffe5d5c tmpfs: miscellane... |
3967 |
static struct file_system_type shmem_fs_type = { |
853ac43ab shmem: unify regu... |
3968 |
.name = "tmpfs", |
3c26ff6e4 convert get_sb_no... |
3969 |
.mount = ramfs_mount, |
853ac43ab shmem: unify regu... |
3970 |
.kill_sb = kill_litter_super, |
2b8576cb0 userns: Allow the... |
3971 |
.fs_flags = FS_USERNS_MOUNT, |
853ac43ab shmem: unify regu... |
3972 |
}; |
41ffe5d5c tmpfs: miscellane... |
3973 |
int __init shmem_init(void) |
853ac43ab shmem: unify regu... |
3974 |
{ |
41ffe5d5c tmpfs: miscellane... |
3975 |
BUG_ON(register_filesystem(&shmem_fs_type) != 0); |
853ac43ab shmem: unify regu... |
3976 |
|
41ffe5d5c tmpfs: miscellane... |
3977 |
shm_mnt = kern_mount(&shmem_fs_type); |
853ac43ab shmem: unify regu... |
3978 3979 3980 3981 |
BUG_ON(IS_ERR(shm_mnt)); return 0; } |
41ffe5d5c tmpfs: miscellane... |
3982 |
int shmem_unuse(swp_entry_t swap, struct page *page) |
853ac43ab shmem: unify regu... |
3983 3984 3985 |
{ return 0; } |
3f96b79ad tmpfs: depend on ... |
3986 3987 3988 3989 |
int shmem_lock(struct file *file, int lock, struct user_struct *user) { return 0; } |
245132643 SHM_UNLOCK: fix U... |
3990 3991 3992 |
void shmem_unlock_mapping(struct address_space *mapping) { } |
c01d5b300 shmem: get_unmapp... |
3993 3994 3995 3996 3997 3998 3999 4000 |
#ifdef CONFIG_MMU unsigned long shmem_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); } #endif |
41ffe5d5c tmpfs: miscellane... |
4001 |
void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) |
94c1e62df tmpfs: take contr... |
4002 |
{ |
41ffe5d5c tmpfs: miscellane... |
4003 |
truncate_inode_pages_range(inode->i_mapping, lstart, lend); |
94c1e62df tmpfs: take contr... |
4004 4005 |
} EXPORT_SYMBOL_GPL(shmem_truncate_range); |
0b0a0806b shmem: fix shared... |
4006 4007 |
#define shmem_vm_ops generic_file_vm_ops #define shmem_file_operations ramfs_file_operations |
454abafe9 ramfs: replace in... |
4008 |
#define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) |
0b0a0806b shmem: fix shared... |
4009 4010 |
#define shmem_acct_size(flags, size) 0 #define shmem_unacct_size(flags, size) do {} while (0) |
853ac43ab shmem: unify regu... |
4011 4012 4013 4014 |
#endif /* CONFIG_SHMEM */ /* common code */ |
1da177e4c Linux-2.6.12-rc2 |
4015 |
|
19938e350 mm/shmem.c: const... |
4016 |
static const struct dentry_operations anon_ops = { |
118b23022 cope with potenti... |
4017 |
.d_dname = simple_dname |
3451538a1 shmem_setup_file(... |
4018 |
}; |
c72770909 security: shmem: ... |
4019 4020 |
static struct file *__shmem_file_setup(const char *name, loff_t size, unsigned long flags, unsigned int i_flags) |
1da177e4c Linux-2.6.12-rc2 |
4021 |
{ |
6b4d0b279 clean shmem_file_... |
4022 |
struct file *res; |
1da177e4c Linux-2.6.12-rc2 |
4023 |
struct inode *inode; |
2c48b9c45 switch alloc_file... |
4024 |
struct path path; |
3451538a1 shmem_setup_file(... |
4025 |
struct super_block *sb; |
1da177e4c Linux-2.6.12-rc2 |
4026 4027 4028 |
struct qstr this; if (IS_ERR(shm_mnt)) |
6b4d0b279 clean shmem_file_... |
4029 |
return ERR_CAST(shm_mnt); |
1da177e4c Linux-2.6.12-rc2 |
4030 |
|
285b2c4fd tmpfs: demolish o... |
4031 |
if (size < 0 || size > MAX_LFS_FILESIZE) |
1da177e4c Linux-2.6.12-rc2 |
4032 4033 4034 4035 |
return ERR_PTR(-EINVAL); if (shmem_acct_size(flags, size)) return ERR_PTR(-ENOMEM); |
6b4d0b279 clean shmem_file_... |
4036 |
res = ERR_PTR(-ENOMEM); |
1da177e4c Linux-2.6.12-rc2 |
4037 4038 4039 |
this.name = name; this.len = strlen(name); this.hash = 0; /* will go */ |
3451538a1 shmem_setup_file(... |
4040 |
sb = shm_mnt->mnt_sb; |
66ee4b888 shmem: fix double... |
4041 |
path.mnt = mntget(shm_mnt); |
3451538a1 shmem_setup_file(... |
4042 |
path.dentry = d_alloc_pseudo(sb, &this); |
2c48b9c45 switch alloc_file... |
4043 |
if (!path.dentry) |
1da177e4c Linux-2.6.12-rc2 |
4044 |
goto put_memory; |
3451538a1 shmem_setup_file(... |
4045 |
d_set_d_op(path.dentry, &anon_ops); |
1da177e4c Linux-2.6.12-rc2 |
4046 |
|
6b4d0b279 clean shmem_file_... |
4047 |
res = ERR_PTR(-ENOSPC); |
3451538a1 shmem_setup_file(... |
4048 |
inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); |
1da177e4c Linux-2.6.12-rc2 |
4049 |
if (!inode) |
66ee4b888 shmem: fix double... |
4050 |
goto put_memory; |
1da177e4c Linux-2.6.12-rc2 |
4051 |
|
c72770909 security: shmem: ... |
4052 |
inode->i_flags |= i_flags; |
2c48b9c45 switch alloc_file... |
4053 |
d_instantiate(path.dentry, inode); |
1da177e4c Linux-2.6.12-rc2 |
4054 |
inode->i_size = size; |
6d6b77f16 filesystems: add ... |
4055 |
clear_nlink(inode); /* It is unlinked */ |
26567cdbb fix nommu breakag... |
4056 4057 |
res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); if (IS_ERR(res)) |
66ee4b888 shmem: fix double... |
4058 |
goto put_path; |
4b42af81f switch shmem_file... |
4059 |
|
6b4d0b279 clean shmem_file_... |
4060 |
res = alloc_file(&path, FMODE_WRITE | FMODE_READ, |
4b42af81f switch shmem_file... |
4061 |
&shmem_file_operations); |
6b4d0b279 clean shmem_file_... |
4062 |
if (IS_ERR(res)) |
66ee4b888 shmem: fix double... |
4063 |
goto put_path; |
4b42af81f switch shmem_file... |
4064 |
|
6b4d0b279 clean shmem_file_... |
4065 |
return res; |
1da177e4c Linux-2.6.12-rc2 |
4066 |
|
1da177e4c Linux-2.6.12-rc2 |
4067 4068 |
put_memory: shmem_unacct_size(flags, size); |
66ee4b888 shmem: fix double... |
4069 4070 |
put_path: path_put(&path); |
6b4d0b279 clean shmem_file_... |
4071 |
return res; |
1da177e4c Linux-2.6.12-rc2 |
4072 |
} |
c72770909 security: shmem: ... |
4073 4074 4075 4076 4077 |
/** * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be * kernel internal. There will be NO LSM permission checks against the * underlying inode. So users of this interface must do LSM checks at a |
e1832f292 ipc: use private ... |
4078 4079 |
* higher layer. The users are the big_key and shm implementations. LSM * checks are provided at the key or shm level rather than the inode. |
c72770909 security: shmem: ... |
4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 |
* @name: name for dentry (to be seen in /proc/<pid>/maps * @size: size to be set for the file * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size */ struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) { return __shmem_file_setup(name, size, flags, S_PRIVATE); } /** * shmem_file_setup - get an unlinked file living in tmpfs * @name: name for dentry (to be seen in /proc/<pid>/maps * @size: size to be set for the file * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size */ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) { return __shmem_file_setup(name, size, flags, 0); } |
395e0ddc4 Export shmem_file... |
4099 |
EXPORT_SYMBOL_GPL(shmem_file_setup); |
1da177e4c Linux-2.6.12-rc2 |
4100 |
|
467118102 mm/shmem and tiny... |
4101 |
/** |
1da177e4c Linux-2.6.12-rc2 |
4102 |
* shmem_zero_setup - setup a shared anonymous mapping |
1da177e4c Linux-2.6.12-rc2 |
4103 4104 4105 4106 4107 4108 |
* @vma: the vma to be mmapped is prepared by do_mmap_pgoff */ int shmem_zero_setup(struct vm_area_struct *vma) { struct file *file; loff_t size = vma->vm_end - vma->vm_start; |
66fc13039 mm: shmem_zero_se... |
4109 4110 4111 4112 4113 4114 4115 |
/* * Cloning a new file under mmap_sem leads to a lock ordering conflict * between XFS directory reading and selinux: since this file is only * accessible to the user through its mapping, use S_PRIVATE flag to * bypass file security, in the same way as shmem_kernel_file_setup(). */ file = __shmem_file_setup("dev/zero", size, vma->vm_flags, S_PRIVATE); |
1da177e4c Linux-2.6.12-rc2 |
4116 4117 4118 4119 4120 4121 4122 |
if (IS_ERR(file)) return PTR_ERR(file); if (vma->vm_file) fput(vma->vm_file); vma->vm_file = file; vma->vm_ops = &shmem_vm_ops; |
f3f0e1d21 khugepaged: add s... |
4123 |
|
e496cf3d7 thp: introduce CO... |
4124 |
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && |
f3f0e1d21 khugepaged: add s... |
4125 4126 4127 4128 |
((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < (vma->vm_end & HPAGE_PMD_MASK)) { khugepaged_enter(vma, vma->vm_flags); } |
1da177e4c Linux-2.6.12-rc2 |
4129 4130 |
return 0; } |
d9d90e5eb tmpfs: add shmem_... |
4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 |
/** * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. * @mapping: the page's address_space * @index: the page index * @gfp: the page allocator flags to use if allocating * * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", * with any new page allocations done using the specified allocation flags. * But read_cache_page_gfp() uses the ->readpage() method: which does not * suit tmpfs, since it may have pages in swapcache, and needs to find those * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. * |
68da9f055 tmpfs: pass gfp t... |
4144 4145 |
* i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. |
d9d90e5eb tmpfs: add shmem_... |
4146 4147 4148 4149 |
*/ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp) { |
68da9f055 tmpfs: pass gfp t... |
4150 4151 |
#ifdef CONFIG_SHMEM struct inode *inode = mapping->host; |
9276aad6c tmpfs: remove_shm... |
4152 |
struct page *page; |
68da9f055 tmpfs: pass gfp t... |
4153 4154 4155 |
int error; BUG_ON(mapping->a_ops != &shmem_aops); |
9e18eb293 tmpfs: mem_cgroup... |
4156 |
error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, |
cfda05267 userfaultfd: shme... |
4157 |
gfp, NULL, NULL, NULL); |
68da9f055 tmpfs: pass gfp t... |
4158 4159 4160 4161 4162 4163 4164 4165 4166 |
if (error) page = ERR_PTR(error); else unlock_page(page); return page; #else /* * The tiny !SHMEM case uses ramfs without swap */ |
d9d90e5eb tmpfs: add shmem_... |
4167 |
return read_cache_page_gfp(mapping, index, gfp); |
68da9f055 tmpfs: pass gfp t... |
4168 |
#endif |
d9d90e5eb tmpfs: add shmem_... |
4169 4170 |
} EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); |