Blame view
mm/swap.c
33.8 KB
457c89965 treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
1da177e4c Linux-2.6.12-rc2 |
2 3 4 5 6 7 8 |
/* * linux/mm/swap.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* |
183ff22bb spelling fixes: mm/ |
9 |
* This file contains the default values for the operation of the |
1da177e4c Linux-2.6.12-rc2 |
10 |
* Linux VM subsystem. Fine-tuning documentation can be found in |
570432470 docs: admin-guide... |
11 |
* Documentation/admin-guide/sysctl/vm.rst. |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 16 17 18 19 20 21 22 23 24 |
* Started 18.12.91 * Swap aging added 23.2.95, Stephen Tweedie. * Buffermem limits added 12.3.98, Rik van Riel. */ #include <linux/mm.h> #include <linux/sched.h> #include <linux/kernel_stat.h> #include <linux/swap.h> #include <linux/mman.h> #include <linux/pagemap.h> #include <linux/pagevec.h> #include <linux/init.h> |
b95f1b31b mm: Map most file... |
25 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
26 |
#include <linux/mm_inline.h> |
1da177e4c Linux-2.6.12-rc2 |
27 |
#include <linux/percpu_counter.h> |
3565fce3a mm, x86: get_user... |
28 |
#include <linux/memremap.h> |
1da177e4c Linux-2.6.12-rc2 |
29 30 31 |
#include <linux/percpu.h> #include <linux/cpu.h> #include <linux/notifier.h> |
e0bf68dde mm: bdi init hooks |
32 |
#include <linux/backing-dev.h> |
66e1707bc Memory controller... |
33 |
#include <linux/memcontrol.h> |
5a0e3ad6a include cleanup: ... |
34 |
#include <linux/gfp.h> |
a27bb332c aio: don't includ... |
35 |
#include <linux/uio.h> |
822fc6136 mm: don't call __... |
36 |
#include <linux/hugetlb.h> |
33c3fc71c mm: introduce idl... |
37 |
#include <linux/page_idle.h> |
b01b21419 mm/swap: Use loca... |
38 |
#include <linux/local_lock.h> |
1da177e4c Linux-2.6.12-rc2 |
39 |
|
64d6519dd swap: cull unevic... |
40 |
#include "internal.h" |
c6286c983 mm: add tracepoin... |
41 42 |
#define CREATE_TRACE_POINTS #include <trace/events/pagemap.h> |
1da177e4c Linux-2.6.12-rc2 |
43 44 |
/* How many pages do we try to swap or page in/out together? */ int page_cluster; |
b01b21419 mm/swap: Use loca... |
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
/* Protecting only lru_rotate.pvec which requires disabling interrupts */ struct lru_rotate { local_lock_t lock; struct pagevec pvec; }; static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { .lock = INIT_LOCAL_LOCK(lock), }; /* * The following struct pagevec are grouped together because they are protected * by disabling preemption (and interrupts remain enabled). */ struct lru_pvecs { local_lock_t lock; struct pagevec lru_add; struct pagevec lru_deactivate_file; struct pagevec lru_deactivate; struct pagevec lru_lazyfree; |
a4a921aa5 mm/swap.c: put ac... |
64 |
#ifdef CONFIG_SMP |
b01b21419 mm/swap: Use loca... |
65 |
struct pagevec activate_page; |
a4a921aa5 mm/swap.c: put ac... |
66 |
#endif |
b01b21419 mm/swap: Use loca... |
67 68 69 70 |
}; static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { .lock = INIT_LOCAL_LOCK(lock), }; |
902aaed0d mm: use pagevec t... |
71 |
|
b221385bc [PATCH] mm/: make... |
72 73 74 75 |
/* * This path almost never happens for VM activity - pages are normally * freed via pagevecs. But it gets used by networking. */ |
920c7a5d0 mm: remove fastca... |
76 |
static void __page_cache_release(struct page *page) |
b221385bc [PATCH] mm/: make... |
77 78 |
{ if (PageLRU(page)) { |
f4b7e272b mm: remove zone_l... |
79 |
pg_data_t *pgdat = page_pgdat(page); |
fa9add641 mm/memcg: apply a... |
80 81 |
struct lruvec *lruvec; unsigned long flags; |
b221385bc [PATCH] mm/: make... |
82 |
|
f4b7e272b mm: remove zone_l... |
83 84 |
spin_lock_irqsave(&pgdat->lru_lock, flags); lruvec = mem_cgroup_page_lruvec(page, pgdat); |
309381fea mm: dump page whe... |
85 |
VM_BUG_ON_PAGE(!PageLRU(page), page); |
b221385bc [PATCH] mm/: make... |
86 |
__ClearPageLRU(page); |
fa9add641 mm/memcg: apply a... |
87 |
del_page_from_lru_list(page, lruvec, page_off_lru(page)); |
f4b7e272b mm: remove zone_l... |
88 |
spin_unlock_irqrestore(&pgdat->lru_lock, flags); |
b221385bc [PATCH] mm/: make... |
89 |
} |
629060270 mm: add PageWaite... |
90 |
__ClearPageWaiters(page); |
918070634 thp: alter compou... |
91 92 93 94 95 |
} static void __put_single_page(struct page *page) { __page_cache_release(page); |
7ae88534c mm: move mem_cgro... |
96 |
mem_cgroup_uncharge(page); |
2d4894b5d mm: remove cold p... |
97 |
free_unref_page(page); |
b221385bc [PATCH] mm/: make... |
98 |
} |
918070634 thp: alter compou... |
99 |
static void __put_compound_page(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
100 |
{ |
822fc6136 mm: don't call __... |
101 102 103 104 105 106 107 108 |
/* * __page_cache_release() is supposed to be called for thp, not for * hugetlb. This is because hugetlb page does never have PageLRU set * (it's never listed to any LRU lists) and no memcg routines should * be called for hugetlb (it has a separate hugetlb_cgroup.) */ if (!PageHuge(page)) __page_cache_release(page); |
ff45fc3ca mm: simplify call... |
109 |
destroy_compound_page(page); |
918070634 thp: alter compou... |
110 |
} |
ddc58f27f mm: drop tail pag... |
111 |
void __put_page(struct page *page) |
8519fb30e [PATCH] mm: compo... |
112 |
{ |
713897038 mm, zone_device: ... |
113 114 115 116 117 118 119 120 121 |
if (is_zone_device_page(page)) { put_dev_pagemap(page->pgmap); /* * The page belongs to the device that created pgmap. Do * not return it to page allocator. */ return; } |
8519fb30e [PATCH] mm: compo... |
122 |
if (unlikely(PageCompound(page))) |
ddc58f27f mm: drop tail pag... |
123 124 |
__put_compound_page(page); else |
918070634 thp: alter compou... |
125 |
__put_single_page(page); |
1da177e4c Linux-2.6.12-rc2 |
126 |
} |
ddc58f27f mm: drop tail pag... |
127 |
EXPORT_SYMBOL(__put_page); |
70b50f94f mm: thp: tail pag... |
128 |
|
1d7ea7324 [PATCH] fuse: fix... |
129 |
/** |
7682486b3 mm: fix various k... |
130 131 |
* put_pages_list() - release a list of pages * @pages: list of pages threaded on page->lru |
1d7ea7324 [PATCH] fuse: fix... |
132 133 134 |
* * Release a list of pages which are strung together on page.lru. Currently * used by read_cache_pages() and related error recovery code. |
1d7ea7324 [PATCH] fuse: fix... |
135 136 137 138 139 |
*/ void put_pages_list(struct list_head *pages) { while (!list_empty(pages)) { struct page *victim; |
f86196ea8 fs: don't open co... |
140 |
victim = lru_to_page(pages); |
1d7ea7324 [PATCH] fuse: fix... |
141 |
list_del(&victim->lru); |
09cbfeaf1 mm, fs: get rid o... |
142 |
put_page(victim); |
1d7ea7324 [PATCH] fuse: fix... |
143 144 145 |
} } EXPORT_SYMBOL(put_pages_list); |
18022c5d8 mm: add get_kerne... |
146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 |
/* * get_kernel_pages() - pin kernel pages in memory * @kiov: An array of struct kvec structures * @nr_segs: number of segments to pin * @write: pinning for read/write, currently ignored * @pages: array that receives pointers to the pages pinned. * Should be at least nr_segs long. * * Returns number of pages pinned. This may be fewer than the number * requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. Each page returned must be released * with a put_page() call when it is finished with. */ int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, struct page **pages) { int seg; for (seg = 0; seg < nr_segs; seg++) { if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) return seg; |
5a178119b mm: add support f... |
167 |
pages[seg] = kmap_to_page(kiov[seg].iov_base); |
09cbfeaf1 mm, fs: get rid o... |
168 |
get_page(pages[seg]); |
18022c5d8 mm: add get_kerne... |
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
} return seg; } EXPORT_SYMBOL_GPL(get_kernel_pages); /* * get_kernel_page() - pin a kernel page in memory * @start: starting kernel address * @write: pinning for read/write, currently ignored * @pages: array that receives pointer to the page pinned. * Must be at least nr_segs long. * * Returns 1 if page is pinned. If the page was not pinned, returns * -errno. The page returned must be released with a put_page() call * when it is finished with. */ int get_kernel_page(unsigned long start, int write, struct page **pages) { const struct kvec kiov = { .iov_base = (void *)start, .iov_len = PAGE_SIZE }; return get_kernel_pages(&kiov, 1, write, pages); } EXPORT_SYMBOL_GPL(get_kernel_page); |
3dd7ae8ec mm: simplify code... |
196 |
static void pagevec_lru_move_fn(struct pagevec *pvec, |
fa9add641 mm/memcg: apply a... |
197 198 |
void (*move_fn)(struct page *page, struct lruvec *lruvec, void *arg), void *arg) |
902aaed0d mm: use pagevec t... |
199 200 |
{ int i; |
68eb0731c mm, pagevec: rele... |
201 |
struct pglist_data *pgdat = NULL; |
fa9add641 mm/memcg: apply a... |
202 |
struct lruvec *lruvec; |
3dd7ae8ec mm: simplify code... |
203 |
unsigned long flags = 0; |
902aaed0d mm: use pagevec t... |
204 205 206 |
for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; |
68eb0731c mm, pagevec: rele... |
207 |
struct pglist_data *pagepgdat = page_pgdat(page); |
902aaed0d mm: use pagevec t... |
208 |
|
68eb0731c mm, pagevec: rele... |
209 210 211 212 213 |
if (pagepgdat != pgdat) { if (pgdat) spin_unlock_irqrestore(&pgdat->lru_lock, flags); pgdat = pagepgdat; spin_lock_irqsave(&pgdat->lru_lock, flags); |
902aaed0d mm: use pagevec t... |
214 |
} |
3dd7ae8ec mm: simplify code... |
215 |
|
68eb0731c mm, pagevec: rele... |
216 |
lruvec = mem_cgroup_page_lruvec(page, pgdat); |
fa9add641 mm/memcg: apply a... |
217 |
(*move_fn)(page, lruvec, arg); |
902aaed0d mm: use pagevec t... |
218 |
} |
68eb0731c mm, pagevec: rele... |
219 220 |
if (pgdat) spin_unlock_irqrestore(&pgdat->lru_lock, flags); |
c6f92f9fb mm: remove cold p... |
221 |
release_pages(pvec->pages, pvec->nr); |
83896fb5e Revert "mm: simpl... |
222 |
pagevec_reinit(pvec); |
d8505dee1 mm: simplify code... |
223 |
} |
fa9add641 mm/memcg: apply a... |
224 225 |
static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec, void *arg) |
3dd7ae8ec mm: simplify code... |
226 227 |
{ int *pgmoved = arg; |
3dd7ae8ec mm: simplify code... |
228 |
|
c55e8d035 mm: vmscan: move ... |
229 230 231 232 |
if (PageLRU(page) && !PageUnevictable(page)) { del_page_from_lru_list(page, lruvec, page_lru(page)); ClearPageActive(page); add_page_to_lru_list_tail(page, lruvec, page_lru(page)); |
6c357848b mm: replace hpage... |
233 |
(*pgmoved) += thp_nr_pages(page); |
3dd7ae8ec mm: simplify code... |
234 235 236 237 238 239 240 241 242 243 244 245 246 247 |
} } /* * pagevec_move_tail() must be called with IRQ disabled. * Otherwise this may cause nasty races. */ static void pagevec_move_tail(struct pagevec *pvec) { int pgmoved = 0; pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved); __count_vm_events(PGROTATED, pgmoved); } |
902aaed0d mm: use pagevec t... |
248 |
/* |
1da177e4c Linux-2.6.12-rc2 |
249 250 |
* Writeback is about to end against a page which has been marked for immediate * reclaim. If it still appears to be reclaimable, move it to the tail of the |
902aaed0d mm: use pagevec t... |
251 |
* inactive list. |
1da177e4c Linux-2.6.12-rc2 |
252 |
*/ |
3dd7ae8ec mm: simplify code... |
253 |
void rotate_reclaimable_page(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
254 |
{ |
c55e8d035 mm: vmscan: move ... |
255 |
if (!PageLocked(page) && !PageDirty(page) && |
894bc3104 Unevictable LRU I... |
256 |
!PageUnevictable(page) && PageLRU(page)) { |
ac6aadb24 mm: rotate_reclai... |
257 258 |
struct pagevec *pvec; unsigned long flags; |
09cbfeaf1 mm, fs: get rid o... |
259 |
get_page(page); |
b01b21419 mm/swap: Use loca... |
260 261 |
local_lock_irqsave(&lru_rotate.lock, flags); pvec = this_cpu_ptr(&lru_rotate.pvec); |
8f182270d mm/swap.c: flush ... |
262 |
if (!pagevec_add(pvec, page) || PageCompound(page)) |
ac6aadb24 mm: rotate_reclai... |
263 |
pagevec_move_tail(pvec); |
b01b21419 mm/swap: Use loca... |
264 |
local_unlock_irqrestore(&lru_rotate.lock, flags); |
ac6aadb24 mm: rotate_reclai... |
265 |
} |
1da177e4c Linux-2.6.12-rc2 |
266 |
} |
96f8bf4fb mm: vmscan: recla... |
267 |
void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages) |
3e2f41f1f memcg: add zone_r... |
268 |
{ |
7cf111bc3 mm: vmscan: deter... |
269 270 271 272 |
do { unsigned long lrusize; /* Record cost event */ |
96f8bf4fb mm: vmscan: recla... |
273 274 |
if (file) lruvec->file_cost += nr_pages; |
7cf111bc3 mm: vmscan: deter... |
275 |
else |
96f8bf4fb mm: vmscan: recla... |
276 |
lruvec->anon_cost += nr_pages; |
7cf111bc3 mm: vmscan: deter... |
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 |
/* * Decay previous events * * Because workloads change over time (and to avoid * overflow) we keep these statistics as a floating * average, which ends up weighing recent refaults * more than old ones. */ lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + lruvec_page_state(lruvec, NR_ACTIVE_ANON) + lruvec_page_state(lruvec, NR_INACTIVE_FILE) + lruvec_page_state(lruvec, NR_ACTIVE_FILE); if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { lruvec->file_cost /= 2; lruvec->anon_cost /= 2; } } while ((lruvec = parent_lruvec(lruvec))); |
3e2f41f1f memcg: add zone_r... |
296 |
} |
96f8bf4fb mm: vmscan: recla... |
297 298 299 |
void lru_note_cost_page(struct page *page) { lru_note_cost(mem_cgroup_page_lruvec(page, page_pgdat(page)), |
6c357848b mm: replace hpage... |
300 |
page_is_file_lru(page), thp_nr_pages(page)); |
96f8bf4fb mm: vmscan: recla... |
301 |
} |
fa9add641 mm/memcg: apply a... |
302 303 |
static void __activate_page(struct page *page, struct lruvec *lruvec, void *arg) |
1da177e4c Linux-2.6.12-rc2 |
304 |
{ |
744ed1442 mm: batch activat... |
305 |
if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { |
7a608572a Revert "mm: batch... |
306 |
int lru = page_lru_base_type(page); |
6c357848b mm: replace hpage... |
307 |
int nr_pages = thp_nr_pages(page); |
744ed1442 mm: batch activat... |
308 |
|
fa9add641 mm/memcg: apply a... |
309 |
del_page_from_lru_list(page, lruvec, lru); |
7a608572a Revert "mm: batch... |
310 311 |
SetPageActive(page); lru += LRU_ACTIVE; |
fa9add641 mm/memcg: apply a... |
312 |
add_page_to_lru_list(page, lruvec, lru); |
24b7e5819 mm: pagemap: avoi... |
313 |
trace_mm_lru_activate(page); |
4f98a2fee vmscan: split LRU... |
314 |
|
21e330fc6 mm: swap: memcg: ... |
315 316 317 |
__count_vm_events(PGACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages); |
1da177e4c Linux-2.6.12-rc2 |
318 |
} |
eb709b0d0 mm: batch activat... |
319 320 321 |
} #ifdef CONFIG_SMP |
eb709b0d0 mm: batch activat... |
322 323 |
static void activate_page_drain(int cpu) { |
b01b21419 mm/swap: Use loca... |
324 |
struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu); |
eb709b0d0 mm: batch activat... |
325 326 327 328 |
if (pagevec_count(pvec)) pagevec_lru_move_fn(pvec, __activate_page, NULL); } |
5fbc46163 mm: make lru_add_... |
329 330 |
static bool need_activate_page_drain(int cpu) { |
b01b21419 mm/swap: Use loca... |
331 |
return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0; |
5fbc46163 mm: make lru_add_... |
332 |
} |
cc2828b21 mm: remove activa... |
333 |
static void activate_page(struct page *page) |
eb709b0d0 mm: batch activat... |
334 |
{ |
800d8c63b shmem: add huge p... |
335 |
page = compound_head(page); |
eb709b0d0 mm: batch activat... |
336 |
if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
337 |
struct pagevec *pvec; |
eb709b0d0 mm: batch activat... |
338 |
|
b01b21419 mm/swap: Use loca... |
339 340 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.activate_page); |
09cbfeaf1 mm, fs: get rid o... |
341 |
get_page(page); |
8f182270d mm/swap.c: flush ... |
342 |
if (!pagevec_add(pvec, page) || PageCompound(page)) |
eb709b0d0 mm: batch activat... |
343 |
pagevec_lru_move_fn(pvec, __activate_page, NULL); |
b01b21419 mm/swap: Use loca... |
344 |
local_unlock(&lru_pvecs.lock); |
eb709b0d0 mm: batch activat... |
345 346 347 348 349 350 351 |
} } #else static inline void activate_page_drain(int cpu) { } |
cc2828b21 mm: remove activa... |
352 |
static void activate_page(struct page *page) |
eb709b0d0 mm: batch activat... |
353 |
{ |
f4b7e272b mm: remove zone_l... |
354 |
pg_data_t *pgdat = page_pgdat(page); |
eb709b0d0 mm: batch activat... |
355 |
|
800d8c63b shmem: add huge p... |
356 |
page = compound_head(page); |
f4b7e272b mm: remove zone_l... |
357 358 359 |
spin_lock_irq(&pgdat->lru_lock); __activate_page(page, mem_cgroup_page_lruvec(page, pgdat), NULL); spin_unlock_irq(&pgdat->lru_lock); |
1da177e4c Linux-2.6.12-rc2 |
360 |
} |
eb709b0d0 mm: batch activat... |
361 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
362 |
|
059285a25 mm: activate !Pag... |
363 364 |
static void __lru_cache_activate_page(struct page *page) { |
b01b21419 mm/swap: Use loca... |
365 |
struct pagevec *pvec; |
059285a25 mm: activate !Pag... |
366 |
int i; |
b01b21419 mm/swap: Use loca... |
367 368 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); |
059285a25 mm: activate !Pag... |
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 |
/* * Search backwards on the optimistic assumption that the page being * activated has just been added to this pagevec. Note that only * the local pagevec is examined as a !PageLRU page could be in the * process of being released, reclaimed, migrated or on a remote * pagevec that is currently being drained. Furthermore, marking * a remote pagevec's page PageActive potentially hits a race where * a page is marked PageActive just after it is added to the inactive * list causing accounting errors and BUG_ON checks to trigger. */ for (i = pagevec_count(pvec) - 1; i >= 0; i--) { struct page *pagevec_page = pvec->pages[i]; if (pagevec_page == page) { SetPageActive(page); break; } } |
b01b21419 mm/swap: Use loca... |
387 |
local_unlock(&lru_pvecs.lock); |
059285a25 mm: activate !Pag... |
388 |
} |
1da177e4c Linux-2.6.12-rc2 |
389 390 391 392 393 394 |
/* * Mark a page as having seen activity. * * inactive,unreferenced -> inactive,referenced * inactive,referenced -> active,unreferenced * active,unreferenced -> active,referenced |
eb39d618f mm: replace init_... |
395 396 397 |
* * When a newly allocated page is not yet visible, so safe for non-atomic ops, * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). |
1da177e4c Linux-2.6.12-rc2 |
398 |
*/ |
920c7a5d0 mm: remove fastca... |
399 |
void mark_page_accessed(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
400 |
{ |
e90309c9f thp: allow mlocke... |
401 |
page = compound_head(page); |
059285a25 mm: activate !Pag... |
402 |
|
a1100a740 mm/swap.c: trivia... |
403 404 405 406 407 408 409 410 411 |
if (!PageReferenced(page)) { SetPageReferenced(page); } else if (PageUnevictable(page)) { /* * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, * this list is never rotated or maintained, so marking an * evictable page accessed has no effect. */ } else if (!PageActive(page)) { |
059285a25 mm: activate !Pag... |
412 413 |
/* * If the page is on the LRU, queue it for activation via |
b01b21419 mm/swap: Use loca... |
414 |
* lru_pvecs.activate_page. Otherwise, assume the page is on a |
059285a25 mm: activate !Pag... |
415 416 417 418 419 420 421 |
* pagevec, mark it active and it'll be moved to the active * LRU on the next drain. */ if (PageLRU(page)) activate_page(page); else __lru_cache_activate_page(page); |
1da177e4c Linux-2.6.12-rc2 |
422 |
ClearPageReferenced(page); |
cb6868832 mm/swap: fix for ... |
423 |
workingset_activation(page); |
1da177e4c Linux-2.6.12-rc2 |
424 |
} |
33c3fc71c mm: introduce idl... |
425 426 |
if (page_is_idle(page)) clear_page_idle(page); |
1da177e4c Linux-2.6.12-rc2 |
427 |
} |
1da177e4c Linux-2.6.12-rc2 |
428 |
EXPORT_SYMBOL(mark_page_accessed); |
f04e9ebbe swap: use an arra... |
429 |
/** |
c53954a09 mm: remove lru pa... |
430 |
* lru_cache_add - add a page to a page list |
f04e9ebbe swap: use an arra... |
431 |
* @page: the page to be added to the LRU. |
2329d3751 mm/swap.c: clean ... |
432 433 434 435 436 |
* * Queue the page for addition to the LRU via pagevec. The decision on whether * to add the page to the [in]active [file|anon] list is deferred until the * pagevec is drained. This gives a chance for the caller of lru_cache_add() * have the page added to the active list using mark_page_accessed(). |
f04e9ebbe swap: use an arra... |
437 |
*/ |
c53954a09 mm: remove lru pa... |
438 |
void lru_cache_add(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
439 |
{ |
6058eaec8 mm: fold and remo... |
440 |
struct pagevec *pvec; |
309381fea mm: dump page whe... |
441 442 |
VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); VM_BUG_ON_PAGE(PageLRU(page), page); |
6058eaec8 mm: fold and remo... |
443 444 445 446 447 448 449 |
get_page(page); local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); if (!pagevec_add(pvec, page) || PageCompound(page)) __pagevec_lru_add(pvec); local_unlock(&lru_pvecs.lock); |
1da177e4c Linux-2.6.12-rc2 |
450 |
} |
6058eaec8 mm: fold and remo... |
451 |
EXPORT_SYMBOL(lru_cache_add); |
1da177e4c Linux-2.6.12-rc2 |
452 |
|
894bc3104 Unevictable LRU I... |
453 |
/** |
b518154e5 mm/vmscan: protec... |
454 |
* lru_cache_add_inactive_or_unevictable |
00501b531 mm: memcontrol: r... |
455 456 457 |
* @page: the page to be added to LRU * @vma: vma in which page is mapped for determining reclaimability * |
b518154e5 mm/vmscan: protec... |
458 |
* Place @page on the inactive or unevictable LRU list, depending on its |
12eab4289 mm/swap.c: fix in... |
459 |
* evictability. |
00501b531 mm: memcontrol: r... |
460 |
*/ |
b518154e5 mm/vmscan: protec... |
461 |
void lru_cache_add_inactive_or_unevictable(struct page *page, |
00501b531 mm: memcontrol: r... |
462 463 |
struct vm_area_struct *vma) { |
b518154e5 mm/vmscan: protec... |
464 |
bool unevictable; |
00501b531 mm: memcontrol: r... |
465 |
VM_BUG_ON_PAGE(PageLRU(page), page); |
b518154e5 mm/vmscan: protec... |
466 467 |
unevictable = (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED; if (unlikely(unevictable) && !TestSetPageMlocked(page)) { |
0964730bf mlock: fix unevic... |
468 |
int nr_pages = thp_nr_pages(page); |
00501b531 mm: memcontrol: r... |
469 470 471 472 473 |
/* * We use the irq-unsafe __mod_zone_page_stat because this * counter is not modified from interrupt context, and the pte * lock is held(spinlock), which implies preemption disabled. */ |
0964730bf mlock: fix unevic... |
474 475 |
__mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); |
00501b531 mm: memcontrol: r... |
476 |
} |
9c4e6b1a7 mm, mlock, vmscan... |
477 |
lru_cache_add(page); |
00501b531 mm: memcontrol: r... |
478 |
} |
902aaed0d mm: use pagevec t... |
479 |
/* |
315601809 mm: deactivate in... |
480 481 482 483 484 |
* If the page can not be invalidated, it is moved to the * inactive list to speed up its reclaim. It is moved to the * head of the list, rather than the tail, to give the flusher * threads some time to write it out, as this is much more * effective than the single-page writeout from reclaim. |
278df9f45 mm: reclaim inval... |
485 486 487 488 489 490 491 492 493 494 495 496 497 498 |
* * If the page isn't page_mapped and dirty/writeback, the page * could reclaim asap using PG_reclaim. * * 1. active, mapped page -> none * 2. active, dirty/writeback page -> inactive, head, PG_reclaim * 3. inactive, mapped page -> none * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim * 5. inactive, clean -> inactive, tail * 6. Others -> none * * In 4, why it moves inactive's head, the VM expects the page would * be write it out by flusher threads as this is much more effective * than the single-page writeout from reclaim. |
315601809 mm: deactivate in... |
499 |
*/ |
cc5993bd7 mm: rename deacti... |
500 |
static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec, |
fa9add641 mm/memcg: apply a... |
501 |
void *arg) |
315601809 mm: deactivate in... |
502 |
{ |
fbbb602e4 mm: deactivations... |
503 |
int lru; |
278df9f45 mm: reclaim inval... |
504 |
bool active; |
6c357848b mm: replace hpage... |
505 |
int nr_pages = thp_nr_pages(page); |
315601809 mm: deactivate in... |
506 |
|
278df9f45 mm: reclaim inval... |
507 |
if (!PageLRU(page)) |
315601809 mm: deactivate in... |
508 |
return; |
bad49d9c8 mm: check PageUne... |
509 510 |
if (PageUnevictable(page)) return; |
315601809 mm: deactivate in... |
511 512 513 |
/* Some processes are using the page */ if (page_mapped(page)) return; |
278df9f45 mm: reclaim inval... |
514 |
active = PageActive(page); |
315601809 mm: deactivate in... |
515 |
lru = page_lru_base_type(page); |
fa9add641 mm/memcg: apply a... |
516 517 |
del_page_from_lru_list(page, lruvec, lru + active); |
315601809 mm: deactivate in... |
518 519 |
ClearPageActive(page); ClearPageReferenced(page); |
315601809 mm: deactivate in... |
520 |
|
278df9f45 mm: reclaim inval... |
521 522 523 524 525 526 |
if (PageWriteback(page) || PageDirty(page)) { /* * PG_reclaim could be raced with end_page_writeback * It can make readahead confusing. But race window * is _really_ small and it's non-critical problem. */ |
e7a1aaf28 mm: replace list_... |
527 |
add_page_to_lru_list(page, lruvec, lru); |
278df9f45 mm: reclaim inval... |
528 529 530 531 532 533 |
SetPageReclaim(page); } else { /* * The page's writeback ends up during pagevec * We moves tha page into tail of inactive. */ |
e7a1aaf28 mm: replace list_... |
534 |
add_page_to_lru_list_tail(page, lruvec, lru); |
5d91f31fa mm: swap: fix vms... |
535 |
__count_vm_events(PGROTATED, nr_pages); |
278df9f45 mm: reclaim inval... |
536 |
} |
21e330fc6 mm: swap: memcg: ... |
537 |
if (active) { |
5d91f31fa mm: swap: fix vms... |
538 |
__count_vm_events(PGDEACTIVATE, nr_pages); |
21e330fc6 mm: swap: memcg: ... |
539 540 541 |
__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); } |
315601809 mm: deactivate in... |
542 |
} |
9c276cc65 mm: introduce MAD... |
543 544 545 546 |
static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec, void *arg) { if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { |
9c276cc65 mm: introduce MAD... |
547 |
int lru = page_lru_base_type(page); |
6c357848b mm: replace hpage... |
548 |
int nr_pages = thp_nr_pages(page); |
9c276cc65 mm: introduce MAD... |
549 550 551 552 553 |
del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE); ClearPageActive(page); ClearPageReferenced(page); add_page_to_lru_list(page, lruvec, lru); |
21e330fc6 mm: swap: memcg: ... |
554 555 556 |
__count_vm_events(PGDEACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); |
9c276cc65 mm: introduce MAD... |
557 558 |
} } |
10853a039 mm: move lazily f... |
559 |
|
f7ad2a6cb mm: move MADV_FRE... |
560 |
static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec, |
10853a039 mm: move lazily f... |
561 562 |
void *arg) { |
f7ad2a6cb mm: move MADV_FRE... |
563 |
if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && |
24c92eb7d mm: avoid marking... |
564 |
!PageSwapCache(page) && !PageUnevictable(page)) { |
f7ad2a6cb mm: move MADV_FRE... |
565 |
bool active = PageActive(page); |
6c357848b mm: replace hpage... |
566 |
int nr_pages = thp_nr_pages(page); |
10853a039 mm: move lazily f... |
567 |
|
f7ad2a6cb mm: move MADV_FRE... |
568 569 |
del_page_from_lru_list(page, lruvec, LRU_INACTIVE_ANON + active); |
10853a039 mm: move lazily f... |
570 571 |
ClearPageActive(page); ClearPageReferenced(page); |
f7ad2a6cb mm: move MADV_FRE... |
572 |
/* |
9de4f22a6 mm: code cleanup ... |
573 574 575 |
* Lazyfree pages are clean anonymous pages. They have * PG_swapbacked flag cleared, to distinguish them from normal * anonymous pages |
f7ad2a6cb mm: move MADV_FRE... |
576 577 578 |
*/ ClearPageSwapBacked(page); add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE); |
10853a039 mm: move lazily f... |
579 |
|
21e330fc6 mm: swap: memcg: ... |
580 581 582 |
__count_vm_events(PGLAZYFREE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages); |
10853a039 mm: move lazily f... |
583 584 |
} } |
315601809 mm: deactivate in... |
585 |
/* |
902aaed0d mm: use pagevec t... |
586 587 588 589 |
* Drain pages out of the cpu's pagevecs. * Either "cpu" is the current CPU, and preemption has already been * disabled; or "cpu" is being hot-unplugged, and is already dead. */ |
f0cb3c76a mm: drain percpu ... |
590 |
void lru_add_drain_cpu(int cpu) |
1da177e4c Linux-2.6.12-rc2 |
591 |
{ |
b01b21419 mm/swap: Use loca... |
592 |
struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu); |
1da177e4c Linux-2.6.12-rc2 |
593 |
|
13f7f7898 mm: pagevec: defe... |
594 |
if (pagevec_count(pvec)) |
a0b8cab3b mm: remove lru pa... |
595 |
__pagevec_lru_add(pvec); |
902aaed0d mm: use pagevec t... |
596 |
|
b01b21419 mm/swap: Use loca... |
597 |
pvec = &per_cpu(lru_rotate.pvec, cpu); |
7e0cc01ea mm/swap.c: annota... |
598 599 |
/* Disabling interrupts below acts as a compiler barrier. */ if (data_race(pagevec_count(pvec))) { |
902aaed0d mm: use pagevec t... |
600 601 602 |
unsigned long flags; /* No harm done if a racing interrupt already did this */ |
b01b21419 mm/swap: Use loca... |
603 |
local_lock_irqsave(&lru_rotate.lock, flags); |
902aaed0d mm: use pagevec t... |
604 |
pagevec_move_tail(pvec); |
b01b21419 mm/swap: Use loca... |
605 |
local_unlock_irqrestore(&lru_rotate.lock, flags); |
902aaed0d mm: use pagevec t... |
606 |
} |
315601809 mm: deactivate in... |
607 |
|
b01b21419 mm/swap: Use loca... |
608 |
pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu); |
315601809 mm: deactivate in... |
609 |
if (pagevec_count(pvec)) |
cc5993bd7 mm: rename deacti... |
610 |
pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); |
eb709b0d0 mm: batch activat... |
611 |
|
b01b21419 mm/swap: Use loca... |
612 |
pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu); |
9c276cc65 mm: introduce MAD... |
613 614 |
if (pagevec_count(pvec)) pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); |
b01b21419 mm/swap: Use loca... |
615 |
pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu); |
10853a039 mm: move lazily f... |
616 |
if (pagevec_count(pvec)) |
f7ad2a6cb mm: move MADV_FRE... |
617 |
pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); |
10853a039 mm: move lazily f... |
618 |
|
eb709b0d0 mm: batch activat... |
619 |
activate_page_drain(cpu); |
315601809 mm: deactivate in... |
620 621 622 |
} /** |
cc5993bd7 mm: rename deacti... |
623 |
* deactivate_file_page - forcefully deactivate a file page |
315601809 mm: deactivate in... |
624 625 626 627 628 629 |
* @page: page to deactivate * * This function hints the VM that @page is a good reclaim candidate, * for example if its invalidation fails due to the page being dirty * or under writeback. */ |
cc5993bd7 mm: rename deacti... |
630 |
void deactivate_file_page(struct page *page) |
315601809 mm: deactivate in... |
631 |
{ |
821ed6bbe mm: filter unevic... |
632 |
/* |
cc5993bd7 mm: rename deacti... |
633 634 |
* In a workload with many unevictable page such as mprotect, * unevictable page deactivation for accelerating reclaim is pointless. |
821ed6bbe mm: filter unevic... |
635 636 637 |
*/ if (PageUnevictable(page)) return; |
315601809 mm: deactivate in... |
638 |
if (likely(get_page_unless_zero(page))) { |
b01b21419 mm/swap: Use loca... |
639 640 641 642 |
struct pagevec *pvec; local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file); |
315601809 mm: deactivate in... |
643 |
|
8f182270d mm/swap.c: flush ... |
644 |
if (!pagevec_add(pvec, page) || PageCompound(page)) |
cc5993bd7 mm: rename deacti... |
645 |
pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); |
b01b21419 mm/swap: Use loca... |
646 |
local_unlock(&lru_pvecs.lock); |
315601809 mm: deactivate in... |
647 |
} |
80bfed904 [PATCH] consolida... |
648 |
} |
9c276cc65 mm: introduce MAD... |
649 650 651 652 653 654 655 656 657 658 659 |
/* * deactivate_page - deactivate a page * @page: page to deactivate * * deactivate_page() moves @page to the inactive list if @page was on the active * list and was not an unevictable page. This is done to accelerate the reclaim * of @page. */ void deactivate_page(struct page *page) { if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
660 |
struct pagevec *pvec; |
9c276cc65 mm: introduce MAD... |
661 |
|
b01b21419 mm/swap: Use loca... |
662 663 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate); |
9c276cc65 mm: introduce MAD... |
664 665 666 |
get_page(page); if (!pagevec_add(pvec, page) || PageCompound(page)) pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL); |
b01b21419 mm/swap: Use loca... |
667 |
local_unlock(&lru_pvecs.lock); |
9c276cc65 mm: introduce MAD... |
668 669 |
} } |
10853a039 mm: move lazily f... |
670 |
/** |
f7ad2a6cb mm: move MADV_FRE... |
671 |
* mark_page_lazyfree - make an anon page lazyfree |
10853a039 mm: move lazily f... |
672 673 |
* @page: page to deactivate * |
f7ad2a6cb mm: move MADV_FRE... |
674 675 |
* mark_page_lazyfree() moves @page to the inactive file list. * This is done to accelerate the reclaim of @page. |
10853a039 mm: move lazily f... |
676 |
*/ |
f7ad2a6cb mm: move MADV_FRE... |
677 |
void mark_page_lazyfree(struct page *page) |
10853a039 mm: move lazily f... |
678 |
{ |
f7ad2a6cb mm: move MADV_FRE... |
679 |
if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && |
24c92eb7d mm: avoid marking... |
680 |
!PageSwapCache(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
681 |
struct pagevec *pvec; |
10853a039 mm: move lazily f... |
682 |
|
b01b21419 mm/swap: Use loca... |
683 684 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree); |
09cbfeaf1 mm, fs: get rid o... |
685 |
get_page(page); |
8f182270d mm/swap.c: flush ... |
686 |
if (!pagevec_add(pvec, page) || PageCompound(page)) |
f7ad2a6cb mm: move MADV_FRE... |
687 |
pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL); |
b01b21419 mm/swap: Use loca... |
688 |
local_unlock(&lru_pvecs.lock); |
10853a039 mm: move lazily f... |
689 690 |
} } |
80bfed904 [PATCH] consolida... |
691 692 |
void lru_add_drain(void) { |
b01b21419 mm/swap: Use loca... |
693 694 695 696 697 698 699 700 701 702 703 |
local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); } void lru_add_drain_cpu_zone(struct zone *zone) { local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); drain_local_pages(zone); local_unlock(&lru_pvecs.lock); |
1da177e4c Linux-2.6.12-rc2 |
704 |
} |
6ea183d60 mm: handle lru_ad... |
705 706 707 |
#ifdef CONFIG_SMP static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); |
c4028958b WorkStruct: make ... |
708 |
static void lru_add_drain_per_cpu(struct work_struct *dummy) |
053837fce [PATCH] mm: migra... |
709 710 711 |
{ lru_add_drain(); } |
9852a7212 mm: drop hotplug ... |
712 713 714 715 716 717 718 719 |
/* * Doesn't need any cpu hotplug locking because we do rely on per-cpu * kworkers being shut down before our page_alloc_cpu_dead callback is * executed on the offlined cpu. * Calling this function with cpu hotplug locks held can actually lead * to obscure indirect dependencies via WQ context. */ void lru_add_drain_all(void) |
053837fce [PATCH] mm: migra... |
720 |
{ |
6446a5131 mm/swap: Do not a... |
721 722 723 724 725 726 727 728 729 730 731 |
/* * lru_drain_gen - Global pages generation number * * (A) Definition: global lru_drain_gen = x implies that all generations * 0 < n <= x are already *scheduled* for draining. * * This is an optimization for the highly-contended use case where a * user space workload keeps constantly generating a flow of pages for * each CPU. */ static unsigned int lru_drain_gen; |
5fbc46163 mm: make lru_add_... |
732 |
static struct cpumask has_work; |
6446a5131 mm/swap: Do not a... |
733 734 |
static DEFINE_MUTEX(lock); unsigned cpu, this_gen; |
5fbc46163 mm: make lru_add_... |
735 |
|
ce612879d mm: move pcp and ... |
736 737 738 739 740 741 |
/* * Make sure nobody triggers this path before mm_percpu_wq is fully * initialized. */ if (WARN_ON(!mm_percpu_wq)) return; |
6446a5131 mm/swap: Do not a... |
742 743 744 745 746 747 748 749 750 751 752 753 754 755 |
/* * Guarantee pagevec counter stores visible by this CPU are visible to * other CPUs before loading the current drain generation. */ smp_mb(); /* * (B) Locally cache global LRU draining generation number * * The read barrier ensures that the counter is loaded before the mutex * is taken. It pairs with smp_mb() inside the mutex critical section * at (D). */ this_gen = smp_load_acquire(&lru_drain_gen); |
eef1a429f mm/swap.c: piggyb... |
756 |
|
5fbc46163 mm: make lru_add_... |
757 |
mutex_lock(&lock); |
eef1a429f mm/swap.c: piggyb... |
758 759 |
/* |
6446a5131 mm/swap: Do not a... |
760 761 |
* (C) Exit the draining operation if a newer generation, from another * lru_add_drain_all(), was already scheduled for draining. Check (A). |
eef1a429f mm/swap.c: piggyb... |
762 |
*/ |
6446a5131 mm/swap: Do not a... |
763 |
if (unlikely(this_gen != lru_drain_gen)) |
eef1a429f mm/swap.c: piggyb... |
764 |
goto done; |
6446a5131 mm/swap: Do not a... |
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 |
/* * (D) Increment global generation number * * Pairs with smp_load_acquire() at (B), outside of the critical * section. Use a full memory barrier to guarantee that the new global * drain generation number is stored before loading pagevec counters. * * This pairing must be done here, before the for_each_online_cpu loop * below which drains the page vectors. * * Let x, y, and z represent some system CPU numbers, where x < y < z. * Assume CPU #z is is in the middle of the for_each_online_cpu loop * below and has already reached CPU #y's per-cpu data. CPU #x comes * along, adds some pages to its per-cpu vectors, then calls * lru_add_drain_all(). * * If the paired barrier is done at any later step, e.g. after the * loop, CPU #x will just exit at (C) and miss flushing out all of its * added pages. */ WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1); smp_mb(); |
eef1a429f mm/swap.c: piggyb... |
787 |
|
5fbc46163 mm: make lru_add_... |
788 |
cpumask_clear(&has_work); |
5fbc46163 mm: make lru_add_... |
789 790 |
for_each_online_cpu(cpu) { struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); |
b01b21419 mm/swap: Use loca... |
791 |
if (pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || |
7e0cc01ea mm/swap.c: annota... |
792 |
data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) || |
b01b21419 mm/swap: Use loca... |
793 794 795 |
pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) || |
5fbc46163 mm: make lru_add_... |
796 797 |
need_activate_page_drain(cpu)) { INIT_WORK(work, lru_add_drain_per_cpu); |
ce612879d mm: move pcp and ... |
798 |
queue_work_on(cpu, mm_percpu_wq, work); |
6446a5131 mm/swap: Do not a... |
799 |
__cpumask_set_cpu(cpu, &has_work); |
5fbc46163 mm: make lru_add_... |
800 801 802 803 804 |
} } for_each_cpu(cpu, &has_work) flush_work(&per_cpu(lru_add_drain_work, cpu)); |
eef1a429f mm/swap.c: piggyb... |
805 |
done: |
5fbc46163 mm: make lru_add_... |
806 |
mutex_unlock(&lock); |
053837fce [PATCH] mm: migra... |
807 |
} |
6ea183d60 mm: handle lru_ad... |
808 809 810 811 812 |
#else void lru_add_drain_all(void) { lru_add_drain(); } |
6446a5131 mm/swap: Do not a... |
813 |
#endif /* CONFIG_SMP */ |
053837fce [PATCH] mm: migra... |
814 |
|
aabfb5729 mm: memcontrol: d... |
815 |
/** |
ea1754a08 mm, fs: remove re... |
816 |
* release_pages - batched put_page() |
aabfb5729 mm: memcontrol: d... |
817 818 |
* @pages: array of pages to release * @nr: number of pages |
1da177e4c Linux-2.6.12-rc2 |
819 |
* |
aabfb5729 mm: memcontrol: d... |
820 821 |
* Decrement the reference count on all the pages in @pages. If it * fell to zero, remove the page from the LRU and free it. |
1da177e4c Linux-2.6.12-rc2 |
822 |
*/ |
c6f92f9fb mm: remove cold p... |
823 |
void release_pages(struct page **pages, int nr) |
1da177e4c Linux-2.6.12-rc2 |
824 825 |
{ int i; |
cc59850ef mm: add free_hot_... |
826 |
LIST_HEAD(pages_to_free); |
599d0c954 mm, vmscan: move ... |
827 |
struct pglist_data *locked_pgdat = NULL; |
fa9add641 mm/memcg: apply a... |
828 |
struct lruvec *lruvec; |
3f649ab72 treewide: Remove ... |
829 830 |
unsigned long flags; unsigned int lock_batch; |
1da177e4c Linux-2.6.12-rc2 |
831 |
|
1da177e4c Linux-2.6.12-rc2 |
832 833 |
for (i = 0; i < nr; i++) { struct page *page = pages[i]; |
1da177e4c Linux-2.6.12-rc2 |
834 |
|
aabfb5729 mm: memcontrol: d... |
835 836 837 |
/* * Make sure the IRQ-safe lock-holding time does not get * excessive with a continuous string of pages from the |
599d0c954 mm, vmscan: move ... |
838 |
* same pgdat. The lock is held only if pgdat != NULL. |
aabfb5729 mm: memcontrol: d... |
839 |
*/ |
599d0c954 mm, vmscan: move ... |
840 841 842 |
if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; |
aabfb5729 mm: memcontrol: d... |
843 |
} |
a9b576f72 mm: move call to ... |
844 |
page = compound_head(page); |
6fcb52a56 thp: reduce usage... |
845 |
if (is_huge_zero_page(page)) |
aa88b68c3 thp: keep huge ze... |
846 |
continue; |
aa88b68c3 thp: keep huge ze... |
847 |
|
c5d6c45e9 mm/swap: fix rele... |
848 |
if (is_zone_device_page(page)) { |
df6ad6983 mm/device-public-... |
849 850 851 852 853 |
if (locked_pgdat) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; } |
c5d6c45e9 mm/swap: fix rele... |
854 855 |
/* * ZONE_DEVICE pages that return 'false' from |
a3e7bea06 mm/swap.c: fix co... |
856 |
* page_is_devmap_managed() do not require special |
c5d6c45e9 mm/swap: fix rele... |
857 858 859 |
* processing, and instead, expect a call to * put_page_testzero(). */ |
07d802699 mm: devmap: refac... |
860 861 |
if (page_is_devmap_managed(page)) { put_devmap_managed_page(page); |
c5d6c45e9 mm/swap: fix rele... |
862 |
continue; |
07d802699 mm: devmap: refac... |
863 |
} |
df6ad6983 mm/device-public-... |
864 |
} |
b5810039a [PATCH] core remo... |
865 |
if (!put_page_testzero(page)) |
1da177e4c Linux-2.6.12-rc2 |
866 |
continue; |
ddc58f27f mm: drop tail pag... |
867 |
if (PageCompound(page)) { |
599d0c954 mm, vmscan: move ... |
868 869 870 |
if (locked_pgdat) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; |
ddc58f27f mm: drop tail pag... |
871 872 873 874 |
} __put_compound_page(page); continue; } |
46453a6e1 [PATCH] mm: never... |
875 |
if (PageLRU(page)) { |
599d0c954 mm, vmscan: move ... |
876 |
struct pglist_data *pgdat = page_pgdat(page); |
894bc3104 Unevictable LRU I... |
877 |
|
599d0c954 mm, vmscan: move ... |
878 879 880 |
if (pgdat != locked_pgdat) { if (locked_pgdat) spin_unlock_irqrestore(&locked_pgdat->lru_lock, |
902aaed0d mm: use pagevec t... |
881 |
flags); |
aabfb5729 mm: memcontrol: d... |
882 |
lock_batch = 0; |
599d0c954 mm, vmscan: move ... |
883 884 |
locked_pgdat = pgdat; spin_lock_irqsave(&locked_pgdat->lru_lock, flags); |
46453a6e1 [PATCH] mm: never... |
885 |
} |
fa9add641 mm/memcg: apply a... |
886 |
|
599d0c954 mm, vmscan: move ... |
887 |
lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); |
309381fea mm: dump page whe... |
888 |
VM_BUG_ON_PAGE(!PageLRU(page), page); |
674539115 [PATCH] mm: less ... |
889 |
__ClearPageLRU(page); |
fa9add641 mm/memcg: apply a... |
890 |
del_page_from_lru_list(page, lruvec, page_off_lru(page)); |
46453a6e1 [PATCH] mm: never... |
891 |
} |
629060270 mm: add PageWaite... |
892 |
__ClearPageWaiters(page); |
c53954a09 mm: remove lru pa... |
893 |
|
cc59850ef mm: add free_hot_... |
894 |
list_add(&page->lru, &pages_to_free); |
1da177e4c Linux-2.6.12-rc2 |
895 |
} |
599d0c954 mm, vmscan: move ... |
896 897 |
if (locked_pgdat) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); |
1da177e4c Linux-2.6.12-rc2 |
898 |
|
747db954c mm: memcontrol: u... |
899 |
mem_cgroup_uncharge_list(&pages_to_free); |
2d4894b5d mm: remove cold p... |
900 |
free_unref_page_list(&pages_to_free); |
1da177e4c Linux-2.6.12-rc2 |
901 |
} |
0be8557bc fuse: use release... |
902 |
EXPORT_SYMBOL(release_pages); |
1da177e4c Linux-2.6.12-rc2 |
903 904 905 906 907 908 909 910 911 912 913 914 915 |
/* * The pages which we're about to release may be in the deferred lru-addition * queues. That would prevent them from really being freed right now. That's * OK from a correctness point of view but is inefficient - those pages may be * cache-warm and we want to give them back to the page allocator ASAP. * * So __pagevec_release() will drain those queues here. __pagevec_lru_add() * and __pagevec_lru_add_active() call release_pages() directly to avoid * mutual recursion. */ void __pagevec_release(struct pagevec *pvec) { |
7f0b5fb95 mm, pagevec: rena... |
916 |
if (!pvec->percpu_pvec_drained) { |
d9ed0d08b mm: only drain pe... |
917 |
lru_add_drain(); |
7f0b5fb95 mm, pagevec: rena... |
918 |
pvec->percpu_pvec_drained = true; |
d9ed0d08b mm: only drain pe... |
919 |
} |
c6f92f9fb mm: remove cold p... |
920 |
release_pages(pvec->pages, pagevec_count(pvec)); |
1da177e4c Linux-2.6.12-rc2 |
921 922 |
pagevec_reinit(pvec); } |
7f2857018 Export __pagevec_... |
923 |
EXPORT_SYMBOL(__pagevec_release); |
12d271078 memcg: fix split_... |
924 |
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
71e3aac07 thp: transparent ... |
925 |
/* used by __split_huge_page_refcount() */ |
fa9add641 mm/memcg: apply a... |
926 |
void lru_add_page_tail(struct page *page, struct page *page_tail, |
5bc7b8aca mm: thp: add spli... |
927 |
struct lruvec *lruvec, struct list_head *list) |
71e3aac07 thp: transparent ... |
928 |
{ |
309381fea mm: dump page whe... |
929 930 931 |
VM_BUG_ON_PAGE(!PageHead(page), page); VM_BUG_ON_PAGE(PageCompound(page_tail), page); VM_BUG_ON_PAGE(PageLRU(page_tail), page); |
35f3aa39f mm: Replace spin_... |
932 |
lockdep_assert_held(&lruvec_pgdat(lruvec)->lru_lock); |
71e3aac07 thp: transparent ... |
933 |
|
5bc7b8aca mm: thp: add spli... |
934 935 |
if (!list) SetPageLRU(page_tail); |
71e3aac07 thp: transparent ... |
936 |
|
12d271078 memcg: fix split_... |
937 938 |
if (likely(PageLRU(page))) list_add_tail(&page_tail->lru, &page->lru); |
5bc7b8aca mm: thp: add spli... |
939 940 941 942 943 |
else if (list) { /* page reclaim is reclaiming a huge page */ get_page(page_tail); list_add_tail(&page_tail->lru, list); } else { |
12d271078 memcg: fix split_... |
944 945 946 947 |
/* * Head page has not yet been counted, as an hpage, * so we must account for each subpage individually. * |
e7a1aaf28 mm: replace list_... |
948 949 |
* Put page_tail on the list at the correct position * so they all end up in order. |
12d271078 memcg: fix split_... |
950 |
*/ |
e7a1aaf28 mm: replace list_... |
951 952 |
add_page_to_lru_list_tail(page_tail, lruvec, page_lru(page_tail)); |
71e3aac07 thp: transparent ... |
953 954 |
} } |
12d271078 memcg: fix split_... |
955 |
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
71e3aac07 thp: transparent ... |
956 |
|
fa9add641 mm/memcg: apply a... |
957 958 |
static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, void *arg) |
3dd7ae8ec mm: simplify code... |
959 |
{ |
9c4e6b1a7 mm, mlock, vmscan... |
960 961 |
enum lru_list lru; int was_unevictable = TestClearPageUnevictable(page); |
6c357848b mm: replace hpage... |
962 |
int nr_pages = thp_nr_pages(page); |
3dd7ae8ec mm: simplify code... |
963 |
|
309381fea mm: dump page whe... |
964 |
VM_BUG_ON_PAGE(PageLRU(page), page); |
3dd7ae8ec mm: simplify code... |
965 |
|
9c4e6b1a7 mm, mlock, vmscan... |
966 967 |
/* * Page becomes evictable in two ways: |
dae966dc8 mm/swap.c: __page... |
968 |
* 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. |
9c4e6b1a7 mm, mlock, vmscan... |
969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 |
* 2) Before acquiring LRU lock to put the page to correct LRU and then * a) do PageLRU check with lock [check_move_unevictable_pages] * b) do PageLRU check before lock [clear_page_mlock] * * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need * following strict ordering: * * #0: __pagevec_lru_add_fn #1: clear_page_mlock * * SetPageLRU() TestClearPageMlocked() * smp_mb() // explicit ordering // above provides strict * // ordering * PageMlocked() PageLRU() * * * if '#1' does not observe setting of PG_lru by '#0' and fails * isolation, the explicit barrier will make sure that page_evictable * check will put the page in correct LRU. Without smp_mb(), SetPageLRU * can be reordered after PageMlocked check and can make '#1' to fail * the isolation of the page whose Mlocked bit is cleared (#0 is also * looking at the same page) and the evictable page will be stranded * in an unevictable LRU. */ |
9a9b6cce6 mm: swap: use smp... |
992 993 |
SetPageLRU(page); smp_mb__after_atomic(); |
9c4e6b1a7 mm, mlock, vmscan... |
994 995 996 |
if (page_evictable(page)) { lru = page_lru(page); |
9c4e6b1a7 mm, mlock, vmscan... |
997 |
if (was_unevictable) |
5d91f31fa mm: swap: fix vms... |
998 |
__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); |
9c4e6b1a7 mm, mlock, vmscan... |
999 1000 1001 1002 1003 |
} else { lru = LRU_UNEVICTABLE; ClearPageActive(page); SetPageUnevictable(page); if (!was_unevictable) |
5d91f31fa mm: swap: fix vms... |
1004 |
__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); |
9c4e6b1a7 mm, mlock, vmscan... |
1005 |
} |
fa9add641 mm/memcg: apply a... |
1006 |
add_page_to_lru_list(page, lruvec, lru); |
24b7e5819 mm: pagemap: avoi... |
1007 |
trace_mm_lru_insertion(page, lru); |
3dd7ae8ec mm: simplify code... |
1008 |
} |
1da177e4c Linux-2.6.12-rc2 |
1009 |
/* |
1da177e4c Linux-2.6.12-rc2 |
1010 1011 1012 |
* Add the passed pages to the LRU, then drop the caller's refcount * on them. Reinitialises the caller's pagevec. */ |
a0b8cab3b mm: remove lru pa... |
1013 |
void __pagevec_lru_add(struct pagevec *pvec) |
1da177e4c Linux-2.6.12-rc2 |
1014 |
{ |
a0b8cab3b mm: remove lru pa... |
1015 |
pagevec_lru_move_fn(pvec, __pagevec_lru_add_fn, NULL); |
1da177e4c Linux-2.6.12-rc2 |
1016 |
} |
1da177e4c Linux-2.6.12-rc2 |
1017 |
|
1da177e4c Linux-2.6.12-rc2 |
1018 |
/** |
0cd6144aa mm + fs: prepare ... |
1019 1020 1021 1022 |
* pagevec_lookup_entries - gang pagecache lookup * @pvec: Where the resulting entries are placed * @mapping: The address_space to search * @start: The starting entry index |
cb6f0f348 mm/swap.c: make f... |
1023 |
* @nr_entries: The maximum number of pages |
0cd6144aa mm + fs: prepare ... |
1024 1025 1026 |
* @indices: The cache indices corresponding to the entries in @pvec * * pagevec_lookup_entries() will search for and return a group of up |
f144c390f mm: docs: fix par... |
1027 |
* to @nr_pages pages and shadow entries in the mapping. All |
0cd6144aa mm + fs: prepare ... |
1028 1029 1030 1031 1032 1033 1034 |
* entries are placed in @pvec. pagevec_lookup_entries() takes a * reference against actual pages in @pvec. * * The search returns a group of mapping-contiguous entries with * ascending indexes. There may be holes in the indices due to * not-present entries. * |
71725ed10 mm: huge tmpfs: t... |
1035 1036 1037 1038 |
* Only one subpage of a Transparent Huge Page is returned in one call: * allowing truncate_inode_pages_range() to evict the whole THP without * cycling through a pagevec of extra references. * |
0cd6144aa mm + fs: prepare ... |
1039 1040 1041 1042 1043 |
* pagevec_lookup_entries() returns the number of entries which were * found. */ unsigned pagevec_lookup_entries(struct pagevec *pvec, struct address_space *mapping, |
e02a9f048 mm/swap.c: make f... |
1044 |
pgoff_t start, unsigned nr_entries, |
0cd6144aa mm + fs: prepare ... |
1045 1046 |
pgoff_t *indices) { |
e02a9f048 mm/swap.c: make f... |
1047 |
pvec->nr = find_get_entries(mapping, start, nr_entries, |
0cd6144aa mm + fs: prepare ... |
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 |
pvec->pages, indices); return pagevec_count(pvec); } /** * pagevec_remove_exceptionals - pagevec exceptionals pruning * @pvec: The pagevec to prune * * pagevec_lookup_entries() fills both pages and exceptional radix * tree entries into the pagevec. This function prunes all * exceptionals from @pvec without leaving holes, so that it can be * passed on to page-only pagevec operations. */ void pagevec_remove_exceptionals(struct pagevec *pvec) { int i, j; for (i = 0, j = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; |
3159f943a xarray: Replace e... |
1067 |
if (!xa_is_value(page)) |
0cd6144aa mm + fs: prepare ... |
1068 1069 1070 1071 1072 1073 |
pvec->pages[j++] = page; } pvec->nr = j; } /** |
b947cee4b mm: implement fin... |
1074 |
* pagevec_lookup_range - gang pagecache lookup |
1da177e4c Linux-2.6.12-rc2 |
1075 1076 1077 |
* @pvec: Where the resulting pages are placed * @mapping: The address_space to search * @start: The starting page index |
b947cee4b mm: implement fin... |
1078 |
* @end: The final page index |
1da177e4c Linux-2.6.12-rc2 |
1079 |
* |
e02a9f048 mm/swap.c: make f... |
1080 |
* pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE |
b947cee4b mm: implement fin... |
1081 1082 |
* pages in the mapping starting from index @start and upto index @end * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a |
1da177e4c Linux-2.6.12-rc2 |
1083 1084 1085 |
* reference against the pages in @pvec. * * The search returns a group of mapping-contiguous pages with ascending |
d72dc8a25 mm: make pagevec_... |
1086 1087 |
* indexes. There may be holes in the indices due to not-present pages. We * also update @start to index the next page for the traversal. |
1da177e4c Linux-2.6.12-rc2 |
1088 |
* |
b947cee4b mm: implement fin... |
1089 |
* pagevec_lookup_range() returns the number of pages which were found. If this |
e02a9f048 mm/swap.c: make f... |
1090 |
* number is smaller than PAGEVEC_SIZE, the end of specified range has been |
b947cee4b mm: implement fin... |
1091 |
* reached. |
1da177e4c Linux-2.6.12-rc2 |
1092 |
*/ |
b947cee4b mm: implement fin... |
1093 |
unsigned pagevec_lookup_range(struct pagevec *pvec, |
397162ffa mm: remove nr_pag... |
1094 |
struct address_space *mapping, pgoff_t *start, pgoff_t end) |
1da177e4c Linux-2.6.12-rc2 |
1095 |
{ |
397162ffa mm: remove nr_pag... |
1096 |
pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, |
b947cee4b mm: implement fin... |
1097 |
pvec->pages); |
1da177e4c Linux-2.6.12-rc2 |
1098 1099 |
return pagevec_count(pvec); } |
b947cee4b mm: implement fin... |
1100 |
EXPORT_SYMBOL(pagevec_lookup_range); |
78539fdfa [XFS] Export page... |
1101 |
|
72b045aec mm: implement fin... |
1102 1103 |
unsigned pagevec_lookup_range_tag(struct pagevec *pvec, struct address_space *mapping, pgoff_t *index, pgoff_t end, |
10bbd2358 pagevec: Use xa_m... |
1104 |
xa_mark_t tag) |
1da177e4c Linux-2.6.12-rc2 |
1105 |
{ |
72b045aec mm: implement fin... |
1106 |
pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, |
67fd707f4 mm: remove nr_pag... |
1107 |
PAGEVEC_SIZE, pvec->pages); |
1da177e4c Linux-2.6.12-rc2 |
1108 1109 |
return pagevec_count(pvec); } |
72b045aec mm: implement fin... |
1110 |
EXPORT_SYMBOL(pagevec_lookup_range_tag); |
1da177e4c Linux-2.6.12-rc2 |
1111 |
|
93d3b7140 mm: add variant o... |
1112 1113 |
unsigned pagevec_lookup_range_nr_tag(struct pagevec *pvec, struct address_space *mapping, pgoff_t *index, pgoff_t end, |
10bbd2358 pagevec: Use xa_m... |
1114 |
xa_mark_t tag, unsigned max_pages) |
93d3b7140 mm: add variant o... |
1115 1116 1117 1118 1119 1120 |
{ pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, min_t(unsigned int, max_pages, PAGEVEC_SIZE), pvec->pages); return pagevec_count(pvec); } EXPORT_SYMBOL(pagevec_lookup_range_nr_tag); |
1da177e4c Linux-2.6.12-rc2 |
1121 1122 1123 1124 1125 |
/* * Perform any setup for the swap system */ void __init swap_setup(void) { |
ca79b0c21 mm: convert total... |
1126 |
unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); |
e0bf68dde mm: bdi init hooks |
1127 |
|
1da177e4c Linux-2.6.12-rc2 |
1128 1129 1130 1131 1132 1133 1134 1135 1136 |
/* Use a smaller cluster for small-memory machines */ if (megs < 16) page_cluster = 2; else page_cluster = 3; /* * Right now other parts of the system means that we * _really_ don't want to cluster much more */ |
1da177e4c Linux-2.6.12-rc2 |
1137 |
} |
07d802699 mm: devmap: refac... |
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 |
#ifdef CONFIG_DEV_PAGEMAP_OPS void put_devmap_managed_page(struct page *page) { int count; if (WARN_ON_ONCE(!page_is_devmap_managed(page))) return; count = page_ref_dec_return(page); /* * devmap page refcounts are 1-based, rather than 0-based: if * refcount is 1, then the page is free and the refcount is * stable because nobody holds a reference on the page. */ if (count == 1) free_devmap_managed_page(page); else if (!count) __put_page(page); } EXPORT_SYMBOL(put_devmap_managed_page); #endif |