Blame view
mm/swap.c
31.6 KB
457c89965 treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
1da177e4c Linux-2.6.12-rc2 |
2 3 4 5 6 7 8 |
/* * linux/mm/swap.c * * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds */ /* |
183ff22bb spelling fixes: mm/ |
9 |
* This file contains the default values for the operation of the |
1da177e4c Linux-2.6.12-rc2 |
10 |
* Linux VM subsystem. Fine-tuning documentation can be found in |
570432470 docs: admin-guide... |
11 |
* Documentation/admin-guide/sysctl/vm.rst. |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 16 17 18 19 20 21 22 23 24 |
* Started 18.12.91 * Swap aging added 23.2.95, Stephen Tweedie. * Buffermem limits added 12.3.98, Rik van Riel. */ #include <linux/mm.h> #include <linux/sched.h> #include <linux/kernel_stat.h> #include <linux/swap.h> #include <linux/mman.h> #include <linux/pagemap.h> #include <linux/pagevec.h> #include <linux/init.h> |
b95f1b31b mm: Map most file... |
25 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
26 |
#include <linux/mm_inline.h> |
1da177e4c Linux-2.6.12-rc2 |
27 |
#include <linux/percpu_counter.h> |
3565fce3a mm, x86: get_user... |
28 |
#include <linux/memremap.h> |
1da177e4c Linux-2.6.12-rc2 |
29 30 31 |
#include <linux/percpu.h> #include <linux/cpu.h> #include <linux/notifier.h> |
e0bf68dde mm: bdi init hooks |
32 |
#include <linux/backing-dev.h> |
66e1707bc Memory controller... |
33 |
#include <linux/memcontrol.h> |
5a0e3ad6a include cleanup: ... |
34 |
#include <linux/gfp.h> |
a27bb332c aio: don't includ... |
35 |
#include <linux/uio.h> |
822fc6136 mm: don't call __... |
36 |
#include <linux/hugetlb.h> |
33c3fc71c mm: introduce idl... |
37 |
#include <linux/page_idle.h> |
b01b21419 mm/swap: Use loca... |
38 |
#include <linux/local_lock.h> |
8cc621d2f mm: fs: invalidat... |
39 |
#include <linux/buffer_head.h> |
1da177e4c Linux-2.6.12-rc2 |
40 |
|
64d6519dd swap: cull unevic... |
41 |
#include "internal.h" |
c6286c983 mm: add tracepoin... |
42 43 |
#define CREATE_TRACE_POINTS #include <trace/events/pagemap.h> |
1da177e4c Linux-2.6.12-rc2 |
44 45 |
/* How many pages do we try to swap or page in/out together? */ int page_cluster; |
b01b21419 mm/swap: Use loca... |
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
/* Protecting only lru_rotate.pvec which requires disabling interrupts */ struct lru_rotate { local_lock_t lock; struct pagevec pvec; }; static DEFINE_PER_CPU(struct lru_rotate, lru_rotate) = { .lock = INIT_LOCAL_LOCK(lock), }; /* * The following struct pagevec are grouped together because they are protected * by disabling preemption (and interrupts remain enabled). */ struct lru_pvecs { local_lock_t lock; struct pagevec lru_add; struct pagevec lru_deactivate_file; struct pagevec lru_deactivate; struct pagevec lru_lazyfree; |
a4a921aa5 mm/swap.c: put ac... |
65 |
#ifdef CONFIG_SMP |
b01b21419 mm/swap: Use loca... |
66 |
struct pagevec activate_page; |
a4a921aa5 mm/swap.c: put ac... |
67 |
#endif |
b01b21419 mm/swap: Use loca... |
68 69 70 71 |
}; static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = { .lock = INIT_LOCAL_LOCK(lock), }; |
902aaed0d mm: use pagevec t... |
72 |
|
b221385bc [PATCH] mm/: make... |
73 74 75 76 |
/* * This path almost never happens for VM activity - pages are normally * freed via pagevecs. But it gets used by networking. */ |
920c7a5d0 mm: remove fastca... |
77 |
static void __page_cache_release(struct page *page) |
b221385bc [PATCH] mm/: make... |
78 79 |
{ if (PageLRU(page)) { |
fa9add641 mm/memcg: apply a... |
80 81 |
struct lruvec *lruvec; unsigned long flags; |
b221385bc [PATCH] mm/: make... |
82 |
|
6168d0da2 mm/lru: replace p... |
83 |
lruvec = lock_page_lruvec_irqsave(page, &flags); |
46ae6b2cc mm/swap.c: don't ... |
84 |
del_page_from_lru_list(page, lruvec); |
875601796 mm: add __clear_p... |
85 |
__clear_page_lru_flags(page); |
6168d0da2 mm/lru: replace p... |
86 |
unlock_page_lruvec_irqrestore(lruvec, flags); |
b221385bc [PATCH] mm/: make... |
87 |
} |
629060270 mm: add PageWaite... |
88 |
__ClearPageWaiters(page); |
918070634 thp: alter compou... |
89 90 91 92 93 |
} static void __put_single_page(struct page *page) { __page_cache_release(page); |
7ae88534c mm: move mem_cgro... |
94 |
mem_cgroup_uncharge(page); |
44042b449 mm/page_alloc: al... |
95 |
free_unref_page(page, 0); |
b221385bc [PATCH] mm/: make... |
96 |
} |
918070634 thp: alter compou... |
97 |
static void __put_compound_page(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
98 |
{ |
822fc6136 mm: don't call __... |
99 100 101 102 103 104 105 106 |
/* * __page_cache_release() is supposed to be called for thp, not for * hugetlb. This is because hugetlb page does never have PageLRU set * (it's never listed to any LRU lists) and no memcg routines should * be called for hugetlb (it has a separate hugetlb_cgroup.) */ if (!PageHuge(page)) __page_cache_release(page); |
ff45fc3ca mm: simplify call... |
107 |
destroy_compound_page(page); |
918070634 thp: alter compou... |
108 |
} |
ddc58f27f mm: drop tail pag... |
109 |
void __put_page(struct page *page) |
8519fb30e [PATCH] mm: compo... |
110 |
{ |
713897038 mm, zone_device: ... |
111 112 113 114 115 116 117 118 119 |
if (is_zone_device_page(page)) { put_dev_pagemap(page->pgmap); /* * The page belongs to the device that created pgmap. Do * not return it to page allocator. */ return; } |
8519fb30e [PATCH] mm: compo... |
120 |
if (unlikely(PageCompound(page))) |
ddc58f27f mm: drop tail pag... |
121 122 |
__put_compound_page(page); else |
918070634 thp: alter compou... |
123 |
__put_single_page(page); |
1da177e4c Linux-2.6.12-rc2 |
124 |
} |
ddc58f27f mm: drop tail pag... |
125 |
EXPORT_SYMBOL(__put_page); |
70b50f94f mm: thp: tail pag... |
126 |
|
1d7ea7324 [PATCH] fuse: fix... |
127 |
/** |
7682486b3 mm: fix various k... |
128 129 |
* put_pages_list() - release a list of pages * @pages: list of pages threaded on page->lru |
1d7ea7324 [PATCH] fuse: fix... |
130 131 132 |
* * Release a list of pages which are strung together on page.lru. Currently * used by read_cache_pages() and related error recovery code. |
1d7ea7324 [PATCH] fuse: fix... |
133 134 135 136 137 |
*/ void put_pages_list(struct list_head *pages) { while (!list_empty(pages)) { struct page *victim; |
f86196ea8 fs: don't open co... |
138 |
victim = lru_to_page(pages); |
1d7ea7324 [PATCH] fuse: fix... |
139 |
list_del(&victim->lru); |
09cbfeaf1 mm, fs: get rid o... |
140 |
put_page(victim); |
1d7ea7324 [PATCH] fuse: fix... |
141 142 143 |
} } EXPORT_SYMBOL(put_pages_list); |
18022c5d8 mm: add get_kerne... |
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 |
/* * get_kernel_pages() - pin kernel pages in memory * @kiov: An array of struct kvec structures * @nr_segs: number of segments to pin * @write: pinning for read/write, currently ignored * @pages: array that receives pointers to the pages pinned. * Should be at least nr_segs long. * * Returns number of pages pinned. This may be fewer than the number * requested. If nr_pages is 0 or negative, returns 0. If no pages * were pinned, returns -errno. Each page returned must be released * with a put_page() call when it is finished with. */ int get_kernel_pages(const struct kvec *kiov, int nr_segs, int write, struct page **pages) { int seg; for (seg = 0; seg < nr_segs; seg++) { if (WARN_ON(kiov[seg].iov_len != PAGE_SIZE)) return seg; |
5a178119b mm: add support f... |
165 |
pages[seg] = kmap_to_page(kiov[seg].iov_base); |
09cbfeaf1 mm, fs: get rid o... |
166 |
get_page(pages[seg]); |
18022c5d8 mm: add get_kerne... |
167 168 169 170 171 |
} return seg; } EXPORT_SYMBOL_GPL(get_kernel_pages); |
3dd7ae8ec mm: simplify code... |
172 |
static void pagevec_lru_move_fn(struct pagevec *pvec, |
c7c7b80c3 mm/swap.c: fold v... |
173 |
void (*move_fn)(struct page *page, struct lruvec *lruvec)) |
902aaed0d mm: use pagevec t... |
174 175 |
{ int i; |
6168d0da2 mm/lru: replace p... |
176 |
struct lruvec *lruvec = NULL; |
3dd7ae8ec mm: simplify code... |
177 |
unsigned long flags = 0; |
902aaed0d mm: use pagevec t... |
178 179 180 |
for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; |
3dd7ae8ec mm: simplify code... |
181 |
|
fc574c235 mm/swap.c: serial... |
182 183 184 |
/* block memcg migration during page moving between lru */ if (!TestClearPageLRU(page)) continue; |
2a5e4e340 mm/lru: introduce... |
185 |
lruvec = relock_page_lruvec_irqsave(page, lruvec, &flags); |
c7c7b80c3 mm/swap.c: fold v... |
186 |
(*move_fn)(page, lruvec); |
fc574c235 mm/swap.c: serial... |
187 188 |
SetPageLRU(page); |
902aaed0d mm: use pagevec t... |
189 |
} |
6168d0da2 mm/lru: replace p... |
190 191 |
if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); |
c6f92f9fb mm: remove cold p... |
192 |
release_pages(pvec->pages, pvec->nr); |
83896fb5e Revert "mm: simpl... |
193 |
pagevec_reinit(pvec); |
d8505dee1 mm: simplify code... |
194 |
} |
c7c7b80c3 mm/swap.c: fold v... |
195 |
static void pagevec_move_tail_fn(struct page *page, struct lruvec *lruvec) |
3dd7ae8ec mm: simplify code... |
196 |
{ |
fc574c235 mm/swap.c: serial... |
197 |
if (!PageUnevictable(page)) { |
46ae6b2cc mm/swap.c: don't ... |
198 |
del_page_from_lru_list(page, lruvec); |
c55e8d035 mm: vmscan: move ... |
199 |
ClearPageActive(page); |
3a9c9788a mm: don't pass "e... |
200 |
add_page_to_lru_list_tail(page, lruvec); |
c7c7b80c3 mm/swap.c: fold v... |
201 |
__count_vm_events(PGROTATED, thp_nr_pages(page)); |
3dd7ae8ec mm: simplify code... |
202 203 |
} } |
d479960e4 mm: disable LRU p... |
204 205 206 207 208 209 210 211 212 213 214 |
/* return true if pagevec needs to drain */ static bool pagevec_add_and_need_flush(struct pagevec *pvec, struct page *page) { bool ret = false; if (!pagevec_add(pvec, page) || PageCompound(page) || lru_cache_disabled()) ret = true; return ret; } |
3dd7ae8ec mm: simplify code... |
215 |
/* |
1da177e4c Linux-2.6.12-rc2 |
216 217 |
* Writeback is about to end against a page which has been marked for immediate * reclaim. If it still appears to be reclaimable, move it to the tail of the |
902aaed0d mm: use pagevec t... |
218 |
* inactive list. |
c7c7b80c3 mm/swap.c: fold v... |
219 220 |
* * rotate_reclaimable_page() must disable IRQs, to prevent nasty races. |
1da177e4c Linux-2.6.12-rc2 |
221 |
*/ |
3dd7ae8ec mm: simplify code... |
222 |
void rotate_reclaimable_page(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
223 |
{ |
c55e8d035 mm: vmscan: move ... |
224 |
if (!PageLocked(page) && !PageDirty(page) && |
894bc3104 Unevictable LRU I... |
225 |
!PageUnevictable(page) && PageLRU(page)) { |
ac6aadb24 mm: rotate_reclai... |
226 227 |
struct pagevec *pvec; unsigned long flags; |
09cbfeaf1 mm, fs: get rid o... |
228 |
get_page(page); |
b01b21419 mm/swap: Use loca... |
229 230 |
local_lock_irqsave(&lru_rotate.lock, flags); pvec = this_cpu_ptr(&lru_rotate.pvec); |
d479960e4 mm: disable LRU p... |
231 |
if (pagevec_add_and_need_flush(pvec, page)) |
c7c7b80c3 mm/swap.c: fold v... |
232 |
pagevec_lru_move_fn(pvec, pagevec_move_tail_fn); |
b01b21419 mm/swap: Use loca... |
233 |
local_unlock_irqrestore(&lru_rotate.lock, flags); |
ac6aadb24 mm: rotate_reclai... |
234 |
} |
1da177e4c Linux-2.6.12-rc2 |
235 |
} |
96f8bf4fb mm: vmscan: recla... |
236 |
void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages) |
3e2f41f1f memcg: add zone_r... |
237 |
{ |
7cf111bc3 mm: vmscan: deter... |
238 239 |
do { unsigned long lrusize; |
6168d0da2 mm/lru: replace p... |
240 241 242 243 244 245 246 247 |
/* * Hold lruvec->lru_lock is safe here, since * 1) The pinned lruvec in reclaim, or * 2) From a pre-LRU page during refault (which also holds the * rcu lock, so would be safe even if the page was on the LRU * and could move simultaneously to a new lruvec). */ spin_lock_irq(&lruvec->lru_lock); |
7cf111bc3 mm: vmscan: deter... |
248 |
/* Record cost event */ |
96f8bf4fb mm: vmscan: recla... |
249 250 |
if (file) lruvec->file_cost += nr_pages; |
7cf111bc3 mm: vmscan: deter... |
251 |
else |
96f8bf4fb mm: vmscan: recla... |
252 |
lruvec->anon_cost += nr_pages; |
7cf111bc3 mm: vmscan: deter... |
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 |
/* * Decay previous events * * Because workloads change over time (and to avoid * overflow) we keep these statistics as a floating * average, which ends up weighing recent refaults * more than old ones. */ lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + lruvec_page_state(lruvec, NR_ACTIVE_ANON) + lruvec_page_state(lruvec, NR_INACTIVE_FILE) + lruvec_page_state(lruvec, NR_ACTIVE_FILE); if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { lruvec->file_cost /= 2; lruvec->anon_cost /= 2; } |
6168d0da2 mm/lru: replace p... |
271 |
spin_unlock_irq(&lruvec->lru_lock); |
7cf111bc3 mm: vmscan: deter... |
272 |
} while ((lruvec = parent_lruvec(lruvec))); |
3e2f41f1f memcg: add zone_r... |
273 |
} |
96f8bf4fb mm: vmscan: recla... |
274 275 |
void lru_note_cost_page(struct page *page) { |
a984226f4 mm: memcontrol: r... |
276 |
lru_note_cost(mem_cgroup_page_lruvec(page), |
6c357848b mm: replace hpage... |
277 |
page_is_file_lru(page), thp_nr_pages(page)); |
96f8bf4fb mm: vmscan: recla... |
278 |
} |
c7c7b80c3 mm/swap.c: fold v... |
279 |
static void __activate_page(struct page *page, struct lruvec *lruvec) |
1da177e4c Linux-2.6.12-rc2 |
280 |
{ |
fc574c235 mm/swap.c: serial... |
281 |
if (!PageActive(page) && !PageUnevictable(page)) { |
6c357848b mm: replace hpage... |
282 |
int nr_pages = thp_nr_pages(page); |
744ed1442 mm: batch activat... |
283 |
|
46ae6b2cc mm/swap.c: don't ... |
284 |
del_page_from_lru_list(page, lruvec); |
7a608572a Revert "mm: batch... |
285 |
SetPageActive(page); |
3a9c9788a mm: don't pass "e... |
286 |
add_page_to_lru_list(page, lruvec); |
24b7e5819 mm: pagemap: avoi... |
287 |
trace_mm_lru_activate(page); |
4f98a2fee vmscan: split LRU... |
288 |
|
21e330fc6 mm: swap: memcg: ... |
289 290 291 |
__count_vm_events(PGACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages); |
1da177e4c Linux-2.6.12-rc2 |
292 |
} |
eb709b0d0 mm: batch activat... |
293 294 295 |
} #ifdef CONFIG_SMP |
eb709b0d0 mm: batch activat... |
296 297 |
static void activate_page_drain(int cpu) { |
b01b21419 mm/swap: Use loca... |
298 |
struct pagevec *pvec = &per_cpu(lru_pvecs.activate_page, cpu); |
eb709b0d0 mm: batch activat... |
299 300 |
if (pagevec_count(pvec)) |
c7c7b80c3 mm/swap.c: fold v... |
301 |
pagevec_lru_move_fn(pvec, __activate_page); |
eb709b0d0 mm: batch activat... |
302 |
} |
5fbc46163 mm: make lru_add_... |
303 304 |
static bool need_activate_page_drain(int cpu) { |
b01b21419 mm/swap: Use loca... |
305 |
return pagevec_count(&per_cpu(lru_pvecs.activate_page, cpu)) != 0; |
5fbc46163 mm: make lru_add_... |
306 |
} |
cc2828b21 mm: remove activa... |
307 |
static void activate_page(struct page *page) |
eb709b0d0 mm: batch activat... |
308 |
{ |
800d8c63b shmem: add huge p... |
309 |
page = compound_head(page); |
eb709b0d0 mm: batch activat... |
310 |
if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
311 |
struct pagevec *pvec; |
eb709b0d0 mm: batch activat... |
312 |
|
b01b21419 mm/swap: Use loca... |
313 314 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.activate_page); |
09cbfeaf1 mm, fs: get rid o... |
315 |
get_page(page); |
d479960e4 mm: disable LRU p... |
316 |
if (pagevec_add_and_need_flush(pvec, page)) |
c7c7b80c3 mm/swap.c: fold v... |
317 |
pagevec_lru_move_fn(pvec, __activate_page); |
b01b21419 mm/swap: Use loca... |
318 |
local_unlock(&lru_pvecs.lock); |
eb709b0d0 mm: batch activat... |
319 320 321 322 323 324 325 |
} } #else static inline void activate_page_drain(int cpu) { } |
cc2828b21 mm: remove activa... |
326 |
static void activate_page(struct page *page) |
eb709b0d0 mm: batch activat... |
327 |
{ |
6168d0da2 mm/lru: replace p... |
328 |
struct lruvec *lruvec; |
eb709b0d0 mm: batch activat... |
329 |
|
800d8c63b shmem: add huge p... |
330 |
page = compound_head(page); |
6168d0da2 mm/lru: replace p... |
331 332 333 334 335 336 |
if (TestClearPageLRU(page)) { lruvec = lock_page_lruvec_irq(page); __activate_page(page, lruvec); unlock_page_lruvec_irq(lruvec); SetPageLRU(page); } |
1da177e4c Linux-2.6.12-rc2 |
337 |
} |
eb709b0d0 mm: batch activat... |
338 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
339 |
|
059285a25 mm: activate !Pag... |
340 341 |
static void __lru_cache_activate_page(struct page *page) { |
b01b21419 mm/swap: Use loca... |
342 |
struct pagevec *pvec; |
059285a25 mm: activate !Pag... |
343 |
int i; |
b01b21419 mm/swap: Use loca... |
344 345 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); |
059285a25 mm: activate !Pag... |
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 |
/* * Search backwards on the optimistic assumption that the page being * activated has just been added to this pagevec. Note that only * the local pagevec is examined as a !PageLRU page could be in the * process of being released, reclaimed, migrated or on a remote * pagevec that is currently being drained. Furthermore, marking * a remote pagevec's page PageActive potentially hits a race where * a page is marked PageActive just after it is added to the inactive * list causing accounting errors and BUG_ON checks to trigger. */ for (i = pagevec_count(pvec) - 1; i >= 0; i--) { struct page *pagevec_page = pvec->pages[i]; if (pagevec_page == page) { SetPageActive(page); break; } } |
b01b21419 mm/swap: Use loca... |
364 |
local_unlock(&lru_pvecs.lock); |
059285a25 mm: activate !Pag... |
365 |
} |
1da177e4c Linux-2.6.12-rc2 |
366 367 368 369 370 371 |
/* * Mark a page as having seen activity. * * inactive,unreferenced -> inactive,referenced * inactive,referenced -> active,unreferenced * active,unreferenced -> active,referenced |
eb39d618f mm: replace init_... |
372 373 374 |
* * When a newly allocated page is not yet visible, so safe for non-atomic ops, * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). |
1da177e4c Linux-2.6.12-rc2 |
375 |
*/ |
920c7a5d0 mm: remove fastca... |
376 |
void mark_page_accessed(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
377 |
{ |
e90309c9f thp: allow mlocke... |
378 |
page = compound_head(page); |
059285a25 mm: activate !Pag... |
379 |
|
a1100a740 mm/swap.c: trivia... |
380 381 382 383 384 385 386 387 388 |
if (!PageReferenced(page)) { SetPageReferenced(page); } else if (PageUnevictable(page)) { /* * Unevictable pages are on the "LRU_UNEVICTABLE" list. But, * this list is never rotated or maintained, so marking an * evictable page accessed has no effect. */ } else if (!PageActive(page)) { |
059285a25 mm: activate !Pag... |
389 390 |
/* * If the page is on the LRU, queue it for activation via |
b01b21419 mm/swap: Use loca... |
391 |
* lru_pvecs.activate_page. Otherwise, assume the page is on a |
059285a25 mm: activate !Pag... |
392 393 394 395 396 397 398 |
* pagevec, mark it active and it'll be moved to the active * LRU on the next drain. */ if (PageLRU(page)) activate_page(page); else __lru_cache_activate_page(page); |
1da177e4c Linux-2.6.12-rc2 |
399 |
ClearPageReferenced(page); |
cb6868832 mm/swap: fix for ... |
400 |
workingset_activation(page); |
1da177e4c Linux-2.6.12-rc2 |
401 |
} |
33c3fc71c mm: introduce idl... |
402 403 |
if (page_is_idle(page)) clear_page_idle(page); |
1da177e4c Linux-2.6.12-rc2 |
404 |
} |
1da177e4c Linux-2.6.12-rc2 |
405 |
EXPORT_SYMBOL(mark_page_accessed); |
f04e9ebbe swap: use an arra... |
406 |
/** |
c53954a09 mm: remove lru pa... |
407 |
* lru_cache_add - add a page to a page list |
f04e9ebbe swap: use an arra... |
408 |
* @page: the page to be added to the LRU. |
2329d3751 mm/swap.c: clean ... |
409 410 411 412 413 |
* * Queue the page for addition to the LRU via pagevec. The decision on whether * to add the page to the [in]active [file|anon] list is deferred until the * pagevec is drained. This gives a chance for the caller of lru_cache_add() * have the page added to the active list using mark_page_accessed(). |
f04e9ebbe swap: use an arra... |
414 |
*/ |
c53954a09 mm: remove lru pa... |
415 |
void lru_cache_add(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
416 |
{ |
6058eaec8 mm: fold and remo... |
417 |
struct pagevec *pvec; |
309381fea mm: dump page whe... |
418 419 |
VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page); VM_BUG_ON_PAGE(PageLRU(page), page); |
6058eaec8 mm: fold and remo... |
420 421 422 423 |
get_page(page); local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_add); |
d479960e4 mm: disable LRU p... |
424 |
if (pagevec_add_and_need_flush(pvec, page)) |
6058eaec8 mm: fold and remo... |
425 426 |
__pagevec_lru_add(pvec); local_unlock(&lru_pvecs.lock); |
1da177e4c Linux-2.6.12-rc2 |
427 |
} |
6058eaec8 mm: fold and remo... |
428 |
EXPORT_SYMBOL(lru_cache_add); |
1da177e4c Linux-2.6.12-rc2 |
429 |
|
894bc3104 Unevictable LRU I... |
430 |
/** |
b518154e5 mm/vmscan: protec... |
431 |
* lru_cache_add_inactive_or_unevictable |
00501b531 mm: memcontrol: r... |
432 433 434 |
* @page: the page to be added to LRU * @vma: vma in which page is mapped for determining reclaimability * |
b518154e5 mm/vmscan: protec... |
435 |
* Place @page on the inactive or unevictable LRU list, depending on its |
12eab4289 mm/swap.c: fix in... |
436 |
* evictability. |
00501b531 mm: memcontrol: r... |
437 |
*/ |
b518154e5 mm/vmscan: protec... |
438 |
void lru_cache_add_inactive_or_unevictable(struct page *page, |
00501b531 mm: memcontrol: r... |
439 440 |
struct vm_area_struct *vma) { |
b518154e5 mm/vmscan: protec... |
441 |
bool unevictable; |
00501b531 mm: memcontrol: r... |
442 |
VM_BUG_ON_PAGE(PageLRU(page), page); |
b518154e5 mm/vmscan: protec... |
443 444 |
unevictable = (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED; if (unlikely(unevictable) && !TestSetPageMlocked(page)) { |
0964730bf mlock: fix unevic... |
445 |
int nr_pages = thp_nr_pages(page); |
00501b531 mm: memcontrol: r... |
446 |
/* |
cb152a1a9 mm: fix some typo... |
447 |
* We use the irq-unsafe __mod_zone_page_state because this |
00501b531 mm: memcontrol: r... |
448 449 450 |
* counter is not modified from interrupt context, and the pte * lock is held(spinlock), which implies preemption disabled. */ |
0964730bf mlock: fix unevic... |
451 452 |
__mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); |
00501b531 mm: memcontrol: r... |
453 |
} |
9c4e6b1a7 mm, mlock, vmscan... |
454 |
lru_cache_add(page); |
00501b531 mm: memcontrol: r... |
455 |
} |
902aaed0d mm: use pagevec t... |
456 |
/* |
315601809 mm: deactivate in... |
457 458 459 460 461 |
* If the page can not be invalidated, it is moved to the * inactive list to speed up its reclaim. It is moved to the * head of the list, rather than the tail, to give the flusher * threads some time to write it out, as this is much more * effective than the single-page writeout from reclaim. |
278df9f45 mm: reclaim inval... |
462 463 464 465 466 467 468 469 470 471 472 473 474 475 |
* * If the page isn't page_mapped and dirty/writeback, the page * could reclaim asap using PG_reclaim. * * 1. active, mapped page -> none * 2. active, dirty/writeback page -> inactive, head, PG_reclaim * 3. inactive, mapped page -> none * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim * 5. inactive, clean -> inactive, tail * 6. Others -> none * * In 4, why it moves inactive's head, the VM expects the page would * be write it out by flusher threads as this is much more effective * than the single-page writeout from reclaim. |
315601809 mm: deactivate in... |
476 |
*/ |
c7c7b80c3 mm/swap.c: fold v... |
477 |
static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec) |
315601809 mm: deactivate in... |
478 |
{ |
46ae6b2cc mm/swap.c: don't ... |
479 |
bool active = PageActive(page); |
6c357848b mm: replace hpage... |
480 |
int nr_pages = thp_nr_pages(page); |
315601809 mm: deactivate in... |
481 |
|
bad49d9c8 mm: check PageUne... |
482 483 |
if (PageUnevictable(page)) return; |
315601809 mm: deactivate in... |
484 485 486 |
/* Some processes are using the page */ if (page_mapped(page)) return; |
46ae6b2cc mm/swap.c: don't ... |
487 |
del_page_from_lru_list(page, lruvec); |
315601809 mm: deactivate in... |
488 489 |
ClearPageActive(page); ClearPageReferenced(page); |
315601809 mm: deactivate in... |
490 |
|
278df9f45 mm: reclaim inval... |
491 492 493 494 495 496 |
if (PageWriteback(page) || PageDirty(page)) { /* * PG_reclaim could be raced with end_page_writeback * It can make readahead confusing. But race window * is _really_ small and it's non-critical problem. */ |
3a9c9788a mm: don't pass "e... |
497 |
add_page_to_lru_list(page, lruvec); |
278df9f45 mm: reclaim inval... |
498 499 500 501 |
SetPageReclaim(page); } else { /* * The page's writeback ends up during pagevec |
c4ffefd16 mm: fix typos and... |
502 |
* We move that page into tail of inactive. |
278df9f45 mm: reclaim inval... |
503 |
*/ |
3a9c9788a mm: don't pass "e... |
504 |
add_page_to_lru_list_tail(page, lruvec); |
5d91f31fa mm: swap: fix vms... |
505 |
__count_vm_events(PGROTATED, nr_pages); |
278df9f45 mm: reclaim inval... |
506 |
} |
21e330fc6 mm: swap: memcg: ... |
507 |
if (active) { |
5d91f31fa mm: swap: fix vms... |
508 |
__count_vm_events(PGDEACTIVATE, nr_pages); |
21e330fc6 mm: swap: memcg: ... |
509 510 511 |
__count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); } |
315601809 mm: deactivate in... |
512 |
} |
c7c7b80c3 mm/swap.c: fold v... |
513 |
static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec) |
9c276cc65 mm: introduce MAD... |
514 |
{ |
fc574c235 mm/swap.c: serial... |
515 |
if (PageActive(page) && !PageUnevictable(page)) { |
6c357848b mm: replace hpage... |
516 |
int nr_pages = thp_nr_pages(page); |
9c276cc65 mm: introduce MAD... |
517 |
|
46ae6b2cc mm/swap.c: don't ... |
518 |
del_page_from_lru_list(page, lruvec); |
9c276cc65 mm: introduce MAD... |
519 520 |
ClearPageActive(page); ClearPageReferenced(page); |
3a9c9788a mm: don't pass "e... |
521 |
add_page_to_lru_list(page, lruvec); |
9c276cc65 mm: introduce MAD... |
522 |
|
21e330fc6 mm: swap: memcg: ... |
523 524 525 |
__count_vm_events(PGDEACTIVATE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages); |
9c276cc65 mm: introduce MAD... |
526 527 |
} } |
10853a039 mm: move lazily f... |
528 |
|
c7c7b80c3 mm/swap.c: fold v... |
529 |
static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec) |
10853a039 mm: move lazily f... |
530 |
{ |
fc574c235 mm/swap.c: serial... |
531 |
if (PageAnon(page) && PageSwapBacked(page) && |
24c92eb7d mm: avoid marking... |
532 |
!PageSwapCache(page) && !PageUnevictable(page)) { |
6c357848b mm: replace hpage... |
533 |
int nr_pages = thp_nr_pages(page); |
10853a039 mm: move lazily f... |
534 |
|
46ae6b2cc mm/swap.c: don't ... |
535 |
del_page_from_lru_list(page, lruvec); |
10853a039 mm: move lazily f... |
536 537 |
ClearPageActive(page); ClearPageReferenced(page); |
f7ad2a6cb mm: move MADV_FRE... |
538 |
/* |
9de4f22a6 mm: code cleanup ... |
539 540 541 |
* Lazyfree pages are clean anonymous pages. They have * PG_swapbacked flag cleared, to distinguish them from normal * anonymous pages |
f7ad2a6cb mm: move MADV_FRE... |
542 543 |
*/ ClearPageSwapBacked(page); |
3a9c9788a mm: don't pass "e... |
544 |
add_page_to_lru_list(page, lruvec); |
10853a039 mm: move lazily f... |
545 |
|
21e330fc6 mm: swap: memcg: ... |
546 547 548 |
__count_vm_events(PGLAZYFREE, nr_pages); __count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages); |
10853a039 mm: move lazily f... |
549 550 |
} } |
315601809 mm: deactivate in... |
551 |
/* |
902aaed0d mm: use pagevec t... |
552 553 554 555 |
* Drain pages out of the cpu's pagevecs. * Either "cpu" is the current CPU, and preemption has already been * disabled; or "cpu" is being hot-unplugged, and is already dead. */ |
f0cb3c76a mm: drain percpu ... |
556 |
void lru_add_drain_cpu(int cpu) |
1da177e4c Linux-2.6.12-rc2 |
557 |
{ |
b01b21419 mm/swap: Use loca... |
558 |
struct pagevec *pvec = &per_cpu(lru_pvecs.lru_add, cpu); |
1da177e4c Linux-2.6.12-rc2 |
559 |
|
13f7f7898 mm: pagevec: defe... |
560 |
if (pagevec_count(pvec)) |
a0b8cab3b mm: remove lru pa... |
561 |
__pagevec_lru_add(pvec); |
902aaed0d mm: use pagevec t... |
562 |
|
b01b21419 mm/swap: Use loca... |
563 |
pvec = &per_cpu(lru_rotate.pvec, cpu); |
7e0cc01ea mm/swap.c: annota... |
564 565 |
/* Disabling interrupts below acts as a compiler barrier. */ if (data_race(pagevec_count(pvec))) { |
902aaed0d mm: use pagevec t... |
566 567 568 |
unsigned long flags; /* No harm done if a racing interrupt already did this */ |
b01b21419 mm/swap: Use loca... |
569 |
local_lock_irqsave(&lru_rotate.lock, flags); |
c7c7b80c3 mm/swap.c: fold v... |
570 |
pagevec_lru_move_fn(pvec, pagevec_move_tail_fn); |
b01b21419 mm/swap: Use loca... |
571 |
local_unlock_irqrestore(&lru_rotate.lock, flags); |
902aaed0d mm: use pagevec t... |
572 |
} |
315601809 mm: deactivate in... |
573 |
|
b01b21419 mm/swap: Use loca... |
574 |
pvec = &per_cpu(lru_pvecs.lru_deactivate_file, cpu); |
315601809 mm: deactivate in... |
575 |
if (pagevec_count(pvec)) |
c7c7b80c3 mm/swap.c: fold v... |
576 |
pagevec_lru_move_fn(pvec, lru_deactivate_file_fn); |
eb709b0d0 mm: batch activat... |
577 |
|
b01b21419 mm/swap: Use loca... |
578 |
pvec = &per_cpu(lru_pvecs.lru_deactivate, cpu); |
9c276cc65 mm: introduce MAD... |
579 |
if (pagevec_count(pvec)) |
c7c7b80c3 mm/swap.c: fold v... |
580 |
pagevec_lru_move_fn(pvec, lru_deactivate_fn); |
9c276cc65 mm: introduce MAD... |
581 |
|
b01b21419 mm/swap: Use loca... |
582 |
pvec = &per_cpu(lru_pvecs.lru_lazyfree, cpu); |
10853a039 mm: move lazily f... |
583 |
if (pagevec_count(pvec)) |
c7c7b80c3 mm/swap.c: fold v... |
584 |
pagevec_lru_move_fn(pvec, lru_lazyfree_fn); |
10853a039 mm: move lazily f... |
585 |
|
eb709b0d0 mm: batch activat... |
586 |
activate_page_drain(cpu); |
315601809 mm: deactivate in... |
587 588 589 |
} /** |
cc5993bd7 mm: rename deacti... |
590 |
* deactivate_file_page - forcefully deactivate a file page |
315601809 mm: deactivate in... |
591 592 593 594 595 596 |
* @page: page to deactivate * * This function hints the VM that @page is a good reclaim candidate, * for example if its invalidation fails due to the page being dirty * or under writeback. */ |
cc5993bd7 mm: rename deacti... |
597 |
void deactivate_file_page(struct page *page) |
315601809 mm: deactivate in... |
598 |
{ |
821ed6bbe mm: filter unevic... |
599 |
/* |
cc5993bd7 mm: rename deacti... |
600 601 |
* In a workload with many unevictable page such as mprotect, * unevictable page deactivation for accelerating reclaim is pointless. |
821ed6bbe mm: filter unevic... |
602 603 604 |
*/ if (PageUnevictable(page)) return; |
315601809 mm: deactivate in... |
605 |
if (likely(get_page_unless_zero(page))) { |
b01b21419 mm/swap: Use loca... |
606 607 608 609 |
struct pagevec *pvec; local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate_file); |
315601809 mm: deactivate in... |
610 |
|
d479960e4 mm: disable LRU p... |
611 |
if (pagevec_add_and_need_flush(pvec, page)) |
c7c7b80c3 mm/swap.c: fold v... |
612 |
pagevec_lru_move_fn(pvec, lru_deactivate_file_fn); |
b01b21419 mm/swap: Use loca... |
613 |
local_unlock(&lru_pvecs.lock); |
315601809 mm: deactivate in... |
614 |
} |
80bfed904 [PATCH] consolida... |
615 |
} |
9c276cc65 mm: introduce MAD... |
616 617 618 619 620 621 622 623 624 625 626 |
/* * deactivate_page - deactivate a page * @page: page to deactivate * * deactivate_page() moves @page to the inactive list if @page was on the active * list and was not an unevictable page. This is done to accelerate the reclaim * of @page. */ void deactivate_page(struct page *page) { if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
627 |
struct pagevec *pvec; |
9c276cc65 mm: introduce MAD... |
628 |
|
b01b21419 mm/swap: Use loca... |
629 630 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_deactivate); |
9c276cc65 mm: introduce MAD... |
631 |
get_page(page); |
d479960e4 mm: disable LRU p... |
632 |
if (pagevec_add_and_need_flush(pvec, page)) |
c7c7b80c3 mm/swap.c: fold v... |
633 |
pagevec_lru_move_fn(pvec, lru_deactivate_fn); |
b01b21419 mm/swap: Use loca... |
634 |
local_unlock(&lru_pvecs.lock); |
9c276cc65 mm: introduce MAD... |
635 636 |
} } |
10853a039 mm: move lazily f... |
637 |
/** |
f7ad2a6cb mm: move MADV_FRE... |
638 |
* mark_page_lazyfree - make an anon page lazyfree |
10853a039 mm: move lazily f... |
639 640 |
* @page: page to deactivate * |
f7ad2a6cb mm: move MADV_FRE... |
641 642 |
* mark_page_lazyfree() moves @page to the inactive file list. * This is done to accelerate the reclaim of @page. |
10853a039 mm: move lazily f... |
643 |
*/ |
f7ad2a6cb mm: move MADV_FRE... |
644 |
void mark_page_lazyfree(struct page *page) |
10853a039 mm: move lazily f... |
645 |
{ |
f7ad2a6cb mm: move MADV_FRE... |
646 |
if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) && |
24c92eb7d mm: avoid marking... |
647 |
!PageSwapCache(page) && !PageUnevictable(page)) { |
b01b21419 mm/swap: Use loca... |
648 |
struct pagevec *pvec; |
10853a039 mm: move lazily f... |
649 |
|
b01b21419 mm/swap: Use loca... |
650 651 |
local_lock(&lru_pvecs.lock); pvec = this_cpu_ptr(&lru_pvecs.lru_lazyfree); |
09cbfeaf1 mm, fs: get rid o... |
652 |
get_page(page); |
d479960e4 mm: disable LRU p... |
653 |
if (pagevec_add_and_need_flush(pvec, page)) |
c7c7b80c3 mm/swap.c: fold v... |
654 |
pagevec_lru_move_fn(pvec, lru_lazyfree_fn); |
b01b21419 mm/swap: Use loca... |
655 |
local_unlock(&lru_pvecs.lock); |
10853a039 mm: move lazily f... |
656 657 |
} } |
80bfed904 [PATCH] consolida... |
658 659 |
void lru_add_drain(void) { |
b01b21419 mm/swap: Use loca... |
660 661 662 663 |
local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); } |
243418e39 mm: fs: invalidat... |
664 665 666 667 668 669 670 671 672 673 674 675 676 |
/* * It's called from per-cpu workqueue context in SMP case so * lru_add_drain_cpu and invalidate_bh_lrus_cpu should run on * the same cpu. It shouldn't be a problem in !SMP case since * the core is only one and the locks will disable preemption. */ static void lru_add_and_bh_lrus_drain(void) { local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); invalidate_bh_lrus_cpu(); } |
b01b21419 mm/swap: Use loca... |
677 678 679 680 681 682 |
void lru_add_drain_cpu_zone(struct zone *zone) { local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); drain_local_pages(zone); local_unlock(&lru_pvecs.lock); |
1da177e4c Linux-2.6.12-rc2 |
683 |
} |
6ea183d60 mm: handle lru_ad... |
684 685 686 |
#ifdef CONFIG_SMP static DEFINE_PER_CPU(struct work_struct, lru_add_drain_work); |
c4028958b WorkStruct: make ... |
687 |
static void lru_add_drain_per_cpu(struct work_struct *dummy) |
053837fce [PATCH] mm: migra... |
688 |
{ |
243418e39 mm: fs: invalidat... |
689 |
lru_add_and_bh_lrus_drain(); |
053837fce [PATCH] mm: migra... |
690 |
} |
9852a7212 mm: drop hotplug ... |
691 692 693 694 695 696 697 |
/* * Doesn't need any cpu hotplug locking because we do rely on per-cpu * kworkers being shut down before our page_alloc_cpu_dead callback is * executed on the offlined cpu. * Calling this function with cpu hotplug locks held can actually lead * to obscure indirect dependencies via WQ context. */ |
d479960e4 mm: disable LRU p... |
698 |
inline void __lru_add_drain_all(bool force_all_cpus) |
053837fce [PATCH] mm: migra... |
699 |
{ |
6446a5131 mm/swap: Do not a... |
700 701 702 703 704 705 706 707 708 709 710 |
/* * lru_drain_gen - Global pages generation number * * (A) Definition: global lru_drain_gen = x implies that all generations * 0 < n <= x are already *scheduled* for draining. * * This is an optimization for the highly-contended use case where a * user space workload keeps constantly generating a flow of pages for * each CPU. */ static unsigned int lru_drain_gen; |
5fbc46163 mm: make lru_add_... |
711 |
static struct cpumask has_work; |
6446a5131 mm/swap: Do not a... |
712 713 |
static DEFINE_MUTEX(lock); unsigned cpu, this_gen; |
5fbc46163 mm: make lru_add_... |
714 |
|
ce612879d mm: move pcp and ... |
715 716 717 718 719 720 |
/* * Make sure nobody triggers this path before mm_percpu_wq is fully * initialized. */ if (WARN_ON(!mm_percpu_wq)) return; |
6446a5131 mm/swap: Do not a... |
721 722 723 724 725 726 727 728 729 730 731 732 733 734 |
/* * Guarantee pagevec counter stores visible by this CPU are visible to * other CPUs before loading the current drain generation. */ smp_mb(); /* * (B) Locally cache global LRU draining generation number * * The read barrier ensures that the counter is loaded before the mutex * is taken. It pairs with smp_mb() inside the mutex critical section * at (D). */ this_gen = smp_load_acquire(&lru_drain_gen); |
eef1a429f mm/swap.c: piggyb... |
735 |
|
5fbc46163 mm: make lru_add_... |
736 |
mutex_lock(&lock); |
eef1a429f mm/swap.c: piggyb... |
737 738 |
/* |
6446a5131 mm/swap: Do not a... |
739 740 |
* (C) Exit the draining operation if a newer generation, from another * lru_add_drain_all(), was already scheduled for draining. Check (A). |
eef1a429f mm/swap.c: piggyb... |
741 |
*/ |
d479960e4 mm: disable LRU p... |
742 |
if (unlikely(this_gen != lru_drain_gen && !force_all_cpus)) |
eef1a429f mm/swap.c: piggyb... |
743 |
goto done; |
6446a5131 mm/swap: Do not a... |
744 745 746 747 748 749 750 751 752 753 754 |
/* * (D) Increment global generation number * * Pairs with smp_load_acquire() at (B), outside of the critical * section. Use a full memory barrier to guarantee that the new global * drain generation number is stored before loading pagevec counters. * * This pairing must be done here, before the for_each_online_cpu loop * below which drains the page vectors. * * Let x, y, and z represent some system CPU numbers, where x < y < z. |
cb152a1a9 mm: fix some typo... |
755 |
* Assume CPU #z is in the middle of the for_each_online_cpu loop |
6446a5131 mm/swap: Do not a... |
756 757 758 759 760 761 762 763 764 765 |
* below and has already reached CPU #y's per-cpu data. CPU #x comes * along, adds some pages to its per-cpu vectors, then calls * lru_add_drain_all(). * * If the paired barrier is done at any later step, e.g. after the * loop, CPU #x will just exit at (C) and miss flushing out all of its * added pages. */ WRITE_ONCE(lru_drain_gen, lru_drain_gen + 1); smp_mb(); |
eef1a429f mm/swap.c: piggyb... |
766 |
|
5fbc46163 mm: make lru_add_... |
767 |
cpumask_clear(&has_work); |
5fbc46163 mm: make lru_add_... |
768 769 |
for_each_online_cpu(cpu) { struct work_struct *work = &per_cpu(lru_add_drain_work, cpu); |
d479960e4 mm: disable LRU p... |
770 771 |
if (force_all_cpus || pagevec_count(&per_cpu(lru_pvecs.lru_add, cpu)) || |
7e0cc01ea mm/swap.c: annota... |
772 |
data_race(pagevec_count(&per_cpu(lru_rotate.pvec, cpu))) || |
b01b21419 mm/swap: Use loca... |
773 774 775 |
pagevec_count(&per_cpu(lru_pvecs.lru_deactivate_file, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_deactivate, cpu)) || pagevec_count(&per_cpu(lru_pvecs.lru_lazyfree, cpu)) || |
8cc621d2f mm: fs: invalidat... |
776 777 |
need_activate_page_drain(cpu) || has_bh_in_lru(cpu, NULL)) { |
5fbc46163 mm: make lru_add_... |
778 |
INIT_WORK(work, lru_add_drain_per_cpu); |
ce612879d mm: move pcp and ... |
779 |
queue_work_on(cpu, mm_percpu_wq, work); |
6446a5131 mm/swap: Do not a... |
780 |
__cpumask_set_cpu(cpu, &has_work); |
5fbc46163 mm: make lru_add_... |
781 782 783 784 785 |
} } for_each_cpu(cpu, &has_work) flush_work(&per_cpu(lru_add_drain_work, cpu)); |
eef1a429f mm/swap.c: piggyb... |
786 |
done: |
5fbc46163 mm: make lru_add_... |
787 |
mutex_unlock(&lock); |
053837fce [PATCH] mm: migra... |
788 |
} |
d479960e4 mm: disable LRU p... |
789 790 791 792 793 |
void lru_add_drain_all(void) { __lru_add_drain_all(false); } |
6ea183d60 mm: handle lru_ad... |
794 795 796 797 798 |
#else void lru_add_drain_all(void) { lru_add_drain(); } |
6446a5131 mm/swap: Do not a... |
799 |
#endif /* CONFIG_SMP */ |
053837fce [PATCH] mm: migra... |
800 |
|
d479960e4 mm: disable LRU p... |
801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 |
atomic_t lru_disable_count = ATOMIC_INIT(0); /* * lru_cache_disable() needs to be called before we start compiling * a list of pages to be migrated using isolate_lru_page(). * It drains pages on LRU cache and then disable on all cpus until * lru_cache_enable is called. * * Must be paired with a call to lru_cache_enable(). */ void lru_cache_disable(void) { atomic_inc(&lru_disable_count); #ifdef CONFIG_SMP /* * lru_add_drain_all in the force mode will schedule draining on * all online CPUs so any calls of lru_cache_disabled wrapped by * local_lock or preemption disabled would be ordered by that. * The atomic operation doesn't need to have stronger ordering * requirements because that is enforeced by the scheduling * guarantees. */ __lru_add_drain_all(true); #else |
243418e39 mm: fs: invalidat... |
825 |
lru_add_and_bh_lrus_drain(); |
d479960e4 mm: disable LRU p... |
826 827 |
#endif } |
aabfb5729 mm: memcontrol: d... |
828 |
/** |
ea1754a08 mm, fs: remove re... |
829 |
* release_pages - batched put_page() |
aabfb5729 mm: memcontrol: d... |
830 831 |
* @pages: array of pages to release * @nr: number of pages |
1da177e4c Linux-2.6.12-rc2 |
832 |
* |
aabfb5729 mm: memcontrol: d... |
833 834 |
* Decrement the reference count on all the pages in @pages. If it * fell to zero, remove the page from the LRU and free it. |
1da177e4c Linux-2.6.12-rc2 |
835 |
*/ |
c6f92f9fb mm: remove cold p... |
836 |
void release_pages(struct page **pages, int nr) |
1da177e4c Linux-2.6.12-rc2 |
837 838 |
{ int i; |
cc59850ef mm: add free_hot_... |
839 |
LIST_HEAD(pages_to_free); |
6168d0da2 mm/lru: replace p... |
840 |
struct lruvec *lruvec = NULL; |
3f649ab72 treewide: Remove ... |
841 842 |
unsigned long flags; unsigned int lock_batch; |
1da177e4c Linux-2.6.12-rc2 |
843 |
|
1da177e4c Linux-2.6.12-rc2 |
844 845 |
for (i = 0; i < nr; i++) { struct page *page = pages[i]; |
1da177e4c Linux-2.6.12-rc2 |
846 |
|
aabfb5729 mm: memcontrol: d... |
847 848 849 |
/* * Make sure the IRQ-safe lock-holding time does not get * excessive with a continuous string of pages from the |
6168d0da2 mm/lru: replace p... |
850 |
* same lruvec. The lock is held only if lruvec != NULL. |
aabfb5729 mm: memcontrol: d... |
851 |
*/ |
6168d0da2 mm/lru: replace p... |
852 853 854 |
if (lruvec && ++lock_batch == SWAP_CLUSTER_MAX) { unlock_page_lruvec_irqrestore(lruvec, flags); lruvec = NULL; |
aabfb5729 mm: memcontrol: d... |
855 |
} |
a9b576f72 mm: move call to ... |
856 |
page = compound_head(page); |
6fcb52a56 thp: reduce usage... |
857 |
if (is_huge_zero_page(page)) |
aa88b68c3 thp: keep huge ze... |
858 |
continue; |
aa88b68c3 thp: keep huge ze... |
859 |
|
c5d6c45e9 mm/swap: fix rele... |
860 |
if (is_zone_device_page(page)) { |
6168d0da2 mm/lru: replace p... |
861 862 863 |
if (lruvec) { unlock_page_lruvec_irqrestore(lruvec, flags); lruvec = NULL; |
df6ad6983 mm/device-public-... |
864 |
} |
c5d6c45e9 mm/swap: fix rele... |
865 866 |
/* * ZONE_DEVICE pages that return 'false' from |
a3e7bea06 mm/swap.c: fix co... |
867 |
* page_is_devmap_managed() do not require special |
c5d6c45e9 mm/swap: fix rele... |
868 869 870 |
* processing, and instead, expect a call to * put_page_testzero(). */ |
07d802699 mm: devmap: refac... |
871 872 |
if (page_is_devmap_managed(page)) { put_devmap_managed_page(page); |
c5d6c45e9 mm/swap: fix rele... |
873 |
continue; |
07d802699 mm: devmap: refac... |
874 |
} |
43fbdeb34 mm: handle zone d... |
875 876 877 |
if (put_page_testzero(page)) put_dev_pagemap(page->pgmap); continue; |
df6ad6983 mm/device-public-... |
878 |
} |
b5810039a [PATCH] core remo... |
879 |
if (!put_page_testzero(page)) |
1da177e4c Linux-2.6.12-rc2 |
880 |
continue; |
ddc58f27f mm: drop tail pag... |
881 |
if (PageCompound(page)) { |
6168d0da2 mm/lru: replace p... |
882 883 884 |
if (lruvec) { unlock_page_lruvec_irqrestore(lruvec, flags); lruvec = NULL; |
ddc58f27f mm: drop tail pag... |
885 886 887 888 |
} __put_compound_page(page); continue; } |
46453a6e1 [PATCH] mm: never... |
889 |
if (PageLRU(page)) { |
2a5e4e340 mm/lru: introduce... |
890 891 892 893 894 |
struct lruvec *prev_lruvec = lruvec; lruvec = relock_page_lruvec_irqsave(page, lruvec, &flags); if (prev_lruvec != lruvec) |
aabfb5729 mm: memcontrol: d... |
895 |
lock_batch = 0; |
fa9add641 mm/memcg: apply a... |
896 |
|
46ae6b2cc mm/swap.c: don't ... |
897 |
del_page_from_lru_list(page, lruvec); |
875601796 mm: add __clear_p... |
898 |
__clear_page_lru_flags(page); |
46453a6e1 [PATCH] mm: never... |
899 |
} |
629060270 mm: add PageWaite... |
900 |
__ClearPageWaiters(page); |
c53954a09 mm: remove lru pa... |
901 |
|
cc59850ef mm: add free_hot_... |
902 |
list_add(&page->lru, &pages_to_free); |
1da177e4c Linux-2.6.12-rc2 |
903 |
} |
6168d0da2 mm/lru: replace p... |
904 905 |
if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); |
1da177e4c Linux-2.6.12-rc2 |
906 |
|
747db954c mm: memcontrol: u... |
907 |
mem_cgroup_uncharge_list(&pages_to_free); |
2d4894b5d mm: remove cold p... |
908 |
free_unref_page_list(&pages_to_free); |
1da177e4c Linux-2.6.12-rc2 |
909 |
} |
0be8557bc fuse: use release... |
910 |
EXPORT_SYMBOL(release_pages); |
1da177e4c Linux-2.6.12-rc2 |
911 912 913 914 915 916 917 918 919 920 921 922 923 |
/* * The pages which we're about to release may be in the deferred lru-addition * queues. That would prevent them from really being freed right now. That's * OK from a correctness point of view but is inefficient - those pages may be * cache-warm and we want to give them back to the page allocator ASAP. * * So __pagevec_release() will drain those queues here. __pagevec_lru_add() * and __pagevec_lru_add_active() call release_pages() directly to avoid * mutual recursion. */ void __pagevec_release(struct pagevec *pvec) { |
7f0b5fb95 mm, pagevec: rena... |
924 |
if (!pvec->percpu_pvec_drained) { |
d9ed0d08b mm: only drain pe... |
925 |
lru_add_drain(); |
7f0b5fb95 mm, pagevec: rena... |
926 |
pvec->percpu_pvec_drained = true; |
d9ed0d08b mm: only drain pe... |
927 |
} |
c6f92f9fb mm: remove cold p... |
928 |
release_pages(pvec->pages, pagevec_count(pvec)); |
1da177e4c Linux-2.6.12-rc2 |
929 930 |
pagevec_reinit(pvec); } |
7f2857018 Export __pagevec_... |
931 |
EXPORT_SYMBOL(__pagevec_release); |
c7c7b80c3 mm/swap.c: fold v... |
932 |
static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec) |
3dd7ae8ec mm: simplify code... |
933 |
{ |
9c4e6b1a7 mm, mlock, vmscan... |
934 |
int was_unevictable = TestClearPageUnevictable(page); |
6c357848b mm: replace hpage... |
935 |
int nr_pages = thp_nr_pages(page); |
3dd7ae8ec mm: simplify code... |
936 |
|
309381fea mm: dump page whe... |
937 |
VM_BUG_ON_PAGE(PageLRU(page), page); |
3dd7ae8ec mm: simplify code... |
938 |
|
9c4e6b1a7 mm, mlock, vmscan... |
939 940 |
/* * Page becomes evictable in two ways: |
dae966dc8 mm/swap.c: __page... |
941 |
* 1) Within LRU lock [munlock_vma_page() and __munlock_pagevec()]. |
9c4e6b1a7 mm, mlock, vmscan... |
942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 |
* 2) Before acquiring LRU lock to put the page to correct LRU and then * a) do PageLRU check with lock [check_move_unevictable_pages] * b) do PageLRU check before lock [clear_page_mlock] * * (1) & (2a) are ok as LRU lock will serialize them. For (2b), we need * following strict ordering: * * #0: __pagevec_lru_add_fn #1: clear_page_mlock * * SetPageLRU() TestClearPageMlocked() * smp_mb() // explicit ordering // above provides strict * // ordering * PageMlocked() PageLRU() * * * if '#1' does not observe setting of PG_lru by '#0' and fails * isolation, the explicit barrier will make sure that page_evictable * check will put the page in correct LRU. Without smp_mb(), SetPageLRU * can be reordered after PageMlocked check and can make '#1' to fail * the isolation of the page whose Mlocked bit is cleared (#0 is also * looking at the same page) and the evictable page will be stranded * in an unevictable LRU. */ |
9a9b6cce6 mm: swap: use smp... |
965 966 |
SetPageLRU(page); smp_mb__after_atomic(); |
9c4e6b1a7 mm, mlock, vmscan... |
967 968 |
if (page_evictable(page)) { |
9c4e6b1a7 mm, mlock, vmscan... |
969 |
if (was_unevictable) |
5d91f31fa mm: swap: fix vms... |
970 |
__count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages); |
9c4e6b1a7 mm, mlock, vmscan... |
971 |
} else { |
9c4e6b1a7 mm, mlock, vmscan... |
972 973 974 |
ClearPageActive(page); SetPageUnevictable(page); if (!was_unevictable) |
5d91f31fa mm: swap: fix vms... |
975 |
__count_vm_events(UNEVICTABLE_PGCULLED, nr_pages); |
9c4e6b1a7 mm, mlock, vmscan... |
976 |
} |
3a9c9788a mm: don't pass "e... |
977 |
add_page_to_lru_list(page, lruvec); |
861404536 mm/swap.c: don't ... |
978 |
trace_mm_lru_insertion(page); |
3dd7ae8ec mm: simplify code... |
979 |
} |
1da177e4c Linux-2.6.12-rc2 |
980 |
/* |
1da177e4c Linux-2.6.12-rc2 |
981 982 983 |
* Add the passed pages to the LRU, then drop the caller's refcount * on them. Reinitialises the caller's pagevec. */ |
a0b8cab3b mm: remove lru pa... |
984 |
void __pagevec_lru_add(struct pagevec *pvec) |
1da177e4c Linux-2.6.12-rc2 |
985 |
{ |
fc574c235 mm/swap.c: serial... |
986 |
int i; |
6168d0da2 mm/lru: replace p... |
987 |
struct lruvec *lruvec = NULL; |
fc574c235 mm/swap.c: serial... |
988 989 990 991 |
unsigned long flags = 0; for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; |
fc574c235 mm/swap.c: serial... |
992 |
|
2a5e4e340 mm/lru: introduce... |
993 |
lruvec = relock_page_lruvec_irqsave(page, lruvec, &flags); |
fc574c235 mm/swap.c: serial... |
994 995 |
__pagevec_lru_add_fn(page, lruvec); } |
6168d0da2 mm/lru: replace p... |
996 997 |
if (lruvec) unlock_page_lruvec_irqrestore(lruvec, flags); |
fc574c235 mm/swap.c: serial... |
998 999 |
release_pages(pvec->pages, pvec->nr); pagevec_reinit(pvec); |
1da177e4c Linux-2.6.12-rc2 |
1000 |
} |
1da177e4c Linux-2.6.12-rc2 |
1001 |
|
1da177e4c Linux-2.6.12-rc2 |
1002 |
/** |
0cd6144aa mm + fs: prepare ... |
1003 1004 1005 |
* pagevec_remove_exceptionals - pagevec exceptionals pruning * @pvec: The pagevec to prune * |
a656a2024 mm: remove pageve... |
1006 1007 |
* find_get_entries() fills both pages and XArray value entries (aka * exceptional entries) into the pagevec. This function prunes all |
0cd6144aa mm + fs: prepare ... |
1008 1009 1010 1011 1012 1013 1014 1015 1016 |
* exceptionals from @pvec without leaving holes, so that it can be * passed on to page-only pagevec operations. */ void pagevec_remove_exceptionals(struct pagevec *pvec) { int i, j; for (i = 0, j = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; |
3159f943a xarray: Replace e... |
1017 |
if (!xa_is_value(page)) |
0cd6144aa mm + fs: prepare ... |
1018 1019 1020 1021 1022 1023 |
pvec->pages[j++] = page; } pvec->nr = j; } /** |
b947cee4b mm: implement fin... |
1024 |
* pagevec_lookup_range - gang pagecache lookup |
1da177e4c Linux-2.6.12-rc2 |
1025 1026 1027 |
* @pvec: Where the resulting pages are placed * @mapping: The address_space to search * @start: The starting page index |
b947cee4b mm: implement fin... |
1028 |
* @end: The final page index |
1da177e4c Linux-2.6.12-rc2 |
1029 |
* |
e02a9f048 mm/swap.c: make f... |
1030 |
* pagevec_lookup_range() will search for & return a group of up to PAGEVEC_SIZE |
b947cee4b mm: implement fin... |
1031 1032 |
* pages in the mapping starting from index @start and upto index @end * (inclusive). The pages are placed in @pvec. pagevec_lookup() takes a |
1da177e4c Linux-2.6.12-rc2 |
1033 1034 1035 |
* reference against the pages in @pvec. * * The search returns a group of mapping-contiguous pages with ascending |
d72dc8a25 mm: make pagevec_... |
1036 1037 |
* indexes. There may be holes in the indices due to not-present pages. We * also update @start to index the next page for the traversal. |
1da177e4c Linux-2.6.12-rc2 |
1038 |
* |
b947cee4b mm: implement fin... |
1039 |
* pagevec_lookup_range() returns the number of pages which were found. If this |
e02a9f048 mm/swap.c: make f... |
1040 |
* number is smaller than PAGEVEC_SIZE, the end of specified range has been |
b947cee4b mm: implement fin... |
1041 |
* reached. |
1da177e4c Linux-2.6.12-rc2 |
1042 |
*/ |
b947cee4b mm: implement fin... |
1043 |
unsigned pagevec_lookup_range(struct pagevec *pvec, |
397162ffa mm: remove nr_pag... |
1044 |
struct address_space *mapping, pgoff_t *start, pgoff_t end) |
1da177e4c Linux-2.6.12-rc2 |
1045 |
{ |
397162ffa mm: remove nr_pag... |
1046 |
pvec->nr = find_get_pages_range(mapping, start, end, PAGEVEC_SIZE, |
b947cee4b mm: implement fin... |
1047 |
pvec->pages); |
1da177e4c Linux-2.6.12-rc2 |
1048 1049 |
return pagevec_count(pvec); } |
b947cee4b mm: implement fin... |
1050 |
EXPORT_SYMBOL(pagevec_lookup_range); |
78539fdfa [XFS] Export page... |
1051 |
|
72b045aec mm: implement fin... |
1052 1053 |
unsigned pagevec_lookup_range_tag(struct pagevec *pvec, struct address_space *mapping, pgoff_t *index, pgoff_t end, |
10bbd2358 pagevec: Use xa_m... |
1054 |
xa_mark_t tag) |
1da177e4c Linux-2.6.12-rc2 |
1055 |
{ |
72b045aec mm: implement fin... |
1056 |
pvec->nr = find_get_pages_range_tag(mapping, index, end, tag, |
67fd707f4 mm: remove nr_pag... |
1057 |
PAGEVEC_SIZE, pvec->pages); |
1da177e4c Linux-2.6.12-rc2 |
1058 1059 |
return pagevec_count(pvec); } |
72b045aec mm: implement fin... |
1060 |
EXPORT_SYMBOL(pagevec_lookup_range_tag); |
1da177e4c Linux-2.6.12-rc2 |
1061 |
|
1da177e4c Linux-2.6.12-rc2 |
1062 1063 1064 1065 1066 |
/* * Perform any setup for the swap system */ void __init swap_setup(void) { |
ca79b0c21 mm: convert total... |
1067 |
unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT); |
e0bf68dde mm: bdi init hooks |
1068 |
|
1da177e4c Linux-2.6.12-rc2 |
1069 1070 1071 1072 1073 1074 1075 1076 1077 |
/* Use a smaller cluster for small-memory machines */ if (megs < 16) page_cluster = 2; else page_cluster = 3; /* * Right now other parts of the system means that we * _really_ don't want to cluster much more */ |
1da177e4c Linux-2.6.12-rc2 |
1078 |
} |
07d802699 mm: devmap: refac... |
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 |
#ifdef CONFIG_DEV_PAGEMAP_OPS void put_devmap_managed_page(struct page *page) { int count; if (WARN_ON_ONCE(!page_is_devmap_managed(page))) return; count = page_ref_dec_return(page); /* * devmap page refcounts are 1-based, rather than 0-based: if * refcount is 1, then the page is free and the refcount is * stable because nobody holds a reference on the page. */ if (count == 1) free_devmap_managed_page(page); else if (!count) __put_page(page); } EXPORT_SYMBOL(put_devmap_managed_page); #endif |