Blame view
mm/swap_cgroup.c
5.21 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
5d1ea48bd mm: page_cgroup: ... |
2 |
#include <linux/swap_cgroup.h> |
4c8210427 mm: page_cgroup n... |
3 |
#include <linux/vmalloc.h> |
5d1ea48bd mm: page_cgroup: ... |
4 |
#include <linux/mm.h> |
27a7faa07 memcg: swap cgrou... |
5 |
|
5d1ea48bd mm: page_cgroup: ... |
6 |
#include <linux/swapops.h> /* depends on mm.h include */ |
27a7faa07 memcg: swap cgrou... |
7 8 9 10 11 |
static DEFINE_MUTEX(swap_cgroup_mutex); struct swap_cgroup_ctrl { struct page **map; unsigned long length; |
e9e58a4ec memcg: avoid use ... |
12 |
spinlock_t lock; |
27a7faa07 memcg: swap cgrou... |
13 |
}; |
61600f578 mm/page_cgroup.c:... |
14 |
static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; |
27a7faa07 memcg: swap cgrou... |
15 |
|
27a7faa07 memcg: swap cgrou... |
16 |
struct swap_cgroup { |
a3b2d6926 cgroups: use css ... |
17 |
unsigned short id; |
27a7faa07 memcg: swap cgrou... |
18 19 |
}; #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup)) |
27a7faa07 memcg: swap cgrou... |
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
/* * SwapCgroup implements "lookup" and "exchange" operations. * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge * against SwapCache. At swap_free(), this is accessed directly from swap. * * This means, * - we have no race in "exchange" when we're accessed via SwapCache because * SwapCache(and its swp_entry) is under lock. * - When called via swap_free(), there is no user of this entry and no race. * Then, we don't need lock around "exchange". * * TODO: we can push these buffers out to HIGHMEM. */ /* * allocate buffer for swap_cgroup. */ static int swap_cgroup_prepare(int type) { struct page *page; struct swap_cgroup_ctrl *ctrl; unsigned long idx, max; |
27a7faa07 memcg: swap cgrou... |
43 44 45 46 47 48 49 |
ctrl = &swap_cgroup_ctrl[type]; for (idx = 0; idx < ctrl->length; idx++) { page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) goto not_enough_page; ctrl->map[idx] = page; |
ef7076294 swap: cond_resche... |
50 51 52 |
if (!(idx % SWAP_CLUSTER_MAX)) cond_resched(); |
27a7faa07 memcg: swap cgrou... |
53 54 55 56 57 58 59 60 61 |
} return 0; not_enough_page: max = idx; for (idx = 0; idx < max; idx++) __free_page(ctrl->map[idx]); return -ENOMEM; } |
38d8b4e6b mm, THP, swap: de... |
62 63 64 65 66 67 68 69 70 71 |
static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl, pgoff_t offset) { struct page *mappage; struct swap_cgroup *sc; mappage = ctrl->map[offset / SC_PER_PAGE]; sc = page_address(mappage); return sc + offset % SC_PER_PAGE; } |
9fb4b7cc0 page_cgroup: add ... |
72 73 74 75 76 |
static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent, struct swap_cgroup_ctrl **ctrlp) { pgoff_t offset = swp_offset(ent); struct swap_cgroup_ctrl *ctrl; |
9fb4b7cc0 page_cgroup: add ... |
77 78 79 80 |
ctrl = &swap_cgroup_ctrl[swp_type(ent)]; if (ctrlp) *ctrlp = ctrl; |
38d8b4e6b mm, THP, swap: de... |
81 |
return __lookup_swap_cgroup(ctrl, offset); |
9fb4b7cc0 page_cgroup: add ... |
82 |
} |
27a7faa07 memcg: swap cgrou... |
83 |
/** |
024914477 memcg: move charg... |
84 |
* swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry. |
dad7557eb mm: fix kernel-do... |
85 |
* @ent: swap entry to be cmpxchged |
024914477 memcg: move charg... |
86 87 88 89 |
* @old: old id * @new: new id * * Returns old id at success, 0 at failure. |
25985edce Fix common misspe... |
90 |
* (There is no mem_cgroup using 0 as its id) |
024914477 memcg: move charg... |
91 92 93 94 |
*/ unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, unsigned short old, unsigned short new) { |
024914477 memcg: move charg... |
95 |
struct swap_cgroup_ctrl *ctrl; |
024914477 memcg: move charg... |
96 |
struct swap_cgroup *sc; |
e9e58a4ec memcg: avoid use ... |
97 98 |
unsigned long flags; unsigned short retval; |
024914477 memcg: move charg... |
99 |
|
9fb4b7cc0 page_cgroup: add ... |
100 |
sc = lookup_swap_cgroup(ent, &ctrl); |
024914477 memcg: move charg... |
101 |
|
e9e58a4ec memcg: avoid use ... |
102 103 104 105 |
spin_lock_irqsave(&ctrl->lock, flags); retval = sc->id; if (retval == old) sc->id = new; |
024914477 memcg: move charg... |
106 |
else |
e9e58a4ec memcg: avoid use ... |
107 108 109 |
retval = 0; spin_unlock_irqrestore(&ctrl->lock, flags); return retval; |
024914477 memcg: move charg... |
110 111 112 |
} /** |
38d8b4e6b mm, THP, swap: de... |
113 114 |
* swap_cgroup_record - record mem_cgroup for a set of swap entries * @ent: the first swap entry to be recorded into |
dad7557eb mm: fix kernel-do... |
115 |
* @id: mem_cgroup to be recorded |
38d8b4e6b mm, THP, swap: de... |
116 |
* @nr_ents: number of swap entries to be recorded |
27a7faa07 memcg: swap cgrou... |
117 |
* |
a3b2d6926 cgroups: use css ... |
118 119 |
* Returns old value at success, 0 at failure. * (Of course, old value can be 0.) |
27a7faa07 memcg: swap cgrou... |
120 |
*/ |
38d8b4e6b mm, THP, swap: de... |
121 122 |
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id, unsigned int nr_ents) |
27a7faa07 memcg: swap cgrou... |
123 |
{ |
27a7faa07 memcg: swap cgrou... |
124 |
struct swap_cgroup_ctrl *ctrl; |
27a7faa07 memcg: swap cgrou... |
125 |
struct swap_cgroup *sc; |
a3b2d6926 cgroups: use css ... |
126 |
unsigned short old; |
e9e58a4ec memcg: avoid use ... |
127 |
unsigned long flags; |
38d8b4e6b mm, THP, swap: de... |
128 129 |
pgoff_t offset = swp_offset(ent); pgoff_t end = offset + nr_ents; |
27a7faa07 memcg: swap cgrou... |
130 |
|
9fb4b7cc0 page_cgroup: add ... |
131 |
sc = lookup_swap_cgroup(ent, &ctrl); |
27a7faa07 memcg: swap cgrou... |
132 |
|
e9e58a4ec memcg: avoid use ... |
133 134 |
spin_lock_irqsave(&ctrl->lock, flags); old = sc->id; |
38d8b4e6b mm, THP, swap: de... |
135 136 137 138 139 140 141 142 143 144 145 |
for (;;) { VM_BUG_ON(sc->id != old); sc->id = id; offset++; if (offset == end) break; if (offset % SC_PER_PAGE) sc++; else sc = __lookup_swap_cgroup(ctrl, offset); } |
e9e58a4ec memcg: avoid use ... |
146 |
spin_unlock_irqrestore(&ctrl->lock, flags); |
27a7faa07 memcg: swap cgrou... |
147 148 149 150 151 |
return old; } /** |
9fb4b7cc0 page_cgroup: add ... |
152 |
* lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry |
27a7faa07 memcg: swap cgrou... |
153 154 |
* @ent: swap entry to be looked up. * |
b3ff8a2f9 cgroup: remove st... |
155 |
* Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) |
27a7faa07 memcg: swap cgrou... |
156 |
*/ |
9fb4b7cc0 page_cgroup: add ... |
157 |
unsigned short lookup_swap_cgroup_id(swp_entry_t ent) |
27a7faa07 memcg: swap cgrou... |
158 |
{ |
9fb4b7cc0 page_cgroup: add ... |
159 |
return lookup_swap_cgroup(ent, NULL)->id; |
27a7faa07 memcg: swap cgrou... |
160 161 162 163 164 165 166 167 168 169 170 |
} int swap_cgroup_swapon(int type, unsigned long max_pages) { void *array; unsigned long array_size; unsigned long length; struct swap_cgroup_ctrl *ctrl; if (!do_swap_account) return 0; |
33278f7f0 memcg: fix off-by... |
171 |
length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); |
27a7faa07 memcg: swap cgrou... |
172 |
array_size = length * sizeof(void *); |
8c1fec1ba mm: Convert vmall... |
173 |
array = vzalloc(array_size); |
27a7faa07 memcg: swap cgrou... |
174 175 |
if (!array) goto nomem; |
27a7faa07 memcg: swap cgrou... |
176 177 178 179 |
ctrl = &swap_cgroup_ctrl[type]; mutex_lock(&swap_cgroup_mutex); ctrl->length = length; ctrl->map = array; |
e9e58a4ec memcg: avoid use ... |
180 |
spin_lock_init(&ctrl->lock); |
27a7faa07 memcg: swap cgrou... |
181 182 183 184 |
if (swap_cgroup_prepare(type)) { /* memory shortage */ ctrl->map = NULL; ctrl->length = 0; |
27a7faa07 memcg: swap cgrou... |
185 |
mutex_unlock(&swap_cgroup_mutex); |
6a5b18d2b memcg: move page-... |
186 |
vfree(array); |
27a7faa07 memcg: swap cgrou... |
187 188 189 |
goto nomem; } mutex_unlock(&swap_cgroup_mutex); |
27a7faa07 memcg: swap cgrou... |
190 191 |
return 0; nomem: |
1170532bb mm: convert print... |
192 193 194 195 |
pr_info("couldn't allocate enough memory for swap_cgroup "); pr_info("swap_cgroup can be disabled by swapaccount=0 boot option "); |
27a7faa07 memcg: swap cgrou... |
196 197 198 199 200 |
return -ENOMEM; } void swap_cgroup_swapoff(int type) { |
6a5b18d2b memcg: move page-... |
201 202 |
struct page **map; unsigned long i, length; |
27a7faa07 memcg: swap cgrou... |
203 204 205 206 207 208 209 |
struct swap_cgroup_ctrl *ctrl; if (!do_swap_account) return; mutex_lock(&swap_cgroup_mutex); ctrl = &swap_cgroup_ctrl[type]; |
6a5b18d2b memcg: move page-... |
210 211 212 213 214 215 216 217 218 |
map = ctrl->map; length = ctrl->length; ctrl->map = NULL; ctrl->length = 0; mutex_unlock(&swap_cgroup_mutex); if (map) { for (i = 0; i < length; i++) { struct page *page = map[i]; |
27a7faa07 memcg: swap cgrou... |
219 220 |
if (page) __free_page(page); |
460bcec84 mm, swap_cgroup: ... |
221 222 |
if (!(i % SWAP_CLUSTER_MAX)) cond_resched(); |
27a7faa07 memcg: swap cgrou... |
223 |
} |
6a5b18d2b memcg: move page-... |
224 |
vfree(map); |
27a7faa07 memcg: swap cgrou... |
225 |
} |
27a7faa07 memcg: swap cgrou... |
226 |
} |