Blame view
mm/swap_cgroup.c
5.21 KB
b24413180
|
1 |
// SPDX-License-Identifier: GPL-2.0 |
5d1ea48bd
|
2 |
#include <linux/swap_cgroup.h> |
4c8210427
|
3 |
#include <linux/vmalloc.h> |
5d1ea48bd
|
4 |
#include <linux/mm.h> |
27a7faa07
|
5 |
|
5d1ea48bd
|
6 |
#include <linux/swapops.h> /* depends on mm.h include */ |
27a7faa07
|
7 8 9 10 11 |
static DEFINE_MUTEX(swap_cgroup_mutex); struct swap_cgroup_ctrl { struct page **map; unsigned long length; |
e9e58a4ec
|
12 |
spinlock_t lock; |
27a7faa07
|
13 |
}; |
61600f578
|
14 |
static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; |
27a7faa07
|
15 |
|
27a7faa07
|
16 |
struct swap_cgroup { |
a3b2d6926
|
17 |
unsigned short id; |
27a7faa07
|
18 19 |
}; #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup)) |
27a7faa07
|
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
/* * SwapCgroup implements "lookup" and "exchange" operations. * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge * against SwapCache. At swap_free(), this is accessed directly from swap. * * This means, * - we have no race in "exchange" when we're accessed via SwapCache because * SwapCache(and its swp_entry) is under lock. * - When called via swap_free(), there is no user of this entry and no race. * Then, we don't need lock around "exchange". * * TODO: we can push these buffers out to HIGHMEM. */ /* * allocate buffer for swap_cgroup. */ static int swap_cgroup_prepare(int type) { struct page *page; struct swap_cgroup_ctrl *ctrl; unsigned long idx, max; |
27a7faa07
|
43 44 45 46 47 48 49 |
ctrl = &swap_cgroup_ctrl[type]; for (idx = 0; idx < ctrl->length; idx++) { page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) goto not_enough_page; ctrl->map[idx] = page; |
ef7076294
|
50 51 52 |
if (!(idx % SWAP_CLUSTER_MAX)) cond_resched(); |
27a7faa07
|
53 54 55 56 57 58 59 60 61 |
} return 0; not_enough_page: max = idx; for (idx = 0; idx < max; idx++) __free_page(ctrl->map[idx]); return -ENOMEM; } |
38d8b4e6b
|
62 63 64 65 66 67 68 69 70 71 |
static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl, pgoff_t offset) { struct page *mappage; struct swap_cgroup *sc; mappage = ctrl->map[offset / SC_PER_PAGE]; sc = page_address(mappage); return sc + offset % SC_PER_PAGE; } |
9fb4b7cc0
|
72 73 74 75 76 |
static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent, struct swap_cgroup_ctrl **ctrlp) { pgoff_t offset = swp_offset(ent); struct swap_cgroup_ctrl *ctrl; |
9fb4b7cc0
|
77 78 79 80 |
ctrl = &swap_cgroup_ctrl[swp_type(ent)]; if (ctrlp) *ctrlp = ctrl; |
38d8b4e6b
|
81 |
return __lookup_swap_cgroup(ctrl, offset); |
9fb4b7cc0
|
82 |
} |
27a7faa07
|
83 |
/** |
024914477
|
84 |
* swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry. |
dad7557eb
|
85 |
* @ent: swap entry to be cmpxchged |
024914477
|
86 87 88 89 |
* @old: old id * @new: new id * * Returns old id at success, 0 at failure. |
25985edce
|
90 |
* (There is no mem_cgroup using 0 as its id) |
024914477
|
91 92 93 94 |
*/ unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, unsigned short old, unsigned short new) { |
024914477
|
95 |
struct swap_cgroup_ctrl *ctrl; |
024914477
|
96 |
struct swap_cgroup *sc; |
e9e58a4ec
|
97 98 |
unsigned long flags; unsigned short retval; |
024914477
|
99 |
|
9fb4b7cc0
|
100 |
sc = lookup_swap_cgroup(ent, &ctrl); |
024914477
|
101 |
|
e9e58a4ec
|
102 103 104 105 |
spin_lock_irqsave(&ctrl->lock, flags); retval = sc->id; if (retval == old) sc->id = new; |
024914477
|
106 |
else |
e9e58a4ec
|
107 108 109 |
retval = 0; spin_unlock_irqrestore(&ctrl->lock, flags); return retval; |
024914477
|
110 111 112 |
} /** |
38d8b4e6b
|
113 114 |
* swap_cgroup_record - record mem_cgroup for a set of swap entries * @ent: the first swap entry to be recorded into |
dad7557eb
|
115 |
* @id: mem_cgroup to be recorded |
38d8b4e6b
|
116 |
* @nr_ents: number of swap entries to be recorded |
27a7faa07
|
117 |
* |
a3b2d6926
|
118 119 |
* Returns old value at success, 0 at failure. * (Of course, old value can be 0.) |
27a7faa07
|
120 |
*/ |
38d8b4e6b
|
121 122 |
unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id, unsigned int nr_ents) |
27a7faa07
|
123 |
{ |
27a7faa07
|
124 |
struct swap_cgroup_ctrl *ctrl; |
27a7faa07
|
125 |
struct swap_cgroup *sc; |
a3b2d6926
|
126 |
unsigned short old; |
e9e58a4ec
|
127 |
unsigned long flags; |
38d8b4e6b
|
128 129 |
pgoff_t offset = swp_offset(ent); pgoff_t end = offset + nr_ents; |
27a7faa07
|
130 |
|
9fb4b7cc0
|
131 |
sc = lookup_swap_cgroup(ent, &ctrl); |
27a7faa07
|
132 |
|
e9e58a4ec
|
133 134 |
spin_lock_irqsave(&ctrl->lock, flags); old = sc->id; |
38d8b4e6b
|
135 136 137 138 139 140 141 142 143 144 145 |
for (;;) { VM_BUG_ON(sc->id != old); sc->id = id; offset++; if (offset == end) break; if (offset % SC_PER_PAGE) sc++; else sc = __lookup_swap_cgroup(ctrl, offset); } |
e9e58a4ec
|
146 |
spin_unlock_irqrestore(&ctrl->lock, flags); |
27a7faa07
|
147 148 149 150 151 |
return old; } /** |
9fb4b7cc0
|
152 |
* lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry |
27a7faa07
|
153 154 |
* @ent: swap entry to be looked up. * |
b3ff8a2f9
|
155 |
* Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) |
27a7faa07
|
156 |
*/ |
9fb4b7cc0
|
157 |
unsigned short lookup_swap_cgroup_id(swp_entry_t ent) |
27a7faa07
|
158 |
{ |
9fb4b7cc0
|
159 |
return lookup_swap_cgroup(ent, NULL)->id; |
27a7faa07
|
160 161 162 163 164 165 166 167 168 169 170 |
} int swap_cgroup_swapon(int type, unsigned long max_pages) { void *array; unsigned long array_size; unsigned long length; struct swap_cgroup_ctrl *ctrl; if (!do_swap_account) return 0; |
33278f7f0
|
171 |
length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); |
27a7faa07
|
172 |
array_size = length * sizeof(void *); |
8c1fec1ba
|
173 |
array = vzalloc(array_size); |
27a7faa07
|
174 175 |
if (!array) goto nomem; |
27a7faa07
|
176 177 178 179 |
ctrl = &swap_cgroup_ctrl[type]; mutex_lock(&swap_cgroup_mutex); ctrl->length = length; ctrl->map = array; |
e9e58a4ec
|
180 |
spin_lock_init(&ctrl->lock); |
27a7faa07
|
181 182 183 184 |
if (swap_cgroup_prepare(type)) { /* memory shortage */ ctrl->map = NULL; ctrl->length = 0; |
27a7faa07
|
185 |
mutex_unlock(&swap_cgroup_mutex); |
6a5b18d2b
|
186 |
vfree(array); |
27a7faa07
|
187 188 189 |
goto nomem; } mutex_unlock(&swap_cgroup_mutex); |
27a7faa07
|
190 191 |
return 0; nomem: |
1170532bb
|
192 193 194 195 |
pr_info("couldn't allocate enough memory for swap_cgroup "); pr_info("swap_cgroup can be disabled by swapaccount=0 boot option "); |
27a7faa07
|
196 197 198 199 200 |
return -ENOMEM; } void swap_cgroup_swapoff(int type) { |
6a5b18d2b
|
201 202 |
struct page **map; unsigned long i, length; |
27a7faa07
|
203 204 205 206 207 208 209 |
struct swap_cgroup_ctrl *ctrl; if (!do_swap_account) return; mutex_lock(&swap_cgroup_mutex); ctrl = &swap_cgroup_ctrl[type]; |
6a5b18d2b
|
210 211 212 213 214 215 216 217 218 |
map = ctrl->map; length = ctrl->length; ctrl->map = NULL; ctrl->length = 0; mutex_unlock(&swap_cgroup_mutex); if (map) { for (i = 0; i < length; i++) { struct page *page = map[i]; |
27a7faa07
|
219 220 |
if (page) __free_page(page); |
460bcec84
|
221 222 |
if (!(i % SWAP_CLUSTER_MAX)) cond_resched(); |
27a7faa07
|
223 |
} |
6a5b18d2b
|
224 |
vfree(map); |
27a7faa07
|
225 |
} |
27a7faa07
|
226 |
} |