Blame view
mm/percpu.c
56.3 KB
fbf59bc9d percpu: implement... |
1 |
/* |
88999a898 percpu: misc prep... |
2 |
* mm/percpu.c - percpu memory allocator |
fbf59bc9d percpu: implement... |
3 4 5 6 7 8 9 |
* * Copyright (C) 2009 SUSE Linux Products GmbH * Copyright (C) 2009 Tejun Heo <tj@kernel.org> * * This file is released under the GPLv2. * * This is percpu allocator which can handle both static and dynamic |
88999a898 percpu: misc prep... |
10 11 12 |
* areas. Percpu areas are allocated in chunks. Each chunk is * consisted of boot-time determined number of units and the first * chunk is used for static percpu variables in the kernel image |
2f39e637e percpu: allow non... |
13 14 15 |
* (special boot time alloc/init handling necessary as these areas * need to be brought up before allocation services are running). * Unit grows as necessary and all units grow or shrink in unison. |
88999a898 percpu: misc prep... |
16 |
* When a chunk is filled up, another chunk is allocated. |
fbf59bc9d percpu: implement... |
17 18 19 20 21 22 23 24 |
* * c0 c1 c2 * ------------------- ------------------- ------------ * | u0 | u1 | u2 | u3 | | u0 | u1 | u2 | u3 | | u0 | u1 | u * ------------------- ...... ------------------- .... ------------ * * Allocation is done in offset-size areas of single unit space. Ie, * an area of 512 bytes at 6k in c1 occupies 512 bytes at 6k of c1:u0, |
2f39e637e percpu: allow non... |
25 26 27 28 |
* c1:u1, c1:u2 and c1:u3. On UMA, units corresponds directly to * cpus. On NUMA, the mapping can be non-linear and even sparse. * Percpu access can be done by configuring percpu base registers * according to cpu to unit mapping and pcpu_unit_size. |
fbf59bc9d percpu: implement... |
29 |
* |
2f39e637e percpu: allow non... |
30 31 |
* There are usually many small percpu allocations many of them being * as small as 4 bytes. The allocator organizes chunks into lists |
fbf59bc9d percpu: implement... |
32 33 |
* according to free size and tries to allocate from the fullest one. * Each chunk keeps the maximum contiguous area size hint which is |
4785879e4 fix a typo on com... |
34 |
* guaranteed to be equal to or larger than the maximum contiguous |
fbf59bc9d percpu: implement... |
35 36 37 38 39 40 41 42 |
* area in the chunk. This helps the allocator not to iterate the * chunk maps unnecessarily. * * Allocation state in each chunk is kept using an array of integers * on chunk->map. A positive value in the map represents a free * region and negative allocated. Allocation inside a chunk is done * by scanning this map sequentially and serving the first matching * entry. This is mostly copied from the percpu_modalloc() allocator. |
e1b9aa3f4 percpu: remove rb... |
43 44 |
* Chunks can be determined from the address using the index field * in the page struct. The index field contains a pointer to the chunk. |
fbf59bc9d percpu: implement... |
45 46 47 |
* * To use this allocator, arch code should do the followings. * |
fbf59bc9d percpu: implement... |
48 |
* - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate |
e01009833 percpu: make x86 ... |
49 50 |
* regular address to percpu pointer and back if they need to be * different from the default |
fbf59bc9d percpu: implement... |
51 |
* |
8d408b4be percpu: give more... |
52 53 |
* - use pcpu_setup_first_chunk() during percpu area initialization to * setup the first chunk containing the kernel static percpu area |
fbf59bc9d percpu: implement... |
54 55 56 57 |
*/ #include <linux/bitmap.h> #include <linux/bootmem.h> |
fd1e8a1fe percpu: introduce... |
58 |
#include <linux/err.h> |
fbf59bc9d percpu: implement... |
59 |
#include <linux/list.h> |
a530b7958 percpu: teach lar... |
60 |
#include <linux/log2.h> |
fbf59bc9d percpu: implement... |
61 62 63 64 65 |
#include <linux/mm.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/percpu.h> #include <linux/pfn.h> |
fbf59bc9d percpu: implement... |
66 |
#include <linux/slab.h> |
ccea34b5d percpu: finer gra... |
67 |
#include <linux/spinlock.h> |
fbf59bc9d percpu: implement... |
68 |
#include <linux/vmalloc.h> |
a56dbddf0 percpu: move full... |
69 |
#include <linux/workqueue.h> |
fbf59bc9d percpu: implement... |
70 71 |
#include <asm/cacheflush.h> |
e01009833 percpu: make x86 ... |
72 |
#include <asm/sections.h> |
fbf59bc9d percpu: implement... |
73 |
#include <asm/tlbflush.h> |
3b034b0d0 percpu: Fix kdump... |
74 |
#include <asm/io.h> |
fbf59bc9d percpu: implement... |
75 |
|
fbf59bc9d percpu: implement... |
76 77 |
#define PCPU_SLOT_BASE_SHIFT 5 /* 1-31 shares the same slot */ #define PCPU_DFL_MAP_ALLOC 16 /* start a map with 16 ents */ |
bbddff054 percpu: use percp... |
78 |
#ifdef CONFIG_SMP |
e01009833 percpu: make x86 ... |
79 80 81 |
/* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */ #ifndef __addr_to_pcpu_ptr #define __addr_to_pcpu_ptr(addr) \ |
43cf38eb5 percpu: add __per... |
82 83 84 |
(void __percpu *)((unsigned long)(addr) - \ (unsigned long)pcpu_base_addr + \ (unsigned long)__per_cpu_start) |
e01009833 percpu: make x86 ... |
85 86 87 |
#endif #ifndef __pcpu_ptr_to_addr #define __pcpu_ptr_to_addr(ptr) \ |
43cf38eb5 percpu: add __per... |
88 89 90 |
(void __force *)((unsigned long)(ptr) + \ (unsigned long)pcpu_base_addr - \ (unsigned long)__per_cpu_start) |
e01009833 percpu: make x86 ... |
91 |
#endif |
bbddff054 percpu: use percp... |
92 93 94 95 96 |
#else /* CONFIG_SMP */ /* on UP, it's always identity mapped */ #define __addr_to_pcpu_ptr(addr) (void __percpu *)(addr) #define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr) #endif /* CONFIG_SMP */ |
e01009833 percpu: make x86 ... |
97 |
|
fbf59bc9d percpu: implement... |
98 99 |
struct pcpu_chunk { struct list_head list; /* linked to pcpu_slot lists */ |
fbf59bc9d percpu: implement... |
100 101 |
int free_size; /* free bytes in the chunk */ int contig_hint; /* max contiguous size hint */ |
bba174f5e percpu: add chunk... |
102 |
void *base_addr; /* base address of this chunk */ |
fbf59bc9d percpu: implement... |
103 104 105 |
int map_used; /* # of map entries used */ int map_alloc; /* # of map entries allocated */ int *map; /* allocation map */ |
88999a898 percpu: misc prep... |
106 |
void *data; /* chunk data */ |
8d408b4be percpu: give more... |
107 |
bool immutable; /* no [de]population allowed */ |
ce3141a27 percpu: drop pcpu... |
108 |
unsigned long populated[]; /* populated bitmap */ |
fbf59bc9d percpu: implement... |
109 |
}; |
40150d37b percpu: add __rea... |
110 111 |
static int pcpu_unit_pages __read_mostly; static int pcpu_unit_size __read_mostly; |
2f39e637e percpu: allow non... |
112 |
static int pcpu_nr_units __read_mostly; |
6563297ce percpu: use group... |
113 |
static int pcpu_atom_size __read_mostly; |
40150d37b percpu: add __rea... |
114 115 |
static int pcpu_nr_slots __read_mostly; static size_t pcpu_chunk_struct_size __read_mostly; |
fbf59bc9d percpu: implement... |
116 |
|
a855b84c3 percpu: fix chunk... |
117 118 119 |
/* cpus with the lowest and highest unit addresses */ static unsigned int pcpu_low_unit_cpu __read_mostly; static unsigned int pcpu_high_unit_cpu __read_mostly; |
2f39e637e percpu: allow non... |
120 |
|
fbf59bc9d percpu: implement... |
121 |
/* the address of the first chunk which starts with the kernel static area */ |
40150d37b percpu: add __rea... |
122 |
void *pcpu_base_addr __read_mostly; |
fbf59bc9d percpu: implement... |
123 |
EXPORT_SYMBOL_GPL(pcpu_base_addr); |
fb435d523 percpu: add pcpu_... |
124 125 |
static const int *pcpu_unit_map __read_mostly; /* cpu -> unit */ const unsigned long *pcpu_unit_offsets __read_mostly; /* cpu -> unit offset */ |
2f39e637e percpu: allow non... |
126 |
|
6563297ce percpu: use group... |
127 128 129 130 |
/* group information, used for vm allocation */ static int pcpu_nr_groups __read_mostly; static const unsigned long *pcpu_group_offsets __read_mostly; static const size_t *pcpu_group_sizes __read_mostly; |
ae9e6bc9f percpu: don't put... |
131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
/* * The first chunk which always exists. Note that unlike other * chunks, this one can be allocated and mapped in several different * ways and thus often doesn't live in the vmalloc area. */ static struct pcpu_chunk *pcpu_first_chunk; /* * Optional reserved chunk. This chunk reserves part of the first * chunk and serves it for reserved allocations. The amount of * reserved offset is in pcpu_reserved_chunk_limit. When reserved * area doesn't exist, the following variables contain NULL and 0 * respectively. */ |
edcb46399 percpu, module: i... |
145 |
static struct pcpu_chunk *pcpu_reserved_chunk; |
edcb46399 percpu, module: i... |
146 |
static int pcpu_reserved_chunk_limit; |
fbf59bc9d percpu: implement... |
147 |
/* |
ccea34b5d percpu: finer gra... |
148 149 150 |
* Synchronization rules. * * There are two locks - pcpu_alloc_mutex and pcpu_lock. The former |
ce3141a27 percpu: drop pcpu... |
151 152 153 |
* protects allocation/reclaim paths, chunks, populated bitmap and * vmalloc mapping. The latter is a spinlock and protects the index * data structures - chunk slots, chunks and area maps in chunks. |
ccea34b5d percpu: finer gra... |
154 155 156 |
* * During allocation, pcpu_alloc_mutex is kept locked all the time and * pcpu_lock is grabbed and released as necessary. All actual memory |
403a91b16 percpu: allow pcp... |
157 158 159 160 |
* allocations are done using GFP_KERNEL with pcpu_lock released. In * general, percpu memory can't be allocated with irq off but * irqsave/restore are still used in alloc path so that it can be used * from early init path - sched_init() specifically. |
ccea34b5d percpu: finer gra... |
161 162 163 164 165 166 167 168 169 |
* * Free path accesses and alters only the index data structures, so it * can be safely called from atomic context. When memory needs to be * returned to the system, free path schedules reclaim_work which * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be * reclaimed, release both locks and frees the chunks. Note that it's * necessary to grab both locks to remove a chunk from circulation as * allocation path might be referencing the chunk with only * pcpu_alloc_mutex locked. |
fbf59bc9d percpu: implement... |
170 |
*/ |
ccea34b5d percpu: finer gra... |
171 172 |
static DEFINE_MUTEX(pcpu_alloc_mutex); /* protects whole alloc and reclaim */ static DEFINE_SPINLOCK(pcpu_lock); /* protects index data structures */ |
fbf59bc9d percpu: implement... |
173 |
|
40150d37b percpu: add __rea... |
174 |
static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */ |
fbf59bc9d percpu: implement... |
175 |
|
a56dbddf0 percpu: move full... |
176 177 178 |
/* reclaim work to release fully free chunks, scheduled from free path */ static void pcpu_reclaim(struct work_struct *work); static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim); |
020ec6537 percpu: factor ou... |
179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
static bool pcpu_addr_in_first_chunk(void *addr) { void *first_start = pcpu_first_chunk->base_addr; return addr >= first_start && addr < first_start + pcpu_unit_size; } static bool pcpu_addr_in_reserved_chunk(void *addr) { void *first_start = pcpu_first_chunk->base_addr; return addr >= first_start && addr < first_start + pcpu_reserved_chunk_limit; } |
d9b55eeb1 percpu: remove un... |
193 |
static int __pcpu_size_to_slot(int size) |
fbf59bc9d percpu: implement... |
194 |
{ |
cae3aeb83 percpu: clean up ... |
195 |
int highbit = fls(size); /* size is in bytes */ |
fbf59bc9d percpu: implement... |
196 197 |
return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1); } |
d9b55eeb1 percpu: remove un... |
198 199 200 201 202 203 |
static int pcpu_size_to_slot(int size) { if (size == pcpu_unit_size) return pcpu_nr_slots - 1; return __pcpu_size_to_slot(size); } |
fbf59bc9d percpu: implement... |
204 205 206 207 208 209 210 |
static int pcpu_chunk_slot(const struct pcpu_chunk *chunk) { if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int)) return 0; return pcpu_size_to_slot(chunk->free_size); } |
88999a898 percpu: misc prep... |
211 212 213 214 215 216 217 218 219 220 221 222 223 |
/* set the pointer to a chunk in a page struct */ static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu) { page->index = (unsigned long)pcpu; } /* obtain pointer to a chunk from a page struct */ static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page) { return (struct pcpu_chunk *)page->index; } static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx) |
fbf59bc9d percpu: implement... |
224 |
{ |
2f39e637e percpu: allow non... |
225 |
return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx; |
fbf59bc9d percpu: implement... |
226 |
} |
9983b6f0c percpu: fix first... |
227 228 |
static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, unsigned int cpu, int page_idx) |
fbf59bc9d percpu: implement... |
229 |
{ |
bba174f5e percpu: add chunk... |
230 |
return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] + |
fb435d523 percpu: add pcpu_... |
231 |
(page_idx << PAGE_SHIFT); |
fbf59bc9d percpu: implement... |
232 |
} |
88999a898 percpu: misc prep... |
233 234 |
static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk, int *rs, int *re, int end) |
ce3141a27 percpu: drop pcpu... |
235 236 237 238 |
{ *rs = find_next_zero_bit(chunk->populated, end, *rs); *re = find_next_bit(chunk->populated, end, *rs + 1); } |
88999a898 percpu: misc prep... |
239 240 |
static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk, int *rs, int *re, int end) |
ce3141a27 percpu: drop pcpu... |
241 242 243 244 245 246 247 |
{ *rs = find_next_bit(chunk->populated, end, *rs); *re = find_next_zero_bit(chunk->populated, end, *rs + 1); } /* * (Un)populated page region iterators. Iterate over (un)populated |
b595076a1 tree-wide: fix co... |
248 |
* page regions between @start and @end in @chunk. @rs and @re should |
ce3141a27 percpu: drop pcpu... |
249 250 251 252 253 254 255 256 257 258 259 260 |
* be integer variables and will be set to start and end page index of * the current region. */ #define pcpu_for_each_unpop_region(chunk, rs, re, start, end) \ for ((rs) = (start), pcpu_next_unpop((chunk), &(rs), &(re), (end)); \ (rs) < (re); \ (rs) = (re) + 1, pcpu_next_unpop((chunk), &(rs), &(re), (end))) #define pcpu_for_each_pop_region(chunk, rs, re, start, end) \ for ((rs) = (start), pcpu_next_pop((chunk), &(rs), &(re), (end)); \ (rs) < (re); \ (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end))) |
fbf59bc9d percpu: implement... |
261 |
/** |
90459ce06 percpu: rename pc... |
262 |
* pcpu_mem_zalloc - allocate memory |
1880d93b8 percpu: replace p... |
263 |
* @size: bytes to allocate |
fbf59bc9d percpu: implement... |
264 |
* |
1880d93b8 percpu: replace p... |
265 |
* Allocate @size bytes. If @size is smaller than PAGE_SIZE, |
90459ce06 percpu: rename pc... |
266 |
* kzalloc() is used; otherwise, vzalloc() is used. The returned |
1880d93b8 percpu: replace p... |
267 |
* memory is always zeroed. |
fbf59bc9d percpu: implement... |
268 |
* |
ccea34b5d percpu: finer gra... |
269 270 271 |
* CONTEXT: * Does GFP_KERNEL allocation. * |
fbf59bc9d percpu: implement... |
272 |
* RETURNS: |
1880d93b8 percpu: replace p... |
273 |
* Pointer to the allocated area on success, NULL on failure. |
fbf59bc9d percpu: implement... |
274 |
*/ |
90459ce06 percpu: rename pc... |
275 |
static void *pcpu_mem_zalloc(size_t size) |
fbf59bc9d percpu: implement... |
276 |
{ |
099a19d91 percpu: allow lim... |
277 278 |
if (WARN_ON_ONCE(!slab_is_available())) return NULL; |
1880d93b8 percpu: replace p... |
279 280 |
if (size <= PAGE_SIZE) return kzalloc(size, GFP_KERNEL); |
7af4c0932 percpu: zero memo... |
281 282 |
else return vzalloc(size); |
1880d93b8 percpu: replace p... |
283 |
} |
fbf59bc9d percpu: implement... |
284 |
|
1880d93b8 percpu: replace p... |
285 286 287 288 289 |
/** * pcpu_mem_free - free memory * @ptr: memory to free * @size: size of the area * |
90459ce06 percpu: rename pc... |
290 |
* Free @ptr. @ptr should have been allocated using pcpu_mem_zalloc(). |
1880d93b8 percpu: replace p... |
291 292 293 |
*/ static void pcpu_mem_free(void *ptr, size_t size) { |
fbf59bc9d percpu: implement... |
294 |
if (size <= PAGE_SIZE) |
1880d93b8 percpu: replace p... |
295 |
kfree(ptr); |
fbf59bc9d percpu: implement... |
296 |
else |
1880d93b8 percpu: replace p... |
297 |
vfree(ptr); |
fbf59bc9d percpu: implement... |
298 299 300 301 302 303 304 305 306 |
} /** * pcpu_chunk_relocate - put chunk in the appropriate chunk slot * @chunk: chunk of interest * @oslot: the previous slot it was on * * This function is called after an allocation or free changed @chunk. * New slot according to the changed state is determined and @chunk is |
edcb46399 percpu, module: i... |
307 308 |
* moved to the slot. Note that the reserved chunk is never put on * chunk slots. |
ccea34b5d percpu: finer gra... |
309 310 311 |
* * CONTEXT: * pcpu_lock. |
fbf59bc9d percpu: implement... |
312 313 314 315 |
*/ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot) { int nslot = pcpu_chunk_slot(chunk); |
edcb46399 percpu, module: i... |
316 |
if (chunk != pcpu_reserved_chunk && oslot != nslot) { |
fbf59bc9d percpu: implement... |
317 318 319 320 321 322 |
if (oslot < nslot) list_move(&chunk->list, &pcpu_slot[nslot]); else list_move_tail(&chunk->list, &pcpu_slot[nslot]); } } |
fbf59bc9d percpu: implement... |
323 |
/** |
833af8427 percpu: restructu... |
324 325 |
* pcpu_need_to_extend - determine whether chunk area map needs to be extended * @chunk: chunk of interest |
9f7dcf224 percpu: move chun... |
326 |
* |
833af8427 percpu: restructu... |
327 |
* Determine whether area map of @chunk needs to be extended to |
25985edce Fix common misspe... |
328 |
* accommodate a new allocation. |
9f7dcf224 percpu: move chun... |
329 |
* |
ccea34b5d percpu: finer gra... |
330 |
* CONTEXT: |
833af8427 percpu: restructu... |
331 |
* pcpu_lock. |
ccea34b5d percpu: finer gra... |
332 |
* |
9f7dcf224 percpu: move chun... |
333 |
* RETURNS: |
833af8427 percpu: restructu... |
334 335 |
* New target map allocation length if extension is necessary, 0 * otherwise. |
9f7dcf224 percpu: move chun... |
336 |
*/ |
833af8427 percpu: restructu... |
337 |
static int pcpu_need_to_extend(struct pcpu_chunk *chunk) |
9f7dcf224 percpu: move chun... |
338 339 |
{ int new_alloc; |
9f7dcf224 percpu: move chun... |
340 |
|
9f7dcf224 percpu: move chun... |
341 342 343 344 345 346 |
if (chunk->map_alloc >= chunk->map_used + 2) return 0; new_alloc = PCPU_DFL_MAP_ALLOC; while (new_alloc < chunk->map_used + 2) new_alloc *= 2; |
833af8427 percpu: restructu... |
347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 |
return new_alloc; } /** * pcpu_extend_area_map - extend area map of a chunk * @chunk: chunk of interest * @new_alloc: new target allocation length of the area map * * Extend area map of @chunk to have @new_alloc entries. * * CONTEXT: * Does GFP_KERNEL allocation. Grabs and releases pcpu_lock. * * RETURNS: * 0 on success, -errno on failure. */ static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc) { int *old = NULL, *new = NULL; size_t old_size = 0, new_size = new_alloc * sizeof(new[0]); unsigned long flags; |
90459ce06 percpu: rename pc... |
368 |
new = pcpu_mem_zalloc(new_size); |
833af8427 percpu: restructu... |
369 |
if (!new) |
9f7dcf224 percpu: move chun... |
370 |
return -ENOMEM; |
ccea34b5d percpu: finer gra... |
371 |
|
833af8427 percpu: restructu... |
372 373 374 375 376 |
/* acquire pcpu_lock and switch to new area map */ spin_lock_irqsave(&pcpu_lock, flags); if (new_alloc <= chunk->map_alloc) goto out_unlock; |
9f7dcf224 percpu: move chun... |
377 |
|
833af8427 percpu: restructu... |
378 |
old_size = chunk->map_alloc * sizeof(chunk->map[0]); |
a002d1484 percpu: fix a mem... |
379 380 381 |
old = chunk->map; memcpy(new, old, old_size); |
9f7dcf224 percpu: move chun... |
382 |
|
9f7dcf224 percpu: move chun... |
383 384 |
chunk->map_alloc = new_alloc; chunk->map = new; |
833af8427 percpu: restructu... |
385 386 387 388 389 390 391 392 393 394 395 |
new = NULL; out_unlock: spin_unlock_irqrestore(&pcpu_lock, flags); /* * pcpu_mem_free() might end up calling vfree() which uses * IRQ-unsafe lock and thus can't be called under pcpu_lock. */ pcpu_mem_free(old, old_size); pcpu_mem_free(new, new_size); |
9f7dcf224 percpu: move chun... |
396 397 398 399 |
return 0; } /** |
fbf59bc9d percpu: implement... |
400 401 402 |
* pcpu_split_block - split a map block * @chunk: chunk of interest * @i: index of map block to split |
cae3aeb83 percpu: clean up ... |
403 404 |
* @head: head size in bytes (can be 0) * @tail: tail size in bytes (can be 0) |
fbf59bc9d percpu: implement... |
405 406 407 408 409 410 411 412 413 |
* * Split the @i'th map block into two or three blocks. If @head is * non-zero, @head bytes block is inserted before block @i moving it * to @i+1 and reducing its size by @head bytes. * * If @tail is non-zero, the target block, which can be @i or @i+1 * depending on @head, is reduced by @tail bytes and @tail byte block * is inserted after the target block. * |
25985edce Fix common misspe... |
414 |
* @chunk->map must have enough free slots to accommodate the split. |
ccea34b5d percpu: finer gra... |
415 416 417 |
* * CONTEXT: * pcpu_lock. |
fbf59bc9d percpu: implement... |
418 |
*/ |
9f7dcf224 percpu: move chun... |
419 420 |
static void pcpu_split_block(struct pcpu_chunk *chunk, int i, int head, int tail) |
fbf59bc9d percpu: implement... |
421 422 |
{ int nr_extra = !!head + !!tail; |
1880d93b8 percpu: replace p... |
423 |
|
9f7dcf224 percpu: move chun... |
424 |
BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra); |
fbf59bc9d percpu: implement... |
425 |
|
9f7dcf224 percpu: move chun... |
426 |
/* insert new subblocks */ |
fbf59bc9d percpu: implement... |
427 428 429 430 431 432 433 434 435 436 437 438 |
memmove(&chunk->map[i + nr_extra], &chunk->map[i], sizeof(chunk->map[0]) * (chunk->map_used - i)); chunk->map_used += nr_extra; if (head) { chunk->map[i + 1] = chunk->map[i] - head; chunk->map[i++] = head; } if (tail) { chunk->map[i++] -= tail; chunk->map[i] = tail; } |
fbf59bc9d percpu: implement... |
439 440 441 442 443 |
} /** * pcpu_alloc_area - allocate area from a pcpu_chunk * @chunk: chunk of interest |
cae3aeb83 percpu: clean up ... |
444 |
* @size: wanted size in bytes |
fbf59bc9d percpu: implement... |
445 446 447 448 449 450 |
* @align: wanted align * * Try to allocate @size bytes area aligned at @align from @chunk. * Note that this function only allocates the offset. It doesn't * populate or map the area. * |
9f7dcf224 percpu: move chun... |
451 452 |
* @chunk->map must have at least two free slots. * |
ccea34b5d percpu: finer gra... |
453 454 455 |
* CONTEXT: * pcpu_lock. * |
fbf59bc9d percpu: implement... |
456 |
* RETURNS: |
9f7dcf224 percpu: move chun... |
457 458 |
* Allocated offset in @chunk on success, -1 if no matching area is * found. |
fbf59bc9d percpu: implement... |
459 460 461 462 463 464 |
*/ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align) { int oslot = pcpu_chunk_slot(chunk); int max_contig = 0; int i, off; |
fbf59bc9d percpu: implement... |
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 |
for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) { bool is_last = i + 1 == chunk->map_used; int head, tail; /* extra for alignment requirement */ head = ALIGN(off, align) - off; BUG_ON(i == 0 && head != 0); if (chunk->map[i] < 0) continue; if (chunk->map[i] < head + size) { max_contig = max(chunk->map[i], max_contig); continue; } /* * If head is small or the previous block is free, * merge'em. Note that 'small' is defined as smaller * than sizeof(int), which is very small but isn't too * uncommon for percpu allocations. */ if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) { if (chunk->map[i - 1] > 0) chunk->map[i - 1] += head; else { chunk->map[i - 1] -= head; chunk->free_size -= head; } chunk->map[i] -= head; off += head; head = 0; } /* if tail is small, just keep it around */ tail = chunk->map[i] - head - size; if (tail < sizeof(int)) tail = 0; /* split if warranted */ if (head || tail) { |
9f7dcf224 percpu: move chun... |
505 |
pcpu_split_block(chunk, i, head, tail); |
fbf59bc9d percpu: implement... |
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 |
if (head) { i++; off += head; max_contig = max(chunk->map[i - 1], max_contig); } if (tail) max_contig = max(chunk->map[i + 1], max_contig); } /* update hint and mark allocated */ if (is_last) chunk->contig_hint = max_contig; /* fully scanned */ else chunk->contig_hint = max(chunk->contig_hint, max_contig); chunk->free_size -= chunk->map[i]; chunk->map[i] = -chunk->map[i]; pcpu_chunk_relocate(chunk, oslot); return off; } chunk->contig_hint = max_contig; /* fully scanned */ pcpu_chunk_relocate(chunk, oslot); |
9f7dcf224 percpu: move chun... |
531 532 |
/* tell the upper layer that this chunk has no matching area */ return -1; |
fbf59bc9d percpu: implement... |
533 534 535 536 537 538 539 540 541 542 |
} /** * pcpu_free_area - free area to a pcpu_chunk * @chunk: chunk of interest * @freeme: offset of area to free * * Free area starting from @freeme to @chunk. Note that this function * only modifies the allocation map. It doesn't depopulate or unmap * the area. |
ccea34b5d percpu: finer gra... |
543 544 545 |
* * CONTEXT: * pcpu_lock. |
fbf59bc9d percpu: implement... |
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 |
*/ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme) { int oslot = pcpu_chunk_slot(chunk); int i, off; for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) if (off == freeme) break; BUG_ON(off != freeme); BUG_ON(chunk->map[i] > 0); chunk->map[i] = -chunk->map[i]; chunk->free_size += chunk->map[i]; /* merge with previous? */ if (i > 0 && chunk->map[i - 1] >= 0) { chunk->map[i - 1] += chunk->map[i]; chunk->map_used--; memmove(&chunk->map[i], &chunk->map[i + 1], (chunk->map_used - i) * sizeof(chunk->map[0])); i--; } /* merge with next? */ if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) { chunk->map[i] += chunk->map[i + 1]; chunk->map_used--; memmove(&chunk->map[i + 1], &chunk->map[i + 2], (chunk->map_used - (i + 1)) * sizeof(chunk->map[0])); } chunk->contig_hint = max(chunk->map[i], chunk->contig_hint); pcpu_chunk_relocate(chunk, oslot); } |
6081089fd percpu: reorganiz... |
580 581 582 |
static struct pcpu_chunk *pcpu_alloc_chunk(void) { struct pcpu_chunk *chunk; |
90459ce06 percpu: rename pc... |
583 |
chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size); |
6081089fd percpu: reorganiz... |
584 585 |
if (!chunk) return NULL; |
90459ce06 percpu: rename pc... |
586 587 |
chunk->map = pcpu_mem_zalloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0])); |
6081089fd percpu: reorganiz... |
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 |
if (!chunk->map) { kfree(chunk); return NULL; } chunk->map_alloc = PCPU_DFL_MAP_ALLOC; chunk->map[chunk->map_used++] = pcpu_unit_size; INIT_LIST_HEAD(&chunk->list); chunk->free_size = pcpu_unit_size; chunk->contig_hint = pcpu_unit_size; return chunk; } static void pcpu_free_chunk(struct pcpu_chunk *chunk) { if (!chunk) return; pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0])); kfree(chunk); } |
9f6455325 percpu: move vmal... |
610 611 612 613 614 615 616 617 618 619 620 621 622 623 |
/* * Chunk management implementation. * * To allow different implementations, chunk alloc/free and * [de]population are implemented in a separate file which is pulled * into this file and compiled together. The following functions * should be implemented. * * pcpu_populate_chunk - populate the specified range of a chunk * pcpu_depopulate_chunk - depopulate the specified range of a chunk * pcpu_create_chunk - create a new chunk * pcpu_destroy_chunk - destroy a chunk, always preceded by full depop * pcpu_addr_to_page - translate address to physical address * pcpu_verify_alloc_info - check alloc_info is acceptable during init |
fbf59bc9d percpu: implement... |
624 |
*/ |
9f6455325 percpu: move vmal... |
625 626 627 628 629 630 |
static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size); static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size); static struct pcpu_chunk *pcpu_create_chunk(void); static void pcpu_destroy_chunk(struct pcpu_chunk *chunk); static struct page *pcpu_addr_to_page(void *addr); static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai); |
fbf59bc9d percpu: implement... |
631 |
|
b0c9778b1 percpu: implement... |
632 633 634 |
#ifdef CONFIG_NEED_PER_CPU_KM #include "percpu-km.c" #else |
9f6455325 percpu: move vmal... |
635 |
#include "percpu-vm.c" |
b0c9778b1 percpu: implement... |
636 |
#endif |
fbf59bc9d percpu: implement... |
637 638 |
/** |
88999a898 percpu: misc prep... |
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 |
* pcpu_chunk_addr_search - determine chunk containing specified address * @addr: address for which the chunk needs to be determined. * * RETURNS: * The address of the found chunk. */ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr) { /* is it in the first chunk? */ if (pcpu_addr_in_first_chunk(addr)) { /* is it in the reserved area? */ if (pcpu_addr_in_reserved_chunk(addr)) return pcpu_reserved_chunk; return pcpu_first_chunk; } /* * The address is relative to unit0 which might be unused and * thus unmapped. Offset the address to the unit space of the * current processor before looking it up in the vmalloc * space. Note that any possible cpu id can be used here, so * there's no need to worry about preemption or cpu hotplug. */ addr += pcpu_unit_offsets[raw_smp_processor_id()]; |
9f6455325 percpu: move vmal... |
663 |
return pcpu_get_page_chunk(pcpu_addr_to_page(addr)); |
88999a898 percpu: misc prep... |
664 665 666 |
} /** |
edcb46399 percpu, module: i... |
667 |
* pcpu_alloc - the percpu allocator |
cae3aeb83 percpu: clean up ... |
668 |
* @size: size of area to allocate in bytes |
fbf59bc9d percpu: implement... |
669 |
* @align: alignment of area (max PAGE_SIZE) |
edcb46399 percpu, module: i... |
670 |
* @reserved: allocate from the reserved chunk if available |
fbf59bc9d percpu: implement... |
671 |
* |
ccea34b5d percpu: finer gra... |
672 673 674 675 |
* Allocate percpu area of @size bytes aligned at @align. * * CONTEXT: * Does GFP_KERNEL allocation. |
fbf59bc9d percpu: implement... |
676 677 678 679 |
* * RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. */ |
43cf38eb5 percpu: add __per... |
680 |
static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved) |
fbf59bc9d percpu: implement... |
681 |
{ |
f2badb0c9 percpu: make allo... |
682 |
static int warn_limit = 10; |
fbf59bc9d percpu: implement... |
683 |
struct pcpu_chunk *chunk; |
f2badb0c9 percpu: make allo... |
684 |
const char *err; |
833af8427 percpu: restructu... |
685 |
int slot, off, new_alloc; |
403a91b16 percpu: allow pcp... |
686 |
unsigned long flags; |
fbf59bc9d percpu: implement... |
687 |
|
8d408b4be percpu: give more... |
688 |
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { |
fbf59bc9d percpu: implement... |
689 690 691 692 693 |
WARN(true, "illegal size (%zu) or align (%zu) for " "percpu allocation ", size, align); return NULL; } |
ccea34b5d percpu: finer gra... |
694 |
mutex_lock(&pcpu_alloc_mutex); |
403a91b16 percpu: allow pcp... |
695 |
spin_lock_irqsave(&pcpu_lock, flags); |
fbf59bc9d percpu: implement... |
696 |
|
edcb46399 percpu, module: i... |
697 698 699 |
/* serve reserved allocations from the reserved chunk if available */ if (reserved && pcpu_reserved_chunk) { chunk = pcpu_reserved_chunk; |
833af8427 percpu: restructu... |
700 701 702 |
if (size > chunk->contig_hint) { err = "alloc from reserved chunk failed"; |
ccea34b5d percpu: finer gra... |
703 |
goto fail_unlock; |
f2badb0c9 percpu: make allo... |
704 |
} |
833af8427 percpu: restructu... |
705 706 707 708 709 710 711 712 713 |
while ((new_alloc = pcpu_need_to_extend(chunk))) { spin_unlock_irqrestore(&pcpu_lock, flags); if (pcpu_extend_area_map(chunk, new_alloc) < 0) { err = "failed to extend area map of reserved chunk"; goto fail_unlock_mutex; } spin_lock_irqsave(&pcpu_lock, flags); } |
edcb46399 percpu, module: i... |
714 715 716 |
off = pcpu_alloc_area(chunk, size, align); if (off >= 0) goto area_found; |
833af8427 percpu: restructu... |
717 |
|
f2badb0c9 percpu: make allo... |
718 |
err = "alloc from reserved chunk failed"; |
ccea34b5d percpu: finer gra... |
719 |
goto fail_unlock; |
edcb46399 percpu, module: i... |
720 |
} |
ccea34b5d percpu: finer gra... |
721 |
restart: |
edcb46399 percpu, module: i... |
722 |
/* search through normal chunks */ |
fbf59bc9d percpu: implement... |
723 724 725 726 |
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) { list_for_each_entry(chunk, &pcpu_slot[slot], list) { if (size > chunk->contig_hint) continue; |
ccea34b5d percpu: finer gra... |
727 |
|
833af8427 percpu: restructu... |
728 729 730 731 732 733 734 735 736 737 738 739 740 741 |
new_alloc = pcpu_need_to_extend(chunk); if (new_alloc) { spin_unlock_irqrestore(&pcpu_lock, flags); if (pcpu_extend_area_map(chunk, new_alloc) < 0) { err = "failed to extend area map"; goto fail_unlock_mutex; } spin_lock_irqsave(&pcpu_lock, flags); /* * pcpu_lock has been dropped, need to * restart cpu_slot list walking. */ goto restart; |
ccea34b5d percpu: finer gra... |
742 |
} |
fbf59bc9d percpu: implement... |
743 744 745 |
off = pcpu_alloc_area(chunk, size, align); if (off >= 0) goto area_found; |
fbf59bc9d percpu: implement... |
746 747 748 749 |
} } /* hmmm... no space left, create a new chunk */ |
403a91b16 percpu: allow pcp... |
750 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
ccea34b5d percpu: finer gra... |
751 |
|
6081089fd percpu: reorganiz... |
752 |
chunk = pcpu_create_chunk(); |
f2badb0c9 percpu: make allo... |
753 754 |
if (!chunk) { err = "failed to allocate new chunk"; |
ccea34b5d percpu: finer gra... |
755 |
goto fail_unlock_mutex; |
f2badb0c9 percpu: make allo... |
756 |
} |
ccea34b5d percpu: finer gra... |
757 |
|
403a91b16 percpu: allow pcp... |
758 |
spin_lock_irqsave(&pcpu_lock, flags); |
fbf59bc9d percpu: implement... |
759 |
pcpu_chunk_relocate(chunk, -1); |
ccea34b5d percpu: finer gra... |
760 |
goto restart; |
fbf59bc9d percpu: implement... |
761 762 |
area_found: |
403a91b16 percpu: allow pcp... |
763 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
ccea34b5d percpu: finer gra... |
764 |
|
fbf59bc9d percpu: implement... |
765 766 |
/* populate, map and clear the area */ if (pcpu_populate_chunk(chunk, off, size)) { |
403a91b16 percpu: allow pcp... |
767 |
spin_lock_irqsave(&pcpu_lock, flags); |
fbf59bc9d percpu: implement... |
768 |
pcpu_free_area(chunk, off); |
f2badb0c9 percpu: make allo... |
769 |
err = "failed to populate"; |
ccea34b5d percpu: finer gra... |
770 |
goto fail_unlock; |
fbf59bc9d percpu: implement... |
771 |
} |
ccea34b5d percpu: finer gra... |
772 |
mutex_unlock(&pcpu_alloc_mutex); |
bba174f5e percpu: add chunk... |
773 774 |
/* return address relative to base address */ return __addr_to_pcpu_ptr(chunk->base_addr + off); |
ccea34b5d percpu: finer gra... |
775 776 |
fail_unlock: |
403a91b16 percpu: allow pcp... |
777 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
ccea34b5d percpu: finer gra... |
778 779 |
fail_unlock_mutex: mutex_unlock(&pcpu_alloc_mutex); |
f2badb0c9 percpu: make allo... |
780 781 782 783 784 785 786 787 788 |
if (warn_limit) { pr_warning("PERCPU: allocation failed, size=%zu align=%zu, " "%s ", size, align, err); dump_stack(); if (!--warn_limit) pr_info("PERCPU: limit reached, disable warning "); } |
ccea34b5d percpu: finer gra... |
789 |
return NULL; |
fbf59bc9d percpu: implement... |
790 |
} |
edcb46399 percpu, module: i... |
791 792 793 794 795 796 |
/** * __alloc_percpu - allocate dynamic percpu area * @size: size of area to allocate in bytes * @align: alignment of area (max PAGE_SIZE) * |
9329ba970 percpu: update co... |
797 798 |
* Allocate zero-filled percpu area of @size bytes aligned at @align. * Might sleep. Might trigger writeouts. |
edcb46399 percpu, module: i... |
799 |
* |
ccea34b5d percpu: finer gra... |
800 801 802 |
* CONTEXT: * Does GFP_KERNEL allocation. * |
edcb46399 percpu, module: i... |
803 804 805 |
* RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. */ |
43cf38eb5 percpu: add __per... |
806 |
void __percpu *__alloc_percpu(size_t size, size_t align) |
edcb46399 percpu, module: i... |
807 808 809 |
{ return pcpu_alloc(size, align, false); } |
fbf59bc9d percpu: implement... |
810 |
EXPORT_SYMBOL_GPL(__alloc_percpu); |
edcb46399 percpu, module: i... |
811 812 813 814 815 |
/** * __alloc_reserved_percpu - allocate reserved percpu area * @size: size of area to allocate in bytes * @align: alignment of area (max PAGE_SIZE) * |
9329ba970 percpu: update co... |
816 817 818 819 |
* Allocate zero-filled percpu area of @size bytes aligned at @align * from reserved percpu area if arch has set it up; otherwise, * allocation is served from the same dynamic area. Might sleep. * Might trigger writeouts. |
edcb46399 percpu, module: i... |
820 |
* |
ccea34b5d percpu: finer gra... |
821 822 823 |
* CONTEXT: * Does GFP_KERNEL allocation. * |
edcb46399 percpu, module: i... |
824 825 826 |
* RETURNS: * Percpu pointer to the allocated area on success, NULL on failure. */ |
43cf38eb5 percpu: add __per... |
827 |
void __percpu *__alloc_reserved_percpu(size_t size, size_t align) |
edcb46399 percpu, module: i... |
828 829 830 |
{ return pcpu_alloc(size, align, true); } |
a56dbddf0 percpu: move full... |
831 832 833 834 835 |
/** * pcpu_reclaim - reclaim fully free chunks, workqueue function * @work: unused * * Reclaim all fully free chunks except for the first one. |
ccea34b5d percpu: finer gra... |
836 837 838 |
* * CONTEXT: * workqueue context. |
a56dbddf0 percpu: move full... |
839 840 |
*/ static void pcpu_reclaim(struct work_struct *work) |
fbf59bc9d percpu: implement... |
841 |
{ |
a56dbddf0 percpu: move full... |
842 843 844 |
LIST_HEAD(todo); struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1]; struct pcpu_chunk *chunk, *next; |
ccea34b5d percpu: finer gra... |
845 846 |
mutex_lock(&pcpu_alloc_mutex); spin_lock_irq(&pcpu_lock); |
a56dbddf0 percpu: move full... |
847 848 849 850 851 852 853 |
list_for_each_entry_safe(chunk, next, head, list) { WARN_ON(chunk->immutable); /* spare the first one */ if (chunk == list_first_entry(head, struct pcpu_chunk, list)) continue; |
a56dbddf0 percpu: move full... |
854 855 |
list_move(&chunk->list, &todo); } |
ccea34b5d percpu: finer gra... |
856 |
spin_unlock_irq(&pcpu_lock); |
a56dbddf0 percpu: move full... |
857 858 |
list_for_each_entry_safe(chunk, next, &todo, list) { |
ce3141a27 percpu: drop pcpu... |
859 |
pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size); |
6081089fd percpu: reorganiz... |
860 |
pcpu_destroy_chunk(chunk); |
a56dbddf0 percpu: move full... |
861 |
} |
971f3918a percpu: fix pcpu_... |
862 863 |
mutex_unlock(&pcpu_alloc_mutex); |
fbf59bc9d percpu: implement... |
864 865 866 867 868 869 |
} /** * free_percpu - free percpu area * @ptr: pointer to area to free * |
ccea34b5d percpu: finer gra... |
870 871 872 873 |
* Free percpu area @ptr. * * CONTEXT: * Can be called from atomic context. |
fbf59bc9d percpu: implement... |
874 |
*/ |
43cf38eb5 percpu: add __per... |
875 |
void free_percpu(void __percpu *ptr) |
fbf59bc9d percpu: implement... |
876 |
{ |
129182e56 percpu: avoid cal... |
877 |
void *addr; |
fbf59bc9d percpu: implement... |
878 |
struct pcpu_chunk *chunk; |
ccea34b5d percpu: finer gra... |
879 |
unsigned long flags; |
fbf59bc9d percpu: implement... |
880 881 882 883 |
int off; if (!ptr) return; |
129182e56 percpu: avoid cal... |
884 |
addr = __pcpu_ptr_to_addr(ptr); |
ccea34b5d percpu: finer gra... |
885 |
spin_lock_irqsave(&pcpu_lock, flags); |
fbf59bc9d percpu: implement... |
886 887 |
chunk = pcpu_chunk_addr_search(addr); |
bba174f5e percpu: add chunk... |
888 |
off = addr - chunk->base_addr; |
fbf59bc9d percpu: implement... |
889 890 |
pcpu_free_area(chunk, off); |
a56dbddf0 percpu: move full... |
891 |
/* if there are more than one fully free chunks, wake up grim reaper */ |
fbf59bc9d percpu: implement... |
892 893 |
if (chunk->free_size == pcpu_unit_size) { struct pcpu_chunk *pos; |
a56dbddf0 percpu: move full... |
894 |
list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list) |
fbf59bc9d percpu: implement... |
895 |
if (pos != chunk) { |
a56dbddf0 percpu: move full... |
896 |
schedule_work(&pcpu_reclaim_work); |
fbf59bc9d percpu: implement... |
897 898 899 |
break; } } |
ccea34b5d percpu: finer gra... |
900 |
spin_unlock_irqrestore(&pcpu_lock, flags); |
fbf59bc9d percpu: implement... |
901 902 |
} EXPORT_SYMBOL_GPL(free_percpu); |
3b034b0d0 percpu: Fix kdump... |
903 |
/** |
10fad5e46 percpu, module: i... |
904 905 906 907 908 909 910 911 912 913 914 915 |
* is_kernel_percpu_address - test whether address is from static percpu area * @addr: address to test * * Test whether @addr belongs to in-kernel static percpu area. Module * static percpu areas are not considered. For those, use * is_module_percpu_address(). * * RETURNS: * %true if @addr is from in-kernel static percpu area, %false otherwise. */ bool is_kernel_percpu_address(unsigned long addr) { |
bbddff054 percpu: use percp... |
916 |
#ifdef CONFIG_SMP |
10fad5e46 percpu, module: i... |
917 918 919 920 921 922 923 924 925 926 |
const size_t static_size = __per_cpu_end - __per_cpu_start; void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr); unsigned int cpu; for_each_possible_cpu(cpu) { void *start = per_cpu_ptr(base, cpu); if ((void *)addr >= start && (void *)addr < start + static_size) return true; } |
bbddff054 percpu: use percp... |
927 928 |
#endif /* on UP, can't distinguish from other static vars, always false */ |
10fad5e46 percpu, module: i... |
929 930 931 932 |
return false; } /** |
3b034b0d0 percpu: Fix kdump... |
933 934 935 936 937 938 939 940 |
* per_cpu_ptr_to_phys - convert translated percpu address to physical address * @addr: the address to be converted to physical address * * Given @addr which is dereferenceable address obtained via one of * percpu access macros, this function translates it into its physical * address. The caller is responsible for ensuring @addr stays valid * until this function finishes. * |
67589c714 percpu: explain w... |
941 942 943 944 945 946 947 948 949 950 951 |
* percpu allocator has special setup for the first chunk, which currently * supports either embedding in linear address space or vmalloc mapping, * and, from the second one, the backing allocator (currently either vm or * km) provides translation. * * The addr can be tranlated simply without checking if it falls into the * first chunk. But the current code reflects better how percpu allocator * actually works, and the verification can discover both bugs in percpu * allocator itself and per_cpu_ptr_to_phys() callers. So we keep current * code. * |
3b034b0d0 percpu: Fix kdump... |
952 953 954 955 956 |
* RETURNS: * The physical address for @addr. */ phys_addr_t per_cpu_ptr_to_phys(void *addr) { |
9983b6f0c percpu: fix first... |
957 958 |
void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr); bool in_first_chunk = false; |
a855b84c3 percpu: fix chunk... |
959 |
unsigned long first_low, first_high; |
9983b6f0c percpu: fix first... |
960 961 962 |
unsigned int cpu; /* |
a855b84c3 percpu: fix chunk... |
963 |
* The following test on unit_low/high isn't strictly |
9983b6f0c percpu: fix first... |
964 965 966 |
* necessary but will speed up lookups of addresses which * aren't in the first chunk. */ |
a855b84c3 percpu: fix chunk... |
967 968 969 970 971 |
first_low = pcpu_chunk_addr(pcpu_first_chunk, pcpu_low_unit_cpu, 0); first_high = pcpu_chunk_addr(pcpu_first_chunk, pcpu_high_unit_cpu, pcpu_unit_pages); if ((unsigned long)addr >= first_low && (unsigned long)addr < first_high) { |
9983b6f0c percpu: fix first... |
972 973 974 975 976 977 978 979 980 981 982 |
for_each_possible_cpu(cpu) { void *start = per_cpu_ptr(base, cpu); if (addr >= start && addr < start + pcpu_unit_size) { in_first_chunk = true; break; } } } if (in_first_chunk) { |
eac522ef4 NOMMU: percpu sho... |
983 |
if (!is_vmalloc_addr(addr)) |
020ec6537 percpu: factor ou... |
984 985 |
return __pa(addr); else |
9f57bd4d6 percpu: fix per_c... |
986 987 |
return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr); |
020ec6537 percpu: factor ou... |
988 |
} else |
9f57bd4d6 percpu: fix per_c... |
989 990 |
return page_to_phys(pcpu_addr_to_page(addr)) + offset_in_page(addr); |
3b034b0d0 percpu: Fix kdump... |
991 |
} |
fbf59bc9d percpu: implement... |
992 |
/** |
fd1e8a1fe percpu: introduce... |
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 |
* pcpu_alloc_alloc_info - allocate percpu allocation info * @nr_groups: the number of groups * @nr_units: the number of units * * Allocate ai which is large enough for @nr_groups groups containing * @nr_units units. The returned ai's groups[0].cpu_map points to the * cpu_map array which is long enough for @nr_units and filled with * NR_CPUS. It's the caller's responsibility to initialize cpu_map * pointer of other groups. * * RETURNS: * Pointer to the allocated pcpu_alloc_info on success, NULL on * failure. */ struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, int nr_units) { struct pcpu_alloc_info *ai; size_t base_size, ai_size; void *ptr; int unit; base_size = ALIGN(sizeof(*ai) + nr_groups * sizeof(ai->groups[0]), __alignof__(ai->groups[0].cpu_map[0])); ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]); ptr = alloc_bootmem_nopanic(PFN_ALIGN(ai_size)); if (!ptr) return NULL; ai = ptr; ptr += base_size; ai->groups[0].cpu_map = ptr; for (unit = 0; unit < nr_units; unit++) ai->groups[0].cpu_map[unit] = NR_CPUS; ai->nr_groups = nr_groups; ai->__ai_size = PFN_ALIGN(ai_size); return ai; } /** * pcpu_free_alloc_info - free percpu allocation info * @ai: pcpu_alloc_info to free * * Free @ai which was allocated by pcpu_alloc_alloc_info(). */ void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai) { free_bootmem(__pa(ai), ai->__ai_size); } /** |
fd1e8a1fe percpu: introduce... |
1048 1049 1050 1051 1052 1053 1054 1055 |
* pcpu_dump_alloc_info - print out information about pcpu_alloc_info * @lvl: loglevel * @ai: allocation info to dump * * Print out information about @ai using loglevel @lvl. */ static void pcpu_dump_alloc_info(const char *lvl, const struct pcpu_alloc_info *ai) |
033e48fb8 percpu: move pcpu... |
1056 |
{ |
fd1e8a1fe percpu: introduce... |
1057 |
int group_width = 1, cpu_width = 1, width; |
033e48fb8 percpu: move pcpu... |
1058 |
char empty_str[] = "--------"; |
fd1e8a1fe percpu: introduce... |
1059 1060 1061 1062 1063 1064 1065 |
int alloc = 0, alloc_end = 0; int group, v; int upa, apl; /* units per alloc, allocs per line */ v = ai->nr_groups; while (v /= 10) group_width++; |
033e48fb8 percpu: move pcpu... |
1066 |
|
fd1e8a1fe percpu: introduce... |
1067 |
v = num_possible_cpus(); |
033e48fb8 percpu: move pcpu... |
1068 |
while (v /= 10) |
fd1e8a1fe percpu: introduce... |
1069 1070 |
cpu_width++; empty_str[min_t(int, cpu_width, sizeof(empty_str) - 1)] = '\0'; |
033e48fb8 percpu: move pcpu... |
1071 |
|
fd1e8a1fe percpu: introduce... |
1072 1073 1074 |
upa = ai->alloc_size / ai->unit_size; width = upa * (cpu_width + 1) + group_width + 3; apl = rounddown_pow_of_two(max(60 / width, 1)); |
033e48fb8 percpu: move pcpu... |
1075 |
|
fd1e8a1fe percpu: introduce... |
1076 1077 1078 |
printk("%spcpu-alloc: s%zu r%zu d%zu u%zu alloc=%zu*%zu", lvl, ai->static_size, ai->reserved_size, ai->dyn_size, ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size); |
033e48fb8 percpu: move pcpu... |
1079 |
|
fd1e8a1fe percpu: introduce... |
1080 1081 1082 1083 1084 1085 1086 1087 |
for (group = 0; group < ai->nr_groups; group++) { const struct pcpu_group_info *gi = &ai->groups[group]; int unit = 0, unit_end = 0; BUG_ON(gi->nr_units % upa); for (alloc_end += gi->nr_units / upa; alloc < alloc_end; alloc++) { if (!(alloc % apl)) { |
033e48fb8 percpu: move pcpu... |
1088 1089 |
printk(" "); |
fd1e8a1fe percpu: introduce... |
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 |
printk("%spcpu-alloc: ", lvl); } printk("[%0*d] ", group_width, group); for (unit_end += upa; unit < unit_end; unit++) if (gi->cpu_map[unit] != NR_CPUS) printk("%0*d ", cpu_width, gi->cpu_map[unit]); else printk("%s ", empty_str); |
033e48fb8 percpu: move pcpu... |
1100 |
} |
033e48fb8 percpu: move pcpu... |
1101 1102 1103 1104 |
} printk(" "); } |
033e48fb8 percpu: move pcpu... |
1105 |
|
fbf59bc9d percpu: implement... |
1106 |
/** |
8d408b4be percpu: give more... |
1107 |
* pcpu_setup_first_chunk - initialize the first percpu chunk |
fd1e8a1fe percpu: introduce... |
1108 |
* @ai: pcpu_alloc_info describing how to percpu area is shaped |
38a6be525 percpu: simplify ... |
1109 |
* @base_addr: mapped address |
8d408b4be percpu: give more... |
1110 1111 1112 |
* * Initialize the first percpu chunk which contains the kernel static * perpcu area. This function is to be called from arch percpu area |
38a6be525 percpu: simplify ... |
1113 |
* setup path. |
8d408b4be percpu: give more... |
1114 |
* |
fd1e8a1fe percpu: introduce... |
1115 1116 1117 1118 1119 1120 |
* @ai contains all information necessary to initialize the first * chunk and prime the dynamic percpu allocator. * * @ai->static_size is the size of static percpu area. * * @ai->reserved_size, if non-zero, specifies the amount of bytes to |
edcb46399 percpu, module: i... |
1121 1122 1123 1124 1125 1126 1127 |
* reserve after the static area in the first chunk. This reserves * the first chunk such that it's available only through reserved * percpu allocation. This is primarily used to serve module percpu * static areas on architectures where the addressing model has * limited offset range for symbol relocations to guarantee module * percpu symbols fall inside the relocatable range. * |
fd1e8a1fe percpu: introduce... |
1128 1129 1130 |
* @ai->dyn_size determines the number of bytes available for dynamic * allocation in the first chunk. The area between @ai->static_size + * @ai->reserved_size + @ai->dyn_size and @ai->unit_size is unused. |
6074d5b0a percpu: more flex... |
1131 |
* |
fd1e8a1fe percpu: introduce... |
1132 1133 1134 |
* @ai->unit_size specifies unit size and must be aligned to PAGE_SIZE * and equal to or larger than @ai->static_size + @ai->reserved_size + * @ai->dyn_size. |
8d408b4be percpu: give more... |
1135 |
* |
fd1e8a1fe percpu: introduce... |
1136 1137 |
* @ai->atom_size is the allocation atom size and used as alignment * for vm areas. |
8d408b4be percpu: give more... |
1138 |
* |
fd1e8a1fe percpu: introduce... |
1139 1140 1141 1142 1143 1144 1145 1146 1147 |
* @ai->alloc_size is the allocation size and always multiple of * @ai->atom_size. This is larger than @ai->atom_size if * @ai->unit_size is larger than @ai->atom_size. * * @ai->nr_groups and @ai->groups describe virtual memory layout of * percpu areas. Units which should be colocated are put into the * same group. Dynamic VM areas will be allocated according to these * groupings. If @ai->nr_groups is zero, a single group containing * all units is assumed. |
8d408b4be percpu: give more... |
1148 |
* |
38a6be525 percpu: simplify ... |
1149 1150 |
* The caller should have mapped the first chunk at @base_addr and * copied static data to each unit. |
fbf59bc9d percpu: implement... |
1151 |
* |
edcb46399 percpu, module: i... |
1152 1153 1154 1155 1156 1157 1158 |
* If the first chunk ends up with both reserved and dynamic areas, it * is served by two chunks - one to serve the core static and reserved * areas and the other for the dynamic area. They share the same vm * and page map but uses different area allocation map to stay away * from each other. The latter chunk is circulated in the chunk slots * and available for dynamic allocation like any other chunks. * |
fbf59bc9d percpu: implement... |
1159 |
* RETURNS: |
fb435d523 percpu: add pcpu_... |
1160 |
* 0 on success, -errno on failure. |
fbf59bc9d percpu: implement... |
1161 |
*/ |
fb435d523 percpu: add pcpu_... |
1162 1163 |
int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, void *base_addr) |
fbf59bc9d percpu: implement... |
1164 |
{ |
635b75fc1 percpu: make pcpu... |
1165 |
static char cpus_buf[4096] __initdata; |
099a19d91 percpu: allow lim... |
1166 1167 |
static int smap[PERCPU_DYNAMIC_EARLY_SLOTS] __initdata; static int dmap[PERCPU_DYNAMIC_EARLY_SLOTS] __initdata; |
fd1e8a1fe percpu: introduce... |
1168 1169 |
size_t dyn_size = ai->dyn_size; size_t size_sum = ai->static_size + ai->reserved_size + dyn_size; |
edcb46399 percpu, module: i... |
1170 |
struct pcpu_chunk *schunk, *dchunk = NULL; |
6563297ce percpu: use group... |
1171 1172 |
unsigned long *group_offsets; size_t *group_sizes; |
fb435d523 percpu: add pcpu_... |
1173 |
unsigned long *unit_off; |
fbf59bc9d percpu: implement... |
1174 |
unsigned int cpu; |
fd1e8a1fe percpu: introduce... |
1175 1176 |
int *unit_map; int group, unit, i; |
fbf59bc9d percpu: implement... |
1177 |
|
635b75fc1 percpu: make pcpu... |
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 |
cpumask_scnprintf(cpus_buf, sizeof(cpus_buf), cpu_possible_mask); #define PCPU_SETUP_BUG_ON(cond) do { \ if (unlikely(cond)) { \ pr_emerg("PERCPU: failed to initialize, %s", #cond); \ pr_emerg("PERCPU: cpu_possible_mask=%s ", cpus_buf); \ pcpu_dump_alloc_info(KERN_EMERG, ai); \ BUG(); \ } \ } while (0) |
2f39e637e percpu: allow non... |
1189 |
/* sanity checks */ |
635b75fc1 percpu: make pcpu... |
1190 |
PCPU_SETUP_BUG_ON(ai->nr_groups <= 0); |
bbddff054 percpu: use percp... |
1191 |
#ifdef CONFIG_SMP |
635b75fc1 percpu: make pcpu... |
1192 |
PCPU_SETUP_BUG_ON(!ai->static_size); |
0415b00d1 percpu: Always al... |
1193 |
PCPU_SETUP_BUG_ON((unsigned long)__per_cpu_start & ~PAGE_MASK); |
bbddff054 percpu: use percp... |
1194 |
#endif |
635b75fc1 percpu: make pcpu... |
1195 |
PCPU_SETUP_BUG_ON(!base_addr); |
0415b00d1 percpu: Always al... |
1196 |
PCPU_SETUP_BUG_ON((unsigned long)base_addr & ~PAGE_MASK); |
635b75fc1 percpu: make pcpu... |
1197 1198 1199 |
PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK); PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); |
099a19d91 percpu: allow lim... |
1200 |
PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE); |
9f6455325 percpu: move vmal... |
1201 |
PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0); |
8d408b4be percpu: give more... |
1202 |
|
6563297ce percpu: use group... |
1203 1204 1205 |
/* process group information and build config tables accordingly */ group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0])); group_sizes = alloc_bootmem(ai->nr_groups * sizeof(group_sizes[0])); |
fd1e8a1fe percpu: introduce... |
1206 |
unit_map = alloc_bootmem(nr_cpu_ids * sizeof(unit_map[0])); |
fb435d523 percpu: add pcpu_... |
1207 |
unit_off = alloc_bootmem(nr_cpu_ids * sizeof(unit_off[0])); |
2f39e637e percpu: allow non... |
1208 |
|
fd1e8a1fe percpu: introduce... |
1209 |
for (cpu = 0; cpu < nr_cpu_ids; cpu++) |
ffe0d5a57 percpu: fix unit_... |
1210 |
unit_map[cpu] = UINT_MAX; |
a855b84c3 percpu: fix chunk... |
1211 1212 1213 |
pcpu_low_unit_cpu = NR_CPUS; pcpu_high_unit_cpu = NR_CPUS; |
2f39e637e percpu: allow non... |
1214 |
|
fd1e8a1fe percpu: introduce... |
1215 1216 |
for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) { const struct pcpu_group_info *gi = &ai->groups[group]; |
2f39e637e percpu: allow non... |
1217 |
|
6563297ce percpu: use group... |
1218 1219 |
group_offsets[group] = gi->base_offset; group_sizes[group] = gi->nr_units * ai->unit_size; |
fd1e8a1fe percpu: introduce... |
1220 1221 1222 1223 |
for (i = 0; i < gi->nr_units; i++) { cpu = gi->cpu_map[i]; if (cpu == NR_CPUS) continue; |
8d408b4be percpu: give more... |
1224 |
|
635b75fc1 percpu: make pcpu... |
1225 1226 1227 |
PCPU_SETUP_BUG_ON(cpu > nr_cpu_ids); PCPU_SETUP_BUG_ON(!cpu_possible(cpu)); PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX); |
fbf59bc9d percpu: implement... |
1228 |
|
fd1e8a1fe percpu: introduce... |
1229 |
unit_map[cpu] = unit + i; |
fb435d523 percpu: add pcpu_... |
1230 |
unit_off[cpu] = gi->base_offset + i * ai->unit_size; |
a855b84c3 percpu: fix chunk... |
1231 1232 1233 1234 1235 1236 1237 |
/* determine low/high unit_cpu */ if (pcpu_low_unit_cpu == NR_CPUS || unit_off[cpu] < unit_off[pcpu_low_unit_cpu]) pcpu_low_unit_cpu = cpu; if (pcpu_high_unit_cpu == NR_CPUS || unit_off[cpu] > unit_off[pcpu_high_unit_cpu]) pcpu_high_unit_cpu = cpu; |
fd1e8a1fe percpu: introduce... |
1238 |
} |
2f39e637e percpu: allow non... |
1239 |
} |
fd1e8a1fe percpu: introduce... |
1240 1241 1242 |
pcpu_nr_units = unit; for_each_possible_cpu(cpu) |
635b75fc1 percpu: make pcpu... |
1243 1244 1245 1246 |
PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX); /* we're done parsing the input, undefine BUG macro and dump config */ #undef PCPU_SETUP_BUG_ON |
bcbea798f percpu: print out... |
1247 |
pcpu_dump_alloc_info(KERN_DEBUG, ai); |
fd1e8a1fe percpu: introduce... |
1248 |
|
6563297ce percpu: use group... |
1249 1250 1251 |
pcpu_nr_groups = ai->nr_groups; pcpu_group_offsets = group_offsets; pcpu_group_sizes = group_sizes; |
fd1e8a1fe percpu: introduce... |
1252 |
pcpu_unit_map = unit_map; |
fb435d523 percpu: add pcpu_... |
1253 |
pcpu_unit_offsets = unit_off; |
2f39e637e percpu: allow non... |
1254 1255 |
/* determine basic parameters */ |
fd1e8a1fe percpu: introduce... |
1256 |
pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT; |
d9b55eeb1 percpu: remove un... |
1257 |
pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT; |
6563297ce percpu: use group... |
1258 |
pcpu_atom_size = ai->atom_size; |
ce3141a27 percpu: drop pcpu... |
1259 1260 |
pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) + BITS_TO_LONGS(pcpu_unit_pages) * sizeof(unsigned long); |
cafe8816b percpu: use negat... |
1261 |
|
d9b55eeb1 percpu: remove un... |
1262 1263 1264 1265 1266 |
/* * Allocate chunk slots. The additional last slot is for * empty chunks. */ pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2; |
fbf59bc9d percpu: implement... |
1267 1268 1269 |
pcpu_slot = alloc_bootmem(pcpu_nr_slots * sizeof(pcpu_slot[0])); for (i = 0; i < pcpu_nr_slots; i++) INIT_LIST_HEAD(&pcpu_slot[i]); |
edcb46399 percpu, module: i... |
1270 1271 1272 1273 1274 1275 1276 |
/* * Initialize static chunk. If reserved_size is zero, the * static chunk covers static area + dynamic allocation area * in the first chunk. If reserved_size is not zero, it * covers static area + reserved area (mostly used for module * static percpu allocation). */ |
2441d15c9 percpu: cosmetic ... |
1277 1278 |
schunk = alloc_bootmem(pcpu_chunk_struct_size); INIT_LIST_HEAD(&schunk->list); |
bba174f5e percpu: add chunk... |
1279 |
schunk->base_addr = base_addr; |
61ace7fa2 percpu: improve f... |
1280 1281 |
schunk->map = smap; schunk->map_alloc = ARRAY_SIZE(smap); |
38a6be525 percpu: simplify ... |
1282 |
schunk->immutable = true; |
ce3141a27 percpu: drop pcpu... |
1283 |
bitmap_fill(schunk->populated, pcpu_unit_pages); |
edcb46399 percpu, module: i... |
1284 |
|
fd1e8a1fe percpu: introduce... |
1285 1286 |
if (ai->reserved_size) { schunk->free_size = ai->reserved_size; |
ae9e6bc9f percpu: don't put... |
1287 |
pcpu_reserved_chunk = schunk; |
fd1e8a1fe percpu: introduce... |
1288 |
pcpu_reserved_chunk_limit = ai->static_size + ai->reserved_size; |
edcb46399 percpu, module: i... |
1289 1290 1291 1292 |
} else { schunk->free_size = dyn_size; dyn_size = 0; /* dynamic area covered */ } |
2441d15c9 percpu: cosmetic ... |
1293 |
schunk->contig_hint = schunk->free_size; |
fbf59bc9d percpu: implement... |
1294 |
|
fd1e8a1fe percpu: introduce... |
1295 |
schunk->map[schunk->map_used++] = -ai->static_size; |
61ace7fa2 percpu: improve f... |
1296 1297 |
if (schunk->free_size) schunk->map[schunk->map_used++] = schunk->free_size; |
edcb46399 percpu, module: i... |
1298 1299 |
/* init dynamic chunk if necessary */ if (dyn_size) { |
ce3141a27 percpu: drop pcpu... |
1300 |
dchunk = alloc_bootmem(pcpu_chunk_struct_size); |
edcb46399 percpu, module: i... |
1301 |
INIT_LIST_HEAD(&dchunk->list); |
bba174f5e percpu: add chunk... |
1302 |
dchunk->base_addr = base_addr; |
edcb46399 percpu, module: i... |
1303 1304 |
dchunk->map = dmap; dchunk->map_alloc = ARRAY_SIZE(dmap); |
38a6be525 percpu: simplify ... |
1305 |
dchunk->immutable = true; |
ce3141a27 percpu: drop pcpu... |
1306 |
bitmap_fill(dchunk->populated, pcpu_unit_pages); |
edcb46399 percpu, module: i... |
1307 1308 1309 1310 1311 |
dchunk->contig_hint = dchunk->free_size = dyn_size; dchunk->map[dchunk->map_used++] = -pcpu_reserved_chunk_limit; dchunk->map[dchunk->map_used++] = dchunk->free_size; } |
2441d15c9 percpu: cosmetic ... |
1312 |
/* link the first chunk in */ |
ae9e6bc9f percpu: don't put... |
1313 1314 |
pcpu_first_chunk = dchunk ?: schunk; pcpu_chunk_relocate(pcpu_first_chunk, -1); |
fbf59bc9d percpu: implement... |
1315 1316 |
/* we're done */ |
bba174f5e percpu: add chunk... |
1317 |
pcpu_base_addr = base_addr; |
fb435d523 percpu: add pcpu_... |
1318 |
return 0; |
fbf59bc9d percpu: implement... |
1319 |
} |
66c3a7577 percpu: generaliz... |
1320 |
|
bbddff054 percpu: use percp... |
1321 |
#ifdef CONFIG_SMP |
f58dc01ba percpu: generaliz... |
1322 1323 1324 1325 |
const char *pcpu_fc_names[PCPU_FC_NR] __initdata = { [PCPU_FC_AUTO] = "auto", [PCPU_FC_EMBED] = "embed", [PCPU_FC_PAGE] = "page", |
f58dc01ba percpu: generaliz... |
1326 |
}; |
66c3a7577 percpu: generaliz... |
1327 |
|
f58dc01ba percpu: generaliz... |
1328 |
enum pcpu_fc pcpu_chosen_fc __initdata = PCPU_FC_AUTO; |
66c3a7577 percpu: generaliz... |
1329 |
|
f58dc01ba percpu: generaliz... |
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 |
static int __init percpu_alloc_setup(char *str) { if (0) /* nada */; #ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK else if (!strcmp(str, "embed")) pcpu_chosen_fc = PCPU_FC_EMBED; #endif #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK else if (!strcmp(str, "page")) pcpu_chosen_fc = PCPU_FC_PAGE; #endif |
f58dc01ba percpu: generaliz... |
1342 1343 1344 |
else pr_warning("PERCPU: unknown allocator %s specified ", str); |
66c3a7577 percpu: generaliz... |
1345 |
|
f58dc01ba percpu: generaliz... |
1346 |
return 0; |
66c3a7577 percpu: generaliz... |
1347 |
} |
f58dc01ba percpu: generaliz... |
1348 |
early_param("percpu_alloc", percpu_alloc_setup); |
66c3a7577 percpu: generaliz... |
1349 |
|
3c9a024fd percpu: fix build... |
1350 1351 1352 1353 1354 |
/* * pcpu_embed_first_chunk() is used by the generic percpu setup. * Build it if needed by the arch config or the generic setup is going * to be used. */ |
08fc45806 percpu: build fir... |
1355 1356 |
#if defined(CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK) || \ !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) |
3c9a024fd percpu: fix build... |
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 |
#define BUILD_EMBED_FIRST_CHUNK #endif /* build pcpu_page_first_chunk() iff needed by the arch config */ #if defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK) #define BUILD_PAGE_FIRST_CHUNK #endif /* pcpu_build_alloc_info() is used by both embed and page first chunk */ #if defined(BUILD_EMBED_FIRST_CHUNK) || defined(BUILD_PAGE_FIRST_CHUNK) /** * pcpu_build_alloc_info - build alloc_info considering distances between CPUs * @reserved_size: the size of reserved percpu area in bytes * @dyn_size: minimum free size for dynamic allocation in bytes * @atom_size: allocation atom size * @cpu_distance_fn: callback to determine distance between cpus, optional * * This function determines grouping of units, their mappings to cpus * and other parameters considering needed percpu size, allocation * atom size and distances between CPUs. * * Groups are always mutliples of atom size and CPUs which are of * LOCAL_DISTANCE both ways are grouped together and share space for * units in the same group. The returned configuration is guaranteed * to have CPUs on different nodes on different groups and >=75% usage * of allocated virtual address space. * * RETURNS: * On success, pointer to the new allocation_info is returned. On * failure, ERR_PTR value is returned. */ static struct pcpu_alloc_info * __init pcpu_build_alloc_info( size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn) { static int group_map[NR_CPUS] __initdata; static int group_cnt[NR_CPUS] __initdata; const size_t static_size = __per_cpu_end - __per_cpu_start; int nr_groups = 1, nr_units = 0; size_t size_sum, min_unit_size, alloc_size; int upa, max_upa, uninitialized_var(best_upa); /* units_per_alloc */ int last_allocs, group, unit; unsigned int cpu, tcpu; struct pcpu_alloc_info *ai; unsigned int *cpu_map; /* this function may be called multiple times */ memset(group_map, 0, sizeof(group_map)); memset(group_cnt, 0, sizeof(group_cnt)); /* calculate size_sum and ensure dyn_size is enough for early alloc */ size_sum = PFN_ALIGN(static_size + reserved_size + max_t(size_t, dyn_size, PERCPU_DYNAMIC_EARLY_SIZE)); dyn_size = size_sum - static_size - reserved_size; /* * Determine min_unit_size, alloc_size and max_upa such that * alloc_size is multiple of atom_size and is the smallest |
25985edce Fix common misspe... |
1416 |
* which can accommodate 4k aligned segments which are equal to |
3c9a024fd percpu: fix build... |
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 |
* or larger than min_unit_size. */ min_unit_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE); alloc_size = roundup(min_unit_size, atom_size); upa = alloc_size / min_unit_size; while (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK)) upa--; max_upa = upa; /* group cpus according to their proximity */ for_each_possible_cpu(cpu) { group = 0; next_group: for_each_possible_cpu(tcpu) { if (cpu == tcpu) break; if (group_map[tcpu] == group && cpu_distance_fn && (cpu_distance_fn(cpu, tcpu) > LOCAL_DISTANCE || cpu_distance_fn(tcpu, cpu) > LOCAL_DISTANCE)) { group++; nr_groups = max(nr_groups, group + 1); goto next_group; } } group_map[cpu] = group; group_cnt[group]++; } /* * Expand unit size until address space usage goes over 75% * and then as much as possible without using more address * space. */ last_allocs = INT_MAX; for (upa = max_upa; upa; upa--) { int allocs = 0, wasted = 0; if (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK)) continue; for (group = 0; group < nr_groups; group++) { int this_allocs = DIV_ROUND_UP(group_cnt[group], upa); allocs += this_allocs; wasted += this_allocs * upa - group_cnt[group]; } /* * Don't accept if wastage is over 1/3. The * greater-than comparison ensures upa==1 always * passes the following check. */ if (wasted > num_possible_cpus() / 3) continue; /* and then don't consume more memory */ if (allocs > last_allocs) break; last_allocs = allocs; best_upa = upa; } upa = best_upa; /* allocate and fill alloc_info */ for (group = 0; group < nr_groups; group++) nr_units += roundup(group_cnt[group], upa); ai = pcpu_alloc_alloc_info(nr_groups, nr_units); if (!ai) return ERR_PTR(-ENOMEM); cpu_map = ai->groups[0].cpu_map; for (group = 0; group < nr_groups; group++) { ai->groups[group].cpu_map = cpu_map; cpu_map += roundup(group_cnt[group], upa); } ai->static_size = static_size; ai->reserved_size = reserved_size; ai->dyn_size = dyn_size; ai->unit_size = alloc_size / upa; ai->atom_size = atom_size; ai->alloc_size = alloc_size; for (group = 0, unit = 0; group_cnt[group]; group++) { struct pcpu_group_info *gi = &ai->groups[group]; /* * Initialize base_offset as if all groups are located * back-to-back. The caller should update this to * reflect actual allocation. */ gi->base_offset = unit * ai->unit_size; for_each_possible_cpu(cpu) if (group_map[cpu] == group) gi->cpu_map[gi->nr_units++] = cpu; gi->nr_units = roundup(gi->nr_units, upa); unit += gi->nr_units; } BUG_ON(unit != nr_units); return ai; } #endif /* BUILD_EMBED_FIRST_CHUNK || BUILD_PAGE_FIRST_CHUNK */ #if defined(BUILD_EMBED_FIRST_CHUNK) |
66c3a7577 percpu: generaliz... |
1524 1525 |
/** * pcpu_embed_first_chunk - embed the first percpu chunk into bootmem |
66c3a7577 percpu: generaliz... |
1526 |
* @reserved_size: the size of reserved percpu area in bytes |
4ba6ce250 percpu: make @dyn... |
1527 |
* @dyn_size: minimum free size for dynamic allocation in bytes |
c8826dd53 percpu: update em... |
1528 1529 1530 |
* @atom_size: allocation atom size * @cpu_distance_fn: callback to determine distance between cpus, optional * @alloc_fn: function to allocate percpu page |
25985edce Fix common misspe... |
1531 |
* @free_fn: function to free percpu page |
66c3a7577 percpu: generaliz... |
1532 1533 1534 1535 1536 |
* * This is a helper to ease setting up embedded first percpu chunk and * can be called where pcpu_setup_first_chunk() is expected. * * If this function is used to setup the first chunk, it is allocated |
c8826dd53 percpu: update em... |
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 |
* by calling @alloc_fn and used as-is without being mapped into * vmalloc area. Allocations are always whole multiples of @atom_size * aligned to @atom_size. * * This enables the first chunk to piggy back on the linear physical * mapping which often uses larger page size. Please note that this * can result in very sparse cpu->unit mapping on NUMA machines thus * requiring large vmalloc address space. Don't use this allocator if * vmalloc space is not orders of magnitude larger than distances * between node memory addresses (ie. 32bit NUMA machines). |
66c3a7577 percpu: generaliz... |
1547 |
* |
4ba6ce250 percpu: make @dyn... |
1548 |
* @dyn_size specifies the minimum dynamic area size. |
66c3a7577 percpu: generaliz... |
1549 1550 |
* * If the needed size is smaller than the minimum or specified unit |
c8826dd53 percpu: update em... |
1551 |
* size, the leftover is returned using @free_fn. |
66c3a7577 percpu: generaliz... |
1552 1553 |
* * RETURNS: |
fb435d523 percpu: add pcpu_... |
1554 |
* 0 on success, -errno on failure. |
66c3a7577 percpu: generaliz... |
1555 |
*/ |
4ba6ce250 percpu: make @dyn... |
1556 |
int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, |
c8826dd53 percpu: update em... |
1557 1558 1559 1560 |
size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn) |
66c3a7577 percpu: generaliz... |
1561 |
{ |
c8826dd53 percpu: update em... |
1562 1563 |
void *base = (void *)ULONG_MAX; void **areas = NULL; |
fd1e8a1fe percpu: introduce... |
1564 |
struct pcpu_alloc_info *ai; |
6ea529a20 percpu: make embe... |
1565 |
size_t size_sum, areas_size, max_distance; |
c8826dd53 percpu: update em... |
1566 |
int group, i, rc; |
66c3a7577 percpu: generaliz... |
1567 |
|
c8826dd53 percpu: update em... |
1568 1569 |
ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, cpu_distance_fn); |
fd1e8a1fe percpu: introduce... |
1570 1571 |
if (IS_ERR(ai)) return PTR_ERR(ai); |
66c3a7577 percpu: generaliz... |
1572 |
|
fd1e8a1fe percpu: introduce... |
1573 |
size_sum = ai->static_size + ai->reserved_size + ai->dyn_size; |
c8826dd53 percpu: update em... |
1574 |
areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *)); |
fa8a7094b x86: implement pe... |
1575 |
|
c8826dd53 percpu: update em... |
1576 1577 |
areas = alloc_bootmem_nopanic(areas_size); if (!areas) { |
fb435d523 percpu: add pcpu_... |
1578 |
rc = -ENOMEM; |
c8826dd53 percpu: update em... |
1579 |
goto out_free; |
fa8a7094b x86: implement pe... |
1580 |
} |
66c3a7577 percpu: generaliz... |
1581 |
|
c8826dd53 percpu: update em... |
1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 |
/* allocate, copy and determine base address */ for (group = 0; group < ai->nr_groups; group++) { struct pcpu_group_info *gi = &ai->groups[group]; unsigned int cpu = NR_CPUS; void *ptr; for (i = 0; i < gi->nr_units && cpu == NR_CPUS; i++) cpu = gi->cpu_map[i]; BUG_ON(cpu == NR_CPUS); /* allocate space for the whole group */ ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size); if (!ptr) { rc = -ENOMEM; goto out_free_areas; } areas[group] = ptr; |
fd1e8a1fe percpu: introduce... |
1599 |
|
c8826dd53 percpu: update em... |
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 |
base = min(ptr, base); for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) { if (gi->cpu_map[i] == NR_CPUS) { /* unused unit, free whole */ free_fn(ptr, ai->unit_size); continue; } /* copy and return the unused part */ memcpy(ptr, __per_cpu_load, ai->static_size); free_fn(ptr + size_sum, ai->unit_size - size_sum); } |
fa8a7094b x86: implement pe... |
1612 |
} |
66c3a7577 percpu: generaliz... |
1613 |
|
c8826dd53 percpu: update em... |
1614 |
/* base address is now known, determine group base offsets */ |
6ea529a20 percpu: make embe... |
1615 1616 |
max_distance = 0; for (group = 0; group < ai->nr_groups; group++) { |
c8826dd53 percpu: update em... |
1617 |
ai->groups[group].base_offset = areas[group] - base; |
1a0c3298d percpu: fix compi... |
1618 1619 |
max_distance = max_t(size_t, max_distance, ai->groups[group].base_offset); |
6ea529a20 percpu: make embe... |
1620 1621 1622 1623 1624 |
} max_distance += ai->unit_size; /* warn if maximum distance is further than 75% of vmalloc space */ if (max_distance > (VMALLOC_END - VMALLOC_START) * 3 / 4) { |
1a0c3298d percpu: fix compi... |
1625 |
pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc " |
787e5b06a percpu: Cast away... |
1626 1627 1628 |
"space 0x%lx ", max_distance, (unsigned long)(VMALLOC_END - VMALLOC_START)); |
6ea529a20 percpu: make embe... |
1629 1630 1631 1632 1633 1634 |
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK /* and fail if we have fallback */ rc = -EINVAL; goto out_free; #endif } |
c8826dd53 percpu: update em... |
1635 |
|
004018e2c percpu: improve b... |
1636 1637 |
pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu ", |
fd1e8a1fe percpu: introduce... |
1638 1639 |
PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size, ai->dyn_size, ai->unit_size); |
d4b95f803 x86,percpu: gener... |
1640 |
|
fb435d523 percpu: add pcpu_... |
1641 |
rc = pcpu_setup_first_chunk(ai, base); |
c8826dd53 percpu: update em... |
1642 1643 1644 1645 1646 1647 1648 |
goto out_free; out_free_areas: for (group = 0; group < ai->nr_groups; group++) free_fn(areas[group], ai->groups[group].nr_units * ai->unit_size); out_free: |
fd1e8a1fe percpu: introduce... |
1649 |
pcpu_free_alloc_info(ai); |
c8826dd53 percpu: update em... |
1650 1651 |
if (areas) free_bootmem(__pa(areas), areas_size); |
fb435d523 percpu: add pcpu_... |
1652 |
return rc; |
d4b95f803 x86,percpu: gener... |
1653 |
} |
3c9a024fd percpu: fix build... |
1654 |
#endif /* BUILD_EMBED_FIRST_CHUNK */ |
d4b95f803 x86,percpu: gener... |
1655 |
|
3c9a024fd percpu: fix build... |
1656 |
#ifdef BUILD_PAGE_FIRST_CHUNK |
d4b95f803 x86,percpu: gener... |
1657 |
/** |
00ae4064b percpu: rename 4k... |
1658 |
* pcpu_page_first_chunk - map the first chunk using PAGE_SIZE pages |
d4b95f803 x86,percpu: gener... |
1659 1660 |
* @reserved_size: the size of reserved percpu area in bytes * @alloc_fn: function to allocate percpu page, always called with PAGE_SIZE |
25985edce Fix common misspe... |
1661 |
* @free_fn: function to free percpu page, always called with PAGE_SIZE |
d4b95f803 x86,percpu: gener... |
1662 1663 |
* @populate_pte_fn: function to populate pte * |
00ae4064b percpu: rename 4k... |
1664 1665 |
* This is a helper to ease setting up page-remapped first percpu * chunk and can be called where pcpu_setup_first_chunk() is expected. |
d4b95f803 x86,percpu: gener... |
1666 1667 1668 1669 1670 |
* * This is the basic allocator. Static percpu area is allocated * page-by-page into vmalloc area. * * RETURNS: |
fb435d523 percpu: add pcpu_... |
1671 |
* 0 on success, -errno on failure. |
d4b95f803 x86,percpu: gener... |
1672 |
*/ |
fb435d523 percpu: add pcpu_... |
1673 1674 1675 1676 |
int __init pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn) |
d4b95f803 x86,percpu: gener... |
1677 |
{ |
8f05a6a65 percpu: make 4k f... |
1678 |
static struct vm_struct vm; |
fd1e8a1fe percpu: introduce... |
1679 |
struct pcpu_alloc_info *ai; |
00ae4064b percpu: rename 4k... |
1680 |
char psize_str[16]; |
ce3141a27 percpu: drop pcpu... |
1681 |
int unit_pages; |
d4b95f803 x86,percpu: gener... |
1682 |
size_t pages_size; |
ce3141a27 percpu: drop pcpu... |
1683 |
struct page **pages; |
fb435d523 percpu: add pcpu_... |
1684 |
int unit, i, j, rc; |
d4b95f803 x86,percpu: gener... |
1685 |
|
00ae4064b percpu: rename 4k... |
1686 |
snprintf(psize_str, sizeof(psize_str), "%luK", PAGE_SIZE >> 10); |
4ba6ce250 percpu: make @dyn... |
1687 |
ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL); |
fd1e8a1fe percpu: introduce... |
1688 1689 1690 1691 1692 1693 |
if (IS_ERR(ai)) return PTR_ERR(ai); BUG_ON(ai->nr_groups != 1); BUG_ON(ai->groups[0].nr_units != num_possible_cpus()); unit_pages = ai->unit_size >> PAGE_SHIFT; |
d4b95f803 x86,percpu: gener... |
1694 1695 |
/* unaligned allocations can't be freed, round up to page size */ |
fd1e8a1fe percpu: introduce... |
1696 1697 |
pages_size = PFN_ALIGN(unit_pages * num_possible_cpus() * sizeof(pages[0])); |
ce3141a27 percpu: drop pcpu... |
1698 |
pages = alloc_bootmem(pages_size); |
d4b95f803 x86,percpu: gener... |
1699 |
|
8f05a6a65 percpu: make 4k f... |
1700 |
/* allocate pages */ |
d4b95f803 x86,percpu: gener... |
1701 |
j = 0; |
fd1e8a1fe percpu: introduce... |
1702 |
for (unit = 0; unit < num_possible_cpus(); unit++) |
ce3141a27 percpu: drop pcpu... |
1703 |
for (i = 0; i < unit_pages; i++) { |
fd1e8a1fe percpu: introduce... |
1704 |
unsigned int cpu = ai->groups[0].cpu_map[unit]; |
d4b95f803 x86,percpu: gener... |
1705 |
void *ptr; |
3cbc85652 percpu: add @alig... |
1706 |
ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE); |
d4b95f803 x86,percpu: gener... |
1707 |
if (!ptr) { |
00ae4064b percpu: rename 4k... |
1708 1709 1710 |
pr_warning("PERCPU: failed to allocate %s page " "for cpu%u ", psize_str, cpu); |
d4b95f803 x86,percpu: gener... |
1711 1712 |
goto enomem; } |
ce3141a27 percpu: drop pcpu... |
1713 |
pages[j++] = virt_to_page(ptr); |
d4b95f803 x86,percpu: gener... |
1714 |
} |
8f05a6a65 percpu: make 4k f... |
1715 1716 |
/* allocate vm area, map the pages and copy static data */ vm.flags = VM_ALLOC; |
fd1e8a1fe percpu: introduce... |
1717 |
vm.size = num_possible_cpus() * ai->unit_size; |
8f05a6a65 percpu: make 4k f... |
1718 |
vm_area_register_early(&vm, PAGE_SIZE); |
fd1e8a1fe percpu: introduce... |
1719 |
for (unit = 0; unit < num_possible_cpus(); unit++) { |
1d9d32572 percpu: make @dyn... |
1720 |
unsigned long unit_addr = |
fd1e8a1fe percpu: introduce... |
1721 |
(unsigned long)vm.addr + unit * ai->unit_size; |
8f05a6a65 percpu: make 4k f... |
1722 |
|
ce3141a27 percpu: drop pcpu... |
1723 |
for (i = 0; i < unit_pages; i++) |
8f05a6a65 percpu: make 4k f... |
1724 1725 1726 |
populate_pte_fn(unit_addr + (i << PAGE_SHIFT)); /* pte already populated, the following shouldn't fail */ |
fb435d523 percpu: add pcpu_... |
1727 1728 1729 1730 1731 |
rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages], unit_pages); if (rc < 0) panic("failed to map percpu area, err=%d ", rc); |
66c3a7577 percpu: generaliz... |
1732 |
|
8f05a6a65 percpu: make 4k f... |
1733 1734 1735 1736 1737 1738 1739 1740 1741 |
/* * FIXME: Archs with virtual cache should flush local * cache for the linear mapping here - something * equivalent to flush_cache_vmap() on the local cpu. * flush_cache_vmap() can't be used as most supporting * data structures are not set up yet. */ /* copy static data */ |
fd1e8a1fe percpu: introduce... |
1742 |
memcpy((void *)unit_addr, __per_cpu_load, ai->static_size); |
66c3a7577 percpu: generaliz... |
1743 1744 1745 |
} /* we're ready, commit */ |
1d9d32572 percpu: make @dyn... |
1746 1747 |
pr_info("PERCPU: %d %s pages/cpu @%p s%zu r%zu d%zu ", |
fd1e8a1fe percpu: introduce... |
1748 1749 |
unit_pages, psize_str, vm.addr, ai->static_size, ai->reserved_size, ai->dyn_size); |
d4b95f803 x86,percpu: gener... |
1750 |
|
fb435d523 percpu: add pcpu_... |
1751 |
rc = pcpu_setup_first_chunk(ai, vm.addr); |
d4b95f803 x86,percpu: gener... |
1752 1753 1754 1755 |
goto out_free_ar; enomem: while (--j >= 0) |
ce3141a27 percpu: drop pcpu... |
1756 |
free_fn(page_address(pages[j]), PAGE_SIZE); |
fb435d523 percpu: add pcpu_... |
1757 |
rc = -ENOMEM; |
d4b95f803 x86,percpu: gener... |
1758 |
out_free_ar: |
ce3141a27 percpu: drop pcpu... |
1759 |
free_bootmem(__pa(pages), pages_size); |
fd1e8a1fe percpu: introduce... |
1760 |
pcpu_free_alloc_info(ai); |
fb435d523 percpu: add pcpu_... |
1761 |
return rc; |
d4b95f803 x86,percpu: gener... |
1762 |
} |
3c9a024fd percpu: fix build... |
1763 |
#endif /* BUILD_PAGE_FIRST_CHUNK */ |
d4b95f803 x86,percpu: gener... |
1764 |
|
bbddff054 percpu: use percp... |
1765 |
#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA |
8c4bfc6e8 x86,percpu: gener... |
1766 |
/* |
bbddff054 percpu: use percp... |
1767 |
* Generic SMP percpu area setup. |
e74e39620 percpu: use dynam... |
1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 |
* * The embedding helper is used because its behavior closely resembles * the original non-dynamic generic percpu area setup. This is * important because many archs have addressing restrictions and might * fail if the percpu area is located far away from the previous * location. As an added bonus, in non-NUMA cases, embedding is * generally a good idea TLB-wise because percpu area can piggy back * on the physical linear memory mapping which uses large page * mappings on applicable archs. */ |
e74e39620 percpu: use dynam... |
1778 1779 |
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); |
c8826dd53 percpu: update em... |
1780 1781 1782 1783 1784 |
static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size, size_t align) { return __alloc_bootmem_nopanic(size, align, __pa(MAX_DMA_ADDRESS)); } |
66c3a7577 percpu: generaliz... |
1785 |
|
c8826dd53 percpu: update em... |
1786 1787 1788 1789 |
static void __init pcpu_dfl_fc_free(void *ptr, size_t size) { free_bootmem(__pa(ptr), size); } |
e74e39620 percpu: use dynam... |
1790 1791 |
void __init setup_per_cpu_areas(void) { |
e74e39620 percpu: use dynam... |
1792 1793 |
unsigned long delta; unsigned int cpu; |
fb435d523 percpu: add pcpu_... |
1794 |
int rc; |
e74e39620 percpu: use dynam... |
1795 1796 1797 1798 1799 |
/* * Always reserve area for module percpu variables. That's * what the legacy allocator did. */ |
fb435d523 percpu: add pcpu_... |
1800 |
rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, |
c8826dd53 percpu: update em... |
1801 1802 |
PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL, pcpu_dfl_fc_alloc, pcpu_dfl_fc_free); |
fb435d523 percpu: add pcpu_... |
1803 |
if (rc < 0) |
bbddff054 percpu: use percp... |
1804 |
panic("Failed to initialize percpu areas."); |
e74e39620 percpu: use dynam... |
1805 1806 1807 |
delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; for_each_possible_cpu(cpu) |
fb435d523 percpu: add pcpu_... |
1808 |
__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; |
66c3a7577 percpu: generaliz... |
1809 |
} |
bbddff054 percpu: use percp... |
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 |
#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */ #else /* CONFIG_SMP */ /* * UP percpu area setup. * * UP always uses km-based percpu allocator with identity mapping. * Static percpu variables are indistinguishable from the usual static * variables and don't require any special preparation. */ void __init setup_per_cpu_areas(void) { const size_t unit_size = roundup_pow_of_two(max_t(size_t, PCPU_MIN_UNIT_SIZE, PERCPU_DYNAMIC_RESERVE)); struct pcpu_alloc_info *ai; void *fc; ai = pcpu_alloc_alloc_info(1, 1); fc = __alloc_bootmem(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); if (!ai || !fc) panic("Failed to allocate memory for percpu areas."); ai->dyn_size = unit_size; ai->unit_size = unit_size; ai->atom_size = unit_size; ai->alloc_size = unit_size; ai->groups[0].nr_units = 1; ai->groups[0].cpu_map[0] = 0; if (pcpu_setup_first_chunk(ai, fc) < 0) panic("Failed to initialize percpu areas."); } #endif /* CONFIG_SMP */ |
099a19d91 percpu: allow lim... |
1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 |
/* * First and reserved chunks are initialized with temporary allocation * map in initdata so that they can be used before slab is online. * This function is called after slab is brought up and replaces those * with properly allocated maps. */ void __init percpu_init_late(void) { struct pcpu_chunk *target_chunks[] = { pcpu_first_chunk, pcpu_reserved_chunk, NULL }; struct pcpu_chunk *chunk; unsigned long flags; int i; for (i = 0; (chunk = target_chunks[i]); i++) { int *map; const size_t size = PERCPU_DYNAMIC_EARLY_SLOTS * sizeof(map[0]); BUILD_BUG_ON(size > PAGE_SIZE); |
90459ce06 percpu: rename pc... |
1866 |
map = pcpu_mem_zalloc(size); |
099a19d91 percpu: allow lim... |
1867 1868 1869 1870 1871 1872 1873 1874 |
BUG_ON(!map); spin_lock_irqsave(&pcpu_lock, flags); memcpy(map, chunk->map, size); chunk->map = map; spin_unlock_irqrestore(&pcpu_lock, flags); } } |