Blame view
mm/nobootmem.c
10.9 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
093258732 bootmem: Separate... |
2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * bootmem - A boot-time physical memory allocator and configurator * * Copyright (C) 1999 Ingo Molnar * 1999 Kanoj Sarcar, SGI * 2008 Johannes Weiner * * Access to this subsystem has to be serialized externally (which is true * for the boot process anyway). */ #include <linux/init.h> #include <linux/pfn.h> #include <linux/slab.h> |
b95f1b31b mm: Map most file... |
15 |
#include <linux/export.h> |
093258732 bootmem: Separate... |
16 17 18 |
#include <linux/kmemleak.h> #include <linux/range.h> #include <linux/memblock.h> |
2382705f2 mm/nobootmem.c: r... |
19 |
#include <linux/bootmem.h> |
093258732 bootmem: Separate... |
20 21 22 |
#include <asm/bug.h> #include <asm/io.h> |
093258732 bootmem: Separate... |
23 24 |
#include "internal.h" |
2382705f2 mm/nobootmem.c: r... |
25 26 27 |
#ifndef CONFIG_HAVE_MEMBLOCK #error CONFIG_HAVE_MEMBLOCK not defined #endif |
e782ab421 bootmem: Move con... |
28 29 30 31 |
#ifndef CONFIG_NEED_MULTIPLE_NODES struct pglist_data __refdata contig_page_data; EXPORT_SYMBOL(contig_page_data); #endif |
093258732 bootmem: Separate... |
32 33 34 |
unsigned long max_low_pfn; unsigned long min_low_pfn; unsigned long max_pfn; |
8dd330300 x86/mm: Introduce... |
35 |
unsigned long long max_possible_pfn; |
093258732 bootmem: Separate... |
36 |
|
8bc1f91e1 bootmem: Move __a... |
37 38 39 40 41 |
static void * __init __alloc_memory_core_early(int nid, u64 size, u64 align, u64 goal, u64 limit) { void *ptr; u64 addr; |
e1720fee2 mm/memblock: add ... |
42 |
enum memblock_flags flags = choose_memblock_flags(); |
8bc1f91e1 bootmem: Move __a... |
43 44 45 |
if (limit > memblock.current_limit) limit = memblock.current_limit; |
a3f5bafcc mm/memblock: allo... |
46 |
again: |
fc6daaf93 mm/memblock: add ... |
47 |
addr = memblock_find_in_range_node(size, align, goal, limit, nid, |
a3f5bafcc mm/memblock: allo... |
48 49 50 51 52 53 54 55 |
flags); if (!addr && (flags & MEMBLOCK_MIRROR)) { flags &= ~MEMBLOCK_MIRROR; pr_warn("Could not allocate %pap bytes of mirrored memory ", &size); goto again; } |
1f5026a7e memblock: Kill ME... |
56 |
if (!addr) |
8bc1f91e1 bootmem: Move __a... |
57 |
return NULL; |
87379ec8c mm/nobootmem.c: a... |
58 59 |
if (memblock_reserve(addr, size)) return NULL; |
8bc1f91e1 bootmem: Move __a... |
60 61 |
ptr = phys_to_virt(addr); memset(ptr, 0, size); |
8bc1f91e1 bootmem: Move __a... |
62 63 64 65 66 67 68 |
/* * The min_count is set to 0 so that bootmem allocated blocks * are never reported as leaks. */ kmemleak_alloc(ptr, size, 0, 0); return ptr; } |
8108ad51f docs/mm: nobootme... |
69 |
/** |
093258732 bootmem: Separate... |
70 71 72 73 74 75 76 77 78 79 80 |
* free_bootmem_late - free bootmem pages directly to page allocator * @addr: starting address of the range * @size: size of the range in bytes * * This is only useful when the bootmem allocator has already been torn * down, but we are still initializing the system. Pages are given directly * to the page allocator, no bootmem metadata is updated because it is gone. */ void __init free_bootmem_late(unsigned long addr, unsigned long size) { unsigned long cursor, end; |
9099daed9 mm: kmemleak: avo... |
81 |
kmemleak_free_part_phys(addr, size); |
093258732 bootmem: Separate... |
82 83 84 85 86 |
cursor = PFN_UP(addr); end = PFN_DOWN(addr + size); for (; cursor < end; cursor++) { |
d70ddd7a5 mm: page_alloc: p... |
87 |
__free_pages_bootmem(pfn_to_page(cursor), cursor, 0); |
093258732 bootmem: Separate... |
88 89 90 91 92 93 |
totalram_pages++; } } static void __init __free_pages_memory(unsigned long start, unsigned long end) { |
309d0b391 mm/nobootmem.c: h... |
94 |
int order; |
093258732 bootmem: Separate... |
95 |
|
309d0b391 mm/nobootmem.c: h... |
96 97 |
while (start < end) { order = min(MAX_ORDER - 1UL, __ffs(start)); |
093258732 bootmem: Separate... |
98 |
|
309d0b391 mm/nobootmem.c: h... |
99 100 |
while (start + (1UL << order) > end) order--; |
093258732 bootmem: Separate... |
101 |
|
d70ddd7a5 mm: page_alloc: p... |
102 |
__free_pages_bootmem(pfn_to_page(start), start, order); |
093258732 bootmem: Separate... |
103 |
|
309d0b391 mm/nobootmem.c: h... |
104 105 |
start += (1UL << order); } |
093258732 bootmem: Separate... |
106 |
} |
29f673860 memblock: free al... |
107 108 109 110 111 112 |
static unsigned long __init __free_memory_core(phys_addr_t start, phys_addr_t end) { unsigned long start_pfn = PFN_UP(start); unsigned long end_pfn = min_t(unsigned long, PFN_DOWN(end), max_low_pfn); |
172ffeb9b mm/nobootmem.c: r... |
113 |
if (start_pfn >= end_pfn) |
29f673860 memblock: free al... |
114 115 116 117 118 119 |
return 0; __free_pages_memory(start_pfn, end_pfn); return end_pfn - start_pfn; } |
b4def3509 mm, nobootmem: cl... |
120 |
static unsigned long __init free_low_memory_core_early(void) |
093258732 bootmem: Separate... |
121 |
{ |
093258732 bootmem: Separate... |
122 |
unsigned long count = 0; |
354f17e1e mm/nobootmem: fre... |
123 |
phys_addr_t start, end; |
8a9ca34c1 memblock, x86: Re... |
124 |
u64 i; |
0a313a998 mem-hotplug: let ... |
125 |
memblock_clear_hotplug(0, -1); |
92923ca3a mm: meminit: only... |
126 127 |
for_each_reserved_mem_region(i, &start, &end) reserve_bootmem_region(start, end); |
914a05165 mm: nobootmem: mo... |
128 129 130 131 132 |
/* * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id * because in some case like Node0 doesn't have RAM installed * low ram will be on Node1 */ |
fc6daaf93 mm/memblock: add ... |
133 134 |
for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end, NULL) |
29f673860 memblock: free al... |
135 |
count += __free_memory_core(start, end); |
093258732 bootmem: Separate... |
136 137 |
return count; } |
7b4b2a0d6 mm: accurately ca... |
138 |
static int reset_managed_pages_done __initdata; |
f784a3f19 mem-hotplug: rese... |
139 |
void reset_node_managed_pages(pg_data_t *pgdat) |
9feedc9d8 mm: introduce new... |
140 141 |
{ struct zone *z; |
9feedc9d8 mm: introduce new... |
142 |
for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) |
7b4b2a0d6 mm: accurately ca... |
143 144 145 146 147 148 |
z->managed_pages = 0; } void __init reset_all_zones_managed_pages(void) { struct pglist_data *pgdat; |
f784a3f19 mem-hotplug: rese... |
149 150 |
if (reset_managed_pages_done) return; |
7b4b2a0d6 mm: accurately ca... |
151 152 |
for_each_online_pgdat(pgdat) reset_node_managed_pages(pgdat); |
f784a3f19 mem-hotplug: rese... |
153 |
|
7b4b2a0d6 mm: accurately ca... |
154 |
reset_managed_pages_done = 1; |
9feedc9d8 mm: introduce new... |
155 |
} |
093258732 bootmem: Separate... |
156 |
/** |
093258732 bootmem: Separate... |
157 158 |
* free_all_bootmem - release free pages to the buddy allocator * |
8108ad51f docs/mm: nobootme... |
159 |
* Return: the number of pages actually released. |
093258732 bootmem: Separate... |
160 161 162 |
*/ unsigned long __init free_all_bootmem(void) { |
0c9885347 mm: concentrate m... |
163 |
unsigned long pages; |
7b4b2a0d6 mm: accurately ca... |
164 |
reset_all_zones_managed_pages(); |
9feedc9d8 mm: introduce new... |
165 |
|
0c9885347 mm: concentrate m... |
166 167 168 169 |
pages = free_low_memory_core_early(); totalram_pages += pages; return pages; |
093258732 bootmem: Separate... |
170 171 172 173 174 |
} /** * free_bootmem_node - mark a page range as usable * @pgdat: node the range resides on |
8108ad51f docs/mm: nobootme... |
175 |
* @physaddr: starting physical address of the range |
093258732 bootmem: Separate... |
176 177 178 179 180 181 182 183 184 |
* @size: size of the range in bytes * * Partial pages will be considered reserved and left as they are. * * The range must reside completely on the specified node. */ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, unsigned long size) { |
24aa07882 memblock, x86: Re... |
185 |
memblock_free(physaddr, size); |
093258732 bootmem: Separate... |
186 187 188 189 |
} /** * free_bootmem - mark a page range as usable |
8108ad51f docs/mm: nobootme... |
190 |
* @addr: starting physical address of the range |
093258732 bootmem: Separate... |
191 192 193 194 195 196 197 198 |
* @size: size of the range in bytes * * Partial pages will be considered reserved and left as they are. * * The range must be contiguous but may span node boundaries. */ void __init free_bootmem(unsigned long addr, unsigned long size) { |
24aa07882 memblock, x86: Re... |
199 |
memblock_free(addr, size); |
093258732 bootmem: Separate... |
200 201 202 203 204 205 206 207 208 209 210 211 212 |
} static void * __init ___alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { void *ptr; if (WARN_ON_ONCE(slab_is_available())) return kzalloc(size, GFP_NOWAIT); restart: |
b11542335 mm/memblock: swit... |
213 |
ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, goal, limit); |
093258732 bootmem: Separate... |
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 |
if (ptr) return ptr; if (goal != 0) { goal = 0; goto restart; } return NULL; } /** * __alloc_bootmem_nopanic - allocate boot memory without panicking * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * |
8108ad51f docs/mm: nobootme... |
237 |
* Return: address of the allocated region or %NULL on failure. |
093258732 bootmem: Separate... |
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 |
*/ void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal) { unsigned long limit = -1UL; return ___alloc_bootmem_nopanic(size, align, goal, limit); } static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit); if (mem) return mem; /* * Whoops, we cannot satisfy the allocation request. */ |
1170532bb mm: convert print... |
257 258 |
pr_alert("bootmem alloc of %lu bytes failed! ", size); |
093258732 bootmem: Separate... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 |
panic("Out of memory"); return NULL; } /** * __alloc_bootmem - allocate boot memory * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * * The function panics if the request can not be satisfied. |
8108ad51f docs/mm: nobootme... |
275 276 |
* * Return: address of the allocated region. |
093258732 bootmem: Separate... |
277 278 279 280 281 282 283 284 |
*/ void * __init __alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal) { unsigned long limit = -1UL; return ___alloc_bootmem(size, align, goal, limit); } |
99ab7b194 mm: sparse: fix u... |
285 |
void * __init ___alloc_bootmem_node_nopanic(pg_data_t *pgdat, |
ba5398683 mm: nobootmem: un... |
286 287 288 289 290 291 292 293 294 295 296 297 |
unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { void *ptr; again: ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, limit); if (ptr) return ptr; |
b11542335 mm/memblock: swit... |
298 |
ptr = __alloc_memory_core_early(NUMA_NO_NODE, size, align, |
ba5398683 mm: nobootmem: un... |
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 |
goal, limit); if (ptr) return ptr; if (goal) { goal = 0; goto again; } return NULL; } void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); return ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, 0); } |
de4985072 mm/nobootmem.c: m... |
319 |
static void * __init ___alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, |
ba5398683 mm: nobootmem: un... |
320 321 322 323 324 325 326 327 |
unsigned long align, unsigned long goal, unsigned long limit) { void *ptr; ptr = ___alloc_bootmem_node_nopanic(pgdat, size, align, goal, limit); if (ptr) return ptr; |
1170532bb mm: convert print... |
328 329 |
pr_alert("bootmem alloc of %lu bytes failed! ", size); |
ba5398683 mm: nobootmem: un... |
330 331 332 |
panic("Out of memory"); return NULL; } |
093258732 bootmem: Separate... |
333 334 335 336 337 338 339 340 341 342 343 344 345 346 |
/** * __alloc_bootmem_node - allocate boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. |
8108ad51f docs/mm: nobootme... |
347 348 |
* * Return: address of the allocated region. |
093258732 bootmem: Separate... |
349 350 351 352 |
*/ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { |
093258732 bootmem: Separate... |
353 354 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); |
ba5398683 mm: nobootmem: un... |
355 |
return ___alloc_bootmem_node(pgdat, size, align, goal, 0); |
093258732 bootmem: Separate... |
356 357 358 359 360 |
} void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { |
093258732 bootmem: Separate... |
361 |
return __alloc_bootmem_node(pgdat, size, align, goal); |
093258732 bootmem: Separate... |
362 |
} |
093258732 bootmem: Separate... |
363 364 365 366 367 368 369 370 371 372 373 374 375 |
/** * __alloc_bootmem_low - allocate low boot memory * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * * The function panics if the request can not be satisfied. |
8108ad51f docs/mm: nobootme... |
376 377 |
* * Return: address of the allocated region. |
093258732 bootmem: Separate... |
378 379 380 381 382 383 |
*/ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, unsigned long goal) { return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT); } |
38fa4175e mm: Add alloc_boo... |
384 385 386 387 388 389 390 |
void * __init __alloc_bootmem_low_nopanic(unsigned long size, unsigned long align, unsigned long goal) { return ___alloc_bootmem_nopanic(size, align, goal, ARCH_LOW_ADDRESS_LIMIT); } |
093258732 bootmem: Separate... |
391 392 393 394 395 396 397 398 399 400 401 402 403 404 |
/** * __alloc_bootmem_low_node - allocate low boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. |
8108ad51f docs/mm: nobootme... |
405 406 |
* * Return: address of the allocated region. |
093258732 bootmem: Separate... |
407 408 409 410 |
*/ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { |
093258732 bootmem: Separate... |
411 412 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); |
ba5398683 mm: nobootmem: un... |
413 414 |
return ___alloc_bootmem_node(pgdat, size, align, goal, ARCH_LOW_ADDRESS_LIMIT); |
093258732 bootmem: Separate... |
415 |
} |