Blame view
mm/bootmem.c
24.1 KB
1da177e4c
|
1 |
/* |
57cfc29ef
|
2 |
* bootmem - A boot-time physical memory allocator and configurator |
1da177e4c
|
3 4 |
* * Copyright (C) 1999 Ingo Molnar |
57cfc29ef
|
5 6 |
* 1999 Kanoj Sarcar, SGI * 2008 Johannes Weiner |
1da177e4c
|
7 |
* |
57cfc29ef
|
8 9 |
* Access to this subsystem has to be serialized externally (which is true * for the boot process anyway). |
1da177e4c
|
10 |
*/ |
1da177e4c
|
11 |
#include <linux/init.h> |
bbc7b92e3
|
12 |
#include <linux/pfn.h> |
5a0e3ad6a
|
13 |
#include <linux/slab.h> |
1da177e4c
|
14 |
#include <linux/bootmem.h> |
1da177e4c
|
15 |
#include <linux/module.h> |
ec3a354bd
|
16 |
#include <linux/kmemleak.h> |
08677214e
|
17 |
#include <linux/range.h> |
e786e86a5
|
18 19 |
#include <asm/bug.h> |
1da177e4c
|
20 |
#include <asm/io.h> |
dfd54cbcc
|
21 |
#include <asm/processor.h> |
e786e86a5
|
22 |
|
1da177e4c
|
23 |
#include "internal.h" |
1da177e4c
|
24 25 26 |
unsigned long max_low_pfn; unsigned long min_low_pfn; unsigned long max_pfn; |
92aa63a5a
|
27 28 29 30 31 32 33 |
#ifdef CONFIG_CRASH_DUMP /* * If we have booted due to a crash, max_pfn will be a very low value. We need * to know the amount of memory that the previous kernel used. */ unsigned long saved_max_pfn; #endif |
08677214e
|
34 |
#ifndef CONFIG_NO_BOOTMEM |
b61bfa3c4
|
35 |
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata; |
636cc40cb
|
36 |
static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list); |
2e5237daf
|
37 38 39 40 41 42 43 44 45 46 47 48 49 |
static int bootmem_debug; static int __init bootmem_debug_setup(char *buf) { bootmem_debug = 1; return 0; } early_param("bootmem_debug", bootmem_debug_setup); #define bdebug(fmt, args...) ({ \ if (unlikely(bootmem_debug)) \ printk(KERN_INFO \ "bootmem::%s " fmt, \ |
80a914dc0
|
50 |
__func__, ## args); \ |
2e5237daf
|
51 |
}) |
df049a5f4
|
52 |
static unsigned long __init bootmap_bytes(unsigned long pages) |
223e8dc92
|
53 |
{ |
df049a5f4
|
54 |
unsigned long bytes = (pages + 7) / 8; |
223e8dc92
|
55 |
|
df049a5f4
|
56 |
return ALIGN(bytes, sizeof(long)); |
223e8dc92
|
57 |
} |
a66fd7dae
|
58 59 60 61 |
/** * bootmem_bootmap_pages - calculate bitmap size in pages * @pages: number of pages the bitmap has to represent */ |
f71bf0cac
|
62 |
unsigned long __init bootmem_bootmap_pages(unsigned long pages) |
1da177e4c
|
63 |
{ |
df049a5f4
|
64 |
unsigned long bytes = bootmap_bytes(pages); |
1da177e4c
|
65 |
|
df049a5f4
|
66 |
return PAGE_ALIGN(bytes) >> PAGE_SHIFT; |
1da177e4c
|
67 |
} |
f71bf0cac
|
68 |
|
679bc9fbb
|
69 70 71 |
/* * link bdata in order */ |
69d49e681
|
72 |
static void __init link_bootmem(bootmem_data_t *bdata) |
679bc9fbb
|
73 |
{ |
636cc40cb
|
74 |
struct list_head *iter; |
f71bf0cac
|
75 |
|
636cc40cb
|
76 77 78 79 |
list_for_each(iter, &bdata_list) { bootmem_data_t *ent; ent = list_entry(iter, bootmem_data_t, list); |
3560e249a
|
80 |
if (bdata->node_min_pfn < ent->node_min_pfn) |
636cc40cb
|
81 |
break; |
679bc9fbb
|
82 |
} |
636cc40cb
|
83 |
list_add_tail(&bdata->list, iter); |
679bc9fbb
|
84 |
} |
bbc7b92e3
|
85 |
/* |
1da177e4c
|
86 87 |
* Called once to set up the allocator itself. */ |
8ae044630
|
88 |
static unsigned long __init init_bootmem_core(bootmem_data_t *bdata, |
1da177e4c
|
89 90 |
unsigned long mapstart, unsigned long start, unsigned long end) { |
bbc7b92e3
|
91 |
unsigned long mapsize; |
1da177e4c
|
92 |
|
2dbb51c49
|
93 |
mminit_validate_memmodel_limits(&start, &end); |
bbc7b92e3
|
94 |
bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart)); |
3560e249a
|
95 |
bdata->node_min_pfn = start; |
1da177e4c
|
96 |
bdata->node_low_pfn = end; |
679bc9fbb
|
97 |
link_bootmem(bdata); |
1da177e4c
|
98 99 100 101 102 |
/* * Initially all pages are reserved - setup_arch() has to * register free RAM areas explicitly. */ |
df049a5f4
|
103 |
mapsize = bootmap_bytes(end - start); |
1da177e4c
|
104 |
memset(bdata->node_bootmem_map, 0xff, mapsize); |
2e5237daf
|
105 106 107 |
bdebug("nid=%td start=%lx map=%lx end=%lx mapsize=%lx ", bdata - bootmem_node_data, start, mapstart, end, mapsize); |
1da177e4c
|
108 109 |
return mapsize; } |
a66fd7dae
|
110 111 112 113 114 115 116 117 118 |
/** * init_bootmem_node - register a node as boot memory * @pgdat: node to register * @freepfn: pfn where the bitmap for this node is to be placed * @startpfn: first pfn on the node * @endpfn: first pfn after the node * * Returns the number of bytes needed to hold the bitmap for this node. */ |
223e8dc92
|
119 120 121 122 123 |
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn, unsigned long startpfn, unsigned long endpfn) { return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn); } |
a66fd7dae
|
124 125 126 127 128 129 130 |
/** * init_bootmem - register boot memory * @start: pfn where the bitmap is to be placed * @pages: number of available physical pages * * Returns the number of bytes needed to hold the bitmap. */ |
223e8dc92
|
131 132 133 134 135 136 |
unsigned long __init init_bootmem(unsigned long start, unsigned long pages) { max_low_pfn = pages; min_low_pfn = start; return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages); } |
08677214e
|
137 |
#endif |
9f993ac3f
|
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 |
/* * free_bootmem_late - free bootmem pages directly to page allocator * @addr: starting address of the range * @size: size of the range in bytes * * This is only useful when the bootmem allocator has already been torn * down, but we are still initializing the system. Pages are given directly * to the page allocator, no bootmem metadata is updated because it is gone. */ void __init free_bootmem_late(unsigned long addr, unsigned long size) { unsigned long cursor, end; kmemleak_free_part(__va(addr), size); cursor = PFN_UP(addr); end = PFN_DOWN(addr + size); for (; cursor < end; cursor++) { __free_pages_bootmem(pfn_to_page(cursor), 0); totalram_pages++; } } |
08677214e
|
161 162 163 164 165 166 167 168 169 170 171 |
#ifdef CONFIG_NO_BOOTMEM static void __init __free_pages_memory(unsigned long start, unsigned long end) { int i; unsigned long start_aligned, end_aligned; int order = ilog2(BITS_PER_LONG); start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1); end_aligned = end & ~(BITS_PER_LONG - 1); if (end_aligned <= start_aligned) { |
08677214e
|
172 173 174 175 176 |
for (i = start; i < end; i++) __free_pages_bootmem(pfn_to_page(i), 0); return; } |
08677214e
|
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 |
for (i = start; i < start_aligned; i++) __free_pages_bootmem(pfn_to_page(i), 0); for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG) __free_pages_bootmem(pfn_to_page(i), order); for (i = end_aligned; i < end; i++) __free_pages_bootmem(pfn_to_page(i), 0); } unsigned long __init free_all_memory_core_early(int nodeid) { int i; u64 start, end; unsigned long count = 0; struct range *range = NULL; int nr_range; nr_range = get_free_all_memory_range(&range, nodeid); for (i = 0; i < nr_range; i++) { start = range[i].start; end = range[i].end; count += end - start; __free_pages_memory(start, end); } return count; } #else |
223e8dc92
|
207 208 |
static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata) { |
41546c174
|
209 |
int aligned; |
223e8dc92
|
210 |
struct page *page; |
41546c174
|
211 212 213 214 |
unsigned long start, end, pages, count = 0; if (!bdata->node_bootmem_map) return 0; |
3560e249a
|
215 |
start = bdata->node_min_pfn; |
41546c174
|
216 |
end = bdata->node_low_pfn; |
223e8dc92
|
217 |
/* |
41546c174
|
218 219 |
* If the start is aligned to the machines wordsize, we might * be able to free pages in bulks of that order. |
223e8dc92
|
220 |
*/ |
41546c174
|
221 |
aligned = !(start & (BITS_PER_LONG - 1)); |
223e8dc92
|
222 |
|
41546c174
|
223 224 225 |
bdebug("nid=%td start=%lx end=%lx aligned=%d ", bdata - bootmem_node_data, start, end, aligned); |
223e8dc92
|
226 |
|
41546c174
|
227 228 |
while (start < end) { unsigned long *map, idx, vec; |
223e8dc92
|
229 |
|
41546c174
|
230 |
map = bdata->node_bootmem_map; |
3560e249a
|
231 |
idx = start - bdata->node_min_pfn; |
41546c174
|
232 233 234 235 236 237 |
vec = ~map[idx / BITS_PER_LONG]; if (aligned && vec == ~0UL && start + BITS_PER_LONG < end) { int order = ilog2(BITS_PER_LONG); __free_pages_bootmem(pfn_to_page(start), order); |
223e8dc92
|
238 |
count += BITS_PER_LONG; |
41546c174
|
239 240 241 242 243 244 |
} else { unsigned long off = 0; while (vec && off < BITS_PER_LONG) { if (vec & 1) { page = pfn_to_page(start + off); |
223e8dc92
|
245 |
__free_pages_bootmem(page, 0); |
41546c174
|
246 |
count++; |
223e8dc92
|
247 |
} |
41546c174
|
248 249 |
vec >>= 1; off++; |
223e8dc92
|
250 |
} |
223e8dc92
|
251 |
} |
41546c174
|
252 |
start += BITS_PER_LONG; |
223e8dc92
|
253 |
} |
223e8dc92
|
254 |
page = virt_to_page(bdata->node_bootmem_map); |
3560e249a
|
255 |
pages = bdata->node_low_pfn - bdata->node_min_pfn; |
41546c174
|
256 257 258 259 |
pages = bootmem_bootmap_pages(pages); count += pages; while (pages--) __free_pages_bootmem(page++, 0); |
223e8dc92
|
260 |
|
2e5237daf
|
261 262 |
bdebug("nid=%td released=%lx ", bdata - bootmem_node_data, count); |
223e8dc92
|
263 264 |
return count; } |
08677214e
|
265 |
#endif |
223e8dc92
|
266 |
|
a66fd7dae
|
267 268 269 270 271 272 |
/** * free_all_bootmem_node - release a node's free pages to the buddy allocator * @pgdat: node to be released * * Returns the number of pages actually released. */ |
223e8dc92
|
273 274 275 |
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat) { register_page_bootmem_info_node(pgdat); |
08677214e
|
276 277 278 279 |
#ifdef CONFIG_NO_BOOTMEM /* free_all_memory_core_early(MAX_NUMNODES) will be called later */ return 0; #else |
223e8dc92
|
280 |
return free_all_bootmem_core(pgdat->bdata); |
08677214e
|
281 |
#endif |
223e8dc92
|
282 |
} |
a66fd7dae
|
283 284 285 286 287 |
/** * free_all_bootmem - release free pages to the buddy allocator * * Returns the number of pages actually released. */ |
223e8dc92
|
288 289 |
unsigned long __init free_all_bootmem(void) { |
08677214e
|
290 |
#ifdef CONFIG_NO_BOOTMEM |
337998587
|
291 292 293 294 295 296 297 298 |
/* * We need to use MAX_NUMNODES instead of NODE_DATA(0)->node_id * because in some case like Node0 doesnt have RAM installed * low ram will be on Node1 * Use MAX_NUMNODES will make sure all ranges in early_node_map[] * will be used instead of only Node0 related */ return free_all_memory_core_early(MAX_NUMNODES); |
08677214e
|
299 |
#else |
aa235fc71
|
300 301 302 303 304 305 306 |
unsigned long total_pages = 0; bootmem_data_t *bdata; list_for_each_entry(bdata, &bdata_list, list) total_pages += free_all_bootmem_core(bdata); return total_pages; |
08677214e
|
307 |
#endif |
223e8dc92
|
308 |
} |
08677214e
|
309 |
#ifndef CONFIG_NO_BOOTMEM |
d747fa4bc
|
310 311 312 313 314 315 316 |
static void __init __free(bootmem_data_t *bdata, unsigned long sidx, unsigned long eidx) { unsigned long idx; bdebug("nid=%td start=%lx end=%lx ", bdata - bootmem_node_data, |
3560e249a
|
317 318 |
sidx + bdata->node_min_pfn, eidx + bdata->node_min_pfn); |
d747fa4bc
|
319 |
|
e2bf3cae5
|
320 321 |
if (bdata->hint_idx > sidx) bdata->hint_idx = sidx; |
d747fa4bc
|
322 323 324 325 326 327 328 329 330 331 332 333 334 335 |
for (idx = sidx; idx < eidx; idx++) if (!test_and_clear_bit(idx, bdata->node_bootmem_map)) BUG(); } static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx, unsigned long eidx, int flags) { unsigned long idx; int exclusive = flags & BOOTMEM_EXCLUSIVE; bdebug("nid=%td start=%lx end=%lx flags=%x ", bdata - bootmem_node_data, |
3560e249a
|
336 337 |
sidx + bdata->node_min_pfn, eidx + bdata->node_min_pfn, |
d747fa4bc
|
338 339 340 341 342 343 344 345 346 347 |
flags); for (idx = sidx; idx < eidx; idx++) if (test_and_set_bit(idx, bdata->node_bootmem_map)) { if (exclusive) { __free(bdata, sidx, idx); return -EBUSY; } bdebug("silent double reserve of PFN %lx ", |
3560e249a
|
348 |
idx + bdata->node_min_pfn); |
d747fa4bc
|
349 350 351 |
} return 0; } |
e2bf3cae5
|
352 353 354 |
static int __init mark_bootmem_node(bootmem_data_t *bdata, unsigned long start, unsigned long end, int reserve, int flags) |
223e8dc92
|
355 356 |
{ unsigned long sidx, eidx; |
223e8dc92
|
357 |
|
e2bf3cae5
|
358 359 360 |
bdebug("nid=%td start=%lx end=%lx reserve=%d flags=%x ", bdata - bootmem_node_data, start, end, reserve, flags); |
223e8dc92
|
361 |
|
3560e249a
|
362 |
BUG_ON(start < bdata->node_min_pfn); |
e2bf3cae5
|
363 |
BUG_ON(end > bdata->node_low_pfn); |
223e8dc92
|
364 |
|
3560e249a
|
365 366 |
sidx = start - bdata->node_min_pfn; eidx = end - bdata->node_min_pfn; |
223e8dc92
|
367 |
|
e2bf3cae5
|
368 369 |
if (reserve) return __reserve(bdata, sidx, eidx, flags); |
223e8dc92
|
370 |
else |
e2bf3cae5
|
371 372 373 374 375 376 377 378 379 380 381 382 383 384 |
__free(bdata, sidx, eidx); return 0; } static int __init mark_bootmem(unsigned long start, unsigned long end, int reserve, int flags) { unsigned long pos; bootmem_data_t *bdata; pos = start; list_for_each_entry(bdata, &bdata_list, list) { int err; unsigned long max; |
3560e249a
|
385 386 |
if (pos < bdata->node_min_pfn || pos >= bdata->node_low_pfn) { |
e2bf3cae5
|
387 388 389 390 391 |
BUG_ON(pos != start); continue; } max = min(bdata->node_low_pfn, end); |
223e8dc92
|
392 |
|
e2bf3cae5
|
393 394 395 396 397 |
err = mark_bootmem_node(bdata, pos, max, reserve, flags); if (reserve && err) { mark_bootmem(start, pos, 0, 0); return err; } |
223e8dc92
|
398 |
|
e2bf3cae5
|
399 400 401 402 403 |
if (max == end) return 0; pos = bdata->node_low_pfn; } BUG(); |
223e8dc92
|
404 |
} |
08677214e
|
405 |
#endif |
223e8dc92
|
406 |
|
a66fd7dae
|
407 408 409 410 411 412 413 414 |
/** * free_bootmem_node - mark a page range as usable * @pgdat: node the range resides on * @physaddr: starting address of the range * @size: size of the range in bytes * * Partial pages will be considered reserved and left as they are. * |
e2bf3cae5
|
415 |
* The range must reside completely on the specified node. |
a66fd7dae
|
416 |
*/ |
223e8dc92
|
417 418 419 |
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, unsigned long size) { |
08677214e
|
420 421 |
#ifdef CONFIG_NO_BOOTMEM free_early(physaddr, physaddr + size); |
08677214e
|
422 |
#else |
e2bf3cae5
|
423 |
unsigned long start, end; |
ec3a354bd
|
424 |
kmemleak_free_part(__va(physaddr), size); |
e2bf3cae5
|
425 426 427 428 |
start = PFN_UP(physaddr); end = PFN_DOWN(physaddr + size); mark_bootmem_node(pgdat->bdata, start, end, 0, 0); |
08677214e
|
429 |
#endif |
223e8dc92
|
430 |
} |
a66fd7dae
|
431 432 433 434 435 436 437 |
/** * free_bootmem - mark a page range as usable * @addr: starting address of the range * @size: size of the range in bytes * * Partial pages will be considered reserved and left as they are. * |
e2bf3cae5
|
438 |
* The range must be contiguous but may span node boundaries. |
a66fd7dae
|
439 |
*/ |
223e8dc92
|
440 441 |
void __init free_bootmem(unsigned long addr, unsigned long size) { |
08677214e
|
442 443 |
#ifdef CONFIG_NO_BOOTMEM free_early(addr, addr + size); |
08677214e
|
444 |
#else |
e2bf3cae5
|
445 |
unsigned long start, end; |
a5645a61b
|
446 |
|
ec3a354bd
|
447 |
kmemleak_free_part(__va(addr), size); |
e2bf3cae5
|
448 449 |
start = PFN_UP(addr); end = PFN_DOWN(addr + size); |
1da177e4c
|
450 |
|
e2bf3cae5
|
451 |
mark_bootmem(start, end, 0, 0); |
08677214e
|
452 |
#endif |
1da177e4c
|
453 |
} |
a66fd7dae
|
454 455 456 457 458 459 460 461 462 |
/** * reserve_bootmem_node - mark a page range as reserved * @pgdat: node the range resides on * @physaddr: starting address of the range * @size: size of the range in bytes * @flags: reservation flags (see linux/bootmem.h) * * Partial pages will be reserved. * |
e2bf3cae5
|
463 |
* The range must reside completely on the specified node. |
a66fd7dae
|
464 |
*/ |
223e8dc92
|
465 466 |
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, unsigned long size, int flags) |
1da177e4c
|
467 |
{ |
08677214e
|
468 469 470 471 |
#ifdef CONFIG_NO_BOOTMEM panic("no bootmem"); return 0; #else |
e2bf3cae5
|
472 |
unsigned long start, end; |
1da177e4c
|
473 |
|
e2bf3cae5
|
474 475 476 477 |
start = PFN_DOWN(physaddr); end = PFN_UP(physaddr + size); return mark_bootmem_node(pgdat->bdata, start, end, 1, flags); |
08677214e
|
478 |
#endif |
223e8dc92
|
479 |
} |
5a982cbc7
|
480 |
|
a66fd7dae
|
481 482 483 484 485 486 487 488 |
/** * reserve_bootmem - mark a page range as usable * @addr: starting address of the range * @size: size of the range in bytes * @flags: reservation flags (see linux/bootmem.h) * * Partial pages will be reserved. * |
e2bf3cae5
|
489 |
* The range must be contiguous but may span node boundaries. |
a66fd7dae
|
490 |
*/ |
223e8dc92
|
491 492 493 |
int __init reserve_bootmem(unsigned long addr, unsigned long size, int flags) { |
08677214e
|
494 495 496 497 |
#ifdef CONFIG_NO_BOOTMEM panic("no bootmem"); return 0; #else |
e2bf3cae5
|
498 |
unsigned long start, end; |
1da177e4c
|
499 |
|
e2bf3cae5
|
500 501 |
start = PFN_DOWN(addr); end = PFN_UP(addr + size); |
223e8dc92
|
502 |
|
e2bf3cae5
|
503 |
return mark_bootmem(start, end, 1, flags); |
08677214e
|
504 |
#endif |
1da177e4c
|
505 |
} |
08677214e
|
506 |
#ifndef CONFIG_NO_BOOTMEM |
8aa043d74
|
507 508 |
static unsigned long __init align_idx(struct bootmem_data *bdata, unsigned long idx, unsigned long step) |
481ebd0d7
|
509 510 511 512 513 514 515 516 517 518 |
{ unsigned long base = bdata->node_min_pfn; /* * Align the index with respect to the node start so that the * combination of both satisfies the requested alignment. */ return ALIGN(base + idx, step) - base; } |
8aa043d74
|
519 520 |
static unsigned long __init align_off(struct bootmem_data *bdata, unsigned long off, unsigned long align) |
481ebd0d7
|
521 522 523 524 525 526 527 |
{ unsigned long base = PFN_PHYS(bdata->node_min_pfn); /* Same as align_idx for byte offsets */ return ALIGN(base + off, align) - base; } |
d0c4f5702
|
528 529 530 |
static void * __init alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) |
1da177e4c
|
531 |
{ |
0f3caba21
|
532 |
unsigned long fallback = 0; |
5f2809e69
|
533 |
unsigned long min, max, start, sidx, midx, step; |
594fe1a04
|
534 535 536 537 |
bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx ", bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT, align, goal, limit); |
5f2809e69
|
538 539 540 |
BUG_ON(!size); BUG_ON(align & (align - 1)); BUG_ON(limit && goal + size > limit); |
1da177e4c
|
541 |
|
7c309a64d
|
542 543 |
if (!bdata->node_bootmem_map) return NULL; |
3560e249a
|
544 |
min = bdata->node_min_pfn; |
5f2809e69
|
545 |
max = bdata->node_low_pfn; |
9a2dc04cf
|
546 |
|
5f2809e69
|
547 548 549 550 551 552 |
goal >>= PAGE_SHIFT; limit >>= PAGE_SHIFT; if (limit && max > limit) max = limit; if (max <= min) |
9a2dc04cf
|
553 |
return NULL; |
5f2809e69
|
554 |
step = max(align >> PAGE_SHIFT, 1UL); |
281dd25cd
|
555 |
|
5f2809e69
|
556 557 558 559 |
if (goal && min < goal && goal < max) start = ALIGN(goal, step); else start = ALIGN(min, step); |
1da177e4c
|
560 |
|
481ebd0d7
|
561 |
sidx = start - bdata->node_min_pfn; |
3560e249a
|
562 |
midx = max - bdata->node_min_pfn; |
1da177e4c
|
563 |
|
5f2809e69
|
564 |
if (bdata->hint_idx > sidx) { |
0f3caba21
|
565 566 567 568 569 |
/* * Handle the valid case of sidx being zero and still * catch the fallback below. */ fallback = sidx + 1; |
481ebd0d7
|
570 |
sidx = align_idx(bdata, bdata->hint_idx, step); |
5f2809e69
|
571 |
} |
1da177e4c
|
572 |
|
5f2809e69
|
573 574 575 576 577 578 |
while (1) { int merge; void *region; unsigned long eidx, i, start_off, end_off; find_block: sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx); |
481ebd0d7
|
579 |
sidx = align_idx(bdata, sidx, step); |
5f2809e69
|
580 |
eidx = sidx + PFN_UP(size); |
ad09315ca
|
581 |
|
5f2809e69
|
582 |
if (sidx >= midx || eidx > midx) |
66d43e98e
|
583 |
break; |
1da177e4c
|
584 |
|
5f2809e69
|
585 586 |
for (i = sidx; i < eidx; i++) if (test_bit(i, bdata->node_bootmem_map)) { |
481ebd0d7
|
587 |
sidx = align_idx(bdata, i, step); |
5f2809e69
|
588 589 590 591 |
if (sidx == i) sidx += step; goto find_block; } |
1da177e4c
|
592 |
|
627240aaa
|
593 |
if (bdata->last_end_off & (PAGE_SIZE - 1) && |
5f2809e69
|
594 |
PFN_DOWN(bdata->last_end_off) + 1 == sidx) |
481ebd0d7
|
595 |
start_off = align_off(bdata, bdata->last_end_off, align); |
5f2809e69
|
596 597 598 599 600 601 602 603 604 605 606 607 |
else start_off = PFN_PHYS(sidx); merge = PFN_DOWN(start_off) < sidx; end_off = start_off + size; bdata->last_end_off = end_off; bdata->hint_idx = PFN_UP(end_off); /* * Reserve the area now: */ |
d747fa4bc
|
608 609 610 |
if (__reserve(bdata, PFN_DOWN(start_off) + merge, PFN_UP(end_off), BOOTMEM_EXCLUSIVE)) BUG(); |
5f2809e69
|
611 |
|
3560e249a
|
612 613 |
region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) + start_off); |
5f2809e69
|
614 |
memset(region, 0, size); |
008139d91
|
615 616 617 618 619 |
/* * The min_count is set to 0 so that bootmem allocated blocks * are never reported as leaks. */ kmemleak_alloc(region, size, 0, 0); |
5f2809e69
|
620 |
return region; |
1da177e4c
|
621 |
} |
0f3caba21
|
622 |
if (fallback) { |
481ebd0d7
|
623 |
sidx = align_idx(bdata, fallback - 1, step); |
0f3caba21
|
624 625 626 627 628 629 |
fallback = 0; goto find_block; } return NULL; } |
d0c4f5702
|
630 631 632 633 |
static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata, unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { |
441c7e0a2
|
634 635 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc(size, GFP_NOWAIT); |
d0c4f5702
|
636 |
#ifdef CONFIG_HAVE_ARCH_BOOTMEM |
433f13a72
|
637 638 639 640 641 642 643 644 645 |
{ bootmem_data_t *p_bdata; p_bdata = bootmem_arch_preferred_node(bdata, size, align, goal, limit); if (p_bdata) return alloc_bootmem_core(p_bdata, size, align, goal, limit); } |
d0c4f5702
|
646 647 648 |
#endif return NULL; } |
08677214e
|
649 |
#endif |
d0c4f5702
|
650 |
|
0f3caba21
|
651 652 653 654 655 |
static void * __init ___alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { |
08677214e
|
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 |
#ifdef CONFIG_NO_BOOTMEM void *ptr; if (WARN_ON_ONCE(slab_is_available())) return kzalloc(size, GFP_NOWAIT); restart: ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit); if (ptr) return ptr; if (goal != 0) { goal = 0; goto restart; } return NULL; #else |
0f3caba21
|
676 |
bootmem_data_t *bdata; |
d0c4f5702
|
677 |
void *region; |
0f3caba21
|
678 679 |
restart: |
d0c4f5702
|
680 681 682 |
region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit); if (region) return region; |
0f3caba21
|
683 |
|
d0c4f5702
|
684 |
list_for_each_entry(bdata, &bdata_list, list) { |
0f3caba21
|
685 686 |
if (goal && bdata->node_low_pfn <= PFN_DOWN(goal)) continue; |
3560e249a
|
687 |
if (limit && bdata->node_min_pfn >= PFN_DOWN(limit)) |
0f3caba21
|
688 689 690 691 692 693 |
break; region = alloc_bootmem_core(bdata, size, align, goal, limit); if (region) return region; } |
5f2809e69
|
694 695 |
if (goal) { goal = 0; |
0f3caba21
|
696 |
goto restart; |
5f2809e69
|
697 |
} |
2e5237daf
|
698 |
|
5f2809e69
|
699 |
return NULL; |
08677214e
|
700 |
#endif |
1da177e4c
|
701 |
} |
a66fd7dae
|
702 703 704 705 706 707 708 709 710 711 712 713 714 |
/** * __alloc_bootmem_nopanic - allocate boot memory without panicking * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * * Returns NULL on failure. */ |
bb0923a66
|
715 |
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, |
0f3caba21
|
716 |
unsigned long goal) |
1da177e4c
|
717 |
{ |
08677214e
|
718 719 720 721 722 723 724 |
unsigned long limit = 0; #ifdef CONFIG_NO_BOOTMEM limit = -1UL; #endif return ___alloc_bootmem_nopanic(size, align, goal, limit); |
0f3caba21
|
725 |
} |
1da177e4c
|
726 |
|
0f3caba21
|
727 728 729 730 731 732 733 734 735 736 737 738 739 |
static void * __init ___alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { void *mem = ___alloc_bootmem_nopanic(size, align, goal, limit); if (mem) return mem; /* * Whoops, we cannot satisfy the allocation request. */ printk(KERN_ALERT "bootmem alloc of %lu bytes failed! ", size); panic("Out of memory"); |
a8062231d
|
740 741 |
return NULL; } |
1da177e4c
|
742 |
|
a66fd7dae
|
743 744 745 746 747 748 749 750 751 752 753 754 755 |
/** * __alloc_bootmem - allocate boot memory * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * * The function panics if the request can not be satisfied. */ |
bb0923a66
|
756 757 |
void * __init __alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal) |
a8062231d
|
758 |
{ |
08677214e
|
759 760 761 762 763 764 765 |
unsigned long limit = 0; #ifdef CONFIG_NO_BOOTMEM limit = -1UL; #endif return ___alloc_bootmem(size, align, goal, limit); |
1da177e4c
|
766 |
} |
08677214e
|
767 |
#ifndef CONFIG_NO_BOOTMEM |
4cc278b72
|
768 769 770 771 772 |
static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata, unsigned long size, unsigned long align, unsigned long goal, unsigned long limit) { void *ptr; |
d0c4f5702
|
773 774 775 |
ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit); if (ptr) return ptr; |
4cc278b72
|
776 777 778 779 780 781 |
ptr = alloc_bootmem_core(bdata, size, align, goal, limit); if (ptr) return ptr; return ___alloc_bootmem(size, align, goal, limit); } |
08677214e
|
782 |
#endif |
4cc278b72
|
783 |
|
a66fd7dae
|
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 |
/** * __alloc_bootmem_node - allocate boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. */ |
bb0923a66
|
799 800 |
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) |
1da177e4c
|
801 |
{ |
b8ab9f820
|
802 |
void *ptr; |
c91c4773b
|
803 804 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); |
08677214e
|
805 |
#ifdef CONFIG_NO_BOOTMEM |
b8ab9f820
|
806 807 808 809 810 811 |
ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, -1ULL); if (ptr) return ptr; ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, |
08677214e
|
812 813 |
goal, -1ULL); #else |
b8ab9f820
|
814 |
ptr = ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0); |
08677214e
|
815 |
#endif |
b8ab9f820
|
816 817 |
return ptr; |
08677214e
|
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 |
} void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { #ifdef MAX_DMA32_PFN unsigned long end_pfn; if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); /* update goal according ...MAX_DMA32_PFN */ end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages; if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) && (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) { void *ptr; unsigned long new_goal; new_goal = MAX_DMA32_PFN << PAGE_SHIFT; #ifdef CONFIG_NO_BOOTMEM ptr = __alloc_memory_core_early(pgdat->node_id, size, align, new_goal, -1ULL); #else ptr = alloc_bootmem_core(pgdat->bdata, size, align, new_goal, 0); #endif if (ptr) return ptr; } #endif return __alloc_bootmem_node(pgdat, size, align, goal); |
1da177e4c
|
851 |
} |
e70260aab
|
852 |
#ifdef CONFIG_SPARSEMEM |
a66fd7dae
|
853 854 855 856 857 858 859 |
/** * alloc_bootmem_section - allocate boot memory from a specific section * @size: size of the request in bytes * @section_nr: sparse map section to allocate from * * Return NULL on failure. */ |
e70260aab
|
860 861 862 |
void * __init alloc_bootmem_section(unsigned long size, unsigned long section_nr) { |
08677214e
|
863 864 865 866 867 868 869 870 871 872 |
#ifdef CONFIG_NO_BOOTMEM unsigned long pfn, goal, limit; pfn = section_nr_to_pfn(section_nr); goal = pfn << PAGE_SHIFT; limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT; return __alloc_memory_core_early(early_pfn_to_nid(pfn), size, SMP_CACHE_BYTES, goal, limit); #else |
75a56cfe9
|
873 874 |
bootmem_data_t *bdata; unsigned long pfn, goal, limit; |
e70260aab
|
875 876 |
pfn = section_nr_to_pfn(section_nr); |
75a56cfe9
|
877 878 879 |
goal = pfn << PAGE_SHIFT; limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT; bdata = &bootmem_node_data[early_pfn_to_nid(pfn)]; |
e70260aab
|
880 |
|
75a56cfe9
|
881 |
return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit); |
08677214e
|
882 |
#endif |
e70260aab
|
883 884 |
} #endif |
b54bbf7b8
|
885 886 887 888 |
void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { void *ptr; |
c91c4773b
|
889 890 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); |
08677214e
|
891 892 893 894 |
#ifdef CONFIG_NO_BOOTMEM ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, -1ULL); #else |
d0c4f5702
|
895 896 897 |
ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0); if (ptr) return ptr; |
b54bbf7b8
|
898 |
ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0); |
08677214e
|
899 |
#endif |
b54bbf7b8
|
900 901 902 903 904 |
if (ptr) return ptr; return __alloc_bootmem_nopanic(size, align, goal); } |
dfd54cbcc
|
905 906 907 |
#ifndef ARCH_LOW_ADDRESS_LIMIT #define ARCH_LOW_ADDRESS_LIMIT 0xffffffffUL #endif |
008857c1a
|
908 |
|
a66fd7dae
|
909 910 911 912 913 914 915 916 917 918 919 920 921 |
/** * __alloc_bootmem_low - allocate low boot memory * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may happen on any node in the system. * * The function panics if the request can not be satisfied. */ |
bb0923a66
|
922 923 |
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, unsigned long goal) |
008857c1a
|
924 |
{ |
0f3caba21
|
925 |
return ___alloc_bootmem(size, align, goal, ARCH_LOW_ADDRESS_LIMIT); |
008857c1a
|
926 |
} |
a66fd7dae
|
927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 |
/** * __alloc_bootmem_low_node - allocate low boot memory from a specific node * @pgdat: node to allocate from * @size: size of the request in bytes * @align: alignment of the region * @goal: preferred starting address of the region * * The goal is dropped if it can not be satisfied and the allocation will * fall back to memory below @goal. * * Allocation may fall back to any node in the system if the specified node * can not hold the requested memory. * * The function panics if the request can not be satisfied. */ |
008857c1a
|
942 943 944 |
void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size, unsigned long align, unsigned long goal) { |
b8ab9f820
|
945 |
void *ptr; |
c91c4773b
|
946 947 |
if (WARN_ON_ONCE(slab_is_available())) return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id); |
08677214e
|
948 |
#ifdef CONFIG_NO_BOOTMEM |
b8ab9f820
|
949 950 951 952 953 |
ptr = __alloc_memory_core_early(pgdat->node_id, size, align, goal, ARCH_LOW_ADDRESS_LIMIT); if (ptr) return ptr; ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, |
08677214e
|
954 955 |
goal, ARCH_LOW_ADDRESS_LIMIT); #else |
b8ab9f820
|
956 |
ptr = ___alloc_bootmem_node(pgdat->bdata, size, align, |
4cc278b72
|
957 |
goal, ARCH_LOW_ADDRESS_LIMIT); |
08677214e
|
958 |
#endif |
b8ab9f820
|
959 |
return ptr; |
008857c1a
|
960 |
} |