Blame view
mm/memory_hotplug.c
50.7 KB
457c89965 treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
3947be196 [PATCH] memory ho... |
2 3 4 5 6 |
/* * linux/mm/memory_hotplug.c * * Copyright (C) */ |
3947be196 [PATCH] memory ho... |
7 8 |
#include <linux/stddef.h> #include <linux/mm.h> |
174cd4b1e sched/headers: Pr... |
9 |
#include <linux/sched/signal.h> |
3947be196 [PATCH] memory ho... |
10 11 12 |
#include <linux/swap.h> #include <linux/interrupt.h> #include <linux/pagemap.h> |
3947be196 [PATCH] memory ho... |
13 |
#include <linux/compiler.h> |
b95f1b31b mm: Map most file... |
14 |
#include <linux/export.h> |
3947be196 [PATCH] memory ho... |
15 |
#include <linux/pagevec.h> |
2d1d43f6a [PATCH] call mm/p... |
16 |
#include <linux/writeback.h> |
3947be196 [PATCH] memory ho... |
17 18 19 20 |
#include <linux/slab.h> #include <linux/sysctl.h> #include <linux/cpu.h> #include <linux/memory.h> |
4b94ffdc4 x86, mm: introduc... |
21 |
#include <linux/memremap.h> |
3947be196 [PATCH] memory ho... |
22 23 24 |
#include <linux/memory_hotplug.h> #include <linux/highmem.h> #include <linux/vmalloc.h> |
0a5470390 [PATCH] register ... |
25 |
#include <linux/ioport.h> |
0c0e61958 memory unplug: pa... |
26 27 28 |
#include <linux/delay.h> #include <linux/migrate.h> #include <linux/page-isolation.h> |
71088785c mm: cleanup to ma... |
29 |
#include <linux/pfn.h> |
6ad696d2c mm: allow memory ... |
30 |
#include <linux/suspend.h> |
6d9c285a6 mm: move inc_zone... |
31 |
#include <linux/mm_inline.h> |
d96ae5309 memory-hotplug: c... |
32 |
#include <linux/firmware-map.h> |
60a5a19e7 memory-hotplug: r... |
33 |
#include <linux/stop_machine.h> |
c8721bbbd mm: memory-hotplu... |
34 |
#include <linux/hugetlb.h> |
c5320926e mem-hotplug: intr... |
35 |
#include <linux/memblock.h> |
698b1b306 mm, compaction: i... |
36 |
#include <linux/compaction.h> |
b15c87263 hwpoison, memory_... |
37 |
#include <linux/rmap.h> |
3947be196 [PATCH] memory ho... |
38 39 |
#include <asm/tlbflush.h> |
1e5ad9a3b mm/memory_hotplug... |
40 |
#include "internal.h" |
e900a918b mm: shuffle initi... |
41 |
#include "shuffle.h" |
1e5ad9a3b mm/memory_hotplug... |
42 |
|
9d0ad8ca4 mm: extend memory... |
43 44 45 46 47 48 |
/* * online_page_callback contains pointer to current page onlining function. * Initially it is generic_online_page(). If it is required it could be * changed by calling set_online_page_callback() for callback registration * and restore_online_page_callback() for generic callback restore. */ |
9d0ad8ca4 mm: extend memory... |
49 |
static online_page_callback_t online_page_callback = generic_online_page; |
bfc8c9013 mem-hotplug: impl... |
50 |
static DEFINE_MUTEX(online_page_callback_lock); |
9d0ad8ca4 mm: extend memory... |
51 |
|
3f906ba23 mm/memory-hotplug... |
52 |
DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock); |
bfc8c9013 mem-hotplug: impl... |
53 |
|
3f906ba23 mm/memory-hotplug... |
54 55 56 57 |
void get_online_mems(void) { percpu_down_read(&mem_hotplug_lock); } |
bfc8c9013 mem-hotplug: impl... |
58 |
|
3f906ba23 mm/memory-hotplug... |
59 60 61 62 |
void put_online_mems(void) { percpu_up_read(&mem_hotplug_lock); } |
bfc8c9013 mem-hotplug: impl... |
63 |
|
4932381ee mm, memory_hotplu... |
64 |
bool movable_node_enabled = false; |
8604d9e53 memory_hotplug: i... |
65 |
#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE |
862919e56 mm/memory_hotplug... |
66 |
int memhp_default_online_type = MMOP_OFFLINE; |
8604d9e53 memory_hotplug: i... |
67 |
#else |
862919e56 mm/memory_hotplug... |
68 |
int memhp_default_online_type = MMOP_ONLINE; |
8604d9e53 memory_hotplug: i... |
69 |
#endif |
31bc3858e memory-hotplug: a... |
70 |
|
86dd995d6 memory_hotplug: i... |
71 72 |
static int __init setup_memhp_default_state(char *str) { |
5f47adf76 mm/memory_hotplug... |
73 74 75 76 |
const int online_type = memhp_online_type_from_str(str); if (online_type >= 0) memhp_default_online_type = online_type; |
86dd995d6 memory_hotplug: i... |
77 78 79 80 |
return 1; } __setup("memhp_default_state=", setup_memhp_default_state); |
30467e0b3 mm, hotplug: fix ... |
81 |
void mem_hotplug_begin(void) |
20d6c96b5 mem-hotplug: intr... |
82 |
{ |
3f906ba23 mm/memory-hotplug... |
83 84 |
cpus_read_lock(); percpu_down_write(&mem_hotplug_lock); |
20d6c96b5 mem-hotplug: intr... |
85 |
} |
30467e0b3 mm, hotplug: fix ... |
86 |
void mem_hotplug_done(void) |
bfc8c9013 mem-hotplug: impl... |
87 |
{ |
3f906ba23 mm/memory-hotplug... |
88 89 |
percpu_up_write(&mem_hotplug_lock); cpus_read_unlock(); |
bfc8c9013 mem-hotplug: impl... |
90 |
} |
20d6c96b5 mem-hotplug: intr... |
91 |
|
357b4da50 x86: respect memo... |
92 |
u64 max_mem_size = U64_MAX; |
45e0b78b0 [PATCH] hot-add-m... |
93 |
/* add this memory to iomem resource */ |
7b7b27214 mm/memory_hotplug... |
94 95 |
static struct resource *register_memory_resource(u64 start, u64 size, const char *resource_name) |
45e0b78b0 [PATCH] hot-add-m... |
96 |
{ |
2794129e9 mm/memory-hotplug... |
97 98 |
struct resource *res; unsigned long flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; |
7b7b27214 mm/memory_hotplug... |
99 100 |
if (strcmp(resource_name, "System RAM")) |
7cf603d17 kernel/resource: ... |
101 |
flags |= IORESOURCE_SYSRAM_DRIVER_MANAGED; |
357b4da50 x86: respect memo... |
102 |
|
f3cd4c865 mm/memory_hotplug... |
103 104 105 106 107 108 109 |
/* * Make sure value parsed from 'mem=' only restricts memory adding * while booting, so that memory hotplug won't be impacted. Please * refer to document of 'mem=' in kernel-parameters.txt for more * details. */ if (start + size > max_mem_size && system_state < SYSTEM_RUNNING) |
357b4da50 x86: respect memo... |
110 |
return ERR_PTR(-E2BIG); |
2794129e9 mm/memory-hotplug... |
111 112 113 114 115 116 117 118 119 120 121 122 |
/* * Request ownership of the new memory range. This might be * a child of an existing resource that was present but * not marked as busy. */ res = __request_region(&iomem_resource, start, size, resource_name, flags); if (!res) { pr_debug("Unable to reserve System RAM region: %016llx->%016llx ", start, start + size); |
6f754ba4c memory-hotplug: d... |
123 |
return ERR_PTR(-EEXIST); |
45e0b78b0 [PATCH] hot-add-m... |
124 125 126 127 128 129 130 131 132 133 |
} return res; } static void release_memory_resource(struct resource *res) { if (!res) return; release_resource(res); kfree(res); |
45e0b78b0 [PATCH] hot-add-m... |
134 |
} |
53947027a [PATCH] hot-add-m... |
135 |
#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE |
46723bfa5 memory-hotplug: i... |
136 137 |
void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) |
047532787 memory hotplug: r... |
138 |
{ |
ddffe98d1 mm/memory_hotplug... |
139 |
page->freelist = (void *)type; |
047532787 memory hotplug: r... |
140 141 |
SetPagePrivate(page); set_page_private(page, info); |
fe896d187 mm: introduce pag... |
142 |
page_ref_inc(page); |
047532787 memory hotplug: r... |
143 |
} |
170a5a7eb mm: make __free_p... |
144 |
void put_page_bootmem(struct page *page) |
047532787 memory hotplug: r... |
145 |
{ |
5f24ce5fd thp: remove PG_buddy |
146 |
unsigned long type; |
047532787 memory hotplug: r... |
147 |
|
ddffe98d1 mm/memory_hotplug... |
148 |
type = (unsigned long) page->freelist; |
5f24ce5fd thp: remove PG_buddy |
149 150 |
BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); |
047532787 memory hotplug: r... |
151 |
|
fe896d187 mm: introduce pag... |
152 |
if (page_ref_dec_return(page) == 1) { |
ddffe98d1 mm/memory_hotplug... |
153 |
page->freelist = NULL; |
047532787 memory hotplug: r... |
154 155 |
ClearPagePrivate(page); set_page_private(page, 0); |
5f24ce5fd thp: remove PG_buddy |
156 |
INIT_LIST_HEAD(&page->lru); |
170a5a7eb mm: make __free_p... |
157 |
free_reserved_page(page); |
047532787 memory hotplug: r... |
158 |
} |
047532787 memory hotplug: r... |
159 |
} |
46723bfa5 memory-hotplug: i... |
160 161 |
#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE #ifndef CONFIG_SPARSEMEM_VMEMMAP |
d92bc3185 mm: make register... |
162 |
static void register_page_bootmem_info_section(unsigned long start_pfn) |
047532787 memory hotplug: r... |
163 |
{ |
f1eca35a0 mm/sparsemem: int... |
164 |
unsigned long mapsize, section_nr, i; |
047532787 memory hotplug: r... |
165 166 |
struct mem_section *ms; struct page *page, *memmap; |
f1eca35a0 mm/sparsemem: int... |
167 |
struct mem_section_usage *usage; |
047532787 memory hotplug: r... |
168 |
|
047532787 memory hotplug: r... |
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 |
section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); /* Get section's memmap address */ memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); /* * Get page for the memmap's phys address * XXX: need more consideration for sparse_vmemmap... */ page = virt_to_page(memmap); mapsize = sizeof(struct page) * PAGES_PER_SECTION; mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; /* remember memmap's page */ for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, SECTION_INFO); |
f1eca35a0 mm/sparsemem: int... |
186 187 |
usage = ms->usage; page = virt_to_page(usage); |
047532787 memory hotplug: r... |
188 |
|
f1eca35a0 mm/sparsemem: int... |
189 |
mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; |
047532787 memory hotplug: r... |
190 191 |
for (i = 0; i < mapsize; i++, page++) |
af370fb8c memory hotplug: s... |
192 |
get_page_bootmem(section_nr, page, MIX_SECTION_INFO); |
047532787 memory hotplug: r... |
193 194 |
} |
46723bfa5 memory-hotplug: i... |
195 196 197 |
#else /* CONFIG_SPARSEMEM_VMEMMAP */ static void register_page_bootmem_info_section(unsigned long start_pfn) { |
f1eca35a0 mm/sparsemem: int... |
198 |
unsigned long mapsize, section_nr, i; |
46723bfa5 memory-hotplug: i... |
199 200 |
struct mem_section *ms; struct page *page, *memmap; |
f1eca35a0 mm/sparsemem: int... |
201 |
struct mem_section_usage *usage; |
46723bfa5 memory-hotplug: i... |
202 |
|
46723bfa5 memory-hotplug: i... |
203 204 205 206 207 208 |
section_nr = pfn_to_section_nr(start_pfn); ms = __nr_to_section(section_nr); memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); |
f1eca35a0 mm/sparsemem: int... |
209 210 |
usage = ms->usage; page = virt_to_page(usage); |
46723bfa5 memory-hotplug: i... |
211 |
|
f1eca35a0 mm/sparsemem: int... |
212 |
mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; |
46723bfa5 memory-hotplug: i... |
213 214 215 216 217 |
for (i = 0; i < mapsize; i++, page++) get_page_bootmem(section_nr, page, MIX_SECTION_INFO); } #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ |
047532787 memory hotplug: r... |
218 |
|
7ded384a1 mm: fix section m... |
219 |
void __init register_page_bootmem_info_node(struct pglist_data *pgdat) |
047532787 memory hotplug: r... |
220 221 222 223 |
{ unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; |
047532787 memory hotplug: r... |
224 225 226 227 228 229 |
nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); |
047532787 memory hotplug: r... |
230 |
pfn = pgdat->node_start_pfn; |
c1f194952 mm/memory_hotplug... |
231 |
end_pfn = pgdat_end_pfn(pgdat); |
047532787 memory hotplug: r... |
232 |
|
7e9f5eb03 mm/memory_hotplug... |
233 |
/* register section info */ |
f14851af0 memory hotplug: f... |
234 235 236 237 238 |
for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { /* * Some platforms can assign the same pfn to multiple nodes - on * node0 as well as nodeN. To avoid registering a pfn against * multiple nodes we check that this pfn does not already |
7e9f5eb03 mm/memory_hotplug... |
239 |
* reside in some other nodes. |
f14851af0 memory hotplug: f... |
240 |
*/ |
f65e91df2 mm: use early_pfn... |
241 |
if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) |
f14851af0 memory hotplug: f... |
242 243 |
register_page_bootmem_info_section(pfn); } |
047532787 memory hotplug: r... |
244 |
} |
46723bfa5 memory-hotplug: i... |
245 |
#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ |
047532787 memory hotplug: r... |
246 |
|
7ea621604 mm/sparsemem: pre... |
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 |
static int check_pfn_span(unsigned long pfn, unsigned long nr_pages, const char *reason) { /* * Disallow all operations smaller than a sub-section and only * allow operations smaller than a section for * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range() * enforces a larger memory_block_size_bytes() granularity for * memory that will be marked online, so this check should only * fire for direct arch_{add,remove}_memory() users outside of * add_memory_resource(). */ unsigned long min_align; if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP)) min_align = PAGES_PER_SUBSECTION; else min_align = PAGES_PER_SECTION; if (!IS_ALIGNED(pfn, min_align) || !IS_ALIGNED(nr_pages, min_align)) { WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx ", reason, pfn, pfn + nr_pages - 1); return -EINVAL; } return 0; } |
dca4436d1 mm/memory_hotplug... |
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 |
static int check_hotplug_memory_addressable(unsigned long pfn, unsigned long nr_pages) { const u64 max_addr = PFN_PHYS(pfn + nr_pages) - 1; if (max_addr >> MAX_PHYSMEM_BITS) { const u64 max_allowed = (1ull << (MAX_PHYSMEM_BITS + 1)) - 1; WARN(1, "Hotplugged memory exceeds maximum addressable address, range=%#llx-%#llx, maximum=%#llx ", (u64)PFN_PHYS(pfn), max_addr, max_allowed); return -E2BIG; } return 0; } |
4edd7ceff mm, hotplug: avoi... |
290 291 292 293 294 295 |
/* * Reasonably generic function for adding memory. It is * expected that archs that support memory hotplug will * call this function after deciding the zone to which to * add the new pages. */ |
7ea621604 mm/sparsemem: pre... |
296 |
int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages, |
f5637d3b4 mm/memory_hotplug... |
297 |
struct mhp_params *params) |
4edd7ceff mm, hotplug: avoi... |
298 |
{ |
6cdd0b30a mm/memory_hotplug... |
299 300 |
const unsigned long end_pfn = pfn + nr_pages; unsigned long cur_nr_pages; |
9a8450304 mm/sparsemem: cle... |
301 |
int err; |
f5637d3b4 mm/memory_hotplug... |
302 |
struct vmem_altmap *altmap = params->altmap; |
4b94ffdc4 x86, mm: introduc... |
303 |
|
bfeb022f8 mm/memory_hotplug... |
304 305 |
if (WARN_ON_ONCE(!params->pgprot.pgprot)) return -EINVAL; |
dca4436d1 mm/memory_hotplug... |
306 307 308 |
err = check_hotplug_memory_addressable(pfn, nr_pages); if (err) return err; |
4b94ffdc4 x86, mm: introduc... |
309 310 311 312 |
if (altmap) { /* * Validate altmap is within bounds of the total request */ |
7ea621604 mm/sparsemem: pre... |
313 |
if (altmap->base_pfn != pfn |
4b94ffdc4 x86, mm: introduc... |
314 315 316 |
|| vmem_altmap_offset(altmap) > nr_pages) { pr_warn_once("memory add fail, invalid altmap "); |
7ea621604 mm/sparsemem: pre... |
317 |
return -EINVAL; |
4b94ffdc4 x86, mm: introduc... |
318 319 320 |
} altmap->alloc = 0; } |
7ea621604 mm/sparsemem: pre... |
321 322 323 |
err = check_pfn_span(pfn, nr_pages, "add"); if (err) return err; |
6cdd0b30a mm/memory_hotplug... |
324 325 326 327 328 |
for (; pfn < end_pfn; pfn += cur_nr_pages) { /* Select all remaining pages up to the next section boundary */ cur_nr_pages = min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); err = sparse_add_section(nid, pfn, cur_nr_pages, altmap); |
ba72b4c8c mm/sparsemem: sup... |
329 330 |
if (err) break; |
f64ac5e6e mm, memory_hotplu... |
331 |
cond_resched(); |
4edd7ceff mm, hotplug: avoi... |
332 |
} |
c435a3905 mm/memory hotplug... |
333 |
vmemmap_populate_print_last(); |
4edd7ceff mm, hotplug: avoi... |
334 335 |
return err; } |
4edd7ceff mm, hotplug: avoi... |
336 |
|
815121d2b memory_hotplug: c... |
337 |
/* find the smallest valid pfn in the range [start_pfn, end_pfn) */ |
d09b0137d mm/memory_hotplug... |
338 |
static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, |
815121d2b memory_hotplug: c... |
339 340 341 |
unsigned long start_pfn, unsigned long end_pfn) { |
49ba3c6b3 mm/hotplug: prepa... |
342 |
for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) { |
7ce700bf1 mm/memory_hotplug... |
343 |
if (unlikely(!pfn_to_online_page(start_pfn))) |
815121d2b memory_hotplug: c... |
344 345 346 347 |
continue; if (unlikely(pfn_to_nid(start_pfn) != nid)) continue; |
9b05158f5 mm/memory_hotplug... |
348 |
if (zone != page_zone(pfn_to_page(start_pfn))) |
815121d2b memory_hotplug: c... |
349 350 351 352 353 354 355 356 357 |
continue; return start_pfn; } return 0; } /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ |
d09b0137d mm/memory_hotplug... |
358 |
static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, |
815121d2b memory_hotplug: c... |
359 360 361 |
unsigned long start_pfn, unsigned long end_pfn) { |
815121d2b memory_hotplug: c... |
362 363 364 365 |
unsigned long pfn; /* pfn is the end pfn of a memory section. */ pfn = end_pfn - 1; |
49ba3c6b3 mm/hotplug: prepa... |
366 |
for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) { |
7ce700bf1 mm/memory_hotplug... |
367 |
if (unlikely(!pfn_to_online_page(pfn))) |
815121d2b memory_hotplug: c... |
368 369 370 371 |
continue; if (unlikely(pfn_to_nid(pfn) != nid)) continue; |
9b05158f5 mm/memory_hotplug... |
372 |
if (zone != page_zone(pfn_to_page(pfn))) |
815121d2b memory_hotplug: c... |
373 374 375 376 377 378 379 380 381 382 383 |
continue; return pfn; } return 0; } static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, unsigned long end_pfn) { |
815121d2b memory_hotplug: c... |
384 |
unsigned long pfn; |
815121d2b memory_hotplug: c... |
385 386 387 |
int nid = zone_to_nid(zone); zone_span_writelock(zone); |
5d12071c5 mm/memory_hotplug... |
388 |
if (zone->zone_start_pfn == start_pfn) { |
815121d2b memory_hotplug: c... |
389 390 391 392 393 394 395 |
/* * If the section is smallest section in the zone, it need * shrink zone->zone_start_pfn and zone->zone_spanned_pages. * In this case, we find second smallest valid mem_section * for shrinking zone. */ pfn = find_smallest_section_pfn(nid, zone, end_pfn, |
5d12071c5 mm/memory_hotplug... |
396 |
zone_end_pfn(zone)); |
815121d2b memory_hotplug: c... |
397 |
if (pfn) { |
5d12071c5 mm/memory_hotplug... |
398 |
zone->spanned_pages = zone_end_pfn(zone) - pfn; |
815121d2b memory_hotplug: c... |
399 |
zone->zone_start_pfn = pfn; |
950b68d91 mm/memory_hotplug... |
400 401 402 |
} else { zone->zone_start_pfn = 0; zone->spanned_pages = 0; |
815121d2b memory_hotplug: c... |
403 |
} |
5d12071c5 mm/memory_hotplug... |
404 |
} else if (zone_end_pfn(zone) == end_pfn) { |
815121d2b memory_hotplug: c... |
405 406 407 408 409 410 |
/* * If the section is biggest section in the zone, it need * shrink zone->spanned_pages. * In this case, we find second biggest valid mem_section for * shrinking zone. */ |
5d12071c5 mm/memory_hotplug... |
411 |
pfn = find_biggest_section_pfn(nid, zone, zone->zone_start_pfn, |
815121d2b memory_hotplug: c... |
412 413 |
start_pfn); if (pfn) |
5d12071c5 mm/memory_hotplug... |
414 |
zone->spanned_pages = pfn - zone->zone_start_pfn + 1; |
950b68d91 mm/memory_hotplug... |
415 416 417 418 |
else { zone->zone_start_pfn = 0; zone->spanned_pages = 0; } |
815121d2b memory_hotplug: c... |
419 |
} |
815121d2b memory_hotplug: c... |
420 421 |
zone_span_writeunlock(zone); } |
00d6c019b mm/memory_hotplug... |
422 |
static void update_pgdat_span(struct pglist_data *pgdat) |
815121d2b memory_hotplug: c... |
423 |
{ |
00d6c019b mm/memory_hotplug... |
424 425 426 427 428 429 430 431 432 |
unsigned long node_start_pfn = 0, node_end_pfn = 0; struct zone *zone; for (zone = pgdat->node_zones; zone < pgdat->node_zones + MAX_NR_ZONES; zone++) { unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; /* No need to lock the zones, they can't change. */ |
656d57119 mm/memory_hotplug... |
433 434 435 436 437 438 439 |
if (!zone->spanned_pages) continue; if (!node_end_pfn) { node_start_pfn = zone->zone_start_pfn; node_end_pfn = zone_end_pfn; continue; } |
00d6c019b mm/memory_hotplug... |
440 441 442 443 |
if (zone_end_pfn > node_end_pfn) node_end_pfn = zone_end_pfn; if (zone->zone_start_pfn < node_start_pfn) node_start_pfn = zone->zone_start_pfn; |
815121d2b memory_hotplug: c... |
444 |
} |
00d6c019b mm/memory_hotplug... |
445 446 |
pgdat->node_start_pfn = node_start_pfn; pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; |
815121d2b memory_hotplug: c... |
447 |
} |
feee6b298 mm/memory_hotplug... |
448 449 450 |
void __ref remove_pfn_range_from_zone(struct zone *zone, unsigned long start_pfn, unsigned long nr_pages) |
815121d2b memory_hotplug: c... |
451 |
{ |
b7e3debdd mm/memory_hotplug... |
452 |
const unsigned long end_pfn = start_pfn + nr_pages; |
815121d2b memory_hotplug: c... |
453 |
struct pglist_data *pgdat = zone->zone_pgdat; |
b7e3debdd mm/memory_hotplug... |
454 |
unsigned long pfn, cur_nr_pages, flags; |
815121d2b memory_hotplug: c... |
455 |
|
d33695b16 mm/memory_hotplug... |
456 |
/* Poison struct pages because they are now uninitialized again. */ |
b7e3debdd mm/memory_hotplug... |
457 458 459 460 461 462 463 464 465 |
for (pfn = start_pfn; pfn < end_pfn; pfn += cur_nr_pages) { cond_resched(); /* Select all remaining pages up to the next section boundary */ cur_nr_pages = min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); page_init_poison(pfn_to_page(pfn), sizeof(struct page) * cur_nr_pages); } |
d33695b16 mm/memory_hotplug... |
466 |
|
7ce700bf1 mm/memory_hotplug... |
467 468 469 470 471 472 473 474 475 |
#ifdef CONFIG_ZONE_DEVICE /* * Zone shrinking code cannot properly deal with ZONE_DEVICE. So * we will not try to shrink the zones - which is okay as * set_zone_contiguous() cannot deal with ZONE_DEVICE either way. */ if (zone_idx(zone) == ZONE_DEVICE) return; #endif |
feee6b298 mm/memory_hotplug... |
476 |
clear_zone_contiguous(zone); |
815121d2b memory_hotplug: c... |
477 478 |
pgdat_resize_lock(zone->zone_pgdat, &flags); shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); |
00d6c019b mm/memory_hotplug... |
479 |
update_pgdat_span(pgdat); |
815121d2b memory_hotplug: c... |
480 |
pgdat_resize_unlock(zone->zone_pgdat, &flags); |
feee6b298 mm/memory_hotplug... |
481 482 |
set_zone_contiguous(zone); |
815121d2b memory_hotplug: c... |
483 |
} |
feee6b298 mm/memory_hotplug... |
484 485 486 |
static void __remove_section(unsigned long pfn, unsigned long nr_pages, unsigned long map_offset, struct vmem_altmap *altmap) |
ea01ea937 hotplug memory re... |
487 |
{ |
104049017 mm/memory_hotplug... |
488 |
struct mem_section *ms = __pfn_to_section(pfn); |
ea01ea937 hotplug memory re... |
489 |
|
9d1d887d7 mm/memory_hotplug... |
490 491 |
if (WARN_ON_ONCE(!valid_section(ms))) return; |
ea01ea937 hotplug memory re... |
492 |
|
ba72b4c8c mm/sparsemem: sup... |
493 |
sparse_remove_section(ms, pfn, nr_pages, map_offset, altmap); |
ea01ea937 hotplug memory re... |
494 |
} |
ea01ea937 hotplug memory re... |
495 |
/** |
feee6b298 mm/memory_hotplug... |
496 |
* __remove_pages() - remove sections of pages |
7ea621604 mm/sparsemem: pre... |
497 |
* @pfn: starting pageframe (must be aligned to start of a section) |
ea01ea937 hotplug memory re... |
498 |
* @nr_pages: number of pages to remove (must be multiple of section size) |
e8b098fc5 mm: kernel-doc: a... |
499 |
* @altmap: alternative device page map or %NULL if default memmap is used |
ea01ea937 hotplug memory re... |
500 501 502 503 504 505 |
* * Generic helper function to remove section mappings and sysfs entries * for the section of the memory we are removing. Caller needs to make * sure that pages are marked reserved and zones are adjust properly by * calling offline_pages(). */ |
feee6b298 mm/memory_hotplug... |
506 507 |
void __remove_pages(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) |
ea01ea937 hotplug memory re... |
508 |
{ |
52fb87c81 mm/memory_hotplug... |
509 510 |
const unsigned long end_pfn = pfn + nr_pages; unsigned long cur_nr_pages; |
4b94ffdc4 x86, mm: introduc... |
511 |
unsigned long map_offset = 0; |
4b94ffdc4 x86, mm: introduc... |
512 |
|
96da43500 mm/hotplug: kill ... |
513 |
map_offset = vmem_altmap_offset(altmap); |
ea01ea937 hotplug memory re... |
514 |
|
7ea621604 mm/sparsemem: pre... |
515 516 |
if (check_pfn_span(pfn, nr_pages, "remove")) return; |
ea01ea937 hotplug memory re... |
517 |
|
52fb87c81 mm/memory_hotplug... |
518 |
for (; pfn < end_pfn; pfn += cur_nr_pages) { |
dd33ad7b2 memory_hotplug: c... |
519 |
cond_resched(); |
52fb87c81 mm/memory_hotplug... |
520 |
/* Select all remaining pages up to the next section boundary */ |
a11b94192 mm/memory_hotplug... |
521 522 |
cur_nr_pages = min(end_pfn - pfn, SECTION_ALIGN_UP(pfn + 1) - pfn); |
52fb87c81 mm/memory_hotplug... |
523 |
__remove_section(pfn, cur_nr_pages, map_offset, altmap); |
4b94ffdc4 x86, mm: introduc... |
524 |
map_offset = 0; |
ea01ea937 hotplug memory re... |
525 |
} |
ea01ea937 hotplug memory re... |
526 |
} |
ea01ea937 hotplug memory re... |
527 |
|
9d0ad8ca4 mm: extend memory... |
528 529 530 |
int set_online_page_callback(online_page_callback_t callback) { int rc = -EINVAL; |
bfc8c9013 mem-hotplug: impl... |
531 532 |
get_online_mems(); mutex_lock(&online_page_callback_lock); |
9d0ad8ca4 mm: extend memory... |
533 534 535 536 537 |
if (online_page_callback == generic_online_page) { online_page_callback = callback; rc = 0; } |
bfc8c9013 mem-hotplug: impl... |
538 539 |
mutex_unlock(&online_page_callback_lock); put_online_mems(); |
9d0ad8ca4 mm: extend memory... |
540 541 542 543 544 545 546 547 |
return rc; } EXPORT_SYMBOL_GPL(set_online_page_callback); int restore_online_page_callback(online_page_callback_t callback) { int rc = -EINVAL; |
bfc8c9013 mem-hotplug: impl... |
548 549 |
get_online_mems(); mutex_lock(&online_page_callback_lock); |
9d0ad8ca4 mm: extend memory... |
550 551 552 553 554 |
if (online_page_callback == callback) { online_page_callback = generic_online_page; rc = 0; } |
bfc8c9013 mem-hotplug: impl... |
555 556 |
mutex_unlock(&online_page_callback_lock); put_online_mems(); |
9d0ad8ca4 mm: extend memory... |
557 558 559 560 |
return rc; } EXPORT_SYMBOL_GPL(restore_online_page_callback); |
18db14912 mm/memory_hotplug... |
561 |
void generic_online_page(struct page *page, unsigned int order) |
9d0ad8ca4 mm: extend memory... |
562 |
{ |
c87cbc1f0 mm, hotplug: fix ... |
563 564 565 566 567 568 569 |
/* * Freeing the page with debug_pagealloc enabled will try to unmap it, * so we should map it first. This is better than introducing a special * case in page freeing fast path. */ if (debug_pagealloc_enabled_static()) kernel_map_pages(page, 1 << order, 1); |
a9cd410a3 mm/page_alloc.c: ... |
570 571 572 573 574 575 576 |
__free_pages_core(page, order); totalram_pages_add(1UL << order); #ifdef CONFIG_HIGHMEM if (PageHighMem(page)) totalhigh_pages_add(1UL << order); #endif } |
18db14912 mm/memory_hotplug... |
577 |
EXPORT_SYMBOL_GPL(generic_online_page); |
a9cd410a3 mm/page_alloc.c: ... |
578 |
|
aac65321b mm/memory_hotplug... |
579 |
static void online_pages_range(unsigned long start_pfn, unsigned long nr_pages) |
3947be196 [PATCH] memory ho... |
580 |
{ |
b2c2ab208 mm/memory_hotplug... |
581 582 |
const unsigned long end_pfn = start_pfn + nr_pages; unsigned long pfn; |
b2c2ab208 mm/memory_hotplug... |
583 584 |
/* |
aac65321b mm/memory_hotplug... |
585 586 587 588 |
* Online the pages in MAX_ORDER - 1 aligned chunks. The callback might * decide to not expose all pages to the buddy (e.g., expose them * later). We account all pages as being online and belonging to this * zone ("present"). |
b2c2ab208 mm/memory_hotplug... |
589 |
*/ |
aac65321b mm/memory_hotplug... |
590 591 |
for (pfn = start_pfn; pfn < end_pfn; pfn += MAX_ORDER_NR_PAGES) (*online_page_callback)(pfn_to_page(pfn), MAX_ORDER - 1); |
2d070eab2 mm: consider zone... |
592 |
|
b2c2ab208 mm/memory_hotplug... |
593 594 |
/* mark all involved sections as online */ online_mem_sections(start_pfn, end_pfn); |
75884fb1c memory unplug: me... |
595 |
} |
d9713679d memory_hotplug: f... |
596 597 598 599 600 |
/* check which state of node_states will be changed when online memory */ static void node_states_check_changes_online(unsigned long nr_pages, struct zone *zone, struct memory_notify *arg) { int nid = zone_to_nid(zone); |
d9713679d memory_hotplug: f... |
601 |
|
98fa15f34 mm: replace all o... |
602 603 604 |
arg->status_change_nid = NUMA_NO_NODE; arg->status_change_nid_normal = NUMA_NO_NODE; arg->status_change_nid_high = NUMA_NO_NODE; |
d9713679d memory_hotplug: f... |
605 |
|
8efe33f40 mm/memory_hotplug... |
606 607 608 |
if (!node_state(nid, N_MEMORY)) arg->status_change_nid = nid; if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY)) |
d9713679d memory_hotplug: f... |
609 |
arg->status_change_nid_normal = nid; |
6715ddf94 hotplug: update n... |
610 |
#ifdef CONFIG_HIGHMEM |
d3ba3ae19 mm/memory_hotplug... |
611 |
if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY)) |
6715ddf94 hotplug: update n... |
612 |
arg->status_change_nid_high = nid; |
6715ddf94 hotplug: update n... |
613 |
#endif |
d9713679d memory_hotplug: f... |
614 615 616 617 618 619 |
} static void node_states_set_node(int node, struct memory_notify *arg) { if (arg->status_change_nid_normal >= 0) node_set_state(node, N_NORMAL_MEMORY); |
6715ddf94 hotplug: update n... |
620 621 |
if (arg->status_change_nid_high >= 0) node_set_state(node, N_HIGH_MEMORY); |
83d83612d mm/memory_hotplug... |
622 623 |
if (arg->status_change_nid >= 0) node_set_state(node, N_MEMORY); |
d9713679d memory_hotplug: f... |
624 |
} |
f1dd2cd13 mm, memory_hotplu... |
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 |
static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn, unsigned long nr_pages) { unsigned long old_end_pfn = zone_end_pfn(zone); if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) zone->zone_start_pfn = start_pfn; zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn; } static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn, unsigned long nr_pages) { unsigned long old_end_pfn = pgdat_end_pfn(pgdat); if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) pgdat->node_start_pfn = start_pfn; pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; |
f1dd2cd13 mm, memory_hotplu... |
645 |
|
3fccb74cf mm/memory_hotplug... |
646 647 648 649 650 |
} /* * Associate the pfn range with the given zone, initializing the memmaps * and resizing the pgdat/zone data to span the added pages. After this * call, all affected pages are PG_reserved. |
d882c0067 mm: pass migratet... |
651 652 653 654 |
* * All aligned pageblocks are initialized to the specified migratetype * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related * zone stats (e.g., nr_isolate_pageblock) are touched. |
3fccb74cf mm/memory_hotplug... |
655 |
*/ |
a99583e78 mm: pass the vmem... |
656 |
void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, |
d882c0067 mm: pass migratet... |
657 658 |
unsigned long nr_pages, struct vmem_altmap *altmap, int migratetype) |
f1dd2cd13 mm, memory_hotplu... |
659 660 661 662 |
{ struct pglist_data *pgdat = zone->zone_pgdat; int nid = pgdat->node_id; unsigned long flags; |
df429ac03 memory-hotplug: m... |
663 |
|
f1dd2cd13 mm, memory_hotplu... |
664 665 666 667 668 |
clear_zone_contiguous(zone); /* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */ pgdat_resize_lock(pgdat, &flags); zone_span_writelock(zone); |
fa004ab73 mm, hotplug: move... |
669 670 |
if (zone_is_empty(zone)) init_currently_empty_zone(zone, start_pfn, nr_pages); |
f1dd2cd13 mm, memory_hotplu... |
671 672 673 674 675 676 677 678 679 680 681 |
resize_zone_range(zone, start_pfn, nr_pages); zone_span_writeunlock(zone); resize_pgdat_range(pgdat, start_pfn, nr_pages); pgdat_resize_unlock(pgdat, &flags); /* * TODO now we have a visible range of pages which are not associated * with their zone properly. Not nice but set_pfnblock_flags_mask * expects the zone spans the pfn range. All the pages in the range * are reserved so nobody should be touching them so we should be safe */ |
98b57685c mm: memmap defer ... |
682 |
memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 0, |
d882c0067 mm: pass migratet... |
683 |
MEMINIT_HOTPLUG, altmap, migratetype); |
f1dd2cd13 mm, memory_hotplu... |
684 685 686 687 688 |
set_zone_contiguous(zone); } /* |
c246a213f mm, memory_hotplu... |
689 690 691 692 |
* Returns a default kernel memory zone for the given pfn range. * If no kernel zone covers this pfn range it will automatically go * to the ZONE_NORMAL. */ |
c6f03e290 mm, memory_hotplu... |
693 |
static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn, |
c246a213f mm, memory_hotplu... |
694 695 696 697 698 699 700 701 702 703 704 705 706 707 |
unsigned long nr_pages) { struct pglist_data *pgdat = NODE_DATA(nid); int zid; for (zid = 0; zid <= ZONE_NORMAL; zid++) { struct zone *zone = &pgdat->node_zones[zid]; if (zone_intersects(zone, start_pfn, nr_pages)) return zone; } return &pgdat->node_zones[ZONE_NORMAL]; } |
c6f03e290 mm, memory_hotplu... |
708 709 |
static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn, unsigned long nr_pages) |
e5e689302 mm, memory_hotplu... |
710 |
{ |
c6f03e290 mm, memory_hotplu... |
711 712 713 714 715 |
struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages); bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages); |
e5e689302 mm, memory_hotplu... |
716 717 |
/* |
c6f03e290 mm, memory_hotplu... |
718 719 |
* We inherit the existing zone in a simple case where zones do not * overlap in the given range |
e5e689302 mm, memory_hotplu... |
720 |
*/ |
c6f03e290 mm, memory_hotplu... |
721 722 |
if (in_kernel ^ in_movable) return (in_kernel) ? kernel_zone : movable_zone; |
9f123ab54 mm, memory_hotplu... |
723 |
|
c6f03e290 mm, memory_hotplu... |
724 725 726 727 728 729 |
/* * If the range doesn't belong to any zone or two zones overlap in the * given range then we use movable zone only if movable_node is * enabled because we always online to a kernel zone by default. */ return movable_node_enabled ? movable_zone : kernel_zone; |
9f123ab54 mm, memory_hotplu... |
730 |
} |
e5e689302 mm, memory_hotplu... |
731 732 |
struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, unsigned long nr_pages) |
f1dd2cd13 mm, memory_hotplu... |
733 |
{ |
c6f03e290 mm, memory_hotplu... |
734 735 |
if (online_type == MMOP_ONLINE_KERNEL) return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); |
f1dd2cd13 mm, memory_hotplu... |
736 |
|
c6f03e290 mm, memory_hotplu... |
737 738 |
if (online_type == MMOP_ONLINE_MOVABLE) return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; |
df429ac03 memory-hotplug: m... |
739 |
|
c6f03e290 mm, memory_hotplu... |
740 |
return default_zone_for_pfn(nid, start_pfn, nr_pages); |
e5e689302 mm, memory_hotplu... |
741 |
} |
bd5c2344f mm/memory_hotplug... |
742 743 |
int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type, int nid) |
75884fb1c memory unplug: me... |
744 |
{ |
aa47228a1 memory_hotplug: u... |
745 |
unsigned long flags; |
3947be196 [PATCH] memory ho... |
746 |
struct zone *zone; |
6811378e7 [PATCH] wait_tabl... |
747 |
int need_zonelists_rebuild = 0; |
7b78d335a memory hotplug: r... |
748 749 |
int ret; struct memory_notify arg; |
d0dc12e86 mm/memory_hotplug... |
750 |
|
4986fac16 mm/memory_hotplug... |
751 752 753 754 |
/* We can only online full sections (e.g., SECTION_IS_ONLINE) */ if (WARN_ON_ONCE(!nr_pages || !IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION))) return -EINVAL; |
381eab4a6 mm/memory_hotplug... |
755 |
mem_hotplug_begin(); |
f1dd2cd13 mm, memory_hotplu... |
756 |
/* associate pfn range with the zone */ |
3fccb74cf mm/memory_hotplug... |
757 |
zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages); |
b30c59279 mm/memory_hotplug... |
758 |
move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE); |
f1dd2cd13 mm, memory_hotplu... |
759 |
|
7b78d335a memory hotplug: r... |
760 761 |
arg.start_pfn = pfn; arg.nr_pages = nr_pages; |
d9713679d memory_hotplug: f... |
762 |
node_states_check_changes_online(nr_pages, zone, &arg); |
7b78d335a memory hotplug: r... |
763 |
|
7b78d335a memory hotplug: r... |
764 765 |
ret = memory_notify(MEM_GOING_ONLINE, &arg); ret = notifier_to_errno(ret); |
e33e33b4d mm, memory hotplu... |
766 767 |
if (ret) goto failed_addition; |
3947be196 [PATCH] memory ho... |
768 |
/* |
b30c59279 mm/memory_hotplug... |
769 770 771 772 773 774 775 776 |
* Fixup the number of isolated pageblocks before marking the sections * onlining, such that undo_isolate_page_range() works correctly. */ spin_lock_irqsave(&zone->lock, flags); zone->nr_isolate_pageblock += nr_pages / pageblock_nr_pages; spin_unlock_irqrestore(&zone->lock, flags); /* |
6811378e7 [PATCH] wait_tabl... |
777 778 779 780 |
* If this zone is not populated, then it is not in zonelist. * This means the page allocator ignores this zone. * So, zonelist must be updated after online. */ |
6dcd73d70 memory-hotplug: a... |
781 |
if (!populated_zone(zone)) { |
6811378e7 [PATCH] wait_tabl... |
782 |
need_zonelists_rebuild = 1; |
72675e131 mm, memory_hotplu... |
783 |
setup_zone_pageset(zone); |
6dcd73d70 memory-hotplug: a... |
784 |
} |
6811378e7 [PATCH] wait_tabl... |
785 |
|
aac65321b mm/memory_hotplug... |
786 787 |
online_pages_range(pfn, nr_pages); zone->present_pages += nr_pages; |
aa47228a1 memory_hotplug: u... |
788 789 |
pgdat_resize_lock(zone->zone_pgdat, &flags); |
aac65321b mm/memory_hotplug... |
790 |
zone->zone_pgdat->node_present_pages += nr_pages; |
aa47228a1 memory_hotplug: u... |
791 |
pgdat_resize_unlock(zone->zone_pgdat, &flags); |
b30c59279 mm/memory_hotplug... |
792 793 794 795 796 797 798 |
node_states_set_node(nid, &arg); if (need_zonelists_rebuild) build_all_zonelists(NULL); zone_pcp_update(zone); /* Basic onlining is complete, allow allocation of onlined pages. */ undo_isolate_page_range(pfn, pfn + nr_pages, MIGRATE_MOVABLE); |
93146d98c mm/memory_hotplug... |
799 |
/* |
b86c5fc4e mm/memory_hotplug... |
800 801 802 803 |
* Freshly onlined pages aren't shuffled (e.g., all pages are placed to * the tail of the freelist when undoing isolation). Shuffle the whole * zone to make sure the just onlined pages are properly distributed * across the whole freelist - to create an initial shuffle. |
93146d98c mm/memory_hotplug... |
804 |
*/ |
e900a918b mm: shuffle initi... |
805 |
shuffle_zone(zone); |
1b79acc91 mm, mem-hotplug: ... |
806 |
init_per_zone_wmark_min(); |
ca9a46f8a mm/memory_hotplug... |
807 808 |
kswapd_run(nid); kcompactd_run(nid); |
61b13993a [PATCH] memory ho... |
809 |
|
2d1d43f6a [PATCH] call mm/p... |
810 |
writeback_set_ratelimit(); |
7b78d335a memory hotplug: r... |
811 |
|
ca9a46f8a mm/memory_hotplug... |
812 |
memory_notify(MEM_ONLINE, &arg); |
381eab4a6 mm/memory_hotplug... |
813 |
mem_hotplug_done(); |
30467e0b3 mm, hotplug: fix ... |
814 |
return 0; |
e33e33b4d mm, memory hotplu... |
815 816 817 818 819 820 821 |
failed_addition: pr_debug("online_pages [mem %#010llx-%#010llx] failed ", (unsigned long long) pfn << PAGE_SHIFT, (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); memory_notify(MEM_CANCEL_ONLINE, &arg); |
feee6b298 mm/memory_hotplug... |
822 |
remove_pfn_range_from_zone(zone, pfn, nr_pages); |
381eab4a6 mm/memory_hotplug... |
823 |
mem_hotplug_done(); |
e33e33b4d mm, memory hotplu... |
824 |
return ret; |
3947be196 [PATCH] memory ho... |
825 |
} |
53947027a [PATCH] hot-add-m... |
826 |
#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ |
bc02af93d [PATCH] pgdat all... |
827 |
|
0bd854200 mem-hotplug: rese... |
828 829 830 831 832 833 834 835 836 |
static void reset_node_present_pages(pg_data_t *pgdat) { struct zone *z; for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) z->present_pages = 0; pgdat->node_present_pages = 0; } |
e13193319 mm/memory_hotplug... |
837 |
/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ |
c68ab18c6 mm/memory_hotplug... |
838 |
static pg_data_t __ref *hotadd_new_pgdat(int nid) |
9af3c2dea [PATCH] pgdat all... |
839 840 |
{ struct pglist_data *pgdat; |
9af3c2dea [PATCH] pgdat all... |
841 |
|
a1e565aa3 memory-hotplug: d... |
842 843 844 845 846 |
pgdat = NODE_DATA(nid); if (!pgdat) { pgdat = arch_alloc_nodedata(nid); if (!pgdat) return NULL; |
9af3c2dea [PATCH] pgdat all... |
847 |
|
33fce0113 mm/memory_hotplug... |
848 849 |
pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); |
a1e565aa3 memory-hotplug: d... |
850 |
arch_refresh_nodedata(nid, pgdat); |
b0dc3a342 mm/memory hotplug... |
851 |
} else { |
33fce0113 mm/memory_hotplug... |
852 |
int cpu; |
e716f2eb2 mm, vmscan: preve... |
853 |
/* |
97a225e69 mm/page_alloc: in... |
854 855 |
* Reset the nr_zones, order and highest_zoneidx before reuse. * Note that kswapd will init kswapd_highest_zoneidx properly |
e716f2eb2 mm, vmscan: preve... |
856 857 |
* when it starts in the near future. */ |
b0dc3a342 mm/memory hotplug... |
858 |
pgdat->nr_zones = 0; |
38087d9b0 mm, vmscan: simpl... |
859 |
pgdat->kswapd_order = 0; |
97a225e69 mm/page_alloc: in... |
860 |
pgdat->kswapd_highest_zoneidx = 0; |
33fce0113 mm/memory_hotplug... |
861 862 863 864 865 866 |
for_each_online_cpu(cpu) { struct per_cpu_nodestat *p; p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); memset(p, 0, sizeof(*p)); } |
a1e565aa3 memory-hotplug: d... |
867 |
} |
9af3c2dea [PATCH] pgdat all... |
868 869 |
/* we can use NODE_DATA(nid) from here */ |
03e85f9d5 mm/page_alloc: In... |
870 |
pgdat->node_id = nid; |
c68ab18c6 mm/memory_hotplug... |
871 |
pgdat->node_start_pfn = 0; |
03e85f9d5 mm/page_alloc: In... |
872 |
|
9af3c2dea [PATCH] pgdat all... |
873 |
/* init node's zones as empty zones, we don't have any present pages.*/ |
03e85f9d5 mm/page_alloc: In... |
874 |
free_area_init_core_hotplug(nid); |
9af3c2dea [PATCH] pgdat all... |
875 |
|
959ecc48f mm/memory_hotplug... |
876 877 878 879 |
/* * The node we allocated has no zone fallback lists. For avoiding * to access not-initialized zonelist, build here. */ |
72675e131 mm, memory_hotplu... |
880 |
build_all_zonelists(pgdat); |
959ecc48f mm/memory_hotplug... |
881 |
|
f784a3f19 mem-hotplug: rese... |
882 |
/* |
0bd854200 mem-hotplug: rese... |
883 884 885 886 |
* When memory is hot-added, all the memory is in offline state. So * clear all zones' present_pages because they will be updated in * online_pages() and offline_pages(). */ |
03e85f9d5 mm/page_alloc: In... |
887 |
reset_node_managed_pages(pgdat); |
0bd854200 mem-hotplug: rese... |
888 |
reset_node_present_pages(pgdat); |
9af3c2dea [PATCH] pgdat all... |
889 890 |
return pgdat; } |
b9ff03608 mm/memory_hotplug... |
891 |
static void rollback_node_hotadd(int nid) |
9af3c2dea [PATCH] pgdat all... |
892 |
{ |
b9ff03608 mm/memory_hotplug... |
893 |
pg_data_t *pgdat = NODE_DATA(nid); |
9af3c2dea [PATCH] pgdat all... |
894 |
arch_refresh_nodedata(nid, NULL); |
5830169f4 mm/memory_hotplug... |
895 |
free_percpu(pgdat->per_cpu_nodestats); |
9af3c2dea [PATCH] pgdat all... |
896 |
arch_free_nodedata(pgdat); |
9af3c2dea [PATCH] pgdat all... |
897 |
} |
0a5470390 [PATCH] register ... |
898 |
|
01b0f1970 cpu/mem hotplug: ... |
899 900 |
/** * try_online_node - online a node if offlined |
e8b098fc5 mm: kernel-doc: a... |
901 |
* @nid: the node ID |
b9ff03608 mm/memory_hotplug... |
902 |
* @set_node_online: Whether we want to online the node |
cf23422b9 cpu/mem hotplug: ... |
903 |
* called by cpu_up() to online a node without onlined memory. |
b9ff03608 mm/memory_hotplug... |
904 905 906 907 908 |
* * Returns: * 1 -> a new node has been allocated * 0 -> the node is already online * -ENOMEM -> the node could not be allocated |
cf23422b9 cpu/mem hotplug: ... |
909 |
*/ |
c68ab18c6 mm/memory_hotplug... |
910 |
static int __try_online_node(int nid, bool set_node_online) |
cf23422b9 cpu/mem hotplug: ... |
911 |
{ |
b9ff03608 mm/memory_hotplug... |
912 913 |
pg_data_t *pgdat; int ret = 1; |
cf23422b9 cpu/mem hotplug: ... |
914 |
|
01b0f1970 cpu/mem hotplug: ... |
915 916 |
if (node_online(nid)) return 0; |
c68ab18c6 mm/memory_hotplug... |
917 |
pgdat = hotadd_new_pgdat(nid); |
7553e8f2d mm, hotplug: fix ... |
918 |
if (!pgdat) { |
01b0f1970 cpu/mem hotplug: ... |
919 920 |
pr_err("Cannot online node %d due to NULL pgdat ", nid); |
cf23422b9 cpu/mem hotplug: ... |
921 922 923 |
ret = -ENOMEM; goto out; } |
b9ff03608 mm/memory_hotplug... |
924 925 926 927 928 929 |
if (set_node_online) { node_set_online(nid); ret = register_one_node(nid); BUG_ON(ret); } |
cf23422b9 cpu/mem hotplug: ... |
930 |
out: |
b9ff03608 mm/memory_hotplug... |
931 932 933 934 935 936 937 938 939 940 941 |
return ret; } /* * Users of this function always want to online/register the node */ int try_online_node(int nid) { int ret; mem_hotplug_begin(); |
c68ab18c6 mm/memory_hotplug... |
942 |
ret = __try_online_node(nid, true); |
bfc8c9013 mem-hotplug: impl... |
943 |
mem_hotplug_done(); |
cf23422b9 cpu/mem hotplug: ... |
944 945 |
return ret; } |
27356f54c mm/hotplug: verif... |
946 947 |
static int check_hotplug_memory_range(u64 start, u64 size) { |
ba3255852 mm/memory_hotplug... |
948 |
/* memory range must be block size aligned */ |
cec3ebd08 mm/memory_hotplug... |
949 950 |
if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) || !IS_ALIGNED(size, memory_block_size_bytes())) { |
ba3255852 mm/memory_hotplug... |
951 |
pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx", |
cec3ebd08 mm/memory_hotplug... |
952 |
memory_block_size_bytes(), start, size); |
27356f54c mm/hotplug: verif... |
953 954 955 956 957 |
return -EINVAL; } return 0; } |
31bc3858e memory-hotplug: a... |
958 959 |
static int online_memory_block(struct memory_block *mem, void *arg) { |
862919e56 mm/memory_hotplug... |
960 |
mem->online_type = memhp_default_online_type; |
dc18d706a memory-hotplug: u... |
961 |
return device_online(&mem->dev); |
31bc3858e memory-hotplug: a... |
962 |
} |
8df1d0e4a mm/memory_hotplug... |
963 964 965 966 967 968 |
/* * NOTE: The caller must call lock_device_hotplug() to serialize hotplug * and online/offline operations (triggered e.g. by sysfs). * * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ |
b61171997 mm/memory_hotplug... |
969 |
int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags) |
bc02af93d [PATCH] pgdat all... |
970 |
{ |
bfeb022f8 mm/memory_hotplug... |
971 |
struct mhp_params params = { .pgprot = PAGE_KERNEL }; |
62cedb9f1 mm: memory hotplu... |
972 |
u64 start, size; |
b9ff03608 mm/memory_hotplug... |
973 |
bool new_node = false; |
bc02af93d [PATCH] pgdat all... |
974 |
int ret; |
62cedb9f1 mm: memory hotplu... |
975 976 |
start = res->start; size = resource_size(res); |
27356f54c mm/hotplug: verif... |
977 978 979 |
ret = check_hotplug_memory_range(start, size); if (ret) return ret; |
fa6d9ec79 mm/memory_hotplug... |
980 981 982 983 984 |
if (!node_possible(nid)) { WARN(1, "node %d was absent from the node_possible_map ", nid); return -EINVAL; } |
bfc8c9013 mem-hotplug: impl... |
985 |
mem_hotplug_begin(); |
ac13c4622 mm/memory_hotplug... |
986 |
|
52219aeaf mm/memory_hotplug... |
987 988 |
if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) memblock_add_node(start, size, nid); |
7f36e3e56 memory-hotplug: a... |
989 |
|
c68ab18c6 mm/memory_hotplug... |
990 |
ret = __try_online_node(nid, false); |
b9ff03608 mm/memory_hotplug... |
991 992 993 |
if (ret < 0) goto error; new_node = ret; |
9af3c2dea [PATCH] pgdat all... |
994 |
|
bc02af93d [PATCH] pgdat all... |
995 |
/* call arch's memory hotadd */ |
f5637d3b4 mm/memory_hotplug... |
996 |
ret = arch_add_memory(nid, start, size, ¶ms); |
9af3c2dea [PATCH] pgdat all... |
997 998 |
if (ret < 0) goto error; |
db051a0da mm/memory_hotplug... |
999 1000 1001 1002 1003 1004 |
/* create memory block devices after memory was added */ ret = create_memory_block_devices(start, size); if (ret) { arch_remove_memory(nid, start, size, NULL); goto error; } |
a1e565aa3 memory-hotplug: d... |
1005 |
if (new_node) { |
d5b6f6a36 mm/memory_hotplug... |
1006 |
/* If sysfs file of new node can't be created, cpu on the node |
0fc44159b [PATCH] Register ... |
1007 1008 |
* can't be hot-added. There is no rollback way now. * So, check by BUG_ON() to catch it reluctantly.. |
d5b6f6a36 mm/memory_hotplug... |
1009 |
* We online node here. We can't roll back from here. |
0fc44159b [PATCH] Register ... |
1010 |
*/ |
d5b6f6a36 mm/memory_hotplug... |
1011 1012 |
node_set_online(nid); ret = __register_one_node(nid); |
0fc44159b [PATCH] Register ... |
1013 1014 |
BUG_ON(ret); } |
d5b6f6a36 mm/memory_hotplug... |
1015 |
/* link memory sections under this node.*/ |
90c7eaeb1 mm: don't panic w... |
1016 1017 |
link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1), MEMINIT_HOTPLUG); |
d5b6f6a36 mm/memory_hotplug... |
1018 |
|
d96ae5309 memory-hotplug: c... |
1019 |
/* create new memmap entry */ |
7b7b27214 mm/memory_hotplug... |
1020 1021 |
if (!strcmp(res->name, "System RAM")) firmware_map_add_hotplug(start, start + size, "System RAM"); |
d96ae5309 memory-hotplug: c... |
1022 |
|
381eab4a6 mm/memory_hotplug... |
1023 1024 |
/* device_online() will take the lock when calling online_pages() */ mem_hotplug_done(); |
9ca6551ee mm/memory_hotplug... |
1025 1026 1027 1028 1029 1030 |
/* * In case we're allowed to merge the resource, flag it and trigger * merging now that adding succeeded. */ if (mhp_flags & MEMHP_MERGE_RESOURCE) merge_system_ram_resource(res); |
31bc3858e memory-hotplug: a... |
1031 |
/* online pages if requested */ |
862919e56 mm/memory_hotplug... |
1032 |
if (memhp_default_online_type != MMOP_OFFLINE) |
fbcf73ce6 mm/memory_hotplug... |
1033 |
walk_memory_blocks(start, size, NULL, online_memory_block); |
31bc3858e memory-hotplug: a... |
1034 |
|
381eab4a6 mm/memory_hotplug... |
1035 |
return ret; |
9af3c2dea [PATCH] pgdat all... |
1036 1037 |
error: /* rollback pgdat allocation and others */ |
b9ff03608 mm/memory_hotplug... |
1038 1039 |
if (new_node) rollback_node_hotadd(nid); |
52219aeaf mm/memory_hotplug... |
1040 1041 |
if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) memblock_remove(start, size); |
bfc8c9013 mem-hotplug: impl... |
1042 |
mem_hotplug_done(); |
bc02af93d [PATCH] pgdat all... |
1043 1044 |
return ret; } |
62cedb9f1 mm: memory hotplu... |
1045 |
|
8df1d0e4a mm/memory_hotplug... |
1046 |
/* requires device_hotplug_lock, see add_memory_resource() */ |
b61171997 mm/memory_hotplug... |
1047 |
int __ref __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) |
62cedb9f1 mm: memory hotplu... |
1048 1049 1050 |
{ struct resource *res; int ret; |
7b7b27214 mm/memory_hotplug... |
1051 |
res = register_memory_resource(start, size, "System RAM"); |
6f754ba4c memory-hotplug: d... |
1052 1053 |
if (IS_ERR(res)) return PTR_ERR(res); |
62cedb9f1 mm: memory hotplu... |
1054 |
|
b61171997 mm/memory_hotplug... |
1055 |
ret = add_memory_resource(nid, res, mhp_flags); |
62cedb9f1 mm: memory hotplu... |
1056 1057 1058 1059 |
if (ret < 0) release_memory_resource(res); return ret; } |
8df1d0e4a mm/memory_hotplug... |
1060 |
|
b61171997 mm/memory_hotplug... |
1061 |
int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags) |
8df1d0e4a mm/memory_hotplug... |
1062 1063 1064 1065 |
{ int rc; lock_device_hotplug(); |
b61171997 mm/memory_hotplug... |
1066 |
rc = __add_memory(nid, start, size, mhp_flags); |
8df1d0e4a mm/memory_hotplug... |
1067 1068 1069 1070 |
unlock_device_hotplug(); return rc; } |
bc02af93d [PATCH] pgdat all... |
1071 |
EXPORT_SYMBOL_GPL(add_memory); |
0c0e61958 memory unplug: pa... |
1072 |
|
417ac617e ANDROID: mm/memor... |
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 |
int add_memory_subsection(int nid, u64 start, u64 size) { struct mhp_params params = { .pgprot = PAGE_KERNEL }; struct resource *res; int ret; if (size == memory_block_size_bytes()) return add_memory(nid, start, size, MHP_NONE); if (!IS_ALIGNED(start, SUBSECTION_SIZE) || !IS_ALIGNED(size, SUBSECTION_SIZE)) { pr_err("%s: start 0x%lx size 0x%lx not aligned to subsection size ", __func__, start, size); return -EINVAL; } res = register_memory_resource(start, size, "System RAM"); if (IS_ERR(res)) return PTR_ERR(res); mem_hotplug_begin(); nid = memory_add_physaddr_to_nid(start); if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) memblock_add_node(start, size, nid); ret = arch_add_memory(nid, start, size, ¶ms); if (ret) { if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) memblock_remove(start, size); pr_err("%s failed to add subsection start 0x%lx size 0x%lx ", __func__, start, size); } mem_hotplug_done(); return ret; } EXPORT_SYMBOL_GPL(add_memory_subsection); |
7b7b27214 mm/memory_hotplug... |
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 |
/* * Add special, driver-managed memory to the system as system RAM. Such * memory is not exposed via the raw firmware-provided memmap as system * RAM, instead, it is detected and added by a driver - during cold boot, * after a reboot, and after kexec. * * Reasons why this memory should not be used for the initial memmap of a * kexec kernel or for placing kexec images: * - The booting kernel is in charge of determining how this memory will be * used (e.g., use persistent memory as system RAM) * - Coordination with a hypervisor is required before this memory * can be used (e.g., inaccessible parts). * * For this memory, no entries in /sys/firmware/memmap ("raw firmware-provided * memory map") are created. Also, the created memory resource is flagged |
7cf603d17 kernel/resource: ... |
1129 |
* with IORESOURCE_SYSRAM_DRIVER_MANAGED, so in-kernel users can special-case |
7b7b27214 mm/memory_hotplug... |
1130 1131 1132 1133 1134 1135 |
* this memory as well (esp., not place kexec images onto it). * * The resource_name (visible via /proc/iomem) has to have the format * "System RAM ($DRIVER)". */ int add_memory_driver_managed(int nid, u64 start, u64 size, |
b61171997 mm/memory_hotplug... |
1136 |
const char *resource_name, mhp_t mhp_flags) |
7b7b27214 mm/memory_hotplug... |
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 |
{ struct resource *res; int rc; if (!resource_name || strstr(resource_name, "System RAM (") != resource_name || resource_name[strlen(resource_name) - 1] != ')') return -EINVAL; lock_device_hotplug(); res = register_memory_resource(start, size, resource_name); if (IS_ERR(res)) { rc = PTR_ERR(res); goto out_unlock; } |
b61171997 mm/memory_hotplug... |
1153 |
rc = add_memory_resource(nid, res, mhp_flags); |
7b7b27214 mm/memory_hotplug... |
1154 1155 1156 1157 1158 1159 1160 1161 |
if (rc < 0) release_memory_resource(res); out_unlock: unlock_device_hotplug(); return rc; } EXPORT_SYMBOL_GPL(add_memory_driver_managed); |
0c0e61958 memory unplug: pa... |
1162 1163 |
#ifdef CONFIG_MEMORY_HOTREMOVE /* |
929179988 mm/memory_hotplug... |
1164 1165 |
* Confirm all pages in a range [start, end) belong to the same zone (skipping * memory holes). When true, return the zone. |
0c0e61958 memory unplug: pa... |
1166 |
*/ |
929179988 mm/memory_hotplug... |
1167 1168 |
struct zone *test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) |
0c0e61958 memory unplug: pa... |
1169 |
{ |
5f0f2887f mm/memory_hotplug... |
1170 |
unsigned long pfn, sec_end_pfn; |
0c0e61958 memory unplug: pa... |
1171 1172 1173 |
struct zone *zone = NULL; struct page *page; int i; |
deb88a2a1 mm/memory_hotplug... |
1174 |
for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1); |
0c0e61958 memory unplug: pa... |
1175 |
pfn < end_pfn; |
deb88a2a1 mm/memory_hotplug... |
1176 |
pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) { |
5f0f2887f mm/memory_hotplug... |
1177 1178 |
/* Make sure the memory section is present first */ if (!present_section_nr(pfn_to_section_nr(pfn))) |
0c0e61958 memory unplug: pa... |
1179 |
continue; |
5f0f2887f mm/memory_hotplug... |
1180 1181 1182 1183 1184 1185 1186 |
for (; pfn < sec_end_pfn && pfn < end_pfn; pfn += MAX_ORDER_NR_PAGES) { i = 0; /* This is just a CONFIG_HOLES_IN_ZONE check.*/ while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i)) i++; |
d6d8c8a48 mm/memory_hotplug... |
1187 |
if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn) |
5f0f2887f mm/memory_hotplug... |
1188 |
continue; |
24feb47c5 mm, memory_hotplu... |
1189 1190 |
/* Check if we got outside of the zone */ if (zone && !zone_spans_pfn(zone, pfn + i)) |
929179988 mm/memory_hotplug... |
1191 |
return NULL; |
5f0f2887f mm/memory_hotplug... |
1192 1193 |
page = pfn_to_page(pfn + i); if (zone && page_zone(page) != zone) |
929179988 mm/memory_hotplug... |
1194 |
return NULL; |
5f0f2887f mm/memory_hotplug... |
1195 1196 |
zone = page_zone(page); } |
0c0e61958 memory unplug: pa... |
1197 |
} |
deb88a2a1 mm/memory_hotplug... |
1198 |
|
929179988 mm/memory_hotplug... |
1199 |
return zone; |
0c0e61958 memory unplug: pa... |
1200 1201 1202 |
} /* |
0efadf48b mm/hotplug: enabl... |
1203 |
* Scan pfn range [start,end) to find movable/migratable pages (LRU pages, |
aa218795c mm: Allow to offl... |
1204 1205 1206 1207 1208 1209 1210 1211 |
* non-lru movable pages and hugepages). Will skip over most unmovable * pages (esp., pages that can be skipped when offlining), but bail out on * definitely unmovable pages. * * Returns: * 0 in case a movable page is found and movable_pfn was updated. * -ENOENT in case no movable page was found. * -EBUSY in case a definitely unmovable page was found. |
0c0e61958 memory unplug: pa... |
1212 |
*/ |
aa218795c mm: Allow to offl... |
1213 1214 |
static int scan_movable_pages(unsigned long start, unsigned long end, unsigned long *movable_pfn) |
0c0e61958 memory unplug: pa... |
1215 1216 |
{ unsigned long pfn; |
eeb0efd07 mm,memory_hotplug... |
1217 |
|
0c0e61958 memory unplug: pa... |
1218 |
for (pfn = start; pfn < end; pfn++) { |
eeb0efd07 mm,memory_hotplug... |
1219 1220 1221 1222 1223 1224 1225 |
struct page *page, *head; unsigned long skip; if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); if (PageLRU(page)) |
aa218795c mm: Allow to offl... |
1226 |
goto found; |
eeb0efd07 mm,memory_hotplug... |
1227 |
if (__PageMovable(page)) |
aa218795c mm: Allow to offl... |
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 |
goto found; /* * PageOffline() pages that are not marked __PageMovable() and * have a reference count > 0 (after MEM_GOING_OFFLINE) are * definitely unmovable. If their reference count would be 0, * they could at least be skipped when offlining memory. */ if (PageOffline(page) && page_count(page)) return -EBUSY; |
eeb0efd07 mm,memory_hotplug... |
1238 1239 1240 1241 |
if (!PageHuge(page)) continue; head = compound_head(page); |
39186cbe6 mm,memory_hotplug... |
1242 |
if (page_huge_active(head)) |
aa218795c mm: Allow to offl... |
1243 |
goto found; |
d8c6546b1 mm: introduce com... |
1244 |
skip = compound_nr(head) - (page - head); |
eeb0efd07 mm,memory_hotplug... |
1245 |
pfn += skip - 1; |
0c0e61958 memory unplug: pa... |
1246 |
} |
aa218795c mm: Allow to offl... |
1247 1248 1249 |
return -ENOENT; found: *movable_pfn = pfn; |
0c0e61958 memory unplug: pa... |
1250 1251 |
return 0; } |
0c0e61958 memory unplug: pa... |
1252 1253 1254 1255 |
static int do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) { unsigned long pfn; |
6c357848b mm: replace hpage... |
1256 |
struct page *page, *head; |
0c0e61958 memory unplug: pa... |
1257 1258 |
int ret = 0; LIST_HEAD(source); |
a85009c37 mm, memory_hotplu... |
1259 |
for (pfn = start_pfn; pfn < end_pfn; pfn++) { |
0c0e61958 memory unplug: pa... |
1260 1261 1262 |
if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); |
6c357848b mm: replace hpage... |
1263 |
head = compound_head(page); |
c8721bbbd mm: memory-hotplu... |
1264 1265 |
if (PageHuge(page)) { |
d8c6546b1 mm: introduce com... |
1266 |
pfn = page_to_pfn(head) + compound_nr(head) - 1; |
daf3538ad mm,memory_hotplug... |
1267 |
isolate_huge_page(head, &source); |
c8721bbbd mm: memory-hotplu... |
1268 |
continue; |
94723aafb mm: unclutter THP... |
1269 |
} else if (PageTransHuge(page)) |
6c357848b mm: replace hpage... |
1270 |
pfn = page_to_pfn(head) + thp_nr_pages(page) - 1; |
c8721bbbd mm: memory-hotplu... |
1271 |
|
b15c87263 hwpoison, memory_... |
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 |
/* * HWPoison pages have elevated reference counts so the migration would * fail on them. It also doesn't make any sense to migrate them in the * first place. Still try to unmap such a page in case it is still mapped * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep * the unmap as the catch all safety net). */ if (PageHWPoison(page)) { if (WARN_ON(PageLRU(page))) isolate_lru_page(page); if (page_mapped(page)) |
dd156e3fc mm/rmap: always d... |
1283 |
try_to_unmap(page, TTU_IGNORE_MLOCK); |
b15c87263 hwpoison, memory_... |
1284 1285 |
continue; } |
700c2a46e mem-hotplug: call... |
1286 |
if (!get_page_unless_zero(page)) |
0c0e61958 memory unplug: pa... |
1287 1288 |
continue; /* |
0efadf48b mm/hotplug: enabl... |
1289 1290 |
* We can skip free pages. And we can deal with pages on * LRU and non-lru movable pages. |
0c0e61958 memory unplug: pa... |
1291 |
*/ |
0efadf48b mm/hotplug: enabl... |
1292 1293 1294 1295 |
if (PageLRU(page)) ret = isolate_lru_page(page); else ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE); |
0c0e61958 memory unplug: pa... |
1296 |
if (!ret) { /* Success */ |
62695a84e vmscan: move isol... |
1297 |
list_add_tail(&page->lru, &source); |
0efadf48b mm/hotplug: enabl... |
1298 1299 |
if (!__PageMovable(page)) inc_node_page_state(page, NR_ISOLATED_ANON + |
9de4f22a6 mm: code cleanup ... |
1300 |
page_is_file_lru(page)); |
6d9c285a6 mm: move inc_zone... |
1301 |
|
0c0e61958 memory unplug: pa... |
1302 |
} else { |
2932c8b05 mm, memory_hotplu... |
1303 1304 |
pr_warn("failed to isolate pfn %lx ", pfn); |
0efadf48b mm/hotplug: enabl... |
1305 |
dump_page(page, "isolation failed"); |
0c0e61958 memory unplug: pa... |
1306 |
} |
1723058ea mm, memory_hotplu... |
1307 |
put_page(page); |
0c0e61958 memory unplug: pa... |
1308 |
} |
f3ab2636c mm: do_migrate_ra... |
1309 |
if (!list_empty(&source)) { |
203e6e5ca mm/memory_hotplug... |
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 |
nodemask_t nmask = node_states[N_MEMORY]; struct migration_target_control mtc = { .nmask = &nmask, .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, }; /* * We have checked that migration range is on a single zone so * we can use the nid of the first page to all the others. */ mtc.nid = page_to_nid(list_first_entry(&source, struct page, lru)); /* * try to allocate from a different node but reuse this node * if there are no other online nodes to be used (e.g. we are * offlining a part of the only existing node) */ node_clear(mtc.nid, nmask); if (nodes_empty(nmask)) node_set(mtc.nid, nmask); ret = migrate_pages(&source, alloc_migration_target, NULL, (unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_HOTPLUG); |
2932c8b05 mm, memory_hotplu... |
1332 1333 1334 1335 1336 1337 |
if (ret) { list_for_each_entry(page, &source, lru) { pr_warn("migrating pfn %lx failed ret:%d ", page_to_pfn(page), ret); dump_page(page, "migration failure"); } |
c8721bbbd mm: memory-hotplu... |
1338 |
putback_movable_pages(&source); |
2932c8b05 mm, memory_hotplu... |
1339 |
} |
0c0e61958 memory unplug: pa... |
1340 |
} |
1723058ea mm, memory_hotplu... |
1341 |
|
0c0e61958 memory unplug: pa... |
1342 1343 |
return ret; } |
c5320926e mem-hotplug: intr... |
1344 1345 |
static int __init cmdline_parse_movable_node(char *p) { |
55ac590c2 memblock, mem_hot... |
1346 |
movable_node_enabled = true; |
c5320926e mem-hotplug: intr... |
1347 1348 1349 |
return 0; } early_param("movable_node", cmdline_parse_movable_node); |
d9713679d memory_hotplug: f... |
1350 1351 1352 1353 1354 1355 |
/* check which state of node_states will be changed when offline memory */ static void node_states_check_changes_offline(unsigned long nr_pages, struct zone *zone, struct memory_notify *arg) { struct pglist_data *pgdat = zone->zone_pgdat; unsigned long present_pages = 0; |
86b27beae mm/memory_hotplug... |
1356 |
enum zone_type zt; |
d9713679d memory_hotplug: f... |
1357 |
|
98fa15f34 mm: replace all o... |
1358 1359 1360 |
arg->status_change_nid = NUMA_NO_NODE; arg->status_change_nid_normal = NUMA_NO_NODE; arg->status_change_nid_high = NUMA_NO_NODE; |
d9713679d memory_hotplug: f... |
1361 1362 |
/* |
86b27beae mm/memory_hotplug... |
1363 1364 1365 1366 1367 1368 |
* Check whether node_states[N_NORMAL_MEMORY] will be changed. * If the memory to be offline is within the range * [0..ZONE_NORMAL], and it is the last present memory there, * the zones in that range will become empty after the offlining, * thus we can determine that we need to clear the node from * node_states[N_NORMAL_MEMORY]. |
d9713679d memory_hotplug: f... |
1369 |
*/ |
86b27beae mm/memory_hotplug... |
1370 |
for (zt = 0; zt <= ZONE_NORMAL; zt++) |
d9713679d memory_hotplug: f... |
1371 |
present_pages += pgdat->node_zones[zt].present_pages; |
86b27beae mm/memory_hotplug... |
1372 |
if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages) |
d9713679d memory_hotplug: f... |
1373 |
arg->status_change_nid_normal = zone_to_nid(zone); |
d9713679d memory_hotplug: f... |
1374 |
|
6715ddf94 hotplug: update n... |
1375 1376 |
#ifdef CONFIG_HIGHMEM /* |
86b27beae mm/memory_hotplug... |
1377 1378 1379 1380 1381 1382 |
* node_states[N_HIGH_MEMORY] contains nodes which * have normal memory or high memory. * Here we add the present_pages belonging to ZONE_HIGHMEM. * If the zone is within the range of [0..ZONE_HIGHMEM), and * we determine that the zones in that range become empty, * we need to clear the node for N_HIGH_MEMORY. |
6715ddf94 hotplug: update n... |
1383 |
*/ |
86b27beae mm/memory_hotplug... |
1384 1385 |
present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages; if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages) |
6715ddf94 hotplug: update n... |
1386 |
arg->status_change_nid_high = zone_to_nid(zone); |
6715ddf94 hotplug: update n... |
1387 |
#endif |
d9713679d memory_hotplug: f... |
1388 |
/* |
86b27beae mm/memory_hotplug... |
1389 1390 1391 1392 1393 1394 1395 1396 |
* We have accounted the pages from [0..ZONE_NORMAL), and * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM * as well. * Here we count the possible pages from ZONE_MOVABLE. * If after having accounted all the pages, we see that the nr_pages * to be offlined is over or equal to the accounted pages, * we know that the node will become empty, and so, we can clear * it for N_MEMORY as well. |
d9713679d memory_hotplug: f... |
1397 |
*/ |
86b27beae mm/memory_hotplug... |
1398 |
present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages; |
d9713679d memory_hotplug: f... |
1399 |
|
d9713679d memory_hotplug: f... |
1400 1401 |
if (nr_pages >= present_pages) arg->status_change_nid = zone_to_nid(zone); |
d9713679d memory_hotplug: f... |
1402 1403 1404 1405 1406 1407 |
} static void node_states_clear_node(int node, struct memory_notify *arg) { if (arg->status_change_nid_normal >= 0) node_clear_state(node, N_NORMAL_MEMORY); |
cf01f6f5e mm/memory_hotplug... |
1408 |
if (arg->status_change_nid_high >= 0) |
d9713679d memory_hotplug: f... |
1409 |
node_clear_state(node, N_HIGH_MEMORY); |
6715ddf94 hotplug: update n... |
1410 |
|
cf01f6f5e mm/memory_hotplug... |
1411 |
if (arg->status_change_nid >= 0) |
6715ddf94 hotplug: update n... |
1412 |
node_clear_state(node, N_MEMORY); |
d9713679d memory_hotplug: f... |
1413 |
} |
c5e79ef56 mm/memory_hotplug... |
1414 1415 1416 1417 1418 1419 1420 1421 |
static int count_system_ram_pages_cb(unsigned long start_pfn, unsigned long nr_pages, void *data) { unsigned long *nr_system_ram_pages = data; *nr_system_ram_pages += nr_pages; return 0; } |
73a11c965 mm/memory_hotplug... |
1422 |
int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages) |
0c0e61958 memory unplug: pa... |
1423 |
{ |
73a11c965 mm/memory_hotplug... |
1424 |
const unsigned long end_pfn = start_pfn + nr_pages; |
0a1a9a000 mm/memory_hotplug... |
1425 |
unsigned long pfn, system_ram_pages = 0; |
d702909f0 memory_hotplug: u... |
1426 |
unsigned long flags; |
0c0e61958 memory unplug: pa... |
1427 |
struct zone *zone; |
7b78d335a memory hotplug: r... |
1428 |
struct memory_notify arg; |
ea15153c3 mm/memory_hotplug... |
1429 |
int ret, node; |
796050932 mm, memory_hotplu... |
1430 |
char *reason; |
0c0e61958 memory unplug: pa... |
1431 |
|
4986fac16 mm/memory_hotplug... |
1432 1433 1434 1435 |
/* We can only offline full sections (e.g., SECTION_IS_ONLINE) */ if (WARN_ON_ONCE(!nr_pages || !IS_ALIGNED(start_pfn | nr_pages, PAGES_PER_SECTION))) return -EINVAL; |
381eab4a6 mm/memory_hotplug... |
1436 |
mem_hotplug_begin(); |
c5e79ef56 mm/memory_hotplug... |
1437 1438 1439 1440 1441 1442 1443 1444 |
/* * Don't allow to offline memory blocks that contain holes. * Consequently, memory blocks with holes can never get onlined * via the hotplug path - online_pages() - as hotplugged memory has * no holes. This way, we e.g., don't have to worry about marking * memory holes PG_reserved, don't need pfn_valid() checks, and can * avoid using walk_system_ram_range() later. */ |
73a11c965 mm/memory_hotplug... |
1445 |
walk_system_ram_range(start_pfn, nr_pages, &system_ram_pages, |
c5e79ef56 mm/memory_hotplug... |
1446 |
count_system_ram_pages_cb); |
73a11c965 mm/memory_hotplug... |
1447 |
if (system_ram_pages != nr_pages) { |
c5e79ef56 mm/memory_hotplug... |
1448 1449 1450 1451 |
ret = -EINVAL; reason = "memory holes"; goto failed_removal; } |
0c0e61958 memory unplug: pa... |
1452 1453 |
/* This makes hotplug much easier...and readable. we assume this for now. .*/ |
929179988 mm/memory_hotplug... |
1454 1455 |
zone = test_pages_in_a_zone(start_pfn, end_pfn); if (!zone) { |
796050932 mm, memory_hotplu... |
1456 1457 1458 |
ret = -EINVAL; reason = "multizone range"; goto failed_removal; |
381eab4a6 mm/memory_hotplug... |
1459 |
} |
7b78d335a memory hotplug: r... |
1460 |
node = zone_to_nid(zone); |
7b78d335a memory hotplug: r... |
1461 |
|
0c0e61958 memory unplug: pa... |
1462 |
/* set above range as isolated */ |
b023f4681 memory-hotplug: s... |
1463 |
ret = start_isolate_page_range(start_pfn, end_pfn, |
d381c5476 mm: only report i... |
1464 |
MIGRATE_MOVABLE, |
756d25be4 mm/page_isolation... |
1465 |
MEMORY_OFFLINE | REPORT_FAILURE); |
3fa0c7c79 mm/page_isolation... |
1466 |
if (ret) { |
796050932 mm, memory_hotplu... |
1467 1468 |
reason = "failure to isolate range"; goto failed_removal; |
381eab4a6 mm/memory_hotplug... |
1469 |
} |
7b78d335a memory hotplug: r... |
1470 1471 1472 |
arg.start_pfn = start_pfn; arg.nr_pages = nr_pages; |
d9713679d memory_hotplug: f... |
1473 |
node_states_check_changes_offline(nr_pages, zone, &arg); |
7b78d335a memory hotplug: r... |
1474 1475 1476 |
ret = memory_notify(MEM_GOING_OFFLINE, &arg); ret = notifier_to_errno(ret); |
796050932 mm, memory_hotplu... |
1477 1478 1479 1480 |
if (ret) { reason = "notifier failure"; goto failed_removal_isolated; } |
7b78d335a memory hotplug: r... |
1481 |
|
bb8965bd8 mm, memory_hotplu... |
1482 |
do { |
aa218795c mm: Allow to offl... |
1483 1484 |
pfn = start_pfn; do { |
bb8965bd8 mm, memory_hotplu... |
1485 1486 1487 1488 1489 |
if (signal_pending(current)) { ret = -EINTR; reason = "signal backoff"; goto failed_removal_isolated; } |
72b39cfc4 mm, memory_hotplu... |
1490 |
|
bb8965bd8 mm, memory_hotplu... |
1491 1492 |
cond_resched(); lru_add_drain_all(); |
bb8965bd8 mm, memory_hotplu... |
1493 |
|
aa218795c mm: Allow to offl... |
1494 1495 |
ret = scan_movable_pages(pfn, end_pfn, &pfn); if (!ret) { |
bb8965bd8 mm, memory_hotplu... |
1496 1497 1498 1499 1500 1501 |
/* * TODO: fatal migration failures should bail * out */ do_migrate_range(pfn, end_pfn); } |
aa218795c mm: Allow to offl... |
1502 1503 1504 1505 1506 |
} while (!ret); if (ret != -ENOENT) { reason = "unmovable page"; goto failed_removal_isolated; |
bb8965bd8 mm, memory_hotplu... |
1507 |
} |
0c0e61958 memory unplug: pa... |
1508 |
|
bb8965bd8 mm, memory_hotplu... |
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 |
/* * Dissolve free hugepages in the memory block before doing * offlining actually in order to make hugetlbfs's object * counting consistent. */ ret = dissolve_free_huge_pages(start_pfn, end_pfn); if (ret) { reason = "failure to dissolve huge pages"; goto failed_removal_isolated; } |
0a1a9a000 mm/memory_hotplug... |
1519 |
|
968318261 mm/memory_hotplug... |
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 |
/* * per-cpu pages are drained in start_isolate_page_range, but if * there are still pages that are not free, make sure that we * drain again, because when we isolated range we might * have raced with another thread that was adding pages to pcp * list. * * Forward progress should be still guaranteed because * pages on the pcp list can only belong to MOVABLE_ZONE * because has_unmovable_pages explicitly checks for * PageBuddy on freed pages on other zones. */ |
0a1a9a000 mm/memory_hotplug... |
1532 |
ret = test_pages_isolated(start_pfn, end_pfn, MEMORY_OFFLINE); |
968318261 mm/memory_hotplug... |
1533 1534 |
if (ret) drain_all_pages(zone); |
5557c766a mm, memory_hotplu... |
1535 |
} while (ret); |
72b39cfc4 mm, memory_hotplu... |
1536 |
|
0a1a9a000 mm/memory_hotplug... |
1537 1538 1539 1540 |
/* Mark all sections offline and remove free pages from the buddy. */ __offline_isolated_pages(start_pfn, end_pfn); pr_info("Offlined Pages %ld ", nr_pages); |
9b7ea46a8 mm/hotplug: fix o... |
1541 |
/* |
b30c59279 mm/memory_hotplug... |
1542 1543 1544 |
* The memory sections are marked offline, and the pageblock flags * effectively stale; nobody should be touching them. Fixup the number * of isolated pageblocks, memory onlining will properly revert this. |
9b7ea46a8 mm/hotplug: fix o... |
1545 1546 |
*/ spin_lock_irqsave(&zone->lock, flags); |
ea15153c3 mm/memory_hotplug... |
1547 |
zone->nr_isolate_pageblock -= nr_pages / pageblock_nr_pages; |
9b7ea46a8 mm/hotplug: fix o... |
1548 |
spin_unlock_irqrestore(&zone->lock, flags); |
0c0e61958 memory unplug: pa... |
1549 |
/* removal success */ |
0a1a9a000 mm/memory_hotplug... |
1550 1551 |
adjust_managed_page_count(pfn_to_page(start_pfn), -nr_pages); zone->present_pages -= nr_pages; |
d702909f0 memory_hotplug: u... |
1552 1553 |
pgdat_resize_lock(zone->zone_pgdat, &flags); |
0a1a9a000 mm/memory_hotplug... |
1554 |
zone->zone_pgdat->node_present_pages -= nr_pages; |
d702909f0 memory_hotplug: u... |
1555 |
pgdat_resize_unlock(zone->zone_pgdat, &flags); |
7b78d335a memory hotplug: r... |
1556 |
|
1b79acc91 mm, mem-hotplug: ... |
1557 |
init_per_zone_wmark_min(); |
1e8537baa memory-hotplug: b... |
1558 |
if (!populated_zone(zone)) { |
340175b7d mm/hotplug: free ... |
1559 |
zone_pcp_reset(zone); |
72675e131 mm, memory_hotplu... |
1560 |
build_all_zonelists(NULL); |
1e8537baa memory-hotplug: b... |
1561 1562 |
} else zone_pcp_update(zone); |
340175b7d mm/hotplug: free ... |
1563 |
|
d9713679d memory_hotplug: f... |
1564 |
node_states_clear_node(node, &arg); |
698b1b306 mm, compaction: i... |
1565 |
if (arg.status_change_nid >= 0) { |
8fe23e057 mm: clear node in... |
1566 |
kswapd_stop(node); |
698b1b306 mm, compaction: i... |
1567 1568 |
kcompactd_stop(node); } |
bce7394a3 page-allocator: r... |
1569 |
|
0c0e61958 memory unplug: pa... |
1570 |
writeback_set_ratelimit(); |
7b78d335a memory hotplug: r... |
1571 1572 |
memory_notify(MEM_OFFLINE, &arg); |
feee6b298 mm/memory_hotplug... |
1573 |
remove_pfn_range_from_zone(zone, start_pfn, nr_pages); |
381eab4a6 mm/memory_hotplug... |
1574 |
mem_hotplug_done(); |
0c0e61958 memory unplug: pa... |
1575 |
return 0; |
796050932 mm, memory_hotplu... |
1576 1577 |
failed_removal_isolated: undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); |
c4efe484b mm/memory_hotplug... |
1578 |
memory_notify(MEM_CANCEL_OFFLINE, &arg); |
0c0e61958 memory unplug: pa... |
1579 |
failed_removal: |
796050932 mm, memory_hotplu... |
1580 1581 |
pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s ", |
e33e33b4d mm, memory hotplu... |
1582 |
(unsigned long long) start_pfn << PAGE_SHIFT, |
796050932 mm, memory_hotplu... |
1583 1584 |
((unsigned long long) end_pfn << PAGE_SHIFT) - 1, reason); |
0c0e61958 memory unplug: pa... |
1585 |
/* pushback to free area */ |
381eab4a6 mm/memory_hotplug... |
1586 |
mem_hotplug_done(); |
0c0e61958 memory unplug: pa... |
1587 1588 |
return ret; } |
71088785c mm: cleanup to ma... |
1589 |
|
d6de9d534 mm/memory_hotplug... |
1590 |
static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) |
bbc76be67 memory-hotplug: r... |
1591 1592 |
{ int ret = !is_memblock_offlined(mem); |
349daa0f9 mm: fix memory_ho... |
1593 1594 1595 1596 |
if (unlikely(ret)) { phys_addr_t beginpa, endpa; beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); |
b6c88d3b9 drivers/base/memo... |
1597 |
endpa = beginpa + memory_block_size_bytes() - 1; |
756a025f0 mm: coalesce spli... |
1598 1599 |
pr_warn("removing memory fails, because memory [%pa-%pa] is onlined ", |
349daa0f9 mm: fix memory_ho... |
1600 |
&beginpa, &endpa); |
bbc76be67 memory-hotplug: r... |
1601 |
|
eca499ab3 mm/hotplug: make ... |
1602 1603 1604 |
return -EBUSY; } return 0; |
bbc76be67 memory-hotplug: r... |
1605 |
} |
0f1cfe9d0 mm/hotplug: remov... |
1606 |
static int check_cpu_on_node(pg_data_t *pgdat) |
60a5a19e7 memory-hotplug: r... |
1607 |
{ |
60a5a19e7 memory-hotplug: r... |
1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 |
int cpu; for_each_present_cpu(cpu) { if (cpu_to_node(cpu) == pgdat->node_id) /* * the cpu on this node isn't removed, and we can't * offline this node. */ return -EBUSY; } return 0; } |
2c91f8fc6 mm/memory_hotplug... |
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 |
static int check_no_memblock_for_node_cb(struct memory_block *mem, void *arg) { int nid = *(int *)arg; /* * If a memory block belongs to multiple nodes, the stored nid is not * reliable. However, such blocks are always online (e.g., cannot get * offlined) and, therefore, are still spanned by the node. */ return mem->nid == nid ? -EEXIST : 0; } |
0f1cfe9d0 mm/hotplug: remov... |
1632 1633 |
/** * try_offline_node |
e8b098fc5 mm: kernel-doc: a... |
1634 |
* @nid: the node ID |
0f1cfe9d0 mm/hotplug: remov... |
1635 1636 1637 1638 1639 1640 |
* * Offline a node if all memory sections and cpus of the node are removed. * * NOTE: The caller must call lock_device_hotplug() to serialize hotplug * and online/offline operations before this call. */ |
90b30cdc1 memory-hotplug: e... |
1641 |
void try_offline_node(int nid) |
60a5a19e7 memory-hotplug: r... |
1642 |
{ |
d822b86a9 memory-hotplug: f... |
1643 |
pg_data_t *pgdat = NODE_DATA(nid); |
2c91f8fc6 mm/memory_hotplug... |
1644 |
int rc; |
60a5a19e7 memory-hotplug: r... |
1645 |
|
2c91f8fc6 mm/memory_hotplug... |
1646 1647 1648 1649 1650 1651 1652 |
/* * If the node still spans pages (especially ZONE_DEVICE), don't * offline it. A node spans memory after move_pfn_range_to_zone(), * e.g., after the memory block was onlined. */ if (pgdat->node_spanned_pages) return; |
60a5a19e7 memory-hotplug: r... |
1653 |
|
2c91f8fc6 mm/memory_hotplug... |
1654 1655 1656 1657 1658 1659 1660 |
/* * Especially offline memory blocks might not be spanned by the * node. They will get spanned by the node once they get onlined. * However, they link to the node in sysfs and can get onlined later. */ rc = for_each_memory_block(&nid, check_no_memblock_for_node_cb); if (rc) |
60a5a19e7 memory-hotplug: r... |
1661 |
return; |
60a5a19e7 memory-hotplug: r... |
1662 |
|
46a3679b8 mm, memory_hotplu... |
1663 |
if (check_cpu_on_node(pgdat)) |
60a5a19e7 memory-hotplug: r... |
1664 1665 1666 1667 1668 1669 1670 1671 1672 |
return; /* * all memory/cpu of this node are removed, we can offline this * node now. */ node_set_offline(nid); unregister_one_node(nid); } |
90b30cdc1 memory-hotplug: e... |
1673 |
EXPORT_SYMBOL(try_offline_node); |
60a5a19e7 memory-hotplug: r... |
1674 |
|
eca499ab3 mm/hotplug: make ... |
1675 |
static int __ref try_remove_memory(int nid, u64 start, u64 size) |
bbc76be67 memory-hotplug: r... |
1676 |
{ |
eca499ab3 mm/hotplug: make ... |
1677 |
int rc = 0; |
993c1aad8 memory-hotplug: t... |
1678 |
|
27356f54c mm/hotplug: verif... |
1679 |
BUG_ON(check_hotplug_memory_range(start, size)); |
6677e3eaf memory-hotplug: c... |
1680 |
/* |
242831eb1 Memory hotplug / ... |
1681 |
* All memory blocks must be offlined before removing memory. Check |
eca499ab3 mm/hotplug: make ... |
1682 |
* whether all memory blocks in question are offline and return error |
242831eb1 Memory hotplug / ... |
1683 |
* if this is not the case. |
6677e3eaf memory-hotplug: c... |
1684 |
*/ |
fbcf73ce6 mm/memory_hotplug... |
1685 |
rc = walk_memory_blocks(start, size, NULL, check_memblock_offlined_cb); |
eca499ab3 mm/hotplug: make ... |
1686 |
if (rc) |
b4223a510 mm/memory_hotplug... |
1687 |
return rc; |
6677e3eaf memory-hotplug: c... |
1688 |
|
46c66c4b7 memory-hotplug: r... |
1689 1690 |
/* remove memmap entry */ firmware_map_remove(start, start + size, "System RAM"); |
4c4b7f9ba mm/memory_hotplug... |
1691 |
|
f1037ec0c mm/memory_hotplug... |
1692 1693 1694 1695 |
/* * Memory block device removal under the device_hotplug_lock is * a barrier against racing online attempts. */ |
4c4b7f9ba mm/memory_hotplug... |
1696 |
remove_memory_block_devices(start, size); |
46c66c4b7 memory-hotplug: r... |
1697 |
|
f1037ec0c mm/memory_hotplug... |
1698 |
mem_hotplug_begin(); |
2c2a5af6f mm, memory_hotplu... |
1699 |
arch_remove_memory(nid, start, size, NULL); |
52219aeaf mm/memory_hotplug... |
1700 1701 1702 1703 1704 |
if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) { memblock_free(start, size); memblock_remove(start, size); } |
cb8e3c8b4 kernel/resource: ... |
1705 |
release_mem_region_adjustable(start, size); |
24d335ca3 memory-hotplug: i... |
1706 |
|
60a5a19e7 memory-hotplug: r... |
1707 |
try_offline_node(nid); |
bfc8c9013 mem-hotplug: impl... |
1708 |
mem_hotplug_done(); |
b4223a510 mm/memory_hotplug... |
1709 |
return 0; |
71088785c mm: cleanup to ma... |
1710 |
} |
d15e59260 mm/memory_hotplug... |
1711 |
|
eca499ab3 mm/hotplug: make ... |
1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 |
/** * remove_memory * @nid: the node ID * @start: physical address of the region to remove * @size: size of the region to remove * * NOTE: The caller must call lock_device_hotplug() to serialize hotplug * and online/offline operations before this call, as required by * try_offline_node(). */ void __remove_memory(int nid, u64 start, u64 size) { /* |
29a90db92 mm/memory_hotplug... |
1726 |
* trigger BUG() if some memory is not offlined prior to calling this |
eca499ab3 mm/hotplug: make ... |
1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 |
* function */ if (try_remove_memory(nid, start, size)) BUG(); } /* * Remove memory if every memory block is offline, otherwise return -EBUSY is * some memory is not offline */ int remove_memory(int nid, u64 start, u64 size) |
d15e59260 mm/memory_hotplug... |
1738 |
{ |
eca499ab3 mm/hotplug: make ... |
1739 |
int rc; |
d15e59260 mm/memory_hotplug... |
1740 |
lock_device_hotplug(); |
eca499ab3 mm/hotplug: make ... |
1741 |
rc = try_remove_memory(nid, start, size); |
d15e59260 mm/memory_hotplug... |
1742 |
unlock_device_hotplug(); |
eca499ab3 mm/hotplug: make ... |
1743 1744 |
return rc; |
d15e59260 mm/memory_hotplug... |
1745 |
} |
71088785c mm: cleanup to ma... |
1746 |
EXPORT_SYMBOL_GPL(remove_memory); |
08b3acd7a mm/memory_hotplug... |
1747 |
|
417ac617e ANDROID: mm/memor... |
1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 |
int remove_memory_subsection(int nid, u64 start, u64 size) { if (size == memory_block_size_bytes()) return remove_memory(nid, start, size); if (!IS_ALIGNED(start, SUBSECTION_SIZE) || !IS_ALIGNED(size, SUBSECTION_SIZE)) { pr_err("%s: start 0x%lx size 0x%lx not aligned to subsection size ", __func__, start, size); return -EINVAL; } mem_hotplug_begin(); if (test_pages_isolated(start, start + size, MEMORY_OFFLINE)) { pr_err("%s: [%lx, %lx) PFNs are not isolated ", __func__, start, start + size); mem_hotplug_done(); return -EBUSY; } arch_remove_memory(nid, start, size, NULL); if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) memblock_remove(start, size); release_mem_region_adjustable(start, size); mem_hotplug_done(); return 0; } EXPORT_SYMBOL_GPL(remove_memory_subsection); |
08b3acd7a mm/memory_hotplug... |
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 |
/* * Try to offline and remove a memory block. Might take a long time to * finish in case memory is still in use. Primarily useful for memory devices * that logically unplugged all memory (so it's no longer in use) and want to * offline + remove the memory block. */ int offline_and_remove_memory(int nid, u64 start, u64 size) { struct memory_block *mem; int rc = -EINVAL; if (!IS_ALIGNED(start, memory_block_size_bytes()) || size != memory_block_size_bytes()) return rc; lock_device_hotplug(); mem = find_memory_block(__pfn_to_section(PFN_DOWN(start))); if (mem) rc = device_offline(&mem->dev); /* Ignore if the device is already offline. */ if (rc > 0) rc = 0; /* * In case we succeeded to offline the memory block, remove it. * This cannot fail as it cannot get onlined in the meantime. */ if (!rc) { rc = try_remove_memory(nid, start, size); WARN_ON_ONCE(rc); } unlock_device_hotplug(); return rc; } EXPORT_SYMBOL_GPL(offline_and_remove_memory); |
aba6efc47 Memory hotplug: M... |
1819 |
#endif /* CONFIG_MEMORY_HOTREMOVE */ |