Blame view
mm/sparse-vmemmap.c
8.05 KB
b24413180
|
1 |
// SPDX-License-Identifier: GPL-2.0 |
8f6aac419
|
2 3 4 |
/* * Virtual Memory Map support * |
cde535359
|
5 |
* (C) 2007 sgi. Christoph Lameter. |
8f6aac419
|
6 7 8 9 10 11 12 |
* * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset * calculation without memory access. * * However, virtual mappings need a page table and TLBs. Many Linux * architectures already map their physical space using 1-1 mappings |
b595076a1
|
13 |
* via TLBs. For those arches the virtual memory map is essentially |
8f6aac419
|
14 15 16 17 |
* for free if we use the same page size as the 1-1 mappings. In that * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * |
29c71111d
|
18 19 |
* The architecture is expected to provide a vmemmap_populate() function * to instantiate the mapping. |
8f6aac419
|
20 21 22 23 |
*/ #include <linux/mm.h> #include <linux/mmzone.h> #include <linux/bootmem.h> |
4b94ffdc4
|
24 |
#include <linux/memremap.h> |
8f6aac419
|
25 |
#include <linux/highmem.h> |
5a0e3ad6a
|
26 |
#include <linux/slab.h> |
8f6aac419
|
27 28 |
#include <linux/spinlock.h> #include <linux/vmalloc.h> |
8bca44bbd
|
29 |
#include <linux/sched.h> |
8f6aac419
|
30 31 32 33 34 35 36 37 38 |
#include <asm/dma.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ |
e0dc3a53d
|
39 |
|
bd721ea73
|
40 |
static void * __ref __earlyonly_bootmem_alloc(int node, |
e0dc3a53d
|
41 42 43 44 |
unsigned long size, unsigned long align, unsigned long goal) { |
bb016b841
|
45 46 |
return memblock_virt_alloc_try_nid(size, align, goal, BOOTMEM_ALLOC_ACCESSIBLE, node); |
e0dc3a53d
|
47 |
} |
9bdac9142
|
48 49 |
static void *vmemmap_buf; static void *vmemmap_buf_end; |
e0dc3a53d
|
50 |
|
8f6aac419
|
51 52 53 54 |
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { |
f52407ce2
|
55 |
struct page *page; |
b95046b04
|
56 57 58 |
page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL, get_order(size)); |
8f6aac419
|
59 60 61 62 |
if (page) return page_address(page); return NULL; } else |
e0dc3a53d
|
63 |
return __earlyonly_bootmem_alloc(node, size, size, |
8f6aac419
|
64 65 |
__pa(MAX_DMA_ADDRESS)); } |
9bdac9142
|
66 |
/* need to make sure size is all the same during early stage */ |
4b94ffdc4
|
67 |
static void * __meminit alloc_block_buf(unsigned long size, int node) |
9bdac9142
|
68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
{ void *ptr; if (!vmemmap_buf) return vmemmap_alloc_block(size, node); /* take the from buf */ ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size); if (ptr + size > vmemmap_buf_end) return vmemmap_alloc_block(size, node); vmemmap_buf = ptr + size; return ptr; } |
4b94ffdc4
|
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap) { return altmap->base_pfn + altmap->reserve + altmap->alloc + altmap->align; } static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap) { unsigned long allocated = altmap->alloc + altmap->align; if (altmap->free > allocated) return altmap->free - allocated; return 0; } /** * vmem_altmap_alloc - allocate pages from the vmem_altmap reservation * @altmap - reserved page pool for the allocation * @nr_pfns - size (in pages) of the allocation * * Allocations are aligned to the size of the request */ static unsigned long __meminit vmem_altmap_alloc(struct vmem_altmap *altmap, unsigned long nr_pfns) { unsigned long pfn = vmem_altmap_next_pfn(altmap); unsigned long nr_align; nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG); nr_align = ALIGN(pfn, nr_align) - pfn; if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap)) return ULONG_MAX; altmap->alloc += nr_pfns; altmap->align += nr_align; return pfn + nr_align; } static void * __meminit altmap_alloc_block_buf(unsigned long size, struct vmem_altmap *altmap) { unsigned long pfn, nr_pfns; void *ptr; if (size & ~PAGE_MASK) { pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld) ", __func__, size); return NULL; } nr_pfns = size >> PAGE_SHIFT; pfn = vmem_altmap_alloc(altmap, nr_pfns); if (pfn < ULONG_MAX) ptr = __va(__pfn_to_phys(pfn)); else ptr = NULL; pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx ", __func__, pfn, altmap->alloc, altmap->align, nr_pfns); return ptr; } /* need to make sure size is all the same during early stage */ void * __meminit __vmemmap_alloc_block_buf(unsigned long size, int node, struct vmem_altmap *altmap) { if (altmap) return altmap_alloc_block_buf(size, altmap); return alloc_block_buf(size, node); } |
8f6aac419
|
155 156 157 158 159 |
void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); |
b41ad14c3
|
160 |
if (node_distance(actual_node, node) > LOCAL_DISTANCE) |
1170532bb
|
161 162 163 |
pr_warn("[%lx-%lx] potential offnode page_structs ", start, end - 1); |
8f6aac419
|
164 |
} |
29c71111d
|
165 |
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) |
8f6aac419
|
166 |
{ |
29c71111d
|
167 168 169 |
pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; |
4b94ffdc4
|
170 |
void *p = alloc_block_buf(PAGE_SIZE, node); |
29c71111d
|
171 |
if (!p) |
9dce07f1a
|
172 |
return NULL; |
29c71111d
|
173 174 175 176 |
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } return pte; |
8f6aac419
|
177 |
} |
29c71111d
|
178 |
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) |
8f6aac419
|
179 |
{ |
29c71111d
|
180 181 182 183 |
pmd_t *pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
184 |
return NULL; |
29c71111d
|
185 |
pmd_populate_kernel(&init_mm, pmd, p); |
8f6aac419
|
186 |
} |
29c71111d
|
187 |
return pmd; |
8f6aac419
|
188 |
} |
8f6aac419
|
189 |
|
c2febafc6
|
190 |
pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) |
8f6aac419
|
191 |
{ |
c2febafc6
|
192 |
pud_t *pud = pud_offset(p4d, addr); |
29c71111d
|
193 194 195 |
if (pud_none(*pud)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
196 |
return NULL; |
29c71111d
|
197 198 199 200 |
pud_populate(&init_mm, pud, p); } return pud; } |
8f6aac419
|
201 |
|
c2febafc6
|
202 203 204 205 206 207 208 209 210 211 212 |
p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node) { p4d_t *p4d = p4d_offset(pgd, addr); if (p4d_none(*p4d)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) return NULL; p4d_populate(&init_mm, p4d, p); } return p4d; } |
29c71111d
|
213 214 215 216 217 218 |
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
219 |
return NULL; |
29c71111d
|
220 |
pgd_populate(&init_mm, pgd, p); |
8f6aac419
|
221 |
} |
29c71111d
|
222 |
return pgd; |
8f6aac419
|
223 |
} |
0aad818b2
|
224 225 |
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, int node) |
8f6aac419
|
226 |
{ |
0aad818b2
|
227 |
unsigned long addr = start; |
29c71111d
|
228 |
pgd_t *pgd; |
c2febafc6
|
229 |
p4d_t *p4d; |
29c71111d
|
230 231 232 |
pud_t *pud; pmd_t *pmd; pte_t *pte; |
8f6aac419
|
233 |
|
29c71111d
|
234 235 236 237 |
for (; addr < end; addr += PAGE_SIZE) { pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; |
c2febafc6
|
238 239 240 241 |
p4d = vmemmap_p4d_populate(pgd, addr, node); if (!p4d) return -ENOMEM; pud = vmemmap_pud_populate(p4d, addr, node); |
29c71111d
|
242 243 244 245 246 247 248 249 250 |
if (!pud) return -ENOMEM; pmd = vmemmap_pmd_populate(pud, addr, node); if (!pmd) return -ENOMEM; pte = vmemmap_pte_populate(pmd, addr, node); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); |
8f6aac419
|
251 |
} |
29c71111d
|
252 253 |
return 0; |
8f6aac419
|
254 |
} |
8f6aac419
|
255 |
|
98f3cfc1d
|
256 |
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
8f6aac419
|
257 |
{ |
0aad818b2
|
258 259 260 261 262 263 264 265 266 |
unsigned long start; unsigned long end; struct page *map; map = pfn_to_page(pnum * PAGES_PER_SECTION); start = (unsigned long)map; end = (unsigned long)(map + PAGES_PER_SECTION); if (vmemmap_populate(start, end, nid)) |
8f6aac419
|
267 268 269 270 |
return NULL; return map; } |
9bdac9142
|
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 |
void __init sparse_mem_maps_populate_node(struct page **map_map, unsigned long pnum_begin, unsigned long pnum_end, unsigned long map_count, int nodeid) { unsigned long pnum; unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; void *vmemmap_buf_start; size = ALIGN(size, PMD_SIZE); vmemmap_buf_start = __earlyonly_bootmem_alloc(nodeid, size * map_count, PMD_SIZE, __pa(MAX_DMA_ADDRESS)); if (vmemmap_buf_start) { vmemmap_buf = vmemmap_buf_start; vmemmap_buf_end = vmemmap_buf_start + size * map_count; } for (pnum = pnum_begin; pnum < pnum_end; pnum++) { struct mem_section *ms; if (!present_section_nr(pnum)) continue; map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); if (map_map[pnum]) continue; ms = __nr_to_section(pnum); |
1170532bb
|
300 301 |
pr_err("%s: sparsemem memory map backing failed some memory will not be available ", |
756a025f0
|
302 |
__func__); |
9bdac9142
|
303 304 305 306 307 |
ms->section_mem_map = 0; } if (vmemmap_buf_start) { /* need to free left buf */ |
bb016b841
|
308 309 |
memblock_free_early(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf); |
9bdac9142
|
310 311 312 313 |
vmemmap_buf = NULL; vmemmap_buf_end = NULL; } } |