Blame view
mm/sparse-vmemmap.c
7.79 KB
8f6aac419 Generic Virtual M... |
1 2 3 |
/* * Virtual Memory Map support * |
cde535359 Christoph has moved |
4 |
* (C) 2007 sgi. Christoph Lameter. |
8f6aac419 Generic Virtual M... |
5 6 7 8 9 10 11 |
* * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset * calculation without memory access. * * However, virtual mappings need a page table and TLBs. Many Linux * architectures already map their physical space using 1-1 mappings |
b595076a1 tree-wide: fix co... |
12 |
* via TLBs. For those arches the virtual memory map is essentially |
8f6aac419 Generic Virtual M... |
13 14 15 16 |
* for free if we use the same page size as the 1-1 mappings. In that * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * |
29c71111d vmemmap: generify... |
17 18 |
* The architecture is expected to provide a vmemmap_populate() function * to instantiate the mapping. |
8f6aac419 Generic Virtual M... |
19 20 21 22 |
*/ #include <linux/mm.h> #include <linux/mmzone.h> #include <linux/bootmem.h> |
4b94ffdc4 x86, mm: introduc... |
23 |
#include <linux/memremap.h> |
8f6aac419 Generic Virtual M... |
24 |
#include <linux/highmem.h> |
5a0e3ad6a include cleanup: ... |
25 |
#include <linux/slab.h> |
8f6aac419 Generic Virtual M... |
26 27 |
#include <linux/spinlock.h> #include <linux/vmalloc.h> |
8bca44bbd mm/sparse-vmemmap... |
28 |
#include <linux/sched.h> |
8f6aac419 Generic Virtual M... |
29 30 31 32 33 34 35 36 37 |
#include <asm/dma.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ |
e0dc3a53d memory hotplug fi... |
38 |
|
bd721ea73 treewide: replace... |
39 |
static void * __ref __earlyonly_bootmem_alloc(int node, |
e0dc3a53d memory hotplug fi... |
40 41 42 43 |
unsigned long size, unsigned long align, unsigned long goal) { |
bb016b841 mm/sparse: use me... |
44 45 |
return memblock_virt_alloc_try_nid(size, align, goal, BOOTMEM_ALLOC_ACCESSIBLE, node); |
e0dc3a53d memory hotplug fi... |
46 |
} |
9bdac9142 sparsemem: Put me... |
47 48 |
static void *vmemmap_buf; static void *vmemmap_buf_end; |
e0dc3a53d memory hotplug fi... |
49 |
|
8f6aac419 Generic Virtual M... |
50 51 52 53 |
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { |
f52407ce2 memory hotplug: a... |
54 55 56 |
struct page *page; if (node_state(node, N_HIGH_MEMORY)) |
055e4fd96 mm: try harder to... |
57 58 59 |
page = alloc_pages_node( node, GFP_KERNEL | __GFP_ZERO | __GFP_REPEAT, get_order(size)); |
f52407ce2 memory hotplug: a... |
60 |
else |
055e4fd96 mm: try harder to... |
61 62 |
page = alloc_pages( GFP_KERNEL | __GFP_ZERO | __GFP_REPEAT, |
f52407ce2 memory hotplug: a... |
63 |
get_order(size)); |
8f6aac419 Generic Virtual M... |
64 65 66 67 |
if (page) return page_address(page); return NULL; } else |
e0dc3a53d memory hotplug fi... |
68 |
return __earlyonly_bootmem_alloc(node, size, size, |
8f6aac419 Generic Virtual M... |
69 70 |
__pa(MAX_DMA_ADDRESS)); } |
9bdac9142 sparsemem: Put me... |
71 |
/* need to make sure size is all the same during early stage */ |
4b94ffdc4 x86, mm: introduc... |
72 |
static void * __meminit alloc_block_buf(unsigned long size, int node) |
9bdac9142 sparsemem: Put me... |
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
{ void *ptr; if (!vmemmap_buf) return vmemmap_alloc_block(size, node); /* take the from buf */ ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size); if (ptr + size > vmemmap_buf_end) return vmemmap_alloc_block(size, node); vmemmap_buf = ptr + size; return ptr; } |
4b94ffdc4 x86, mm: introduc... |
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 |
static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap) { return altmap->base_pfn + altmap->reserve + altmap->alloc + altmap->align; } static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap) { unsigned long allocated = altmap->alloc + altmap->align; if (altmap->free > allocated) return altmap->free - allocated; return 0; } /** * vmem_altmap_alloc - allocate pages from the vmem_altmap reservation * @altmap - reserved page pool for the allocation * @nr_pfns - size (in pages) of the allocation * * Allocations are aligned to the size of the request */ static unsigned long __meminit vmem_altmap_alloc(struct vmem_altmap *altmap, unsigned long nr_pfns) { unsigned long pfn = vmem_altmap_next_pfn(altmap); unsigned long nr_align; nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG); nr_align = ALIGN(pfn, nr_align) - pfn; if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap)) return ULONG_MAX; altmap->alloc += nr_pfns; altmap->align += nr_align; return pfn + nr_align; } static void * __meminit altmap_alloc_block_buf(unsigned long size, struct vmem_altmap *altmap) { unsigned long pfn, nr_pfns; void *ptr; if (size & ~PAGE_MASK) { pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld) ", __func__, size); return NULL; } nr_pfns = size >> PAGE_SHIFT; pfn = vmem_altmap_alloc(altmap, nr_pfns); if (pfn < ULONG_MAX) ptr = __va(__pfn_to_phys(pfn)); else ptr = NULL; pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx ", __func__, pfn, altmap->alloc, altmap->align, nr_pfns); return ptr; } /* need to make sure size is all the same during early stage */ void * __meminit __vmemmap_alloc_block_buf(unsigned long size, int node, struct vmem_altmap *altmap) { if (altmap) return altmap_alloc_block_buf(size, altmap); return alloc_block_buf(size, node); } |
8f6aac419 Generic Virtual M... |
160 161 162 163 164 |
void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); |
b41ad14c3 vmemmap: warn abo... |
165 |
if (node_distance(actual_node, node) > LOCAL_DISTANCE) |
1170532bb mm: convert print... |
166 167 168 |
pr_warn("[%lx-%lx] potential offnode page_structs ", start, end - 1); |
8f6aac419 Generic Virtual M... |
169 |
} |
29c71111d vmemmap: generify... |
170 |
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
171 |
{ |
29c71111d vmemmap: generify... |
172 173 174 |
pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; |
4b94ffdc4 x86, mm: introduc... |
175 |
void *p = alloc_block_buf(PAGE_SIZE, node); |
29c71111d vmemmap: generify... |
176 |
if (!p) |
9dce07f1a NULL noise: fs/*,... |
177 |
return NULL; |
29c71111d vmemmap: generify... |
178 179 180 181 |
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } return pte; |
8f6aac419 Generic Virtual M... |
182 |
} |
29c71111d vmemmap: generify... |
183 |
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
184 |
{ |
29c71111d vmemmap: generify... |
185 186 187 188 |
pmd_t *pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
189 |
return NULL; |
29c71111d vmemmap: generify... |
190 |
pmd_populate_kernel(&init_mm, pmd, p); |
8f6aac419 Generic Virtual M... |
191 |
} |
29c71111d vmemmap: generify... |
192 |
return pmd; |
8f6aac419 Generic Virtual M... |
193 |
} |
8f6aac419 Generic Virtual M... |
194 |
|
29c71111d vmemmap: generify... |
195 |
pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
196 |
{ |
29c71111d vmemmap: generify... |
197 198 199 200 |
pud_t *pud = pud_offset(pgd, addr); if (pud_none(*pud)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
201 |
return NULL; |
29c71111d vmemmap: generify... |
202 203 204 205 |
pud_populate(&init_mm, pud, p); } return pud; } |
8f6aac419 Generic Virtual M... |
206 |
|
29c71111d vmemmap: generify... |
207 208 209 210 211 212 |
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
213 |
return NULL; |
29c71111d vmemmap: generify... |
214 |
pgd_populate(&init_mm, pgd, p); |
8f6aac419 Generic Virtual M... |
215 |
} |
29c71111d vmemmap: generify... |
216 |
return pgd; |
8f6aac419 Generic Virtual M... |
217 |
} |
0aad818b2 sparse-vmemmap: s... |
218 219 |
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, int node) |
8f6aac419 Generic Virtual M... |
220 |
{ |
0aad818b2 sparse-vmemmap: s... |
221 |
unsigned long addr = start; |
29c71111d vmemmap: generify... |
222 223 224 225 |
pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; |
8f6aac419 Generic Virtual M... |
226 |
|
29c71111d vmemmap: generify... |
227 228 229 230 231 232 233 234 235 236 237 238 239 240 |
for (; addr < end; addr += PAGE_SIZE) { pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; pud = vmemmap_pud_populate(pgd, addr, node); if (!pud) return -ENOMEM; pmd = vmemmap_pmd_populate(pud, addr, node); if (!pmd) return -ENOMEM; pte = vmemmap_pte_populate(pmd, addr, node); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); |
8f6aac419 Generic Virtual M... |
241 |
} |
29c71111d vmemmap: generify... |
242 243 |
return 0; |
8f6aac419 Generic Virtual M... |
244 |
} |
8f6aac419 Generic Virtual M... |
245 |
|
98f3cfc1d memory hotplug: H... |
246 |
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
8f6aac419 Generic Virtual M... |
247 |
{ |
0aad818b2 sparse-vmemmap: s... |
248 249 250 251 252 253 254 255 256 |
unsigned long start; unsigned long end; struct page *map; map = pfn_to_page(pnum * PAGES_PER_SECTION); start = (unsigned long)map; end = (unsigned long)(map + PAGES_PER_SECTION); if (vmemmap_populate(start, end, nid)) |
8f6aac419 Generic Virtual M... |
257 258 259 260 |
return NULL; return map; } |
9bdac9142 sparsemem: Put me... |
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 |
void __init sparse_mem_maps_populate_node(struct page **map_map, unsigned long pnum_begin, unsigned long pnum_end, unsigned long map_count, int nodeid) { unsigned long pnum; unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; void *vmemmap_buf_start; size = ALIGN(size, PMD_SIZE); vmemmap_buf_start = __earlyonly_bootmem_alloc(nodeid, size * map_count, PMD_SIZE, __pa(MAX_DMA_ADDRESS)); if (vmemmap_buf_start) { vmemmap_buf = vmemmap_buf_start; vmemmap_buf_end = vmemmap_buf_start + size * map_count; } for (pnum = pnum_begin; pnum < pnum_end; pnum++) { struct mem_section *ms; if (!present_section_nr(pnum)) continue; map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); if (map_map[pnum]) continue; ms = __nr_to_section(pnum); |
1170532bb mm: convert print... |
290 291 |
pr_err("%s: sparsemem memory map backing failed some memory will not be available ", |
756a025f0 mm: coalesce spli... |
292 |
__func__); |
9bdac9142 sparsemem: Put me... |
293 294 295 296 297 |
ms->section_mem_map = 0; } if (vmemmap_buf_start) { /* need to free left buf */ |
bb016b841 mm/sparse: use me... |
298 299 |
memblock_free_early(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf); |
9bdac9142 sparsemem: Put me... |
300 301 302 303 |
vmemmap_buf = NULL; vmemmap_buf_end = NULL; } } |