Blame view
mm/sparse-vmemmap.c
5.95 KB
8f6aac419
|
1 2 3 |
/* * Virtual Memory Map support * |
cde535359
|
4 |
* (C) 2007 sgi. Christoph Lameter. |
8f6aac419
|
5 6 7 8 9 10 11 |
* * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset * calculation without memory access. * * However, virtual mappings need a page table and TLBs. Many Linux * architectures already map their physical space using 1-1 mappings |
b595076a1
|
12 |
* via TLBs. For those arches the virtual memory map is essentially |
8f6aac419
|
13 14 15 16 |
* for free if we use the same page size as the 1-1 mappings. In that * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * |
29c71111d
|
17 18 |
* The architecture is expected to provide a vmemmap_populate() function * to instantiate the mapping. |
8f6aac419
|
19 20 21 22 23 |
*/ #include <linux/mm.h> #include <linux/mmzone.h> #include <linux/bootmem.h> #include <linux/highmem.h> |
5a0e3ad6a
|
24 |
#include <linux/slab.h> |
8f6aac419
|
25 26 |
#include <linux/spinlock.h> #include <linux/vmalloc.h> |
8bca44bbd
|
27 |
#include <linux/sched.h> |
8f6aac419
|
28 29 30 31 32 33 34 35 36 |
#include <asm/dma.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ |
e0dc3a53d
|
37 38 39 40 41 42 |
static void * __init_refok __earlyonly_bootmem_alloc(int node, unsigned long size, unsigned long align, unsigned long goal) { |
bb016b841
|
43 44 |
return memblock_virt_alloc_try_nid(size, align, goal, BOOTMEM_ALLOC_ACCESSIBLE, node); |
e0dc3a53d
|
45 |
} |
9bdac9142
|
46 47 |
static void *vmemmap_buf; static void *vmemmap_buf_end; |
e0dc3a53d
|
48 |
|
8f6aac419
|
49 50 51 52 |
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { |
f52407ce2
|
53 54 55 |
struct page *page; if (node_state(node, N_HIGH_MEMORY)) |
055e4fd96
|
56 57 58 |
page = alloc_pages_node( node, GFP_KERNEL | __GFP_ZERO | __GFP_REPEAT, get_order(size)); |
f52407ce2
|
59 |
else |
055e4fd96
|
60 61 |
page = alloc_pages( GFP_KERNEL | __GFP_ZERO | __GFP_REPEAT, |
f52407ce2
|
62 |
get_order(size)); |
8f6aac419
|
63 64 65 66 |
if (page) return page_address(page); return NULL; } else |
e0dc3a53d
|
67 |
return __earlyonly_bootmem_alloc(node, size, size, |
8f6aac419
|
68 69 |
__pa(MAX_DMA_ADDRESS)); } |
9bdac9142
|
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 |
/* need to make sure size is all the same during early stage */ void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node) { void *ptr; if (!vmemmap_buf) return vmemmap_alloc_block(size, node); /* take the from buf */ ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size); if (ptr + size > vmemmap_buf_end) return vmemmap_alloc_block(size, node); vmemmap_buf = ptr + size; return ptr; } |
8f6aac419
|
87 88 89 90 91 |
void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); |
b41ad14c3
|
92 |
if (node_distance(actual_node, node) > LOCAL_DISTANCE) |
8f6aac419
|
93 94 95 96 |
printk(KERN_WARNING "[%lx-%lx] potential offnode " "page_structs ", start, end - 1); } |
29c71111d
|
97 |
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) |
8f6aac419
|
98 |
{ |
29c71111d
|
99 100 101 |
pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; |
9bdac9142
|
102 |
void *p = vmemmap_alloc_block_buf(PAGE_SIZE, node); |
29c71111d
|
103 |
if (!p) |
9dce07f1a
|
104 |
return NULL; |
29c71111d
|
105 106 107 108 |
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } return pte; |
8f6aac419
|
109 |
} |
29c71111d
|
110 |
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) |
8f6aac419
|
111 |
{ |
29c71111d
|
112 113 114 115 |
pmd_t *pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
116 |
return NULL; |
29c71111d
|
117 |
pmd_populate_kernel(&init_mm, pmd, p); |
8f6aac419
|
118 |
} |
29c71111d
|
119 |
return pmd; |
8f6aac419
|
120 |
} |
8f6aac419
|
121 |
|
29c71111d
|
122 |
pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node) |
8f6aac419
|
123 |
{ |
29c71111d
|
124 125 126 127 |
pud_t *pud = pud_offset(pgd, addr); if (pud_none(*pud)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
128 |
return NULL; |
29c71111d
|
129 130 131 132 |
pud_populate(&init_mm, pud, p); } return pud; } |
8f6aac419
|
133 |
|
29c71111d
|
134 135 136 137 138 139 |
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
140 |
return NULL; |
29c71111d
|
141 |
pgd_populate(&init_mm, pgd, p); |
8f6aac419
|
142 |
} |
29c71111d
|
143 |
return pgd; |
8f6aac419
|
144 |
} |
0aad818b2
|
145 146 |
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, int node) |
8f6aac419
|
147 |
{ |
0aad818b2
|
148 |
unsigned long addr = start; |
29c71111d
|
149 150 151 152 |
pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; |
8f6aac419
|
153 |
|
29c71111d
|
154 155 156 157 158 159 160 161 162 163 164 165 166 167 |
for (; addr < end; addr += PAGE_SIZE) { pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; pud = vmemmap_pud_populate(pgd, addr, node); if (!pud) return -ENOMEM; pmd = vmemmap_pmd_populate(pud, addr, node); if (!pmd) return -ENOMEM; pte = vmemmap_pte_populate(pmd, addr, node); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); |
8f6aac419
|
168 |
} |
29c71111d
|
169 170 |
return 0; |
8f6aac419
|
171 |
} |
8f6aac419
|
172 |
|
98f3cfc1d
|
173 |
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
8f6aac419
|
174 |
{ |
0aad818b2
|
175 176 177 178 179 180 181 182 183 |
unsigned long start; unsigned long end; struct page *map; map = pfn_to_page(pnum * PAGES_PER_SECTION); start = (unsigned long)map; end = (unsigned long)(map + PAGES_PER_SECTION); if (vmemmap_populate(start, end, nid)) |
8f6aac419
|
184 185 186 187 |
return NULL; return map; } |
9bdac9142
|
188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
void __init sparse_mem_maps_populate_node(struct page **map_map, unsigned long pnum_begin, unsigned long pnum_end, unsigned long map_count, int nodeid) { unsigned long pnum; unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; void *vmemmap_buf_start; size = ALIGN(size, PMD_SIZE); vmemmap_buf_start = __earlyonly_bootmem_alloc(nodeid, size * map_count, PMD_SIZE, __pa(MAX_DMA_ADDRESS)); if (vmemmap_buf_start) { vmemmap_buf = vmemmap_buf_start; vmemmap_buf_end = vmemmap_buf_start + size * map_count; } for (pnum = pnum_begin; pnum < pnum_end; pnum++) { struct mem_section *ms; if (!present_section_nr(pnum)) continue; map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); if (map_map[pnum]) continue; ms = __nr_to_section(pnum); printk(KERN_ERR "%s: sparsemem memory map backing failed " "some memory will not be available. ", __func__); ms->section_mem_map = 0; } if (vmemmap_buf_start) { /* need to free left buf */ |
bb016b841
|
225 226 |
memblock_free_early(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf); |
9bdac9142
|
227 228 229 230 |
vmemmap_buf = NULL; vmemmap_buf_end = NULL; } } |