Blame view
mm/sparse-vmemmap.c
5.86 KB
8f6aac419
|
1 2 3 |
/* * Virtual Memory Map support * |
cde535359
|
4 |
* (C) 2007 sgi. Christoph Lameter. |
8f6aac419
|
5 6 7 8 9 10 11 |
* * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset * calculation without memory access. * * However, virtual mappings need a page table and TLBs. Many Linux * architectures already map their physical space using 1-1 mappings |
b595076a1
|
12 |
* via TLBs. For those arches the virtual memory map is essentially |
8f6aac419
|
13 14 15 16 |
* for free if we use the same page size as the 1-1 mappings. In that * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * |
29c71111d
|
17 18 |
* The architecture is expected to provide a vmemmap_populate() function * to instantiate the mapping. |
8f6aac419
|
19 20 21 22 23 |
*/ #include <linux/mm.h> #include <linux/mmzone.h> #include <linux/bootmem.h> #include <linux/highmem.h> |
5a0e3ad6a
|
24 |
#include <linux/slab.h> |
8f6aac419
|
25 26 |
#include <linux/spinlock.h> #include <linux/vmalloc.h> |
8bca44bbd
|
27 |
#include <linux/sched.h> |
8f6aac419
|
28 29 30 31 32 33 34 35 36 |
#include <asm/dma.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ |
e0dc3a53d
|
37 38 39 40 41 42 |
static void * __init_refok __earlyonly_bootmem_alloc(int node, unsigned long size, unsigned long align, unsigned long goal) { |
08677214e
|
43 |
return __alloc_bootmem_node_high(NODE_DATA(node), size, align, goal); |
e0dc3a53d
|
44 |
} |
9bdac9142
|
45 46 |
static void *vmemmap_buf; static void *vmemmap_buf_end; |
e0dc3a53d
|
47 |
|
8f6aac419
|
48 49 50 51 |
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { |
f52407ce2
|
52 53 54 55 |
struct page *page; if (node_state(node, N_HIGH_MEMORY)) page = alloc_pages_node(node, |
8f6aac419
|
56 |
GFP_KERNEL | __GFP_ZERO, get_order(size)); |
f52407ce2
|
57 58 59 |
else page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(size)); |
8f6aac419
|
60 61 62 63 |
if (page) return page_address(page); return NULL; } else |
e0dc3a53d
|
64 |
return __earlyonly_bootmem_alloc(node, size, size, |
8f6aac419
|
65 66 |
__pa(MAX_DMA_ADDRESS)); } |
9bdac9142
|
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
/* need to make sure size is all the same during early stage */ void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node) { void *ptr; if (!vmemmap_buf) return vmemmap_alloc_block(size, node); /* take the from buf */ ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size); if (ptr + size > vmemmap_buf_end) return vmemmap_alloc_block(size, node); vmemmap_buf = ptr + size; return ptr; } |
8f6aac419
|
84 85 86 87 88 |
void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); |
b41ad14c3
|
89 |
if (node_distance(actual_node, node) > LOCAL_DISTANCE) |
8f6aac419
|
90 91 92 93 |
printk(KERN_WARNING "[%lx-%lx] potential offnode " "page_structs ", start, end - 1); } |
29c71111d
|
94 |
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) |
8f6aac419
|
95 |
{ |
29c71111d
|
96 97 98 |
pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; |
9bdac9142
|
99 |
void *p = vmemmap_alloc_block_buf(PAGE_SIZE, node); |
29c71111d
|
100 |
if (!p) |
9dce07f1a
|
101 |
return NULL; |
29c71111d
|
102 103 104 105 |
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } return pte; |
8f6aac419
|
106 |
} |
29c71111d
|
107 |
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) |
8f6aac419
|
108 |
{ |
29c71111d
|
109 110 111 112 |
pmd_t *pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
113 |
return NULL; |
29c71111d
|
114 |
pmd_populate_kernel(&init_mm, pmd, p); |
8f6aac419
|
115 |
} |
29c71111d
|
116 |
return pmd; |
8f6aac419
|
117 |
} |
8f6aac419
|
118 |
|
29c71111d
|
119 |
pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node) |
8f6aac419
|
120 |
{ |
29c71111d
|
121 122 123 124 |
pud_t *pud = pud_offset(pgd, addr); if (pud_none(*pud)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
125 |
return NULL; |
29c71111d
|
126 127 128 129 |
pud_populate(&init_mm, pud, p); } return pud; } |
8f6aac419
|
130 |
|
29c71111d
|
131 132 133 134 135 136 |
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a
|
137 |
return NULL; |
29c71111d
|
138 |
pgd_populate(&init_mm, pgd, p); |
8f6aac419
|
139 |
} |
29c71111d
|
140 |
return pgd; |
8f6aac419
|
141 |
} |
29c71111d
|
142 143 |
int __meminit vmemmap_populate_basepages(struct page *start_page, unsigned long size, int node) |
8f6aac419
|
144 |
{ |
8f6aac419
|
145 |
unsigned long addr = (unsigned long)start_page; |
29c71111d
|
146 147 148 149 150 |
unsigned long end = (unsigned long)(start_page + size); pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; |
8f6aac419
|
151 |
|
29c71111d
|
152 153 154 155 156 157 158 159 160 161 162 163 164 165 |
for (; addr < end; addr += PAGE_SIZE) { pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; pud = vmemmap_pud_populate(pgd, addr, node); if (!pud) return -ENOMEM; pmd = vmemmap_pmd_populate(pud, addr, node); if (!pmd) return -ENOMEM; pte = vmemmap_pte_populate(pmd, addr, node); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); |
8f6aac419
|
166 |
} |
29c71111d
|
167 168 |
return 0; |
8f6aac419
|
169 |
} |
8f6aac419
|
170 |
|
98f3cfc1d
|
171 |
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
8f6aac419
|
172 173 174 175 176 177 178 179 |
{ struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); int error = vmemmap_populate(map, PAGES_PER_SECTION, nid); if (error) return NULL; return map; } |
9bdac9142
|
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
void __init sparse_mem_maps_populate_node(struct page **map_map, unsigned long pnum_begin, unsigned long pnum_end, unsigned long map_count, int nodeid) { unsigned long pnum; unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; void *vmemmap_buf_start; size = ALIGN(size, PMD_SIZE); vmemmap_buf_start = __earlyonly_bootmem_alloc(nodeid, size * map_count, PMD_SIZE, __pa(MAX_DMA_ADDRESS)); if (vmemmap_buf_start) { vmemmap_buf = vmemmap_buf_start; vmemmap_buf_end = vmemmap_buf_start + size * map_count; } for (pnum = pnum_begin; pnum < pnum_end; pnum++) { struct mem_section *ms; if (!present_section_nr(pnum)) continue; map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); if (map_map[pnum]) continue; ms = __nr_to_section(pnum); printk(KERN_ERR "%s: sparsemem memory map backing failed " "some memory will not be available. ", __func__); ms->section_mem_map = 0; } if (vmemmap_buf_start) { /* need to free left buf */ |
9bdac9142
|
217 |
free_bootmem(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf); |
9bdac9142
|
218 219 220 221 |
vmemmap_buf = NULL; vmemmap_buf_end = NULL; } } |