Blame view
mm/sparse-vmemmap.c
4.17 KB
8f6aac419 Generic Virtual M... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
/* * Virtual Memory Map support * * (C) 2007 sgi. Christoph Lameter <clameter@sgi.com>. * * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn, * virt_to_page, page_address() to be implemented as a base offset * calculation without memory access. * * However, virtual mappings need a page table and TLBs. Many Linux * architectures already map their physical space using 1-1 mappings * via TLBs. For those arches the virtual memmory map is essentially * for free if we use the same page size as the 1-1 mappings. In that * case the overhead consists of a few additional pages that are * allocated to create a view of memory for vmemmap. * |
29c71111d vmemmap: generify... |
17 18 |
* The architecture is expected to provide a vmemmap_populate() function * to instantiate the mapping. |
8f6aac419 Generic Virtual M... |
19 20 21 22 23 24 25 26 |
*/ #include <linux/mm.h> #include <linux/mmzone.h> #include <linux/bootmem.h> #include <linux/highmem.h> #include <linux/module.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> |
8bca44bbd mm/sparse-vmemmap... |
27 |
#include <linux/sched.h> |
8f6aac419 Generic Virtual M... |
28 29 30 31 32 33 34 35 36 |
#include <asm/dma.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> /* * Allocate a block of memory to be used to back the virtual memory map * or to back the page tables that are used to create the mapping. * Uses the main allocators if they are available, else bootmem. */ |
e0dc3a53d memory hotplug fi... |
37 38 39 40 41 42 43 44 |
static void * __init_refok __earlyonly_bootmem_alloc(int node, unsigned long size, unsigned long align, unsigned long goal) { return __alloc_bootmem_node(NODE_DATA(node), size, align, goal); } |
8f6aac419 Generic Virtual M... |
45 46 47 48 49 50 51 52 53 54 |
void * __meminit vmemmap_alloc_block(unsigned long size, int node) { /* If the main allocator is up use that, fallback to bootmem. */ if (slab_is_available()) { struct page *page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size)); if (page) return page_address(page); return NULL; } else |
e0dc3a53d memory hotplug fi... |
55 |
return __earlyonly_bootmem_alloc(node, size, size, |
8f6aac419 Generic Virtual M... |
56 57 |
__pa(MAX_DMA_ADDRESS)); } |
8f6aac419 Generic Virtual M... |
58 59 60 61 62 63 64 65 66 67 68 |
void __meminit vmemmap_verify(pte_t *pte, int node, unsigned long start, unsigned long end) { unsigned long pfn = pte_pfn(*pte); int actual_node = early_pfn_to_nid(pfn); if (actual_node != node) printk(KERN_WARNING "[%lx-%lx] potential offnode " "page_structs ", start, end - 1); } |
29c71111d vmemmap: generify... |
69 |
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
70 |
{ |
29c71111d vmemmap: generify... |
71 72 73 74 75 |
pte_t *pte = pte_offset_kernel(pmd, addr); if (pte_none(*pte)) { pte_t entry; void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
76 |
return NULL; |
29c71111d vmemmap: generify... |
77 78 79 80 |
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL); set_pte_at(&init_mm, addr, pte, entry); } return pte; |
8f6aac419 Generic Virtual M... |
81 |
} |
29c71111d vmemmap: generify... |
82 |
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
83 |
{ |
29c71111d vmemmap: generify... |
84 85 86 87 |
pmd_t *pmd = pmd_offset(pud, addr); if (pmd_none(*pmd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
88 |
return NULL; |
29c71111d vmemmap: generify... |
89 |
pmd_populate_kernel(&init_mm, pmd, p); |
8f6aac419 Generic Virtual M... |
90 |
} |
29c71111d vmemmap: generify... |
91 |
return pmd; |
8f6aac419 Generic Virtual M... |
92 |
} |
8f6aac419 Generic Virtual M... |
93 |
|
29c71111d vmemmap: generify... |
94 |
pud_t * __meminit vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node) |
8f6aac419 Generic Virtual M... |
95 |
{ |
29c71111d vmemmap: generify... |
96 97 98 99 |
pud_t *pud = pud_offset(pgd, addr); if (pud_none(*pud)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
100 |
return NULL; |
29c71111d vmemmap: generify... |
101 102 103 104 |
pud_populate(&init_mm, pud, p); } return pud; } |
8f6aac419 Generic Virtual M... |
105 |
|
29c71111d vmemmap: generify... |
106 107 108 109 110 111 |
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) { void *p = vmemmap_alloc_block(PAGE_SIZE, node); if (!p) |
9dce07f1a NULL noise: fs/*,... |
112 |
return NULL; |
29c71111d vmemmap: generify... |
113 |
pgd_populate(&init_mm, pgd, p); |
8f6aac419 Generic Virtual M... |
114 |
} |
29c71111d vmemmap: generify... |
115 |
return pgd; |
8f6aac419 Generic Virtual M... |
116 |
} |
29c71111d vmemmap: generify... |
117 118 |
int __meminit vmemmap_populate_basepages(struct page *start_page, unsigned long size, int node) |
8f6aac419 Generic Virtual M... |
119 |
{ |
8f6aac419 Generic Virtual M... |
120 |
unsigned long addr = (unsigned long)start_page; |
29c71111d vmemmap: generify... |
121 122 123 124 125 |
unsigned long end = (unsigned long)(start_page + size); pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; |
8f6aac419 Generic Virtual M... |
126 |
|
29c71111d vmemmap: generify... |
127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
for (; addr < end; addr += PAGE_SIZE) { pgd = vmemmap_pgd_populate(addr, node); if (!pgd) return -ENOMEM; pud = vmemmap_pud_populate(pgd, addr, node); if (!pud) return -ENOMEM; pmd = vmemmap_pmd_populate(pud, addr, node); if (!pmd) return -ENOMEM; pte = vmemmap_pte_populate(pmd, addr, node); if (!pte) return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); |
8f6aac419 Generic Virtual M... |
141 |
} |
29c71111d vmemmap: generify... |
142 143 |
return 0; |
8f6aac419 Generic Virtual M... |
144 |
} |
8f6aac419 Generic Virtual M... |
145 |
|
98f3cfc1d memory hotplug: H... |
146 |
struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid) |
8f6aac419 Generic Virtual M... |
147 148 149 150 151 152 153 154 |
{ struct page *map = pfn_to_page(pnum * PAGES_PER_SECTION); int error = vmemmap_populate(map, PAGES_PER_SECTION, nid); if (error) return NULL; return map; } |