Eric Lee / linux-smarc-t335x-v3.2

Commit 0d9ea75443dc7e37843e656b8ebc947a6d16d618

Authored by Jon Tollefson 2008-07-24 12:27:56 +0800

Committed by Linus Torvalds 2008-07-25 01:47:19 +0800

Exists in master and in 4 other branches

powerpc: support multiple hugepage sizes

Instead of using the variable mmu_huge_psize to keep track of the huge
page size we use an array of MMU_PAGE_* values. For each supported huge
page size we need to know the hugepte_shift value and have a
pgtable_cache. The hstate or an mmu_huge_psizes index is passed to
functions so that they know which huge page size they should use.

The hugepage sizes 16M and 64K are setup(if available on the hardware) so
that they don't have to be set on the boot cmd line in order to use them.
The number of 16G pages have to be specified at boot-time though (e.g.
hugepagesz=16G hugepages=5).

Signed-off-by: Jon Tollefson <kniht@linux.vnet.ibm.com>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 9 changed files with 199 additions and 118 deletions Inline Diff

Documentation/kernel-parameters.txt
arch/powerpc/mm/hash_utils_64.c
arch/powerpc/mm/hugetlbpage.c
arch/powerpc/mm/init_64.c
arch/powerpc/mm/tlb_64.c
include/asm-powerpc/hugetlb.h
include/asm-powerpc/mmu-hash64.h
include/asm-powerpc/page_64.h
include/asm-powerpc/pgalloc-64.h

Documentation/kernel-parameters.txt

Diff comments View file @ 0d9ea75

arch/powerpc/mm/hash_utils_64.c

Diff comments View file @ 0d9ea75

 /*
  * PowerPC64 port by Mike Corrigan and Dave Engebretsen
  *   {mikejc|engebret}@us.ibm.com
  *
  *    Copyright (c) 2000 Mike Corrigan <mikejc@us.ibm.com>
  *
  * SMP scalability work:
  *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
  *
  *    Module name: htab.c
  *
  *    Description:
  *      PowerPC Hashed Page Table functions
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #undef DEBUG
 #undef DEBUG_LOW
 #include <linux/spinlock.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/proc_fs.h>
 #include <linux/stat.h>
 #include <linux/sysctl.h>
 #include <linux/ctype.h>
 #include <linux/cache.h>
 #include <linux/init.h>
 #include <linux/signal.h>
 #include <linux/lmb.h>
 #include <asm/processor.h>
 #include <asm/pgtable.h>
 #include <asm/mmu.h>
 #include <asm/mmu_context.h>
 #include <asm/page.h>
 #include <asm/types.h>
 #include <asm/system.h>
 #include <asm/uaccess.h>
 #include <asm/machdep.h>
 #include <asm/prom.h>
 #include <asm/abs_addr.h>
 #include <asm/tlbflush.h>
 #include <asm/io.h>
 #include <asm/eeh.h>
 #include <asm/tlb.h>
 #include <asm/cacheflush.h>
 #include <asm/cputable.h>
 #include <asm/sections.h>
 #include <asm/spu.h>
 #include <asm/udbg.h>
 #ifdef DEBUG
 #define DBG(fmt...) udbg_printf(fmt)
 #else
 #define DBG(fmt...)
 #endif
 #ifdef DEBUG_LOW
 #define DBG_LOW(fmt...) udbg_printf(fmt)
 #else
 #define DBG_LOW(fmt...)
 #endif
 #define KB (1024)
 #define MB (1024*KB)
 #define GB (1024L*MB)
 /*
  * Note:  pte   --> Linux PTE
  *        HPTE  --> PowerPC Hashed Page Table Entry
  *
  * Execution context:
  *   htab_initialize is called with the MMU off (of course), but
  *   the kernel has been copied down to zero so it can directly
  *   reference global data.  At this point it is very difficult
  *   to print debug info.
  *
  */
 #ifdef CONFIG_U3_DART
 extern unsigned long dart_tablebase;
 #endif /* CONFIG_U3_DART */
 static unsigned long _SDR1;
 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
 struct hash_pte *htab_address;
 unsigned long htab_size_bytes;
 unsigned long htab_hash_mask;
 int mmu_linear_psize = MMU_PAGE_4K;
 int mmu_virtual_psize = MMU_PAGE_4K;
 int mmu_vmalloc_psize = MMU_PAGE_4K;
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 int mmu_vmemmap_psize = MMU_PAGE_4K;
 #endif
 int mmu_io_psize = MMU_PAGE_4K;
 int mmu_kernel_ssize = MMU_SEGSIZE_256M;
 int mmu_highuser_ssize = MMU_SEGSIZE_256M;
 u16 mmu_slb_size = 64;
 #ifdef CONFIG_HUGETLB_PAGE
-int mmu_huge_psize = MMU_PAGE_16M;
 unsigned int HPAGE_SHIFT;
 #endif
 #ifdef CONFIG_PPC_64K_PAGES
 int mmu_ci_restrictions;
 #endif
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static u8 *linear_map_hash_slots;
 static unsigned long linear_map_hash_count;
 static DEFINE_SPINLOCK(linear_map_hash_lock);
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 /* There are definitions of page sizes arrays to be used when none
  * is provided by the firmware.
  */
 /* Pre-POWER4 CPUs (4k pages only)
  */
 static struct mmu_psize_def mmu_psize_defaults_old[] = {
 	[MMU_PAGE_4K] = {
 		.shift	= 12,
 		.sllp	= 0,
 		.penc	= 0,
 		.avpnm	= 0,
 		.tlbiel = 0,
 	},
 };
 /* POWER4, GPUL, POWER5
  *
  * Support for 16Mb large pages
  */
 static struct mmu_psize_def mmu_psize_defaults_gp[] = {
 	[MMU_PAGE_4K] = {
 		.shift	= 12,
 		.sllp	= 0,
 		.penc	= 0,
 		.avpnm	= 0,
 		.tlbiel = 1,
 	},
 	[MMU_PAGE_16M] = {
 		.shift	= 24,
 		.sllp	= SLB_VSID_L,
 		.penc	= 0,
 		.avpnm	= 0x1UL,
 		.tlbiel = 0,
 	},
 };
 int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
 		      unsigned long pstart, unsigned long mode,
 		      int psize, int ssize)
 {
 	unsigned long vaddr, paddr;
 	unsigned int step, shift;
 	unsigned long tmp_mode;
 	int ret = 0;
 	shift = mmu_psize_defs[psize].shift;
 	step = 1 << shift;
 	for (vaddr = vstart, paddr = pstart; vaddr < vend;
 	     vaddr += step, paddr += step) {
 		unsigned long hash, hpteg;
 		unsigned long vsid = get_kernel_vsid(vaddr, ssize);
 		unsigned long va = hpt_va(vaddr, vsid, ssize);
 		tmp_mode = mode;
 		/* Make non-kernel text non-executable */
 		if (!in_kernel_text(vaddr))
 			tmp_mode = mode | HPTE_R_N;
 		hash = hpt_hash(va, shift, ssize);
 		hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
 		DBG("htab_bolt_mapping: calling %p\n", ppc_md.hpte_insert);
 		BUG_ON(!ppc_md.hpte_insert);
 		ret = ppc_md.hpte_insert(hpteg, va, paddr,
 				tmp_mode, HPTE_V_BOLTED, psize, ssize);
 		if (ret < 0)
 			break;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 		if ((paddr >> PAGE_SHIFT) < linear_map_hash_count)
 			linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80;
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 	}
 	return ret < 0 ? ret : 0;
 }
 #ifdef CONFIG_MEMORY_HOTPLUG
 static int htab_remove_mapping(unsigned long vstart, unsigned long vend,
 		      int psize, int ssize)
 {
 	unsigned long vaddr;
 	unsigned int step, shift;
 	shift = mmu_psize_defs[psize].shift;
 	step = 1 << shift;
 	if (!ppc_md.hpte_removebolted) {
 		printk(KERN_WARNING "Platform doesn't implement "
 				"hpte_removebolted\n");
 		return -EINVAL;
 	}
 	for (vaddr = vstart; vaddr < vend; vaddr += step)
 		ppc_md.hpte_removebolted(vaddr, psize, ssize);
 	return 0;
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
 static int __init htab_dt_scan_seg_sizes(unsigned long node,
 					 const char *uname, int depth,
 					 void *data)
 {
 	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 	u32 *prop;
 	unsigned long size = 0;
 	/* We are scanning "cpu" nodes only */
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,processor-segment-sizes",
 					  &size);
 	if (prop == NULL)
 		return 0;
 	for (; size >= 4; size -= 4, ++prop) {
 		if (prop[0] == 40) {
 			DBG("1T segment support detected\n");
 			cur_cpu_spec->cpu_features |= CPU_FTR_1T_SEGMENT;
 			return 1;
 		}
 	}
 	cur_cpu_spec->cpu_features &= ~CPU_FTR_NO_SLBIE_B;
 	return 0;
 }
 static void __init htab_init_seg_sizes(void)
 {
 	of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL);
 }
 static int __init htab_dt_scan_page_sizes(unsigned long node,
 					  const char *uname, int depth,
 					  void *data)
 {
 	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 	u32 *prop;
 	unsigned long size = 0;
 	/* We are scanning "cpu" nodes only */
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 	prop = (u32 *)of_get_flat_dt_prop(node,
 					  "ibm,segment-page-sizes", &size);
 	if (prop != NULL) {
 		DBG("Page sizes from device-tree:\n");
 		size /= 4;
 		cur_cpu_spec->cpu_features &= ~(CPU_FTR_16M_PAGE);
 		while(size > 0) {
 			unsigned int shift = prop[0];
 			unsigned int slbenc = prop[1];
 			unsigned int lpnum = prop[2];
 			unsigned int lpenc = 0;
 			struct mmu_psize_def *def;
 			int idx = -1;
 			size -= 3; prop += 3;
 			while(size > 0 && lpnum) {
 				if (prop[0] == shift)
 					lpenc = prop[1];
 				prop += 2; size -= 2;
 				lpnum--;
 			}
 			switch(shift) {
 			case 0xc:
 				idx = MMU_PAGE_4K;
 				break;
 			case 0x10:
 				idx = MMU_PAGE_64K;
 				break;
 			case 0x14:
 				idx = MMU_PAGE_1M;
 				break;
 			case 0x18:
 				idx = MMU_PAGE_16M;
 				cur_cpu_spec->cpu_features |= CPU_FTR_16M_PAGE;
 				break;
 			case 0x22:
 				idx = MMU_PAGE_16G;
 				break;
 			}
 			if (idx < 0)
 				continue;
 			def = &mmu_psize_defs[idx];
 			def->shift = shift;
 			if (shift <= 23)
 				def->avpnm = 0;
 			else
 				def->avpnm = (1 << (shift - 23)) - 1;
 			def->sllp = slbenc;
 			def->penc = lpenc;
 			/* We don't know for sure what's up with tlbiel, so
 			 * for now we only set it for 4K and 64K pages
 			 */
 			if (idx == MMU_PAGE_4K || idx == MMU_PAGE_64K)
 				def->tlbiel = 1;
 			else
 				def->tlbiel = 0;
 			DBG(" %d: shift=%02x, sllp=%04x, avpnm=%08x, "
 			    "tlbiel=%d, penc=%d\n",
 			    idx, shift, def->sllp, def->avpnm, def->tlbiel,
 			    def->penc);
 		}
 		return 1;
 	}
 	return 0;
 }
 /* Scan for 16G memory blocks that have been set aside for huge pages
  * and reserve those blocks for 16G huge pages.
  */
 static int __init htab_dt_scan_hugepage_blocks(unsigned long node,
 					const char *uname, int depth,
 					void *data) {
 	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 	unsigned long *addr_prop;
 	u32 *page_count_prop;
 	unsigned int expected_pages;
 	long unsigned int phys_addr;
 	long unsigned int block_size;
 	/* We are scanning "memory" nodes only */
 	if (type == NULL || strcmp(type, "memory") != 0)
 		return 0;
 	/* This property is the log base 2 of the number of virtual pages that
 	 * will represent this memory block. */
 	page_count_prop = of_get_flat_dt_prop(node, "ibm,expected#pages", NULL);
 	if (page_count_prop == NULL)
 		return 0;
 	expected_pages = (1 << page_count_prop[0]);
 	addr_prop = of_get_flat_dt_prop(node, "reg", NULL);
 	if (addr_prop == NULL)
 		return 0;
 	phys_addr = addr_prop[0];
 	block_size = addr_prop[1];
 	if (block_size != (16 * GB))
 		return 0;
 	printk(KERN_INFO "Huge page(16GB) memory: "
 			"addr = 0x%lX size = 0x%lX pages = %d\n",
 			phys_addr, block_size, expected_pages);
 	lmb_reserve(phys_addr, block_size * expected_pages);
 	add_gpage(phys_addr, block_size, expected_pages);
 	return 0;
 }
 static void __init htab_init_page_sizes(void)
 {
 	int rc;
 	/* Default to 4K pages only */
 	memcpy(mmu_psize_defs, mmu_psize_defaults_old,
 	       sizeof(mmu_psize_defaults_old));
 	/*
 	 * Try to find the available page sizes in the device-tree
 	 */
 	rc = of_scan_flat_dt(htab_dt_scan_page_sizes, NULL);
 	if (rc != 0)  /* Found */
 		goto found;
 	/*
 	 * Not in the device-tree, let's fallback on known size
 	 * list for 16M capable GP & GR
 	 */
 	if (cpu_has_feature(CPU_FTR_16M_PAGE))
 		memcpy(mmu_psize_defs, mmu_psize_defaults_gp,
 		       sizeof(mmu_psize_defaults_gp));
  found:
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	/*
 	 * Pick a size for the linear mapping. Currently, we only support
 	 * 16M, 1M and 4K which is the default
 	 */
 	if (mmu_psize_defs[MMU_PAGE_16M].shift)
 		mmu_linear_psize = MMU_PAGE_16M;
 	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
 		mmu_linear_psize = MMU_PAGE_1M;
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 #ifdef CONFIG_PPC_64K_PAGES
 	/*
 	 * Pick a size for the ordinary pages. Default is 4K, we support
 	 * 64K for user mappings and vmalloc if supported by the processor.
 	 * We only use 64k for ioremap if the processor
 	 * (and firmware) support cache-inhibited large pages.
 	 * If not, we use 4k and set mmu_ci_restrictions so that
 	 * hash_page knows to switch processes that use cache-inhibited
 	 * mappings to 4k pages.
 	 */
 	if (mmu_psize_defs[MMU_PAGE_64K].shift) {
 		mmu_virtual_psize = MMU_PAGE_64K;
 		mmu_vmalloc_psize = MMU_PAGE_64K;
 		if (mmu_linear_psize == MMU_PAGE_4K)
 			mmu_linear_psize = MMU_PAGE_64K;
 		if (cpu_has_feature(CPU_FTR_CI_LARGE_PAGE)) {
 			/*
 			 * Don't use 64k pages for ioremap on pSeries, since
 			 * that would stop us accessing the HEA ethernet.
 			 */
 			if (!machine_is(pseries))
 				mmu_io_psize = MMU_PAGE_64K;
 		} else
 			mmu_ci_restrictions = 1;
 	}
 #endif /* CONFIG_PPC_64K_PAGES */
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 	/* We try to use 16M pages for vmemmap if that is supported
 	 * and we have at least 1G of RAM at boot
 	 */
 	if (mmu_psize_defs[MMU_PAGE_16M].shift &&
 	    lmb_phys_mem_size() >= 0x40000000)
 		mmu_vmemmap_psize = MMU_PAGE_16M;
 	else if (mmu_psize_defs[MMU_PAGE_64K].shift)
 		mmu_vmemmap_psize = MMU_PAGE_64K;
 	else
 		mmu_vmemmap_psize = MMU_PAGE_4K;
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 	printk(KERN_DEBUG "Page orders: linear mapping = %d, "
 	       "virtual = %d, io = %d"
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 	       ", vmemmap = %d"
 #endif
 	       "\n",
 	       mmu_psize_defs[mmu_linear_psize].shift,
 	       mmu_psize_defs[mmu_virtual_psize].shift,
 	       mmu_psize_defs[mmu_io_psize].shift
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 	       ,mmu_psize_defs[mmu_vmemmap_psize].shift
 #endif
 	       );
 #ifdef CONFIG_HUGETLB_PAGE
 	/* Reserve 16G huge page memory sections for huge pages */
 	of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
-/* Init large page size. Currently, we pick 16M or 1M depending
+/* Set default large page size. Currently, we pick 16M or 1M depending
 	 * on what is available
 	 */
 	if (mmu_psize_defs[MMU_PAGE_16M].shift)
-		set_huge_psize(MMU_PAGE_16M);
+		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
 	/* With 4k/4level pagetables, we can't (for now) cope with a
 	 * huge page size < PMD_SIZE */
 	else if (mmu_psize_defs[MMU_PAGE_1M].shift)
-		set_huge_psize(MMU_PAGE_1M);
+		HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
 #endif /* CONFIG_HUGETLB_PAGE */
 }
 static int __init htab_dt_scan_pftsize(unsigned long node,
 				       const char *uname, int depth,
 				       void *data)
 {
 	char *type = of_get_flat_dt_prop(node, "device_type", NULL);
 	u32 *prop;
 	/* We are scanning "cpu" nodes only */
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 	prop = (u32 *)of_get_flat_dt_prop(node, "ibm,pft-size", NULL);
 	if (prop != NULL) {
 		/* pft_size[0] is the NUMA CEC cookie */
 		ppc64_pft_size = prop[1];
 		return 1;
 	}
 	return 0;
 }
 static unsigned long __init htab_get_table_size(void)
 {
 	unsigned long mem_size, rnd_mem_size, pteg_count;
 	/* If hash size isn't already provided by the platform, we try to
 	 * retrieve it from the device-tree. If it's not there neither, we
 	 * calculate it now based on the total RAM size
 	 */
 	if (ppc64_pft_size == 0)
 		of_scan_flat_dt(htab_dt_scan_pftsize, NULL);
 	if (ppc64_pft_size)
 		return 1UL << ppc64_pft_size;
 	/* round mem_size up to next power of 2 */
 	mem_size = lmb_phys_mem_size();
 	rnd_mem_size = 1UL << __ilog2(mem_size);
 	if (rnd_mem_size < mem_size)
 		rnd_mem_size <<= 1;
 	/* # pages / 2 */
 	pteg_count = max(rnd_mem_size >> (12 + 1), 1UL << 11);
 	return pteg_count << 7;
 }
 #ifdef CONFIG_MEMORY_HOTPLUG
 void create_section_mapping(unsigned long start, unsigned long end)
 {
 		BUG_ON(htab_bolt_mapping(start, end, __pa(start),
 			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX,
 			mmu_linear_psize, mmu_kernel_ssize));
 }
 int remove_section_mapping(unsigned long start, unsigned long end)
 {
 	return htab_remove_mapping(start, end, mmu_linear_psize,
 			mmu_kernel_ssize);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
 static inline void make_bl(unsigned int *insn_addr, void *func)
 {
 	unsigned long funcp = *((unsigned long *)func);
 	int offset = funcp - (unsigned long)insn_addr;
 	*insn_addr = (unsigned int)(0x48000001 | (offset & 0x03fffffc));
 	flush_icache_range((unsigned long)insn_addr, 4+
 			   (unsigned long)insn_addr);
 }
 static void __init htab_finish_init(void)
 {
 	extern unsigned int *htab_call_hpte_insert1;
 	extern unsigned int *htab_call_hpte_insert2;
 	extern unsigned int *htab_call_hpte_remove;
 	extern unsigned int *htab_call_hpte_updatepp;
 #ifdef CONFIG_PPC_HAS_HASH_64K
 	extern unsigned int *ht64_call_hpte_insert1;
 	extern unsigned int *ht64_call_hpte_insert2;
 	extern unsigned int *ht64_call_hpte_remove;
 	extern unsigned int *ht64_call_hpte_updatepp;
 	make_bl(ht64_call_hpte_insert1, ppc_md.hpte_insert);
 	make_bl(ht64_call_hpte_insert2, ppc_md.hpte_insert);
 	make_bl(ht64_call_hpte_remove, ppc_md.hpte_remove);
 	make_bl(ht64_call_hpte_updatepp, ppc_md.hpte_updatepp);
 #endif /* CONFIG_PPC_HAS_HASH_64K */
 	make_bl(htab_call_hpte_insert1, ppc_md.hpte_insert);
 	make_bl(htab_call_hpte_insert2, ppc_md.hpte_insert);
 	make_bl(htab_call_hpte_remove, ppc_md.hpte_remove);
 	make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
 }
 void __init htab_initialize(void)
 {
 	unsigned long table;
 	unsigned long pteg_count;
 	unsigned long mode_rw;
 	unsigned long base = 0, size = 0, limit;
 	int i;
 	DBG(" -> htab_initialize()\n");
 	/* Initialize segment sizes */
 	htab_init_seg_sizes();
 	/* Initialize page sizes */
 	htab_init_page_sizes();
 	if (cpu_has_feature(CPU_FTR_1T_SEGMENT)) {
 		mmu_kernel_ssize = MMU_SEGSIZE_1T;
 		mmu_highuser_ssize = MMU_SEGSIZE_1T;
 		printk(KERN_INFO "Using 1TB segments\n");
 	}
 	/*
 	 * Calculate the required size of the htab.  We want the number of
 	 * PTEGs to equal one half the number of real pages.
 	 */
 	htab_size_bytes = htab_get_table_size();
 	pteg_count = htab_size_bytes >> 7;
 	htab_hash_mask = pteg_count - 1;
 	if (firmware_has_feature(FW_FEATURE_LPAR)) {
 		/* Using a hypervisor which owns the htab */
 		htab_address = NULL;
 		_SDR1 = 0;
 	} else {
 		/* Find storage for the HPT.  Must be contiguous in
 		 * the absolute address space. On cell we want it to be
 		 * in the first 2 Gig so we can use it for IOMMU hacks.
 		 */
 		if (machine_is(cell))
 			limit = 0x80000000;
 		else
 			limit = 0;
 		table = lmb_alloc_base(htab_size_bytes, htab_size_bytes, limit);
 		DBG("Hash table allocated at %lx, size: %lx\n", table,
 		    htab_size_bytes);
 		htab_address = abs_to_virt(table);
 		/* htab absolute addr + encoded htabsize */
 		_SDR1 = table + __ilog2(pteg_count) - 11;
 		/* Initialize the HPT with no entries */
 		memset((void *)table, 0, htab_size_bytes);
 		/* Set SDR1 */
 		mtspr(SPRN_SDR1, _SDR1);
 	}
 	mode_rw = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	linear_map_hash_count = lmb_end_of_DRAM() >> PAGE_SHIFT;
 	linear_map_hash_slots = __va(lmb_alloc_base(linear_map_hash_count,
 						    1, lmb.rmo_size));
 	memset(linear_map_hash_slots, 0, linear_map_hash_count);
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 	/* On U3 based machines, we need to reserve the DART area and
 	 * _NOT_ map it to avoid cache paradoxes as it's remapped non
 	 * cacheable later on
 	 */
 	/* create bolted the linear mapping in the hash table */
 	for (i=0; i < lmb.memory.cnt; i++) {
 		base = (unsigned long)__va(lmb.memory.region[i].base);
 		size = lmb.memory.region[i].size;
 		DBG("creating mapping for region: %lx : %lx\n", base, size);
 #ifdef CONFIG_U3_DART
 		/* Do not map the DART space. Fortunately, it will be aligned
 		 * in such a way that it will not cross two lmb regions and
 		 * will fit within a single 16Mb page.
 		 * The DART space is assumed to be a full 16Mb region even if
 		 * we only use 2Mb of that space. We will use more of it later
 		 * for AGP GART. We have to use a full 16Mb large page.
 		 */
 		DBG("DART base: %lx\n", dart_tablebase);
 		if (dart_tablebase != 0 && dart_tablebase >= base
 		    && dart_tablebase < (base + size)) {
 			unsigned long dart_table_end = dart_tablebase + 16 * MB;
 			if (base != dart_tablebase)
 				BUG_ON(htab_bolt_mapping(base, dart_tablebase,
 							__pa(base), mode_rw,
 							mmu_linear_psize,
 							mmu_kernel_ssize));
 			if ((base + size) > dart_table_end)
 				BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB,
 							base + size,
 							__pa(dart_table_end),
 							 mode_rw,
 							 mmu_linear_psize,
 							 mmu_kernel_ssize));
 			continue;
 		}
 #endif /* CONFIG_U3_DART */
 		BUG_ON(htab_bolt_mapping(base, base + size, __pa(base),
 				mode_rw, mmu_linear_psize, mmu_kernel_ssize));
        }
 	/*
 	 * If we have a memory_limit and we've allocated TCEs then we need to
 	 * explicitly map the TCE area at the top of RAM. We also cope with the
 	 * case that the TCEs start below memory_limit.
 	 * tce_alloc_start/end are 16MB aligned so the mapping should work
 	 * for either 4K or 16MB pages.
 	 */
 	if (tce_alloc_start) {
 		tce_alloc_start = (unsigned long)__va(tce_alloc_start);
 		tce_alloc_end = (unsigned long)__va(tce_alloc_end);
 		if (base + size >= tce_alloc_start)
 			tce_alloc_start = base + size + 1;
 		BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end,
 					 __pa(tce_alloc_start), mode_rw,
 					 mmu_linear_psize, mmu_kernel_ssize));
 	}
 	htab_finish_init();
 	DBG(" <- htab_initialize()\n");
 }
 #undef KB
 #undef MB
 void htab_initialize_secondary(void)
 {
 	if (!firmware_has_feature(FW_FEATURE_LPAR))
 		mtspr(SPRN_SDR1, _SDR1);
 }
 /*
  * Called by asm hashtable.S for doing lazy icache flush
  */
 unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
 {
 	struct page *page;
 	if (!pfn_valid(pte_pfn(pte)))
 		return pp;
 	page = pte_page(pte);
 	/* page is dirty */
 	if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
 		if (trap == 0x400) {
 			__flush_dcache_icache(page_address(page));
 			set_bit(PG_arch_1, &page->flags);
 		} else
 			pp |= HPTE_R_N;
 	}
 	return pp;
 }
 #ifdef CONFIG_PPC_MM_SLICES
 unsigned int get_paca_psize(unsigned long addr)
 {
 	unsigned long index, slices;
 	if (addr < SLICE_LOW_TOP) {
 		slices = get_paca()->context.low_slices_psize;
 		index = GET_LOW_SLICE_INDEX(addr);
 	} else {
 		slices = get_paca()->context.high_slices_psize;
 		index = GET_HIGH_SLICE_INDEX(addr);
 	}
 	return (slices >> (index * 4)) & 0xF;
 }
 #else
 unsigned int get_paca_psize(unsigned long addr)
 {
 	return get_paca()->context.user_psize;
 }
 #endif
 /*
  * Demote a segment to using 4k pages.
  * For now this makes the whole process use 4k pages.
  */
 #ifdef CONFIG_PPC_64K_PAGES
 void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
 {
 	if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
 		return;
 	slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K);
 #ifdef CONFIG_SPU_BASE
 	spu_flush_all_slbs(mm);
 #endif
 	if (get_paca_psize(addr) != MMU_PAGE_4K) {
 		get_paca()->context = mm->context;
 		slb_flush_and_rebolt();
 	}
 }
 #endif /* CONFIG_PPC_64K_PAGES */
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 /*
  * This looks up a 2-bit protection code for a 4k subpage of a 64k page.
  * Userspace sets the subpage permissions using the subpage_prot system call.
  *
  * Result is 0: full permissions, _PAGE_RW: read-only,
  * _PAGE_USER or _PAGE_USER|_PAGE_RW: no access.
  */
 static int subpage_protection(pgd_t *pgdir, unsigned long ea)
 {
 	struct subpage_prot_table *spt = pgd_subpage_prot(pgdir);
 	u32 spp = 0;
 	u32 **sbpm, *sbpp;
 	if (ea >= spt->maxaddr)
 		return 0;
 	if (ea < 0x100000000) {
 		/* addresses below 4GB use spt->low_prot */
 		sbpm = spt->low_prot;
 	} else {
 		sbpm = spt->protptrs[ea >> SBP_L3_SHIFT];
 		if (!sbpm)
 			return 0;
 	}
 	sbpp = sbpm[(ea >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)];
 	if (!sbpp)
 		return 0;
 	spp = sbpp[(ea >> PAGE_SHIFT) & (SBP_L1_COUNT - 1)];
 	/* extract 2-bit bitfield for this 4k subpage */
 	spp >>= 30 - 2 * ((ea >> 12) & 0xf);
 	/* turn 0,1,2,3 into combination of _PAGE_USER and _PAGE_RW */
 	spp = ((spp & 2) ? _PAGE_USER : 0) | ((spp & 1) ? _PAGE_RW : 0);
 	return spp;
 }
 #else /* CONFIG_PPC_SUBPAGE_PROT */
 static inline int subpage_protection(pgd_t *pgdir, unsigned long ea)
 {
 	return 0;
 }
 #endif
 /* Result code is:
  *  0 - handled
  *  1 - normal page fault
  * -1 - critical hash insertion error
  * -2 - access not permitted by subpage protection mechanism
  */
 int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 {
 	void *pgdir;
 	unsigned long vsid;
 	struct mm_struct *mm;
 	pte_t *ptep;
 	cpumask_t tmp;
 	int rc, user_region = 0, local = 0;
 	int psize, ssize;
 	DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n",
 		ea, access, trap);
 	if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) {
 		DBG_LOW(" out of pgtable range !\n");
  		return 1;
 	}
 	/* Get region & vsid */
  	switch (REGION_ID(ea)) {
 	case USER_REGION_ID:
 		user_region = 1;
 		mm = current->mm;
 		if (! mm) {
 			DBG_LOW(" user region with no mm !\n");
 			return 1;
 		}
 		psize = get_slice_psize(mm, ea);
 		ssize = user_segment_size(ea);
 		vsid = get_vsid(mm->context.id, ea, ssize);
 		break;
 	case VMALLOC_REGION_ID:
 		mm = &init_mm;
 		vsid = get_kernel_vsid(ea, mmu_kernel_ssize);
 		if (ea < VMALLOC_END)
 			psize = mmu_vmalloc_psize;
 		else
 			psize = mmu_io_psize;
 		ssize = mmu_kernel_ssize;
 		break;
 	default:
 		/* Not a valid range
 		 * Send the problem up to do_page_fault
 		 */
 		return 1;
 	}
 	DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
 	/* Get pgdir */
 	pgdir = mm->pgd;
 	if (pgdir == NULL)
 		return 1;
 	/* Check CPU locality */
 	tmp = cpumask_of_cpu(smp_processor_id());
 	if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
 		local = 1;
 #ifdef CONFIG_HUGETLB_PAGE
 	/* Handle hugepage regions */
-	if (HPAGE_SHIFT && psize == mmu_huge_psize) {
+	if (HPAGE_SHIFT && mmu_huge_psizes[psize]) {
 		DBG_LOW(" -> huge page !\n");
 		return hash_huge_page(mm, access, ea, vsid, local, trap);
 	}
 #endif /* CONFIG_HUGETLB_PAGE */
 #ifndef CONFIG_PPC_64K_PAGES
 	/* If we use 4K pages and our psize is not 4K, then we are hitting
 	 * a special driver mapping, we need to align the address before
 	 * we fetch the PTE
 	 */
 	if (psize != MMU_PAGE_4K)
 		ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
 #endif /* CONFIG_PPC_64K_PAGES */
 	/* Get PTE and page size from page tables */
 	ptep = find_linux_pte(pgdir, ea);
 	if (ptep == NULL || !pte_present(*ptep)) {
 		DBG_LOW(" no PTE !\n");
 		return 1;
 	}
 #ifndef CONFIG_PPC_64K_PAGES
 	DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
 #else
 	DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),
 		pte_val(*(ptep + PTRS_PER_PTE)));
 #endif
 	/* Pre-check access permissions (will be re-checked atomically
 	 * in __hash_page_XX but this pre-check is a fast path
 	 */
 	if (access & ~pte_val(*ptep)) {
 		DBG_LOW(" no access !\n");
 		return 1;
 	}
 	/* Do actual hashing */
 #ifdef CONFIG_PPC_64K_PAGES
 	/* If _PAGE_4K_PFN is set, make sure this is a 4k segment */
 	if ((pte_val(*ptep) & _PAGE_4K_PFN) && psize == MMU_PAGE_64K) {
 		demote_segment_4k(mm, ea);
 		psize = MMU_PAGE_4K;
 	}
 	/* If this PTE is non-cacheable and we have restrictions on
 	 * using non cacheable large pages, then we switch to 4k
 	 */
 	if (mmu_ci_restrictions && psize == MMU_PAGE_64K &&
 	    (pte_val(*ptep) & _PAGE_NO_CACHE)) {
 		if (user_region) {
 			demote_segment_4k(mm, ea);
 			psize = MMU_PAGE_4K;
 		} else if (ea < VMALLOC_END) {
 			/*
 			 * some driver did a non-cacheable mapping
 			 * in vmalloc space, so switch vmalloc
 			 * to 4k pages
 			 */
 			printk(KERN_ALERT "Reducing vmalloc segment "
 			       "to 4kB pages because of "
 			       "non-cacheable mapping\n");
 			psize = mmu_vmalloc_psize = MMU_PAGE_4K;
 #ifdef CONFIG_SPU_BASE
 			spu_flush_all_slbs(mm);
 #endif
 		}
 	}
 	if (user_region) {
 		if (psize != get_paca_psize(ea)) {
 			get_paca()->context = mm->context;
 			slb_flush_and_rebolt();
 		}
 	} else if (get_paca()->vmalloc_sllp !=
 		   mmu_psize_defs[mmu_vmalloc_psize].sllp) {
 		get_paca()->vmalloc_sllp =
 			mmu_psize_defs[mmu_vmalloc_psize].sllp;
 		slb_vmalloc_update();
 	}
 #endif /* CONFIG_PPC_64K_PAGES */
 #ifdef CONFIG_PPC_HAS_HASH_64K
 	if (psize == MMU_PAGE_64K)
 		rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
 	else
 #endif /* CONFIG_PPC_HAS_HASH_64K */
 	{
 		int spp = subpage_protection(pgdir, ea);
 		if (access & spp)
 			rc = -2;
 		else
 			rc = __hash_page_4K(ea, access, vsid, ptep, trap,
 					    local, ssize, spp);
 	}
 #ifndef CONFIG_PPC_64K_PAGES
 	DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));
 #else
 	DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),
 		pte_val(*(ptep + PTRS_PER_PTE)));
 #endif
 	DBG_LOW(" -> rc=%d\n", rc);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(hash_page);
 void hash_preload(struct mm_struct *mm, unsigned long ea,
 		  unsigned long access, unsigned long trap)
 {
 	unsigned long vsid;
 	void *pgdir;
 	pte_t *ptep;
 	cpumask_t mask;
 	unsigned long flags;
 	int local = 0;
 	int ssize;
 	BUG_ON(REGION_ID(ea) != USER_REGION_ID);
 #ifdef CONFIG_PPC_MM_SLICES
 	/* We only prefault standard pages for now */
 	if (unlikely(get_slice_psize(mm, ea) != mm->context.user_psize))
 		return;
 #endif
 	DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"
 		" trap=%lx\n", mm, mm->pgd, ea, access, trap);
 	/* Get Linux PTE if available */
 	pgdir = mm->pgd;
 	if (pgdir == NULL)
 		return;
 	ptep = find_linux_pte(pgdir, ea);
 	if (!ptep)
 		return;
 #ifdef CONFIG_PPC_64K_PAGES
 	/* If either _PAGE_4K_PFN or _PAGE_NO_CACHE is set (and we are on
 	 * a 64K kernel), then we don't preload, hash_page() will take
 	 * care of it once we actually try to access the page.
 	 * That way we don't have to duplicate all of the logic for segment
 	 * page size demotion here
 	 */
 	if (pte_val(*ptep) & (_PAGE_4K_PFN | _PAGE_NO_CACHE))
 		return;
 #endif /* CONFIG_PPC_64K_PAGES */
 	/* Get VSID */
 	ssize = user_segment_size(ea);
 	vsid = get_vsid(mm->context.id, ea, ssize);
 	/* Hash doesn't like irqs */
 	local_irq_save(flags);
 	/* Is that local to this CPU ? */
 	mask = cpumask_of_cpu(smp_processor_id());
 	if (cpus_equal(mm->cpu_vm_mask, mask))
 		local = 1;
 	/* Hash it in */
 #ifdef CONFIG_PPC_HAS_HASH_64K
 	if (mm->context.user_psize == MMU_PAGE_64K)
 		__hash_page_64K(ea, access, vsid, ptep, trap, local, ssize);
 	else
 #endif /* CONFIG_PPC_HAS_HASH_64K */
 		__hash_page_4K(ea, access, vsid, ptep, trap, local, ssize,
 			       subpage_protection(pgdir, ea));
 	local_irq_restore(flags);
 }
 /* WARNING: This is called from hash_low_64.S, if you change this prototype,
  *          do not forget to update the assembly call site !
  */
 void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int ssize,
 		     int local)
 {
 	unsigned long hash, index, shift, hidx, slot;
 	DBG_LOW("flush_hash_page(va=%016x)\n", va);
 	pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
 		hash = hpt_hash(va, shift, ssize);
 		hidx = __rpte_to_hidx(pte, index);
 		if (hidx & _PTEIDX_SECONDARY)
 			hash = ~hash;
 		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 		slot += hidx & _PTEIDX_GROUP_IX;
 		DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx);
 		ppc_md.hpte_invalidate(slot, va, psize, ssize, local);
 	} pte_iterate_hashed_end();
 }
 void flush_hash_range(unsigned long number, int local)
 {
 	if (ppc_md.flush_hash_range)
 		ppc_md.flush_hash_range(number, local);
 	else {
 		int i;
 		struct ppc64_tlb_batch *batch =
 			&__get_cpu_var(ppc64_tlb_batch);
 		for (i = 0; i < number; i++)
 			flush_hash_page(batch->vaddr[i], batch->pte[i],
 					batch->psize, batch->ssize, local);
 	}
 }
 /*
  * low_hash_fault is called when we the low level hash code failed
  * to instert a PTE due to an hypervisor error
  */
 void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
 {
 	if (user_mode(regs)) {
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 		if (rc == -2)
 			_exception(SIGSEGV, regs, SEGV_ACCERR, address);
 		else
 #endif
 			_exception(SIGBUS, regs, BUS_ADRERR, address);
 	} else
 		bad_page_fault(regs, address, SIGBUS);
 }
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi)
 {
 	unsigned long hash, hpteg;
 	unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
 	unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
 	unsigned long mode = _PAGE_ACCESSED | _PAGE_DIRTY |
 		_PAGE_COHERENT | PP_RWXX | HPTE_R_N;
 	int ret;
 	hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
 	hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP);
 	ret = ppc_md.hpte_insert(hpteg, va, __pa(vaddr),
 				 mode, HPTE_V_BOLTED,
 				 mmu_linear_psize, mmu_kernel_ssize);
 	BUG_ON (ret < 0);
 	spin_lock(&linear_map_hash_lock);
 	BUG_ON(linear_map_hash_slots[lmi] & 0x80);
 	linear_map_hash_slots[lmi] = ret | 0x80;
 	spin_unlock(&linear_map_hash_lock);
 }
 static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
 {
 	unsigned long hash, hidx, slot;
 	unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize);
 	unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize);
 	hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize);
 	spin_lock(&linear_map_hash_lock);
 	BUG_ON(!(linear_map_hash_slots[lmi] & 0x80));
 	hidx = linear_map_hash_slots[lmi] & 0x7f;
 	linear_map_hash_slots[lmi] = 0;
 	spin_unlock(&linear_map_hash_lock);
 	if (hidx & _PTEIDX_SECONDARY)
 		hash = ~hash;
 	slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 	slot += hidx & _PTEIDX_GROUP_IX;
 	ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, mmu_kernel_ssize, 0);
 }
 void kernel_map_pages(struct page *page, int numpages, int enable)
 {
 	unsigned long flags, vaddr, lmi;
 	int i;
 	local_irq_save(flags);
 	for (i = 0; i < numpages; i++, page++) {
 		vaddr = (unsigned long)page_address(page);
 		lmi = __pa(vaddr) >> PAGE_SHIFT;
 		if (lmi >= linear_map_hash_count)
 			continue;
 		if (enable)
 			kernel_map_linear_page(vaddr, lmi);
 		else
 			kernel_unmap_linear_page(vaddr, lmi);
 	}
 	local_irq_restore(flags);
 }
 #endif /* CONFIG_DEBUG_PAGEALLOC */

arch/powerpc/mm/hugetlbpage.c

Diff comments View file @ 0d9ea75

 /*
  * PPC64 (POWER4) Huge TLB Page Support for Kernel.
  *
  * Copyright (C) 2003 David Gibson, IBM Corporation.
  *
  * Based on the IA-32 version:
  * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
  */
 #include <linux/init.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/hugetlb.h>
 #include <linux/pagemap.h>
 #include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/sysctl.h>
 #include <asm/mman.h>
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
 #include <asm/machdep.h>
 #include <asm/cputable.h>
 #include <asm/spu.h>
 #define PAGE_SHIFT_64K	16
 #define PAGE_SHIFT_16M	24
 #define PAGE_SHIFT_16G	34
 #define NUM_LOW_AREAS	(0x100000000UL >> SID_SHIFT)
 #define NUM_HIGH_AREAS	(PGTABLE_RANGE >> HTLB_AREA_SHIFT)
 #define MAX_NUMBER_GPAGES	1024
 /* Tracks the 16G pages after the device tree is scanned and before the
  * huge_boot_pages list is ready.  */
 static unsigned long gpage_freearray[MAX_NUMBER_GPAGES];
 static unsigned nr_gpages;
-unsigned int hugepte_shift;
+/* Array of valid huge page sizes - non-zero value(hugepte_shift) is
-#define PTRS_PER_HUGEPTE	(1 << hugepte_shift)
+ * stored for the huge page sizes that are valid.
-#define HUGEPTE_TABLE_SIZE	(sizeof(pte_t) << hugepte_shift)
+ */
+unsigned int mmu_huge_psizes[MMU_PAGE_COUNT] = { }; /* initialize all to 0 */
-#define HUGEPD_SHIFT		(HPAGE_SHIFT + hugepte_shift)
+#define hugepte_shift			mmu_huge_psizes
-#define HUGEPD_SIZE		(1UL << HUGEPD_SHIFT)
+#define PTRS_PER_HUGEPTE(psize)		(1 << hugepte_shift[psize])
-#define HUGEPD_MASK		(~(HUGEPD_SIZE-1))
+#define HUGEPTE_TABLE_SIZE(psize)	(sizeof(pte_t) << hugepte_shift[psize])
-#define huge_pgtable_cache	(pgtable_cache[HUGEPTE_CACHE_NUM])
+#define HUGEPD_SHIFT(psize)		(mmu_psize_to_shift(psize) \
+						+ hugepte_shift[psize])
+#define HUGEPD_SIZE(psize)		(1UL << HUGEPD_SHIFT(psize))
+#define HUGEPD_MASK(psize)		(~(HUGEPD_SIZE(psize)-1))
+/* Subtract one from array size because we don't need a cache for 4K since
+ * is not a huge page size */
+#define huge_pgtable_cache(psize)	(pgtable_cache[HUGEPTE_CACHE_NUM \
+							+ psize-1])
+#define HUGEPTE_CACHE_NAME(psize)	(huge_pgtable_cache_name[psize])
+static const char *huge_pgtable_cache_name[MMU_PAGE_COUNT] = {
+	"unused_4K", "hugepte_cache_64K", "unused_64K_AP",
+	"hugepte_cache_1M", "hugepte_cache_16M", "hugepte_cache_16G"
+};
 /* Flag to mark huge PD pointers.  This means pmd_bad() and pud_bad()
  * will choke on pointers to hugepte tables, which is handy for
  * catching screwups early. */
 #define HUGEPD_OK	0x1
 typedef struct { unsigned long pd; } hugepd_t;
 #define hugepd_none(hpd)	((hpd).pd == 0)
+static inline int shift_to_mmu_psize(unsigned int shift)
+{
+	switch (shift) {
+#ifndef CONFIG_PPC_64K_PAGES
+	case PAGE_SHIFT_64K:
+	    return MMU_PAGE_64K;
+#endif
+	case PAGE_SHIFT_16M:
+	    return MMU_PAGE_16M;
+	case PAGE_SHIFT_16G:
+	    return MMU_PAGE_16G;
+	}
+	return -1;
+}
+static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
+{
+	if (mmu_psize_defs[mmu_psize].shift)
+		return mmu_psize_defs[mmu_psize].shift;
+	BUG();
+}
 static inline pte_t *hugepd_page(hugepd_t hpd)
 {
 	BUG_ON(!(hpd.pd & HUGEPD_OK));
 	return (pte_t *)(hpd.pd & ~HUGEPD_OK);
 }
-static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr)
+static inline pte_t *hugepte_offset(hugepd_t *hpdp, unsigned long addr,
+				    struct hstate *hstate)
 {
-	unsigned long idx = ((addr >> HPAGE_SHIFT) & (PTRS_PER_HUGEPTE-1));
+	unsigned int shift = huge_page_shift(hstate);
+	int psize = shift_to_mmu_psize(shift);
+	unsigned long idx = ((addr >> shift) & (PTRS_PER_HUGEPTE(psize)-1));
 	pte_t *dir = hugepd_page(*hpdp);
 	return dir + idx;
 }
 static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
-			   unsigned long address)
+			   unsigned long address, unsigned int psize)
 {
-	pte_t *new = kmem_cache_alloc(huge_pgtable_cache,
+	pte_t *new = kmem_cache_alloc(huge_pgtable_cache(psize),
 				      GFP_KERNEL|__GFP_REPEAT);
 	if (! new)
 		return -ENOMEM;
 	spin_lock(&mm->page_table_lock);
 	if (!hugepd_none(*hpdp))
-		kmem_cache_free(huge_pgtable_cache, new);
+		kmem_cache_free(huge_pgtable_cache(psize), new);
 	else
 		hpdp->pd = (unsigned long)new | HUGEPD_OK;
 	spin_unlock(&mm->page_table_lock);
 	return 0;
 }
 /* Base page size affects how we walk hugetlb page tables */
 #ifdef CONFIG_PPC_64K_PAGES
-#define hpmd_offset(pud, addr)		pmd_offset(pud, addr)
+#define hpmd_offset(pud, addr, h)	pmd_offset(pud, addr)
-#define hpmd_alloc(mm, pud, addr)	pmd_alloc(mm, pud, addr)
+#define hpmd_alloc(mm, pud, addr, h)	pmd_alloc(mm, pud, addr)
 #else
 static inline
-pmd_t *hpmd_offset(pud_t *pud, unsigned long addr)
+pmd_t *hpmd_offset(pud_t *pud, unsigned long addr, struct hstate *hstate)
 {
-	if (HPAGE_SHIFT == PAGE_SHIFT_64K)
+	if (huge_page_shift(hstate) == PAGE_SHIFT_64K)
 		return pmd_offset(pud, addr);
 	else
 		return (pmd_t *) pud;
 }
 static inline
-pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr)
+pmd_t *hpmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long addr,
+		  struct hstate *hstate)
 {
-	if (HPAGE_SHIFT == PAGE_SHIFT_64K)
+	if (huge_page_shift(hstate) == PAGE_SHIFT_64K)
 		return pmd_alloc(mm, pud, addr);
 	else
 		return (pmd_t *) pud;
 }
 #endif
 /* Build list of addresses of gigantic pages.  This function is used in early
  * boot before the buddy or bootmem allocator is setup.
  */
 void add_gpage(unsigned long addr, unsigned long page_size,
 	unsigned long number_of_pages)
 {
 	if (!addr)
 		return;
 	while (number_of_pages > 0) {
 		gpage_freearray[nr_gpages] = addr;
 		nr_gpages++;
 		number_of_pages--;
 		addr += page_size;
 	}
 }
 /* Moves the gigantic page addresses from the temporary list to the
- * huge_boot_pages list.  */
+ * huge_boot_pages list.
-int alloc_bootmem_huge_page(struct hstate *h)
+ */
+int alloc_bootmem_huge_page(struct hstate *hstate)
 {
 	struct huge_bootmem_page *m;
 	if (nr_gpages == 0)
 		return 0;
 	m = phys_to_virt(gpage_freearray[--nr_gpages]);
 	gpage_freearray[nr_gpages] = 0;
 	list_add(&m->list, &huge_boot_pages);
-	m->hstate = h;
+	m->hstate = hstate;
 	return 1;
 }
 /* Modelled after find_linux_pte() */
 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 {
 	pgd_t *pg;
 	pud_t *pu;
 	pmd_t *pm;
-	BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize);
+	unsigned int psize;
+	unsigned int shift;
+	unsigned long sz;
+	struct hstate *hstate;
+	psize = get_slice_psize(mm, addr);
+	shift = mmu_psize_to_shift(psize);
+	sz = ((1UL) << shift);
+	hstate = size_to_hstate(sz);
-	addr &= HPAGE_MASK;
+	addr &= hstate->mask;
 	pg = pgd_offset(mm, addr);
 	if (!pgd_none(*pg)) {
 		pu = pud_offset(pg, addr);
 		if (!pud_none(*pu)) {
-			pm = hpmd_offset(pu, addr);
+			pm = hpmd_offset(pu, addr, hstate);
 			if (!pmd_none(*pm))
-				return hugepte_offset((hugepd_t *)pm, addr);
+				return hugepte_offset((hugepd_t *)pm, addr,
+						      hstate);
 		}
 	}
 	return NULL;
 }
 pte_t *huge_pte_alloc(struct mm_struct *mm,
 			unsigned long addr, unsigned long sz)
 {
 	pgd_t *pg;
 	pud_t *pu;
 	pmd_t *pm;
 	hugepd_t *hpdp = NULL;
+	struct hstate *hstate;
+	unsigned int psize;
+	hstate = size_to_hstate(sz);
-	BUG_ON(get_slice_psize(mm, addr) != mmu_huge_psize);
+	psize = get_slice_psize(mm, addr);
+	BUG_ON(!mmu_huge_psizes[psize]);
-	addr &= HPAGE_MASK;
+	addr &= hstate->mask;
 	pg = pgd_offset(mm, addr);
 	pu = pud_alloc(mm, pg, addr);
 	if (pu) {
-		pm = hpmd_alloc(mm, pu, addr);
+		pm = hpmd_alloc(mm, pu, addr, hstate);
 		if (pm)
 			hpdp = (hugepd_t *)pm;
 	}
 	if (! hpdp)
 		return NULL;
-	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr))
+	if (hugepd_none(*hpdp) && __hugepte_alloc(mm, hpdp, addr, psize))
 		return NULL;
-	return hugepte_offset(hpdp, addr);
+	return hugepte_offset(hpdp, addr, hstate);
 }
 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 {
 	return 0;
 }
-static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp)
+static void free_hugepte_range(struct mmu_gather *tlb, hugepd_t *hpdp,
+			       unsigned int psize)
 {
 	pte_t *hugepte = hugepd_page(*hpdp);
 	hpdp->pd = 0;
 	tlb->need_flush = 1;
-	pgtable_free_tlb(tlb, pgtable_free_cache(hugepte, HUGEPTE_CACHE_NUM,
+	pgtable_free_tlb(tlb, pgtable_free_cache(hugepte,
+						 HUGEPTE_CACHE_NUM+psize-1,
 						 PGF_CACHENUM_MASK));
 }
 static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
 				   unsigned long addr, unsigned long end,
-				   unsigned long floor, unsigned long ceiling)
+				   unsigned long floor, unsigned long ceiling,
+				   unsigned int psize)
 {
 	pmd_t *pmd;
 	unsigned long next;
 	unsigned long start;
 	start = addr;
 	pmd = pmd_offset(pud, addr);
 	do {
 		next = pmd_addr_end(addr, end);
 		if (pmd_none(*pmd))
 			continue;
-		free_hugepte_range(tlb, (hugepd_t *)pmd);
+		free_hugepte_range(tlb, (hugepd_t *)pmd, psize);
 	} while (pmd++, addr = next, addr != end);
 	start &= PUD_MASK;
 	if (start < floor)
 		return;
 	if (ceiling) {
 		ceiling &= PUD_MASK;
 		if (!ceiling)
 			return;
 	}
 	if (end - 1 > ceiling - 1)
 		return;
 	pmd = pmd_offset(pud, start);
 	pud_clear(pud);
 	pmd_free_tlb(tlb, pmd);
 }
 static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
 				   unsigned long addr, unsigned long end,
 				   unsigned long floor, unsigned long ceiling)
 {
 	pud_t *pud;
 	unsigned long next;
 	unsigned long start;
+	unsigned int shift;
+	unsigned int psize = get_slice_psize(tlb->mm, addr);
+	shift = mmu_psize_to_shift(psize);
 	start = addr;
 	pud = pud_offset(pgd, addr);
 	do {
 		next = pud_addr_end(addr, end);
 #ifdef CONFIG_PPC_64K_PAGES
 		if (pud_none_or_clear_bad(pud))
 			continue;
-		hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling);
+		hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling,
+				       psize);
 #else
-		if (HPAGE_SHIFT == PAGE_SHIFT_64K) {
+		if (shift == PAGE_SHIFT_64K) {
 			if (pud_none_or_clear_bad(pud))
 				continue;
-			hugetlb_free_pmd_range(tlb, pud, addr, next, floor, ceiling);
+			hugetlb_free_pmd_range(tlb, pud, addr, next, floor,
+					       ceiling, psize);
 		} else {
 			if (pud_none(*pud))
 				continue;
-			free_hugepte_range(tlb, (hugepd_t *)pud);
+			free_hugepte_range(tlb, (hugepd_t *)pud, psize);
 		}
 #endif
 	} while (pud++, addr = next, addr != end);
 	start &= PGDIR_MASK;
 	if (start < floor)
 		return;
 	if (ceiling) {
 		ceiling &= PGDIR_MASK;
 		if (!ceiling)
 			return;
 	}
 	if (end - 1 > ceiling - 1)
 		return;
 	pud = pud_offset(pgd, start);
 	pgd_clear(pgd);
 	pud_free_tlb(tlb, pud);
 }
 /*
  * This function frees user-level page tables of a process.
  *
  * Must be called with pagetable lock held.
  */
 void hugetlb_free_pgd_range(struct mmu_gather *tlb,
 			    unsigned long addr, unsigned long end,
 			    unsigned long floor, unsigned long ceiling)
 {
 	pgd_t *pgd;
 	unsigned long next;
 	unsigned long start;
 	/*
 	 * Comments below take from the normal free_pgd_range().  They
 	 * apply here too.  The tests against HUGEPD_MASK below are
 	 * essential, because we *don't* test for this at the bottom
 	 * level.  Without them we'll attempt to free a hugepte table
 	 * when we unmap just part of it, even if there are other
 	 * active mappings using it.
 	 *
 	 * The next few lines have given us lots of grief...
 	 *
 	 * Why are we testing HUGEPD* at this top level?  Because
 	 * often there will be no work to do at all, and we'd prefer
 	 * not to go all the way down to the bottom just to discover
 	 * that.
 	 *
 	 * Why all these "- 1"s?  Because 0 represents both the bottom
 	 * of the address space and the top of it (using -1 for the
 	 * top wouldn't help much: the masks would do the wrong thing).
 	 * The rule is that addr 0 and floor 0 refer to the bottom of
 	 * the address space, but end 0 and ceiling 0 refer to the top
 	 * Comparisons need to use "end - 1" and "ceiling - 1" (though
 	 * that end 0 case should be mythical).
 	 *
 	 * Wherever addr is brought up or ceiling brought down, we
 	 * must be careful to reject "the opposite 0" before it
 	 * confuses the subsequent tests.  But what about where end is
 	 * brought down by HUGEPD_SIZE below? no, end can't go down to
 	 * 0 there.
 	 *
 	 * Whereas we round start (addr) and ceiling down, by different
 	 * masks at different levels, in order to test whether a table
 	 * now has no other vmas using it, so can be freed, we don't
 	 * bother to round floor or end up - the tests don't need that.
 	 */
+	unsigned int psize = get_slice_psize(tlb->mm, addr);
-	addr &= HUGEPD_MASK;
+	addr &= HUGEPD_MASK(psize);
 	if (addr < floor) {
-		addr += HUGEPD_SIZE;
+		addr += HUGEPD_SIZE(psize);
 		if (!addr)
 			return;
 	}
 	if (ceiling) {
-		ceiling &= HUGEPD_MASK;
+		ceiling &= HUGEPD_MASK(psize);
 		if (!ceiling)
 			return;
 	}
 	if (end - 1 > ceiling - 1)
-		end -= HUGEPD_SIZE;
+		end -= HUGEPD_SIZE(psize);
 	if (addr > end - 1)
 		return;
 	start = addr;
 	pgd = pgd_offset(tlb->mm, addr);
 	do {
-		BUG_ON(get_slice_psize(tlb->mm, addr) != mmu_huge_psize);
+		psize = get_slice_psize(tlb->mm, addr);
+		BUG_ON(!mmu_huge_psizes[psize]);
 		next = pgd_addr_end(addr, end);
 		if (pgd_none_or_clear_bad(pgd))
 			continue;
 		hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling);
 	} while (pgd++, addr = next, addr != end);
 }
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 		     pte_t *ptep, pte_t pte)
 {
 	if (pte_present(*ptep)) {
 		/* We open-code pte_clear because we need to pass the right
 		 * argument to hpte_need_flush (huge / !huge). Might not be
 		 * necessary anymore if we make hpte_need_flush() get the
 		 * page size from the slices
 		 */
-		pte_update(mm, addr & HPAGE_MASK, ptep, ~0UL, 1);
+		unsigned int psize = get_slice_psize(mm, addr);
+		unsigned int shift = mmu_psize_to_shift(psize);
+		unsigned long sz = ((1UL) << shift);
+		struct hstate *hstate = size_to_hstate(sz);
+		pte_update(mm, addr & hstate->mask, ptep, ~0UL, 1);
 	}
 	*ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
 }
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep)
 {
 	unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
 	return __pte(old);
 }
 struct page *
 follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
 {
 	pte_t *ptep;
 	struct page *page;
+	unsigned int mmu_psize = get_slice_psize(mm, address);
-	if (get_slice_psize(mm, address) != mmu_huge_psize)
+	/* Verify it is a huge page else bail. */
+	if (!mmu_huge_psizes[mmu_psize])
 		return ERR_PTR(-EINVAL);
 	ptep = huge_pte_offset(mm, address);
 	page = pte_page(*ptep);
-	if (page)
+	if (page) {
-		page += (address % HPAGE_SIZE) / PAGE_SIZE;
+		unsigned int shift = mmu_psize_to_shift(mmu_psize);
+		unsigned long sz = ((1UL) << shift);
+		page += (address % sz) / PAGE_SIZE;
+	}
 	return page;
 }
 int pmd_huge(pmd_t pmd)
 {
 	return 0;
 }
 int pud_huge(pud_t pud)
 {
 	return 0;
 }
 struct page *
 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
 		pmd_t *pmd, int write)
 {
 	BUG();
 	return NULL;
 }
 unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 					unsigned long len, unsigned long pgoff,
 					unsigned long flags)
 {
-	return slice_get_unmapped_area(addr, len, flags,
+	struct hstate *hstate = hstate_file(file);
-				       mmu_huge_psize, 1, 0);
+	int mmu_psize = shift_to_mmu_psize(huge_page_shift(hstate));
+	return slice_get_unmapped_area(addr, len, flags, mmu_psize, 1, 0);
 }
 /*
  * Called by asm hashtable.S for doing lazy icache flush
  */
 static unsigned int hash_huge_page_do_lazy_icache(unsigned long rflags,
-						  pte_t pte, int trap)
+					pte_t pte, int trap, unsigned long sz)
 {
 	struct page *page;
 	int i;
 	if (!pfn_valid(pte_pfn(pte)))
 		return rflags;
 	page = pte_page(pte);
 	/* page is dirty */
 	if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
 		if (trap == 0x400) {
-			for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++)
+			for (i = 0; i < (sz / PAGE_SIZE); i++)
 				__flush_dcache_icache(page_address(page+i));
 			set_bit(PG_arch_1, &page->flags);
 		} else {
 			rflags |= HPTE_R_N;
 		}
 	}
 	return rflags;
 }
 int hash_huge_page(struct mm_struct *mm, unsigned long access,
 		   unsigned long ea, unsigned long vsid, int local,
 		   unsigned long trap)
 {
 	pte_t *ptep;
 	unsigned long old_pte, new_pte;
-	unsigned long va, rflags, pa;
+	unsigned long va, rflags, pa, sz;
 	long slot;
 	int err = 1;
 	int ssize = user_segment_size(ea);
+	unsigned int mmu_psize;
+	int shift;
+	mmu_psize = get_slice_psize(mm, ea);
+	if (!mmu_huge_psizes[mmu_psize])
+		goto out;
 	ptep = huge_pte_offset(mm, ea);
 	/* Search the Linux page table for a match with va */
 	va = hpt_va(ea, vsid, ssize);
 	/*
 	 * If no pte found or not present, send the problem up to
 	 * do_page_fault
 	 */
 	if (unlikely(!ptep || pte_none(*ptep)))
 		goto out;
 	/*
 	 * Check the user's access rights to the page.  If access should be
 	 * prevented then send the problem up to do_page_fault.
 	 */
 	if (unlikely(access & ~pte_val(*ptep)))
 		goto out;
 	/*
 	 * At this point, we have a pte (old_pte) which can be used to build
 	 * or update an HPTE. There are 2 cases:
 	 *
 	 * 1. There is a valid (present) pte with no associated HPTE (this is
 	 *	the most common case)
 	 * 2. There is a valid (present) pte with an associated HPTE. The
 	 *	current values of the pp bits in the HPTE prevent access
 	 *	because we are doing software DIRTY bit management and the
 	 *	page is currently not DIRTY.
 	 */
 	do {
 		old_pte = pte_val(*ptep);
 		if (old_pte & _PAGE_BUSY)
 			goto out;
 		new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED;
 	} while(old_pte != __cmpxchg_u64((unsigned long *)ptep,
 					 old_pte, new_pte));
 	rflags = 0x2 | (!(new_pte & _PAGE_RW));
  	/* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */
 	rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N);
+	shift = mmu_psize_to_shift(mmu_psize);
+	sz = ((1UL) << shift);
 	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
 		/* No CPU has hugepages but lacks no execute, so we
 		 * don't need to worry about that case */
 		rflags = hash_huge_page_do_lazy_icache(rflags, __pte(old_pte),
-						       trap);
+						       trap, sz);
 	/* Check if pte already has an hpte (case 2) */
 	if (unlikely(old_pte & _PAGE_HASHPTE)) {
 		/* There MIGHT be an HPTE for this pte */
 		unsigned long hash, slot;
-		hash = hpt_hash(va, HPAGE_SHIFT, ssize);
+		hash = hpt_hash(va, shift, ssize);
 		if (old_pte & _PAGE_F_SECOND)
 			hash = ~hash;
 		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
 		slot += (old_pte & _PAGE_F_GIX) >> 12;
-		if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize,
+		if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize,
 					 ssize, local) == -1)
 			old_pte &= ~_PAGE_HPTEFLAGS;
 	}
 	if (likely(!(old_pte & _PAGE_HASHPTE))) {
-		unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize);
+		unsigned long hash = hpt_hash(va, shift, ssize);
 		unsigned long hpte_group;
 		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
 repeat:
 		hpte_group = ((hash & htab_hash_mask) *
 			      HPTES_PER_GROUP) & ~0x7UL;
 		/* clear HPTE slot informations in new PTE */
 #ifdef CONFIG_PPC_64K_PAGES
 		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0;
 #else
 		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE;
 #endif
 		/* Add in WIMG bits */
 		rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE |
 				      _PAGE_COHERENT | _PAGE_GUARDED));
 		/* Insert into the hash table, primary slot */
 		slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0,
-					  mmu_huge_psize, ssize);
+					  mmu_psize, ssize);
 		/* Primary is full, try the secondary */
 		if (unlikely(slot == -1)) {
 			hpte_group = ((~hash & htab_hash_mask) *
 				      HPTES_PER_GROUP) & ~0x7UL;
 			slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags,
 						  HPTE_V_SECONDARY,
-						  mmu_huge_psize, ssize);
+						  mmu_psize, ssize);
 			if (slot == -1) {
 				if (mftb() & 0x1)
 					hpte_group = ((hash & htab_hash_mask) *
 						      HPTES_PER_GROUP)&~0x7UL;
 				ppc_md.hpte_remove(hpte_group);
 				goto repeat;
                         }
 		}
 		if (unlikely(slot == -2))
 			panic("hash_huge_page: pte_insert failed\n");
 		new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX);
 	}
 	/*
 	 * No need to use ldarx/stdcx here
 	 */
 	*ptep = __pte(new_pte & ~_PAGE_BUSY);
 	err = 0;
  out:
 	return err;
 }
 void set_huge_psize(int psize)
 {
 	/* Check that it is a page size supported by the hardware and
 	 * that it fits within pagetable limits. */
 	if (mmu_psize_defs[psize].shift &&
 		mmu_psize_defs[psize].shift < SID_SHIFT_1T &&
 		(mmu_psize_defs[psize].shift > MIN_HUGEPTE_SHIFT ||
 		 mmu_psize_defs[psize].shift == PAGE_SHIFT_64K ||
 		 mmu_psize_defs[psize].shift == PAGE_SHIFT_16G)) {
-		/* Return if huge page size is the same as the
+		/* Return if huge page size has already been setup or is the
-		 * base page size. */
+		 * same as the base page size. */
-		if (mmu_psize_defs[psize].shift == PAGE_SHIFT)
+		if (mmu_huge_psizes[psize] ||
+		   mmu_psize_defs[psize].shift == PAGE_SHIFT)
 			return;
+		hugetlb_add_hstate(mmu_psize_defs[psize].shift - PAGE_SHIFT);
-		HPAGE_SHIFT = mmu_psize_defs[psize].shift;
+		switch (mmu_psize_defs[psize].shift) {
-		mmu_huge_psize = psize;
-		switch (HPAGE_SHIFT) {
 		case PAGE_SHIFT_64K:
 		    /* We only allow 64k hpages with 4k base page,
 		     * which was checked above, and always put them
 		     * at the PMD */
-		    hugepte_shift = PMD_SHIFT;
+		    hugepte_shift[psize] = PMD_SHIFT;
 		    break;
 		case PAGE_SHIFT_16M:
 		    /* 16M pages can be at two different levels
 		     * of pagestables based on base page size */
 		    if (PAGE_SHIFT == PAGE_SHIFT_64K)
-			    hugepte_shift = PMD_SHIFT;
+			    hugepte_shift[psize] = PMD_SHIFT;
 		    else /* 4k base page */
-			    hugepte_shift = PUD_SHIFT;
+			    hugepte_shift[psize] = PUD_SHIFT;
 		    break;
 		case PAGE_SHIFT_16G:
 		    /* 16G pages are always at PGD level */
-		    hugepte_shift = PGDIR_SHIFT;
+		    hugepte_shift[psize] = PGDIR_SHIFT;
 		    break;
 		}
-		hugepte_shift -= HPAGE_SHIFT;
+		hugepte_shift[psize] -= mmu_psize_defs[psize].shift;
 	} else
-		HPAGE_SHIFT = 0;
+		hugepte_shift[psize] = 0;
 }
 static int __init hugepage_setup_sz(char *str)
 {
 	unsigned long long size;
-	int mmu_psize = -1;
+	int mmu_psize;
 	int shift;
 	size = memparse(str, &str);
 	shift = __ffs(size);
-	switch (shift) {
+	mmu_psize = shift_to_mmu_psize(shift);
-#ifndef CONFIG_PPC_64K_PAGES
+	if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift)
-	case PAGE_SHIFT_64K:
-		mmu_psize = MMU_PAGE_64K;
-		break;
-#endif
-	case PAGE_SHIFT_16M:
-		mmu_psize = MMU_PAGE_16M;
-		break;
-	case PAGE_SHIFT_16G:
-		mmu_psize = MMU_PAGE_16G;
-		break;
-	}
-	if (mmu_psize >= 0 && mmu_psize_defs[mmu_psize].shift) {
 		set_huge_psize(mmu_psize);
-		hugetlb_add_hstate(shift - PAGE_SHIFT);
-	}
 	else
 		printk(KERN_WARNING "Invalid huge page size specified(%llu)\n", size);
 	return 1;
 }
 __setup("hugepagesz=", hugepage_setup_sz);
 static void zero_ctor(struct kmem_cache *cache, void *addr)
 {
 	memset(addr, 0, kmem_cache_size(cache));
 }
 static int __init hugetlbpage_init(void)
 {
+	unsigned int psize;
 	if (!cpu_has_feature(CPU_FTR_16M_PAGE))
 		return -ENODEV;
+	/* Add supported huge page sizes.  Need to change HUGE_MAX_HSTATE
+	 * and adjust PTE_NONCACHE_NUM if the number of supported huge page
+	 * sizes changes.
+	 */
+	set_huge_psize(MMU_PAGE_16M);
+	set_huge_psize(MMU_PAGE_64K);
+	set_huge_psize(MMU_PAGE_16G);
-	huge_pgtable_cache = kmem_cache_create("hugepte_cache",
+	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {

arch/powerpc/mm/init_64.c

Diff comments View file @ 0d9ea75

1	/*	1	/*
2	* PowerPC version	2	* PowerPC version
3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4	*	4	*
5	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)	5	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6	* and Cort Dougan (PReP) (cort@cs.nmt.edu)	6	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
7	* Copyright (C) 1996 Paul Mackerras	7	* Copyright (C) 1996 Paul Mackerras
8	*	8	*
9	* Derived from "arch/i386/mm/init.c"	9	* Derived from "arch/i386/mm/init.c"
10	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds	10	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
11	*	11	*
12	* Dave Engebretsen <engebret@us.ibm.com>	12	* Dave Engebretsen <engebret@us.ibm.com>
13	* Rework for PPC64 port.	13	* Rework for PPC64 port.
14	*	14	*
15	* This program is free software; you can redistribute it and/or	15	* This program is free software; you can redistribute it and/or
16	* modify it under the terms of the GNU General Public License	16	* modify it under the terms of the GNU General Public License
17	* as published by the Free Software Foundation; either version	17	* as published by the Free Software Foundation; either version
18	* 2 of the License, or (at your option) any later version.	18	* 2 of the License, or (at your option) any later version.
19	*	19	*
20	*/	20	*/
21		21
22	#undef DEBUG	22	#undef DEBUG
23		23
24	#include <linux/signal.h>	24	#include <linux/signal.h>
25	#include <linux/sched.h>	25	#include <linux/sched.h>
26	#include <linux/kernel.h>	26	#include <linux/kernel.h>
27	#include <linux/errno.h>	27	#include <linux/errno.h>
28	#include <linux/string.h>	28	#include <linux/string.h>
29	#include <linux/types.h>	29	#include <linux/types.h>
30	#include <linux/mman.h>	30	#include <linux/mman.h>
31	#include <linux/mm.h>	31	#include <linux/mm.h>
32	#include <linux/swap.h>	32	#include <linux/swap.h>
33	#include <linux/stddef.h>	33	#include <linux/stddef.h>
34	#include <linux/vmalloc.h>	34	#include <linux/vmalloc.h>
35	#include <linux/init.h>	35	#include <linux/init.h>
36	#include <linux/delay.h>	36	#include <linux/delay.h>
37	#include <linux/bootmem.h>	37	#include <linux/bootmem.h>
38	#include <linux/highmem.h>	38	#include <linux/highmem.h>
39	#include <linux/idr.h>	39	#include <linux/idr.h>
40	#include <linux/nodemask.h>	40	#include <linux/nodemask.h>
41	#include <linux/module.h>	41	#include <linux/module.h>
42	#include <linux/poison.h>	42	#include <linux/poison.h>
43	#include <linux/lmb.h>	43	#include <linux/lmb.h>
44		44
45	#include <asm/pgalloc.h>	45	#include <asm/pgalloc.h>
46	#include <asm/page.h>	46	#include <asm/page.h>
47	#include <asm/prom.h>	47	#include <asm/prom.h>
48	#include <asm/rtas.h>	48	#include <asm/rtas.h>
49	#include <asm/io.h>	49	#include <asm/io.h>
50	#include <asm/mmu_context.h>	50	#include <asm/mmu_context.h>
51	#include <asm/pgtable.h>	51	#include <asm/pgtable.h>
52	#include <asm/mmu.h>	52	#include <asm/mmu.h>
53	#include <asm/uaccess.h>	53	#include <asm/uaccess.h>
54	#include <asm/smp.h>	54	#include <asm/smp.h>
55	#include <asm/machdep.h>	55	#include <asm/machdep.h>
56	#include <asm/tlb.h>	56	#include <asm/tlb.h>
57	#include <asm/eeh.h>	57	#include <asm/eeh.h>
58	#include <asm/processor.h>	58	#include <asm/processor.h>
59	#include <asm/mmzone.h>	59	#include <asm/mmzone.h>
60	#include <asm/cputable.h>	60	#include <asm/cputable.h>
61	#include <asm/sections.h>	61	#include <asm/sections.h>
62	#include <asm/system.h>	62	#include <asm/system.h>
63	#include <asm/iommu.h>	63	#include <asm/iommu.h>
64	#include <asm/abs_addr.h>	64	#include <asm/abs_addr.h>
65	#include <asm/vdso.h>	65	#include <asm/vdso.h>
66		66
67	#include "mmu_decl.h"	67	#include "mmu_decl.h"
68		68
69	#if PGTABLE_RANGE > USER_VSID_RANGE	69	#if PGTABLE_RANGE > USER_VSID_RANGE
70	#warning Limited user VSID range means pagetable space is wasted	70	#warning Limited user VSID range means pagetable space is wasted
71	#endif	71	#endif
72		72
73	#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)	73	#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
74	#warning TASK_SIZE is smaller than it needs to be.	74	#warning TASK_SIZE is smaller than it needs to be.
75	#endif	75	#endif
76		76
77	phys_addr_t memstart_addr = ~0;	77	phys_addr_t memstart_addr = ~0;
78	phys_addr_t kernstart_addr;	78	phys_addr_t kernstart_addr;
79		79
80	void free_initmem(void)	80	void free_initmem(void)
81	{	81	{
82	unsigned long addr;	82	unsigned long addr;
83		83
84	addr = (unsigned long)__init_begin;	84	addr = (unsigned long)__init_begin;
85	for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {	85	for (; addr < (unsigned long)__init_end; addr += PAGE_SIZE) {
86	memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);	86	memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
87	ClearPageReserved(virt_to_page(addr));	87	ClearPageReserved(virt_to_page(addr));
88	init_page_count(virt_to_page(addr));	88	init_page_count(virt_to_page(addr));
89	free_page(addr);	89	free_page(addr);
90	totalram_pages++;	90	totalram_pages++;
91	}	91	}
92	printk ("Freeing unused kernel memory: %luk freed\n",	92	printk ("Freeing unused kernel memory: %luk freed\n",
93	((unsigned long)__init_end - (unsigned long)__init_begin) >> 10);	93	((unsigned long)__init_end - (unsigned long)__init_begin) >> 10);
94	}	94	}
95		95
96	#ifdef CONFIG_BLK_DEV_INITRD	96	#ifdef CONFIG_BLK_DEV_INITRD
97	void free_initrd_mem(unsigned long start, unsigned long end)	97	void free_initrd_mem(unsigned long start, unsigned long end)
98	{	98	{
99	if (start < end)	99	if (start < end)
100	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);	100	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
101	for (; start < end; start += PAGE_SIZE) {	101	for (; start < end; start += PAGE_SIZE) {
102	ClearPageReserved(virt_to_page(start));	102	ClearPageReserved(virt_to_page(start));
103	init_page_count(virt_to_page(start));	103	init_page_count(virt_to_page(start));
104	free_page(start);	104	free_page(start);
105	totalram_pages++;	105	totalram_pages++;
106	}	106	}
107	}	107	}
108	#endif	108	#endif
109		109
110	#ifdef CONFIG_PROC_KCORE	110	#ifdef CONFIG_PROC_KCORE
111	static struct kcore_list kcore_vmem;	111	static struct kcore_list kcore_vmem;
112		112
113	static int __init setup_kcore(void)	113	static int __init setup_kcore(void)
114	{	114	{
115	int i;	115	int i;
116		116
117	for (i=0; i < lmb.memory.cnt; i++) {	117	for (i=0; i < lmb.memory.cnt; i++) {
118	unsigned long base, size;	118	unsigned long base, size;
119	struct kcore_list *kcore_mem;	119	struct kcore_list *kcore_mem;
120		120
121	base = lmb.memory.region[i].base;	121	base = lmb.memory.region[i].base;
122	size = lmb.memory.region[i].size;	122	size = lmb.memory.region[i].size;
123		123
124	/* GFP_ATOMIC to avoid might_sleep warnings during boot */	124	/* GFP_ATOMIC to avoid might_sleep warnings during boot */
125	kcore_mem = kmalloc(sizeof(struct kcore_list), GFP_ATOMIC);	125	kcore_mem = kmalloc(sizeof(struct kcore_list), GFP_ATOMIC);
126	if (!kcore_mem)	126	if (!kcore_mem)
127	panic("%s: kmalloc failed\n", __func__);	127	panic("%s: kmalloc failed\n", __func__);
128		128
129	kclist_add(kcore_mem, __va(base), size);	129	kclist_add(kcore_mem, __va(base), size);
130	}	130	}
131		131
132	kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START);	132	kclist_add(&kcore_vmem, (void *)VMALLOC_START, VMALLOC_END-VMALLOC_START);
133		133
134	return 0;	134	return 0;
135	}	135	}
136	module_init(setup_kcore);	136	module_init(setup_kcore);
137	#endif	137	#endif
138		138
139	static void zero_ctor(struct kmem_cache cache, void addr)	139	static void zero_ctor(struct kmem_cache cache, void addr)
140	{	140	{
141	memset(addr, 0, kmem_cache_size(cache));	141	memset(addr, 0, kmem_cache_size(cache));
142	}	142	}
143		143
144	static const unsigned int pgtable_cache_size[2] = {	144	static const unsigned int pgtable_cache_size[2] = {
145	PGD_TABLE_SIZE, PMD_TABLE_SIZE	145	PGD_TABLE_SIZE, PMD_TABLE_SIZE
146	};	146	};
147	static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {	147	static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {
148	#ifdef CONFIG_PPC_64K_PAGES	148	#ifdef CONFIG_PPC_64K_PAGES
149	"pgd_cache", "pmd_cache",	149	"pgd_cache", "pmd_cache",
150	#else	150	#else
151	"pgd_cache", "pud_pmd_cache",	151	"pgd_cache", "pud_pmd_cache",
152	#endif /* CONFIG_PPC_64K_PAGES */	152	#endif /* CONFIG_PPC_64K_PAGES */
153	};	153	};
154		154
155	#ifdef CONFIG_HUGETLB_PAGE	155	#ifdef CONFIG_HUGETLB_PAGE
156	/* Hugepages need one extra cache, initialized in hugetlbpage.c. We	156	/* Hugepages need an extra cache per hugepagesize, initialized in
157	* can't put into the tables above, because HPAGE_SHIFT is not compile	157	* hugetlbpage.c. We can't put into the tables above, because HPAGE_SHIFT
158	* time constant. */	158	* is not compile time constant. */
159	struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)+1];	159	struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)+MMU_PAGE_COUNT];
160	#else	160	#else
161	struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];	161	struct kmem_cache *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];
162	#endif	162	#endif
163		163
164	void pgtable_cache_init(void)	164	void pgtable_cache_init(void)
165	{	165	{
166	int i;	166	int i;
167		167
168	for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) {	168	for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) {
169	int size = pgtable_cache_size[i];	169	int size = pgtable_cache_size[i];
170	const char *name = pgtable_cache_name[i];	170	const char *name = pgtable_cache_name[i];
171		171
172	pr_debug("Allocating page table cache %s (#%d) "	172	pr_debug("Allocating page table cache %s (#%d) "
173	"for size: %08x...\n", name, i, size);	173	"for size: %08x...\n", name, i, size);
174	pgtable_cache[i] = kmem_cache_create(name,	174	pgtable_cache[i] = kmem_cache_create(name,
175	size, size,	175	size, size,
176	SLAB_PANIC,	176	SLAB_PANIC,
177	zero_ctor);	177	zero_ctor);
178	}	178	}
179	}	179	}
180		180
181	#ifdef CONFIG_SPARSEMEM_VMEMMAP	181	#ifdef CONFIG_SPARSEMEM_VMEMMAP
182	/*	182	/*
183	* Given an address within the vmemmap, determine the pfn of the page that	183	* Given an address within the vmemmap, determine the pfn of the page that
184	* represents the start of the section it is within. Note that we have to	184	* represents the start of the section it is within. Note that we have to
185	* do this by hand as the proffered address may not be correctly aligned.	185	* do this by hand as the proffered address may not be correctly aligned.
186	* Subtraction of non-aligned pointers produces undefined results.	186	* Subtraction of non-aligned pointers produces undefined results.
187	*/	187	*/
188	static unsigned long __meminit vmemmap_section_start(unsigned long page)	188	static unsigned long __meminit vmemmap_section_start(unsigned long page)
189	{	189	{
190	unsigned long offset = page - ((unsigned long)(vmemmap));	190	unsigned long offset = page - ((unsigned long)(vmemmap));
191		191
192	/* Return the pfn of the start of the section. */	192	/* Return the pfn of the start of the section. */
193	return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;	193	return (offset / sizeof(struct page)) & PAGE_SECTION_MASK;
194	}	194	}
195		195
196	/*	196	/*
197	* Check if this vmemmap page is already initialised. If any section	197	* Check if this vmemmap page is already initialised. If any section
198	* which overlaps this vmemmap page is initialised then this page is	198	* which overlaps this vmemmap page is initialised then this page is
199	* initialised already.	199	* initialised already.
200	*/	200	*/
201	static int __meminit vmemmap_populated(unsigned long start, int page_size)	201	static int __meminit vmemmap_populated(unsigned long start, int page_size)
202	{	202	{
203	unsigned long end = start + page_size;	203	unsigned long end = start + page_size;
204		204
205	for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))	205	for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
206	if (pfn_valid(vmemmap_section_start(start)))	206	if (pfn_valid(vmemmap_section_start(start)))
207	return 1;	207	return 1;
208		208
209	return 0;	209	return 0;
210	}	210	}
211		211
212	int __meminit vmemmap_populate(struct page *start_page,	212	int __meminit vmemmap_populate(struct page *start_page,
213	unsigned long nr_pages, int node)	213	unsigned long nr_pages, int node)
214	{	214	{
215	unsigned long mode_rw;	215	unsigned long mode_rw;
216	unsigned long start = (unsigned long)start_page;	216	unsigned long start = (unsigned long)start_page;
217	unsigned long end = (unsigned long)(start_page + nr_pages);	217	unsigned long end = (unsigned long)(start_page + nr_pages);
218	unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;	218	unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
219		219
220	mode_rw = _PAGE_ACCESSED \| _PAGE_DIRTY \| _PAGE_COHERENT \| PP_RWXX;	220	mode_rw = _PAGE_ACCESSED \| _PAGE_DIRTY \| _PAGE_COHERENT \| PP_RWXX;
221		221
222	/* Align to the page size of the linear mapping. */	222	/* Align to the page size of the linear mapping. */
223	start = _ALIGN_DOWN(start, page_size);	223	start = _ALIGN_DOWN(start, page_size);
224		224
225	for (; start < end; start += page_size) {	225	for (; start < end; start += page_size) {
226	int mapped;	226	int mapped;
227	void *p;	227	void *p;
228		228
229	if (vmemmap_populated(start, page_size))	229	if (vmemmap_populated(start, page_size))
230	continue;	230	continue;
231		231
232	p = vmemmap_alloc_block(page_size, node);	232	p = vmemmap_alloc_block(page_size, node);
233	if (!p)	233	if (!p)
234	return -ENOMEM;	234	return -ENOMEM;
235		235
236	pr_debug("vmemmap %08lx allocated at %p, physical %08lx.\n",	236	pr_debug("vmemmap %08lx allocated at %p, physical %08lx.\n",
237	start, p, __pa(p));	237	start, p, __pa(p));
238		238
239	mapped = htab_bolt_mapping(start, start + page_size,	239	mapped = htab_bolt_mapping(start, start + page_size,
240	__pa(p), mode_rw, mmu_vmemmap_psize,	240	__pa(p), mode_rw, mmu_vmemmap_psize,
241	mmu_kernel_ssize);	241	mmu_kernel_ssize);
242	BUG_ON(mapped < 0);	242	BUG_ON(mapped < 0);
243	}	243	}
244		244
245	return 0;	245	return 0;
246	}	246	}
247	#endif /* CONFIG_SPARSEMEM_VMEMMAP */	247	#endif /* CONFIG_SPARSEMEM_VMEMMAP */
248		248

arch/powerpc/mm/tlb_64.c

Diff comments View file @ 0d9ea75

1	/*	1	/*
2	* This file contains the routines for flushing entries from the	2	* This file contains the routines for flushing entries from the
3	* TLB and MMU hash table.	3	* TLB and MMU hash table.
4	*	4	*
5	* Derived from arch/ppc64/mm/init.c:	5	* Derived from arch/ppc64/mm/init.c:
6	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)	6	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7	*	7	*
8	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)	8	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
9	* and Cort Dougan (PReP) (cort@cs.nmt.edu)	9	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
10	* Copyright (C) 1996 Paul Mackerras	10	* Copyright (C) 1996 Paul Mackerras
11	*	11	*
12	* Derived from "arch/i386/mm/init.c"	12	* Derived from "arch/i386/mm/init.c"
13	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds	13	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
14	*	14	*
15	* Dave Engebretsen <engebret@us.ibm.com>	15	* Dave Engebretsen <engebret@us.ibm.com>
16	* Rework for PPC64 port.	16	* Rework for PPC64 port.
17	*	17	*
18	* This program is free software; you can redistribute it and/or	18	* This program is free software; you can redistribute it and/or
19	* modify it under the terms of the GNU General Public License	19	* modify it under the terms of the GNU General Public License
20	* as published by the Free Software Foundation; either version	20	* as published by the Free Software Foundation; either version
21	* 2 of the License, or (at your option) any later version.	21	* 2 of the License, or (at your option) any later version.
22	*/	22	*/
23		23
24	#include <linux/kernel.h>	24	#include <linux/kernel.h>
25	#include <linux/mm.h>	25	#include <linux/mm.h>
26	#include <linux/init.h>	26	#include <linux/init.h>
27	#include <linux/percpu.h>	27	#include <linux/percpu.h>
28	#include <linux/hardirq.h>	28	#include <linux/hardirq.h>
29	#include <asm/pgalloc.h>	29	#include <asm/pgalloc.h>
30	#include <asm/tlbflush.h>	30	#include <asm/tlbflush.h>
31	#include <asm/tlb.h>	31	#include <asm/tlb.h>
32	#include <asm/bug.h>	32	#include <asm/bug.h>
33		33
34	DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);	34	DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
35		35
36	/* This is declared as we are using the more or less generic	36	/* This is declared as we are using the more or less generic
37	* include/asm-powerpc/tlb.h file -- tgall	37	* include/asm-powerpc/tlb.h file -- tgall
38	*/	38	*/
39	DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);	39	DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
40	static DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);	40	static DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
41	static unsigned long pte_freelist_forced_free;	41	static unsigned long pte_freelist_forced_free;
42		42
43	struct pte_freelist_batch	43	struct pte_freelist_batch
44	{	44	{
45	struct rcu_head rcu;	45	struct rcu_head rcu;
46	unsigned int index;	46	unsigned int index;
47	pgtable_free_t tables[0];	47	pgtable_free_t tables[0];
48	};	48	};
49		49
50	#define PTE_FREELIST_SIZE \	50	#define PTE_FREELIST_SIZE \
51	((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \	51	((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \
52	/ sizeof(pgtable_free_t))	52	/ sizeof(pgtable_free_t))
53		53
54	static void pte_free_smp_sync(void *arg)	54	static void pte_free_smp_sync(void *arg)
55	{	55	{
56	/* Do nothing, just ensure we sync with all CPUs */	56	/* Do nothing, just ensure we sync with all CPUs */
57	}	57	}
58		58
59	/* This is only called when we are critically out of memory	59	/* This is only called when we are critically out of memory
60	* (and fail to get a page in pte_free_tlb).	60	* (and fail to get a page in pte_free_tlb).
61	*/	61	*/
62	static void pgtable_free_now(pgtable_free_t pgf)	62	static void pgtable_free_now(pgtable_free_t pgf)
63	{	63	{
64	pte_freelist_forced_free++;	64	pte_freelist_forced_free++;
65		65
66	smp_call_function(pte_free_smp_sync, NULL, 1);	66	smp_call_function(pte_free_smp_sync, NULL, 1);
67		67
68	pgtable_free(pgf);	68	pgtable_free(pgf);
69	}	69	}
70		70
71	static void pte_free_rcu_callback(struct rcu_head *head)	71	static void pte_free_rcu_callback(struct rcu_head *head)
72	{	72	{
73	struct pte_freelist_batch *batch =	73	struct pte_freelist_batch *batch =
74	container_of(head, struct pte_freelist_batch, rcu);	74	container_of(head, struct pte_freelist_batch, rcu);
75	unsigned int i;	75	unsigned int i;
76		76
77	for (i = 0; i < batch->index; i++)	77	for (i = 0; i < batch->index; i++)
78	pgtable_free(batch->tables[i]);	78	pgtable_free(batch->tables[i]);
79		79
80	free_page((unsigned long)batch);	80	free_page((unsigned long)batch);
81	}	81	}
82		82
83	static void pte_free_submit(struct pte_freelist_batch *batch)	83	static void pte_free_submit(struct pte_freelist_batch *batch)
84	{	84	{
85	INIT_RCU_HEAD(&batch->rcu);	85	INIT_RCU_HEAD(&batch->rcu);
86	call_rcu(&batch->rcu, pte_free_rcu_callback);	86	call_rcu(&batch->rcu, pte_free_rcu_callback);
87	}	87	}
88		88
89	void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf)	89	void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf)
90	{	90	{
91	/* This is safe since tlb_gather_mmu has disabled preemption */	91	/* This is safe since tlb_gather_mmu has disabled preemption */
92	cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id());	92	cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id());
93	struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);	93	struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);
94		94
95	if (atomic_read(&tlb->mm->mm_users) < 2 \|\|	95	if (atomic_read(&tlb->mm->mm_users) < 2 \|\|
96	cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) {	96	cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) {
97	pgtable_free(pgf);	97	pgtable_free(pgf);
98	return;	98	return;
99	}	99	}
100		100
101	if (*batchp == NULL) {	101	if (*batchp == NULL) {
102	batchp = (struct pte_freelist_batch )__get_free_page(GFP_ATOMIC);	102	batchp = (struct pte_freelist_batch )__get_free_page(GFP_ATOMIC);
103	if (*batchp == NULL) {	103	if (*batchp == NULL) {
104	pgtable_free_now(pgf);	104	pgtable_free_now(pgf);
105	return;	105	return;
106	}	106	}
107	(*batchp)->index = 0;	107	(*batchp)->index = 0;
108	}	108	}
109	(batchp)->tables[(batchp)->index++] = pgf;	109	(batchp)->tables[(batchp)->index++] = pgf;
110	if ((*batchp)->index == PTE_FREELIST_SIZE) {	110	if ((*batchp)->index == PTE_FREELIST_SIZE) {
111	pte_free_submit(*batchp);	111	pte_free_submit(*batchp);
112	*batchp = NULL;	112	*batchp = NULL;
113	}	113	}
114	}	114	}
115		115
116	/*	116	/*
117	* A linux PTE was changed and the corresponding hash table entry	117	* A linux PTE was changed and the corresponding hash table entry
118	* neesd to be flushed. This function will either perform the flush	118	* neesd to be flushed. This function will either perform the flush
119	* immediately or will batch it up if the current CPU has an active	119	* immediately or will batch it up if the current CPU has an active
120	* batch on it.	120	* batch on it.
121	*	121	*
122	* Must be called from within some kind of spinlock/non-preempt region...	122	* Must be called from within some kind of spinlock/non-preempt region...
123	*/	123	*/
124	void hpte_need_flush(struct mm_struct *mm, unsigned long addr,	124	void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
125	pte_t *ptep, unsigned long pte, int huge)	125	pte_t *ptep, unsigned long pte, int huge)
126	{	126	{
127	struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);	127	struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
128	unsigned long vsid, vaddr;	128	unsigned long vsid, vaddr;
129	unsigned int psize;	129	unsigned int psize;
130	int ssize;	130	int ssize;
131	real_pte_t rpte;	131	real_pte_t rpte;
132	int i;	132	int i;
133		133
134	i = batch->index;	134	i = batch->index;
135		135
136	/* We mask the address for the base page size. Huge pages will	136	/* We mask the address for the base page size. Huge pages will
137	* have applied their own masking already	137	* have applied their own masking already
138	*/	138	*/
139	addr &= PAGE_MASK;	139	addr &= PAGE_MASK;
140		140
141	/* Get page size (maybe move back to caller).	141	/* Get page size (maybe move back to caller).
142	*	142	*
143	* NOTE: when using special 64K mappings in 4K environment like	143	* NOTE: when using special 64K mappings in 4K environment like
144	* for SPEs, we obtain the page size from the slice, which thus	144	* for SPEs, we obtain the page size from the slice, which thus
145	* must still exist (and thus the VMA not reused) at the time	145	* must still exist (and thus the VMA not reused) at the time
146	* of this call	146	* of this call
147	*/	147	*/
148	if (huge) {	148	if (huge) {
149	#ifdef CONFIG_HUGETLB_PAGE	149	#ifdef CONFIG_HUGETLB_PAGE
150	psize = mmu_huge_psize;	150	psize = get_slice_psize(mm, addr);;
151	#else	151	#else
152	BUG();	152	BUG();
153	psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */	153	psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */
154	#endif	154	#endif
155	} else	155	} else
156	psize = pte_pagesize_index(mm, addr, pte);	156	psize = pte_pagesize_index(mm, addr, pte);
157		157
158	/* Build full vaddr */	158	/* Build full vaddr */
159	if (!is_kernel_addr(addr)) {	159	if (!is_kernel_addr(addr)) {
160	ssize = user_segment_size(addr);	160	ssize = user_segment_size(addr);
161	vsid = get_vsid(mm->context.id, addr, ssize);	161	vsid = get_vsid(mm->context.id, addr, ssize);
162	WARN_ON(vsid == 0);	162	WARN_ON(vsid == 0);
163	} else {	163	} else {
164	vsid = get_kernel_vsid(addr, mmu_kernel_ssize);	164	vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
165	ssize = mmu_kernel_ssize;	165	ssize = mmu_kernel_ssize;
166	}	166	}
167	vaddr = hpt_va(addr, vsid, ssize);	167	vaddr = hpt_va(addr, vsid, ssize);
168	rpte = __real_pte(__pte(pte), ptep);	168	rpte = __real_pte(__pte(pte), ptep);
169		169
170	/*	170	/*
171	* Check if we have an active batch on this CPU. If not, just	171	* Check if we have an active batch on this CPU. If not, just
172	* flush now and return. For now, we don global invalidates	172	* flush now and return. For now, we don global invalidates
173	* in that case, might be worth testing the mm cpu mask though	173	* in that case, might be worth testing the mm cpu mask though
174	* and decide to use local invalidates instead...	174	* and decide to use local invalidates instead...
175	*/	175	*/
176	if (!batch->active) {	176	if (!batch->active) {
177	flush_hash_page(vaddr, rpte, psize, ssize, 0);	177	flush_hash_page(vaddr, rpte, psize, ssize, 0);
178	return;	178	return;
179	}	179	}
180		180
181	/*	181	/*
182	* This can happen when we are in the middle of a TLB batch and	182	* This can happen when we are in the middle of a TLB batch and
183	* we encounter memory pressure (eg copy_page_range when it tries	183	* we encounter memory pressure (eg copy_page_range when it tries
184	* to allocate a new pte). If we have to reclaim memory and end	184	* to allocate a new pte). If we have to reclaim memory and end
185	* up scanning and resetting referenced bits then our batch context	185	* up scanning and resetting referenced bits then our batch context
186	* will change mid stream.	186	* will change mid stream.
187	*	187	*
188	* We also need to ensure only one page size is present in a given	188	* We also need to ensure only one page size is present in a given
189	* batch	189	* batch
190	*/	190	*/
191	if (i != 0 && (mm != batch->mm \|\| batch->psize != psize \|\|	191	if (i != 0 && (mm != batch->mm \|\| batch->psize != psize \|\|
192	batch->ssize != ssize)) {	192	batch->ssize != ssize)) {
193	__flush_tlb_pending(batch);	193	__flush_tlb_pending(batch);
194	i = 0;	194	i = 0;
195	}	195	}
196	if (i == 0) {	196	if (i == 0) {
197	batch->mm = mm;	197	batch->mm = mm;
198	batch->psize = psize;	198	batch->psize = psize;
199	batch->ssize = ssize;	199	batch->ssize = ssize;
200	}	200	}
201	batch->pte[i] = rpte;	201	batch->pte[i] = rpte;
202	batch->vaddr[i] = vaddr;	202	batch->vaddr[i] = vaddr;
203	batch->index = ++i;	203	batch->index = ++i;
204	if (i >= PPC64_TLB_BATCH_NR)	204	if (i >= PPC64_TLB_BATCH_NR)
205	__flush_tlb_pending(batch);	205	__flush_tlb_pending(batch);
206	}	206	}
207		207
208	/*	208	/*
209	* This function is called when terminating an mmu batch or when a batch	209	* This function is called when terminating an mmu batch or when a batch
210	* is full. It will perform the flush of all the entries currently stored	210	* is full. It will perform the flush of all the entries currently stored
211	* in a batch.	211	* in a batch.
212	*	212	*
213	* Must be called from within some kind of spinlock/non-preempt region...	213	* Must be called from within some kind of spinlock/non-preempt region...
214	*/	214	*/
215	void __flush_tlb_pending(struct ppc64_tlb_batch *batch)	215	void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
216	{	216	{
217	cpumask_t tmp;	217	cpumask_t tmp;
218	int i, local = 0;	218	int i, local = 0;
219		219
220	i = batch->index;	220	i = batch->index;
221	tmp = cpumask_of_cpu(smp_processor_id());	221	tmp = cpumask_of_cpu(smp_processor_id());
222	if (cpus_equal(batch->mm->cpu_vm_mask, tmp))	222	if (cpus_equal(batch->mm->cpu_vm_mask, tmp))
223	local = 1;	223	local = 1;
224	if (i == 1)	224	if (i == 1)
225	flush_hash_page(batch->vaddr[0], batch->pte[0],	225	flush_hash_page(batch->vaddr[0], batch->pte[0],
226	batch->psize, batch->ssize, local);	226	batch->psize, batch->ssize, local);
227	else	227	else
228	flush_hash_range(i, local);	228	flush_hash_range(i, local);
229	batch->index = 0;	229	batch->index = 0;
230	}	230	}
231		231
232	void pte_free_finish(void)	232	void pte_free_finish(void)
233	{	233	{
234	/* This is safe since tlb_gather_mmu has disabled preemption */	234	/* This is safe since tlb_gather_mmu has disabled preemption */
235	struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);	235	struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur);
236		236
237	if (*batchp == NULL)	237	if (*batchp == NULL)
238	return;	238	return;
239	pte_free_submit(*batchp);	239	pte_free_submit(*batchp);
240	*batchp = NULL;	240	*batchp = NULL;
241	}	241	}
242		242
243	/**	243	/**
244	* __flush_hash_table_range - Flush all HPTEs for a given address range	244	* __flush_hash_table_range - Flush all HPTEs for a given address range
245	* from the hash table (and the TLB). But keeps	245	* from the hash table (and the TLB). But keeps
246	* the linux PTEs intact.	246	* the linux PTEs intact.
247	*	247	*
248	* @mm : mm_struct of the target address space (generally init_mm)	248	* @mm : mm_struct of the target address space (generally init_mm)
249	* @start : starting address	249	* @start : starting address
250	* @end : ending address (not included in the flush)	250	* @end : ending address (not included in the flush)
251	*	251	*
252	* This function is mostly to be used by some IO hotplug code in order	252	* This function is mostly to be used by some IO hotplug code in order
253	* to remove all hash entries from a given address range used to map IO	253	* to remove all hash entries from a given address range used to map IO
254	* space on a removed PCI-PCI bidge without tearing down the full mapping	254	* space on a removed PCI-PCI bidge without tearing down the full mapping
255	* since 64K pages may overlap with other bridges when using 64K pages	255	* since 64K pages may overlap with other bridges when using 64K pages
256	* with 4K HW pages on IO space.	256	* with 4K HW pages on IO space.
257	*	257	*
258	* Because of that usage pattern, it's only available with CONFIG_HOTPLUG	258	* Because of that usage pattern, it's only available with CONFIG_HOTPLUG
259	* and is implemented for small size rather than speed.	259	* and is implemented for small size rather than speed.
260	*/	260	*/
261	#ifdef CONFIG_HOTPLUG	261	#ifdef CONFIG_HOTPLUG
262		262
263	void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,	263	void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
264	unsigned long end)	264	unsigned long end)
265	{	265	{
266	unsigned long flags;	266	unsigned long flags;
267		267
268	start = _ALIGN_DOWN(start, PAGE_SIZE);	268	start = _ALIGN_DOWN(start, PAGE_SIZE);
269	end = _ALIGN_UP(end, PAGE_SIZE);	269	end = _ALIGN_UP(end, PAGE_SIZE);
270		270
271	BUG_ON(!mm->pgd);	271	BUG_ON(!mm->pgd);
272		272
273	/* Note: Normally, we should only ever use a batch within a	273	/* Note: Normally, we should only ever use a batch within a
274	* PTE locked section. This violates the rule, but will work	274	* PTE locked section. This violates the rule, but will work
275	* since we don't actually modify the PTEs, we just flush the	275	* since we don't actually modify the PTEs, we just flush the
276	* hash while leaving the PTEs intact (including their reference	276	* hash while leaving the PTEs intact (including their reference
277	* to being hashed). This is not the most performance oriented	277	* to being hashed). This is not the most performance oriented
278	* way to do things but is fine for our needs here.	278	* way to do things but is fine for our needs here.
279	*/	279	*/
280	local_irq_save(flags);	280	local_irq_save(flags);
281	arch_enter_lazy_mmu_mode();	281	arch_enter_lazy_mmu_mode();
282	for (; start < end; start += PAGE_SIZE) {	282	for (; start < end; start += PAGE_SIZE) {
283	pte_t *ptep = find_linux_pte(mm->pgd, start);	283	pte_t *ptep = find_linux_pte(mm->pgd, start);
284	unsigned long pte;	284	unsigned long pte;
285		285
286	if (ptep == NULL)	286	if (ptep == NULL)
287	continue;	287	continue;
288	pte = pte_val(*ptep);	288	pte = pte_val(*ptep);
289	if (!(pte & _PAGE_HASHPTE))	289	if (!(pte & _PAGE_HASHPTE))
290	continue;	290	continue;
291	hpte_need_flush(mm, start, ptep, pte, 0);	291	hpte_need_flush(mm, start, ptep, pte, 0);
292	}	292	}
293	arch_leave_lazy_mmu_mode();	293	arch_leave_lazy_mmu_mode();
294	local_irq_restore(flags);	294	local_irq_restore(flags);
295	}	295	}
296		296
297	#endif /* CONFIG_HOTPLUG */	297	#endif /* CONFIG_HOTPLUG */
298		298

include/asm-powerpc/hugetlb.h

Diff comments View file @ 0d9ea75

 #ifndef _ASM_POWERPC_HUGETLB_H
 #define _ASM_POWERPC_HUGETLB_H
 #include <asm/page.h>
 int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
 			   unsigned long len);
 void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
 			    unsigned long end, unsigned long floor,
 			    unsigned long ceiling);
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 		     pte_t *ptep, pte_t pte);
 pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep);
 /*
  * If the arch doesn't supply something else, assume that hugepage
  * size aligned regions are ok without further preparation.
  */
 static inline int prepare_hugepage_range(struct file *file,
 			unsigned long addr, unsigned long len)
 {
-	if (len & ~HPAGE_MASK)
+	struct hstate *h = hstate_file(file);
+	if (len & ~huge_page_mask(h))
 		return -EINVAL;
-	if (addr & ~HPAGE_MASK)
+	if (addr & ~huge_page_mask(h))
 		return -EINVAL;
 	return 0;
 }
 static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm)
 {
 }
 static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
 					 unsigned long addr, pte_t *ptep)
 {
 }
 static inline int huge_pte_none(pte_t pte)
 {
 	return pte_none(pte);
 }
 static inline pte_t huge_pte_wrprotect(pte_t pte)
 {
 	return pte_wrprotect(pte);
 }
 static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 					     unsigned long addr, pte_t *ptep,
 					     pte_t pte, int dirty)
 {
 	return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
 }
 static inline pte_t huge_ptep_get(pte_t *ptep)
 {
 	return *ptep;
 }
 static inline int arch_prepare_hugepage(struct page *page)
 {
 	return 0;
 }
 static inline void arch_release_hugepage(struct page *page)
 {
 }
 #endif /* _ASM_POWERPC_HUGETLB_H */

include/asm-powerpc/mmu-hash64.h

Diff comments View file @ 0d9ea75

include/asm-powerpc/page_64.h

Diff comments View file @ 0d9ea75

 #ifndef _ASM_POWERPC_PAGE_64_H
 #define _ASM_POWERPC_PAGE_64_H
 /*
  * Copyright (C) 2001 PPC64 Team, IBM Corp
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 /*
  * We always define HW_PAGE_SHIFT to 12 as use of 64K pages remains Linux
  * specific, every notion of page number shared with the firmware, TCEs,
  * iommu, etc... still uses a page size of 4K.
  */
 #define HW_PAGE_SHIFT		12
 #define HW_PAGE_SIZE		(ASM_CONST(1) << HW_PAGE_SHIFT)
 #define HW_PAGE_MASK		(~(HW_PAGE_SIZE-1))
 /*
  * PAGE_FACTOR is the number of bits factor between PAGE_SHIFT and
  * HW_PAGE_SHIFT, that is 4K pages.
  */
 #define PAGE_FACTOR		(PAGE_SHIFT - HW_PAGE_SHIFT)
 /* Segment size; normal 256M segments */
 #define SID_SHIFT		28
 #define SID_MASK		ASM_CONST(0xfffffffff)
 #define ESID_MASK		0xfffffffff0000000UL
 #define GET_ESID(x)		(((x) >> SID_SHIFT) & SID_MASK)
 /* 1T segments */
 #define SID_SHIFT_1T		40
 #define SID_MASK_1T		0xffffffUL
 #define ESID_MASK_1T		0xffffff0000000000UL
 #define GET_ESID_1T(x)		(((x) >> SID_SHIFT_1T) & SID_MASK_1T)
 #ifndef __ASSEMBLY__
 #include <asm/cache.h>
 typedef unsigned long pte_basic_t;
 static __inline__ void clear_page(void *addr)
 {
 	unsigned long lines, line_size;
 	line_size = ppc64_caches.dline_size;
 	lines = ppc64_caches.dlines_per_page;
 	__asm__ __volatile__(
 	"mtctr	%1	# clear_page\n\
 1:      dcbz	0,%0\n\
 	add	%0,%0,%3\n\
 	bdnz+	1b"
         : "=r" (addr)
         : "r" (lines), "0" (addr), "r" (line_size)
 	: "ctr", "memory");
 }
 extern void copy_4K_page(void *to, void *from);
 #ifdef CONFIG_PPC_64K_PAGES
 static inline void copy_page(void *to, void *from)
 {
 	unsigned int i;
 	for (i=0; i < (1 << (PAGE_SHIFT - 12)); i++) {
 		copy_4K_page(to, from);
 		to += 4096;
 		from += 4096;
 	}
 }
 #else /* CONFIG_PPC_64K_PAGES */
 static inline void copy_page(void *to, void *from)
 {
 	copy_4K_page(to, from);
 }
 #endif /* CONFIG_PPC_64K_PAGES */
 /* Log 2 of page table size */
 extern u64 ppc64_pft_size;
 /* Large pages size */
 #ifdef CONFIG_HUGETLB_PAGE
 extern unsigned int HPAGE_SHIFT;
 #else
 #define HPAGE_SHIFT PAGE_SHIFT
 #endif
 #define HPAGE_SIZE		((1UL) << HPAGE_SHIFT)
 #define HPAGE_MASK		(~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+#define HUGE_MAX_HSTATE		3
 #endif /* __ASSEMBLY__ */
 #ifdef CONFIG_PPC_MM_SLICES
 #define SLICE_LOW_SHIFT		28
 #define SLICE_HIGH_SHIFT	40
 #define SLICE_LOW_TOP		(0x100000000ul)
 #define SLICE_NUM_LOW		(SLICE_LOW_TOP >> SLICE_LOW_SHIFT)
 #define SLICE_NUM_HIGH		(PGTABLE_RANGE >> SLICE_HIGH_SHIFT)
 #define GET_LOW_SLICE_INDEX(addr)	((addr) >> SLICE_LOW_SHIFT)
 #define GET_HIGH_SLICE_INDEX(addr)	((addr) >> SLICE_HIGH_SHIFT)
 #ifndef __ASSEMBLY__
 struct slice_mask {
 	u16 low_slices;
 	u16 high_slices;
 };
 struct mm_struct;
 extern unsigned long slice_get_unmapped_area(unsigned long addr,
 					     unsigned long len,
 					     unsigned long flags,
 					     unsigned int psize,
 					     int topdown,
 					     int use_cache);
 extern unsigned int get_slice_psize(struct mm_struct *mm,
 				    unsigned long addr);
 extern void slice_init_context(struct mm_struct *mm, unsigned int psize);
 extern void slice_set_user_psize(struct mm_struct *mm, unsigned int psize);
 extern void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
 				  unsigned long len, unsigned int psize);
 #define slice_mm_new_context(mm)	((mm)->context.id == 0)
 #endif /* __ASSEMBLY__ */
 #else
 #define slice_init()
 #define get_slice_psize(mm, addr)	((mm)->context.user_psize)
 #define slice_set_user_psize(mm, psize)		\
 do {						\
 	(mm)->context.user_psize = (psize);	\
 	(mm)->context.sllp = SLB_VSID_USER | mmu_psize_defs[(psize)].sllp; \
 } while (0)
 #define slice_set_range_psize(mm, start, len, psize)	\
 	slice_set_user_psize((mm), (psize))
 #define slice_mm_new_context(mm)	1
 #endif /* CONFIG_PPC_MM_SLICES */
 #ifdef CONFIG_HUGETLB_PAGE
 #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
 #endif /* !CONFIG_HUGETLB_PAGE */
 #ifdef MODULE
 #define __page_aligned __attribute__((__aligned__(PAGE_SIZE)))
 #else
 #define __page_aligned \
 	__attribute__((__aligned__(PAGE_SIZE), \
 		__section__(".data.page_aligned")))
 #endif
 #define VM_DATA_DEFAULT_FLAGS \
 	(test_thread_flag(TIF_32BIT) ? \
 	 VM_DATA_DEFAULT_FLAGS32 : VM_DATA_DEFAULT_FLAGS64)
 /*
  * This is the default if a program doesn't have a PT_GNU_STACK
  * program header entry. The PPC64 ELF ABI has a non executable stack
  * stack by default, so in the absense of a PT_GNU_STACK program header
  * we turn execute permission off.
  */
 #define VM_STACK_DEFAULT_FLAGS32	(VM_READ | VM_WRITE | VM_EXEC | \
 					 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 #define VM_STACK_DEFAULT_FLAGS64	(VM_READ | VM_WRITE | \
 					 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 #define VM_STACK_DEFAULT_FLAGS \
 	(test_thread_flag(TIF_32BIT) ? \
 	 VM_STACK_DEFAULT_FLAGS32 : VM_STACK_DEFAULT_FLAGS64)
 #include <asm-generic/page.h>
 #endif /* _ASM_POWERPC_PAGE_64_H */

include/asm-powerpc/pgalloc-64.h

Diff comments View file @ 0d9ea75

1	#ifndef _ASM_POWERPC_PGALLOC_64_H	1	#ifndef _ASM_POWERPC_PGALLOC_64_H
2	#define _ASM_POWERPC_PGALLOC_64_H	2	#define _ASM_POWERPC_PGALLOC_64_H
3	/*	3	/*
4	* This program is free software; you can redistribute it and/or	4	* This program is free software; you can redistribute it and/or
5	* modify it under the terms of the GNU General Public License	5	* modify it under the terms of the GNU General Public License
6	* as published by the Free Software Foundation; either version	6	* as published by the Free Software Foundation; either version
7	* 2 of the License, or (at your option) any later version.	7	* 2 of the License, or (at your option) any later version.
8	*/	8	*/
9		9
10	#include <linux/mm.h>	10	#include <linux/mm.h>
11	#include <linux/slab.h>	11	#include <linux/slab.h>
12	#include <linux/cpumask.h>	12	#include <linux/cpumask.h>
13	#include <linux/percpu.h>	13	#include <linux/percpu.h>
14		14
15	#ifndef CONFIG_PPC_SUBPAGE_PROT	15	#ifndef CONFIG_PPC_SUBPAGE_PROT
16	static inline void subpage_prot_free(pgd_t *pgd) {}	16	static inline void subpage_prot_free(pgd_t *pgd) {}
17	#endif	17	#endif
18		18
19	extern struct kmem_cache *pgtable_cache[];	19	extern struct kmem_cache *pgtable_cache[];
20		20
21	#define PGD_CACHE_NUM 0	21	#define PGD_CACHE_NUM 0
22	#define PUD_CACHE_NUM 1	22	#define PUD_CACHE_NUM 1
23	#define PMD_CACHE_NUM 1	23	#define PMD_CACHE_NUM 1
24	#define HUGEPTE_CACHE_NUM 2	24	#define HUGEPTE_CACHE_NUM 2
25	#define PTE_NONCACHE_NUM 3 /* from GFP rather than kmem_cache */	25	#define PTE_NONCACHE_NUM 7 /* from GFP rather than kmem_cache */
26		26
27	static inline pgd_t pgd_alloc(struct mm_struct mm)	27	static inline pgd_t pgd_alloc(struct mm_struct mm)
28	{	28	{
29	return kmem_cache_alloc(pgtable_cache[PGD_CACHE_NUM], GFP_KERNEL);	29	return kmem_cache_alloc(pgtable_cache[PGD_CACHE_NUM], GFP_KERNEL);
30	}	30	}
31		31
32	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)	32	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)
33	{	33	{
34	subpage_prot_free(pgd);	34	subpage_prot_free(pgd);
35	kmem_cache_free(pgtable_cache[PGD_CACHE_NUM], pgd);	35	kmem_cache_free(pgtable_cache[PGD_CACHE_NUM], pgd);
36	}	36	}
37		37
38	#ifndef CONFIG_PPC_64K_PAGES	38	#ifndef CONFIG_PPC_64K_PAGES
39		39
40	#define pgd_populate(MM, PGD, PUD) pgd_set(PGD, PUD)	40	#define pgd_populate(MM, PGD, PUD) pgd_set(PGD, PUD)
41		41
42	static inline pud_t pud_alloc_one(struct mm_struct mm, unsigned long addr)	42	static inline pud_t pud_alloc_one(struct mm_struct mm, unsigned long addr)
43	{	43	{
44	return kmem_cache_alloc(pgtable_cache[PUD_CACHE_NUM],	44	return kmem_cache_alloc(pgtable_cache[PUD_CACHE_NUM],
45	GFP_KERNEL\|__GFP_REPEAT);	45	GFP_KERNEL\|__GFP_REPEAT);
46	}	46	}
47		47
48	static inline void pud_free(struct mm_struct mm, pud_t pud)	48	static inline void pud_free(struct mm_struct mm, pud_t pud)
49	{	49	{
50	kmem_cache_free(pgtable_cache[PUD_CACHE_NUM], pud);	50	kmem_cache_free(pgtable_cache[PUD_CACHE_NUM], pud);
51	}	51	}
52		52
53	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)	53	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
54	{	54	{
55	pud_set(pud, (unsigned long)pmd);	55	pud_set(pud, (unsigned long)pmd);
56	}	56	}
57		57
58	#define pmd_populate(mm, pmd, pte_page) \	58	#define pmd_populate(mm, pmd, pte_page) \
59	pmd_populate_kernel(mm, pmd, page_address(pte_page))	59	pmd_populate_kernel(mm, pmd, page_address(pte_page))
60	#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))	60	#define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
61	#define pmd_pgtable(pmd) pmd_page(pmd)	61	#define pmd_pgtable(pmd) pmd_page(pmd)
62		62
63		63
64	#else /* CONFIG_PPC_64K_PAGES */	64	#else /* CONFIG_PPC_64K_PAGES */
65		65
66	#define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd)	66	#define pud_populate(mm, pud, pmd) pud_set(pud, (unsigned long)pmd)
67		67
68	static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,	68	static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,
69	pte_t *pte)	69	pte_t *pte)
70	{	70	{
71	pmd_set(pmd, (unsigned long)pte);	71	pmd_set(pmd, (unsigned long)pte);
72	}	72	}
73		73
74	#define pmd_populate(mm, pmd, pte_page) \	74	#define pmd_populate(mm, pmd, pte_page) \
75	pmd_populate_kernel(mm, pmd, page_address(pte_page))	75	pmd_populate_kernel(mm, pmd, page_address(pte_page))
76	#define pmd_pgtable(pmd) pmd_page(pmd)	76	#define pmd_pgtable(pmd) pmd_page(pmd)
77		77
78	#endif /* CONFIG_PPC_64K_PAGES */	78	#endif /* CONFIG_PPC_64K_PAGES */
79		79
80	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)	80	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)
81	{	81	{
82	return kmem_cache_alloc(pgtable_cache[PMD_CACHE_NUM],	82	return kmem_cache_alloc(pgtable_cache[PMD_CACHE_NUM],
83	GFP_KERNEL\|__GFP_REPEAT);	83	GFP_KERNEL\|__GFP_REPEAT);
84	}	84	}
85		85
86	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)	86	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
87	{	87	{
88	kmem_cache_free(pgtable_cache[PMD_CACHE_NUM], pmd);	88	kmem_cache_free(pgtable_cache[PMD_CACHE_NUM], pmd);
89	}	89	}
90		90
91	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,	91	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,
92	unsigned long address)	92	unsigned long address)
93	{	93	{
94	return (pte_t *)__get_free_page(GFP_KERNEL \| __GFP_REPEAT \| __GFP_ZERO);	94	return (pte_t *)__get_free_page(GFP_KERNEL \| __GFP_REPEAT \| __GFP_ZERO);
95	}	95	}
96		96
97	static inline pgtable_t pte_alloc_one(struct mm_struct *mm,	97	static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
98	unsigned long address)	98	unsigned long address)
99	{	99	{
100	struct page *page;	100	struct page *page;
101	pte_t *pte;	101	pte_t *pte;
102		102
103	pte = pte_alloc_one_kernel(mm, address);	103	pte = pte_alloc_one_kernel(mm, address);
104	if (!pte)	104	if (!pte)
105	return NULL;	105	return NULL;
106	page = virt_to_page(pte);	106	page = virt_to_page(pte);
107	pgtable_page_ctor(page);	107	pgtable_page_ctor(page);
108	return page;	108	return page;
109	}	109	}
110		110
111	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)	111	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)
112	{	112	{
113	free_page((unsigned long)pte);	113	free_page((unsigned long)pte);
114	}	114	}
115		115
116	static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)	116	static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
117	{	117	{
118	pgtable_page_dtor(ptepage);	118	pgtable_page_dtor(ptepage);
119	__free_page(ptepage);	119	__free_page(ptepage);
120	}	120	}
121		121
122	#define PGF_CACHENUM_MASK 0x3	122	#define PGF_CACHENUM_MASK 0x7
123		123
124	typedef struct pgtable_free {	124	typedef struct pgtable_free {
125	unsigned long val;	125	unsigned long val;
126	} pgtable_free_t;	126	} pgtable_free_t;
127		127
128	static inline pgtable_free_t pgtable_free_cache(void *p, int cachenum,	128	static inline pgtable_free_t pgtable_free_cache(void *p, int cachenum,
129	unsigned long mask)	129	unsigned long mask)
130	{	130	{
131	BUG_ON(cachenum > PGF_CACHENUM_MASK);	131	BUG_ON(cachenum > PGF_CACHENUM_MASK);
132		132
133	return (pgtable_free_t){.val = ((unsigned long) p & ~mask) \| cachenum};	133	return (pgtable_free_t){.val = ((unsigned long) p & ~mask) \| cachenum};
134	}	134	}
135		135
136	static inline void pgtable_free(pgtable_free_t pgf)	136	static inline void pgtable_free(pgtable_free_t pgf)
137	{	137	{
138	void p = (void )(pgf.val & ~PGF_CACHENUM_MASK);	138	void p = (void )(pgf.val & ~PGF_CACHENUM_MASK);
139	int cachenum = pgf.val & PGF_CACHENUM_MASK;	139	int cachenum = pgf.val & PGF_CACHENUM_MASK;
140		140
141	if (cachenum == PTE_NONCACHE_NUM)	141	if (cachenum == PTE_NONCACHE_NUM)
142	free_page((unsigned long)p);	142	free_page((unsigned long)p);
143	else	143	else
144	kmem_cache_free(pgtable_cache[cachenum], p);	144	kmem_cache_free(pgtable_cache[cachenum], p);
145	}	145	}
146		146
147	extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf);	147	extern void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf);
148		148
149	#define __pte_free_tlb(tlb,ptepage) \	149	#define __pte_free_tlb(tlb,ptepage) \
150	do { \	150	do { \
151	pgtable_page_dtor(ptepage); \	151	pgtable_page_dtor(ptepage); \
152	pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \	152	pgtable_free_tlb(tlb, pgtable_free_cache(page_address(ptepage), \
153	PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \	153	PTE_NONCACHE_NUM, PTE_TABLE_SIZE-1)); \
154	} while (0)	154	} while (0)
155	#define __pmd_free_tlb(tlb, pmd) \	155	#define __pmd_free_tlb(tlb, pmd) \
156	pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \	156	pgtable_free_tlb(tlb, pgtable_free_cache(pmd, \
157	PMD_CACHE_NUM, PMD_TABLE_SIZE-1))	157	PMD_CACHE_NUM, PMD_TABLE_SIZE-1))
158	#ifndef CONFIG_PPC_64K_PAGES	158	#ifndef CONFIG_PPC_64K_PAGES
159	#define __pud_free_tlb(tlb, pud) \	159	#define __pud_free_tlb(tlb, pud) \
160	pgtable_free_tlb(tlb, pgtable_free_cache(pud, \	160	pgtable_free_tlb(tlb, pgtable_free_cache(pud, \
161	PUD_CACHE_NUM, PUD_TABLE_SIZE-1))	161	PUD_CACHE_NUM, PUD_TABLE_SIZE-1))
162	#endif /* CONFIG_PPC_64K_PAGES */	162	#endif /* CONFIG_PPC_64K_PAGES */
163		163
164	#define check_pgt_cache() do { } while (0)	164	#define check_pgt_cache() do { } while (0)
165		165
166	#endif /* _ASM_POWERPC_PGALLOC_64_H */	166	#endif /* _ASM_POWERPC_PGALLOC_64_H */
167		167

1	Kernel Parameters	1	Kernel Parameters
2	~~~~~~~~~~~~~~~~~	2	~~~~~~~~~~~~~~~~~
3		3
4	The following is a consolidated list of the kernel parameters as implemented	4	The following is a consolidated list of the kernel parameters as implemented
5	(mostly) by the __setup() macro and sorted into English Dictionary order	5	(mostly) by the __setup() macro and sorted into English Dictionary order
6	(defined as ignoring all punctuation and sorting digits before letters in a	6	(defined as ignoring all punctuation and sorting digits before letters in a
7	case insensitive manner), and with descriptions where known.	7	case insensitive manner), and with descriptions where known.
8		8
9	Module parameters for loadable modules are specified only as the	9	Module parameters for loadable modules are specified only as the
10	parameter name with optional '=' and value as appropriate, such as:	10	parameter name with optional '=' and value as appropriate, such as:
11		11
12	modprobe usbcore blinkenlights=1	12	modprobe usbcore blinkenlights=1
13		13
14	Module parameters for modules that are built into the kernel image	14	Module parameters for modules that are built into the kernel image
15	are specified on the kernel command line with the module name plus	15	are specified on the kernel command line with the module name plus
16	'.' plus parameter name, with '=' and value if appropriate, such as:	16	'.' plus parameter name, with '=' and value if appropriate, such as:
17		17
18	usbcore.blinkenlights=1	18	usbcore.blinkenlights=1
19		19
20	This document may not be entirely up to date and comprehensive. The command	20	This document may not be entirely up to date and comprehensive. The command
21	"modinfo -p ${modulename}" shows a current list of all parameters of a loadable	21	"modinfo -p ${modulename}" shows a current list of all parameters of a loadable
22	module. Loadable modules, after being loaded into the running kernel, also	22	module. Loadable modules, after being loaded into the running kernel, also
23	reveal their parameters in /sys/module/${modulename}/parameters/. Some of these	23	reveal their parameters in /sys/module/${modulename}/parameters/. Some of these
24	parameters may be changed at runtime by the command	24	parameters may be changed at runtime by the command
25	"echo -n ${value} > /sys/module/${modulename}/parameters/${parm}".	25	"echo -n ${value} > /sys/module/${modulename}/parameters/${parm}".
26		26
27	The parameters listed below are only valid if certain kernel build options were	27	The parameters listed below are only valid if certain kernel build options were
28	enabled and if respective hardware is present. The text in square brackets at	28	enabled and if respective hardware is present. The text in square brackets at
29	the beginning of each description states the restrictions within which a	29	the beginning of each description states the restrictions within which a
30	parameter is applicable:	30	parameter is applicable:
31		31
32	ACPI ACPI support is enabled.	32	ACPI ACPI support is enabled.
33	AGP AGP (Accelerated Graphics Port) is enabled.	33	AGP AGP (Accelerated Graphics Port) is enabled.
34	ALSA ALSA sound support is enabled.	34	ALSA ALSA sound support is enabled.
35	APIC APIC support is enabled.	35	APIC APIC support is enabled.
36	APM Advanced Power Management support is enabled.	36	APM Advanced Power Management support is enabled.
37	AVR32 AVR32 architecture is enabled.	37	AVR32 AVR32 architecture is enabled.
38	AX25 Appropriate AX.25 support is enabled.	38	AX25 Appropriate AX.25 support is enabled.
39	BLACKFIN Blackfin architecture is enabled.	39	BLACKFIN Blackfin architecture is enabled.
40	DRM Direct Rendering Management support is enabled.	40	DRM Direct Rendering Management support is enabled.