Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 1ae1c1d09f220ded48ee9a7d91a65e94f95c4af1

Authored by Gerald Schaefer 2012-10-09 07:30:24 +0800

Committed by Linus Torvalds 2012-10-09 15:22:31 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

thp, s390: architecture backend for thp on s390

This implements the architecture backend for transparent hugepages
on s390.

Signed-off-by: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Hillf Danton <dhillf@gmail.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 6 changed files with 220 additions and 18 deletions Inline Diff

arch/s390/include/asm/hugetlb.h
arch/s390/include/asm/pgtable.h
arch/s390/include/asm/setup.h
arch/s390/include/asm/tlb.h
arch/s390/kernel/early.c
arch/s390/mm/pgtable.c

arch/s390/include/asm/hugetlb.h

Diff comments View file @ 1ae1c1d

 /*
  *  IBM System z Huge TLB Page Support for Kernel.
  *
  *    Copyright IBM Corp. 2008
  *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
  */
 #ifndef _ASM_S390_HUGETLB_H
 #define _ASM_S390_HUGETLB_H
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #define is_hugepage_only_range(mm, addr, len)	0
 #define hugetlb_free_pgd_range			free_pgd_range
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 		     pte_t *ptep, pte_t pte);
 /*
  * If the arch doesn't supply something else, assume that hugepage
  * size aligned regions are ok without further preparation.
  */
 static inline int prepare_hugepage_range(struct file *file,
 			unsigned long addr, unsigned long len)
 {
 	if (len & ~HPAGE_MASK)
 		return -EINVAL;
 	if (addr & ~HPAGE_MASK)
 		return -EINVAL;
 	return 0;
 }
 #define hugetlb_prefault_arch_hook(mm)		do { } while (0)
 #define arch_clear_hugepage_flags(page)		do { } while (0)
 int arch_prepare_hugepage(struct page *page);
 void arch_release_hugepage(struct page *page);
 static inline pte_t huge_pte_wrprotect(pte_t pte)
 {
 	pte_val(pte) |= _PAGE_RO;
 	return pte;
 }
 static inline int huge_pte_none(pte_t pte)
 {
 	return (pte_val(pte) & _SEGMENT_ENTRY_INV) &&
 		!(pte_val(pte) & _SEGMENT_ENTRY_RO);
 }
 static inline pte_t huge_ptep_get(pte_t *ptep)
 {
 	pte_t pte = *ptep;
 	unsigned long mask;
 	if (!MACHINE_HAS_HPAGE) {
 		ptep = (pte_t *) (pte_val(pte) & _SEGMENT_ENTRY_ORIGIN);
 		if (ptep) {
 			mask = pte_val(pte) &
 				(_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO);
 			pte = pte_mkhuge(*ptep);
 			pte_val(pte) |= mask;
 		}
 	}
 	return pte;
 }
 static inline void __pmd_csp(pmd_t *pmdp)
 {
 	register unsigned long reg2 asm("2") = pmd_val(*pmdp);
 	register unsigned long reg3 asm("3") = pmd_val(*pmdp) |
 					       _SEGMENT_ENTRY_INV;
 	register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5;
 	asm volatile(
 		"	csp %1,%3"
 		: "=m" (*pmdp)
 		: "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc");
-	pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
-static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
-{
-	unsigned long sto = (unsigned long) pmdp -
-				pmd_index(address) * sizeof(pmd_t);
-	if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
-		asm volatile(
-			"	.insn	rrf,0xb98e0000,%2,%3,0,0"
-			: "=m" (*pmdp)
-			: "m" (*pmdp), "a" (sto),
-			  "a" ((address & HPAGE_MASK))
-		);
-	}
-	pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
-}
 static inline void huge_ptep_invalidate(struct mm_struct *mm,
 					unsigned long address, pte_t *ptep)
 {
 	pmd_t *pmdp = (pmd_t *) ptep;
 	if (MACHINE_HAS_IDTE)
 		__pmd_idte(address, pmdp);
 	else
 		__pmd_csp(pmdp);
+	pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 					    unsigned long addr, pte_t *ptep)
 {
 	pte_t pte = huge_ptep_get(ptep);
 	huge_ptep_invalidate(mm, addr, ptep);
 	return pte;
 }
 #define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \
 ({									    \
 	int __changed = !pte_same(huge_ptep_get(__ptep), __entry);	    \
 	if (__changed) {						    \
 		huge_ptep_invalidate((__vma)->vm_mm, __addr, __ptep);	    \
 		set_huge_pte_at((__vma)->vm_mm, __addr, __ptep, __entry);   \
 	}								    \
 	__changed;							    \
 })
 #define huge_ptep_set_wrprotect(__mm, __addr, __ptep)			\
 ({									\
 	pte_t __pte = huge_ptep_get(__ptep);				\
 	if (pte_write(__pte)) {						\
 		huge_ptep_invalidate(__mm, __addr, __ptep);		\
 		set_huge_pte_at(__mm, __addr, __ptep,			\
 				huge_pte_wrprotect(__pte));		\
 	}								\
 })
 static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
 					 unsigned long address, pte_t *ptep)
 {
 	huge_ptep_invalidate(vma->vm_mm, address, ptep);
 }
 #endif /* _ASM_S390_HUGETLB_H */

arch/s390/include/asm/pgtable.h

Diff comments View file @ 1ae1c1d

 /*
  *  S390 version
  *    Copyright IBM Corp. 1999, 2000
  *    Author(s): Hartmut Penner (hp@de.ibm.com)
  *               Ulrich Weigand (weigand@de.ibm.com)
  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
  *
  *  Derived from "include/asm-i386/pgtable.h"
  */
 #ifndef _ASM_S390_PGTABLE_H
 #define _ASM_S390_PGTABLE_H
 /*
  * The Linux memory management assumes a three-level page table setup. For
  * s390 31 bit we "fold" the mid level into the top-level page table, so
  * that we physically have the same two-level page table as the s390 mmu
  * expects in 31 bit mode. For s390 64 bit we use three of the five levels
  * the hardware provides (region first and region second tables are not
  * used).
  *
  * The "pgd_xxx()" functions are trivial for a folded two-level
  * setup: the pgd is never bad, and a pmd always exists (as it's folded
  * into the pgd entry)
  *
  * This file contains the functions and defines necessary to modify and use
  * the S390 page table tree.
  */
 #ifndef __ASSEMBLY__
 #include <linux/sched.h>
 #include <linux/mm_types.h>
 #include <asm/bug.h>
 #include <asm/page.h>
 extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
 extern void paging_init(void);
 extern void vmem_map_init(void);
 extern void fault_init(void);
 /*
  * The S390 doesn't have any external MMU info: the kernel page
  * tables contain all the necessary information.
  */
 #define update_mmu_cache(vma, address, ptep)     do { } while (0)
 /*
  * ZERO_PAGE is a global shared page that is always zero; used
  * for zero-mapped memory areas etc..
  */
 extern unsigned long empty_zero_page;
 extern unsigned long zero_page_mask;
 #define ZERO_PAGE(vaddr) \
 	(virt_to_page((void *)(empty_zero_page + \
 	 (((unsigned long)(vaddr)) &zero_page_mask))))
 #define is_zero_pfn is_zero_pfn
 static inline int is_zero_pfn(unsigned long pfn)
 {
 	extern unsigned long zero_pfn;
 	unsigned long offset_from_zero_pfn = pfn - zero_pfn;
 	return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT);
 }
 #define my_zero_pfn(addr)	page_to_pfn(ZERO_PAGE(addr))
 #endif /* !__ASSEMBLY__ */
 /*
  * PMD_SHIFT determines the size of the area a second-level page
  * table can map
  * PGDIR_SHIFT determines what a third-level page table entry can map
  */
 #ifndef CONFIG_64BIT
 # define PMD_SHIFT	20
 # define PUD_SHIFT	20
 # define PGDIR_SHIFT	20
 #else /* CONFIG_64BIT */
 # define PMD_SHIFT	20
 # define PUD_SHIFT	31
 # define PGDIR_SHIFT	42
 #endif /* CONFIG_64BIT */
 #define PMD_SIZE        (1UL << PMD_SHIFT)
 #define PMD_MASK        (~(PMD_SIZE-1))
 #define PUD_SIZE	(1UL << PUD_SHIFT)
 #define PUD_MASK	(~(PUD_SIZE-1))
 #define PGDIR_SIZE	(1UL << PGDIR_SHIFT)
 #define PGDIR_MASK	(~(PGDIR_SIZE-1))
 /*
  * entries per page directory level: the S390 is two-level, so
  * we don't really have any PMD directory physically.
  * for S390 segment-table entries are combined to one PGD
  * that leads to 1024 pte per pgd
  */
 #define PTRS_PER_PTE	256
 #ifndef CONFIG_64BIT
 #define PTRS_PER_PMD	1
 #define PTRS_PER_PUD	1
 #else /* CONFIG_64BIT */
 #define PTRS_PER_PMD	2048
 #define PTRS_PER_PUD	2048
 #endif /* CONFIG_64BIT */
 #define PTRS_PER_PGD	2048
 #define FIRST_USER_ADDRESS  0
 #define pte_ERROR(e) \
 	printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
 #define pmd_ERROR(e) \
 	printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
 #define pud_ERROR(e) \
 	printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
 #define pgd_ERROR(e) \
 	printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
 #ifndef __ASSEMBLY__
 /*
  * The vmalloc area will always be on the topmost area of the kernel
  * mapping. We reserve 96MB (31bit) / 128GB (64bit) for vmalloc,
  * which should be enough for any sane case.
  * By putting vmalloc at the top, we maximise the gap between physical
  * memory and vmalloc to catch misplaced memory accesses. As a side
  * effect, this also makes sure that 64 bit module code cannot be used
  * as system call address.
  */
 extern unsigned long VMALLOC_START;
 extern unsigned long VMALLOC_END;
 extern struct page *vmemmap;
 #define VMEM_MAX_PHYS ((unsigned long) vmemmap)
 /*
  * A 31 bit pagetable entry of S390 has following format:
  *  |   PFRA          |    |  OS  |
  * 0                   0IP0
  * 00000000001111111111222222222233
  * 01234567890123456789012345678901
  *
  * I Page-Invalid Bit:    Page is not available for address-translation
  * P Page-Protection Bit: Store access not possible for page
  *
  * A 31 bit segmenttable entry of S390 has following format:
  *  |   P-table origin      |  |PTL
  * 0                         IC
  * 00000000001111111111222222222233
  * 01234567890123456789012345678901
  *
  * I Segment-Invalid Bit:    Segment is not available for address-translation
  * C Common-Segment Bit:     Segment is not private (PoP 3-30)
  * PTL Page-Table-Length:    Page-table length (PTL+1*16 entries -> up to 256)
  *
  * The 31 bit segmenttable origin of S390 has following format:
  *
  *  |S-table origin   |     | STL |
  * X                   **GPS
  * 00000000001111111111222222222233
  * 01234567890123456789012345678901
  *
  * X Space-Switch event:
  * G Segment-Invalid Bit:     *
  * P Private-Space Bit:       Segment is not private (PoP 3-30)
  * S Storage-Alteration:
  * STL Segment-Table-Length:  Segment-table length (STL+1*16 entries -> up to 2048)
  *
  * A 64 bit pagetable entry of S390 has following format:
  * |			 PFRA			      |0IPC|  OS  |
  * 0000000000111111111122222222223333333333444444444455555555556666
  * 0123456789012345678901234567890123456789012345678901234567890123
  *
  * I Page-Invalid Bit:    Page is not available for address-translation
  * P Page-Protection Bit: Store access not possible for page
  * C Change-bit override: HW is not required to set change bit
  *
  * A 64 bit segmenttable entry of S390 has following format:
  * |        P-table origin                              |      TT
  * 0000000000111111111122222222223333333333444444444455555555556666
  * 0123456789012345678901234567890123456789012345678901234567890123
  *
  * I Segment-Invalid Bit:    Segment is not available for address-translation
  * C Common-Segment Bit:     Segment is not private (PoP 3-30)
  * P Page-Protection Bit: Store access not possible for page
  * TT Type 00
  *
  * A 64 bit region table entry of S390 has following format:
  * |        S-table origin                             |   TF  TTTL
  * 0000000000111111111122222222223333333333444444444455555555556666
  * 0123456789012345678901234567890123456789012345678901234567890123
  *
  * I Segment-Invalid Bit:    Segment is not available for address-translation
  * TT Type 01
  * TF
  * TL Table length
  *
  * The 64 bit regiontable origin of S390 has following format:
  * |      region table origon                          |       DTTL
  * 0000000000111111111122222222223333333333444444444455555555556666
  * 0123456789012345678901234567890123456789012345678901234567890123
  *
  * X Space-Switch event:
  * G Segment-Invalid Bit:
  * P Private-Space Bit:
  * S Storage-Alteration:
  * R Real space
  * TL Table-Length:
  *
  * A storage key has the following format:
  * | ACC |F|R|C|0|
  *  0   3 4 5 6 7
  * ACC: access key
  * F  : fetch protection bit
  * R  : referenced bit
  * C  : changed bit
  */
 /* Hardware bits in the page table entry */
 #define _PAGE_CO	0x100		/* HW Change-bit override */
 #define _PAGE_RO	0x200		/* HW read-only bit  */
 #define _PAGE_INVALID	0x400		/* HW invalid bit    */
 /* Software bits in the page table entry */
 #define _PAGE_SWT	0x001		/* SW pte type bit t */
 #define _PAGE_SWX	0x002		/* SW pte type bit x */
 #define _PAGE_SWC	0x004		/* SW pte changed bit (for KVM) */
 #define _PAGE_SWR	0x008		/* SW pte referenced bit (for KVM) */
 #define _PAGE_SPECIAL	0x010		/* SW associated with special page */
 #define __HAVE_ARCH_PTE_SPECIAL
 /* Set of bits not changed in pte_modify */
 #define _PAGE_CHG_MASK	(PAGE_MASK | _PAGE_SPECIAL | _PAGE_SWC | _PAGE_SWR)
 /* Six different types of pages. */
 #define _PAGE_TYPE_EMPTY	0x400
 #define _PAGE_TYPE_NONE		0x401
 #define _PAGE_TYPE_SWAP		0x403
 #define _PAGE_TYPE_FILE		0x601	/* bit 0x002 is used for offset !! */
 #define _PAGE_TYPE_RO		0x200
 #define _PAGE_TYPE_RW		0x000
 /*
  * Only four types for huge pages, using the invalid bit and protection bit
  * of a segment table entry.
  */
 #define _HPAGE_TYPE_EMPTY	0x020	/* _SEGMENT_ENTRY_INV */
 #define _HPAGE_TYPE_NONE	0x220
 #define _HPAGE_TYPE_RO		0x200	/* _SEGMENT_ENTRY_RO  */
 #define _HPAGE_TYPE_RW		0x000
 /*
  * PTE type bits are rather complicated. handle_pte_fault uses pte_present,
  * pte_none and pte_file to find out the pte type WITHOUT holding the page
  * table lock. ptep_clear_flush on the other hand uses ptep_clear_flush to
  * invalidate a given pte. ipte sets the hw invalid bit and clears all tlbs
  * for the page. The page table entry is set to _PAGE_TYPE_EMPTY afterwards.
  * This change is done while holding the lock, but the intermediate step
  * of a previously valid pte with the hw invalid bit set can be observed by
  * handle_pte_fault. That makes it necessary that all valid pte types with
  * the hw invalid bit set must be distinguishable from the four pte types
  * empty, none, swap and file.
  *
  *			irxt  ipte  irxt
  * _PAGE_TYPE_EMPTY	1000   ->   1000
  * _PAGE_TYPE_NONE	1001   ->   1001
  * _PAGE_TYPE_SWAP	1011   ->   1011
  * _PAGE_TYPE_FILE	11?1   ->   11?1
  * _PAGE_TYPE_RO	0100   ->   1100
  * _PAGE_TYPE_RW	0000   ->   1000
  *
  * pte_none is true for bits combinations 1000, 1010, 1100, 1110
  * pte_present is true for bits combinations 0000, 0010, 0100, 0110, 1001
  * pte_file is true for bits combinations 1101, 1111
  * swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid.
  */
 #ifndef CONFIG_64BIT
 /* Bits in the segment table address-space-control-element */
 #define _ASCE_SPACE_SWITCH	0x80000000UL	/* space switch event	    */
 #define _ASCE_ORIGIN_MASK	0x7ffff000UL	/* segment table origin	    */
 #define _ASCE_PRIVATE_SPACE	0x100	/* private space control	    */
 #define _ASCE_ALT_EVENT		0x80	/* storage alteration event control */
 #define _ASCE_TABLE_LENGTH	0x7f	/* 128 x 64 entries = 8k	    */
 /* Bits in the segment table entry */
 #define _SEGMENT_ENTRY_ORIGIN	0x7fffffc0UL	/* page table origin	    */
 #define _SEGMENT_ENTRY_RO	0x200	/* page protection bit		    */
 #define _SEGMENT_ENTRY_INV	0x20	/* invalid segment table entry	    */
 #define _SEGMENT_ENTRY_COMMON	0x10	/* common segment bit		    */
 #define _SEGMENT_ENTRY_PTL	0x0f	/* page table length		    */
 #define _SEGMENT_ENTRY		(_SEGMENT_ENTRY_PTL)
 #define _SEGMENT_ENTRY_EMPTY	(_SEGMENT_ENTRY_INV)
 /* Page status table bits for virtualization */
 #define RCP_ACC_BITS	0xf0000000UL
 #define RCP_FP_BIT	0x08000000UL
 #define RCP_PCL_BIT	0x00800000UL
 #define RCP_HR_BIT	0x00400000UL
 #define RCP_HC_BIT	0x00200000UL
 #define RCP_GR_BIT	0x00040000UL
 #define RCP_GC_BIT	0x00020000UL
 /* User dirty / referenced bit for KVM's migration feature */
 #define KVM_UR_BIT	0x00008000UL
 #define KVM_UC_BIT	0x00004000UL
 #else /* CONFIG_64BIT */
 /* Bits in the segment/region table address-space-control-element */
 #define _ASCE_ORIGIN		~0xfffUL/* segment table origin		    */
 #define _ASCE_PRIVATE_SPACE	0x100	/* private space control	    */
 #define _ASCE_ALT_EVENT		0x80	/* storage alteration event control */
 #define _ASCE_SPACE_SWITCH	0x40	/* space switch event		    */
 #define _ASCE_REAL_SPACE	0x20	/* real space control		    */
 #define _ASCE_TYPE_MASK		0x0c	/* asce table type mask		    */
 #define _ASCE_TYPE_REGION1	0x0c	/* region first table type	    */
 #define _ASCE_TYPE_REGION2	0x08	/* region second table type	    */
 #define _ASCE_TYPE_REGION3	0x04	/* region third table type	    */
 #define _ASCE_TYPE_SEGMENT	0x00	/* segment table type		    */
 #define _ASCE_TABLE_LENGTH	0x03	/* region table length		    */
 /* Bits in the region table entry */
 #define _REGION_ENTRY_ORIGIN	~0xfffUL/* region/segment table origin	    */
 #define _REGION_ENTRY_INV	0x20	/* invalid region table entry	    */
 #define _REGION_ENTRY_TYPE_MASK	0x0c	/* region/segment table type mask   */
 #define _REGION_ENTRY_TYPE_R1	0x0c	/* region first table type	    */
 #define _REGION_ENTRY_TYPE_R2	0x08	/* region second table type	    */
 #define _REGION_ENTRY_TYPE_R3	0x04	/* region third table type	    */
 #define _REGION_ENTRY_LENGTH	0x03	/* region third length		    */
 #define _REGION1_ENTRY		(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
 #define _REGION1_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV)
 #define _REGION2_ENTRY		(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
 #define _REGION2_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV)
 #define _REGION3_ENTRY		(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
 #define _REGION3_ENTRY_EMPTY	(_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV)
 /* Bits in the segment table entry */
 #define _SEGMENT_ENTRY_ORIGIN	~0x7ffUL/* segment table origin		    */
 #define _SEGMENT_ENTRY_RO	0x200	/* page protection bit		    */
 #define _SEGMENT_ENTRY_INV	0x20	/* invalid segment table entry	    */
 #define _SEGMENT_ENTRY		(0)
 #define _SEGMENT_ENTRY_EMPTY	(_SEGMENT_ENTRY_INV)
 #define _SEGMENT_ENTRY_LARGE	0x400	/* STE-format control, large page   */
 #define _SEGMENT_ENTRY_CO	0x100	/* change-recording override   */
 #define _SEGMENT_ENTRY_SPLIT_BIT 0	/* THP splitting bit number */
 #define _SEGMENT_ENTRY_SPLIT	(1UL << _SEGMENT_ENTRY_SPLIT_BIT)
+/* Set of bits not changed in pmd_modify */
+#define _SEGMENT_CHG_MASK	(_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
+				 | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
 /* Page status table bits for virtualization */
 #define RCP_ACC_BITS	0xf000000000000000UL
 #define RCP_FP_BIT	0x0800000000000000UL
 #define RCP_PCL_BIT	0x0080000000000000UL
 #define RCP_HR_BIT	0x0040000000000000UL
 #define RCP_HC_BIT	0x0020000000000000UL
 #define RCP_GR_BIT	0x0004000000000000UL
 #define RCP_GC_BIT	0x0002000000000000UL
 /* User dirty / referenced bit for KVM's migration feature */
 #define KVM_UR_BIT	0x0000800000000000UL
 #define KVM_UC_BIT	0x0000400000000000UL
 #endif /* CONFIG_64BIT */
 /*
  * A user page table pointer has the space-switch-event bit, the
  * private-space-control bit and the storage-alteration-event-control
  * bit set. A kernel page table pointer doesn't need them.
  */
 #define _ASCE_USER_BITS		(_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
 				 _ASCE_ALT_EVENT)
 /*
  * Page protection definitions.
  */
 #define PAGE_NONE	__pgprot(_PAGE_TYPE_NONE)
 #define PAGE_RO		__pgprot(_PAGE_TYPE_RO)
 #define PAGE_RW		__pgprot(_PAGE_TYPE_RW)
 #define PAGE_KERNEL	PAGE_RW
 #define PAGE_COPY	PAGE_RO
 /*
  * On s390 the page table entry has an invalid bit and a read-only bit.
  * Read permission implies execute permission and write permission
  * implies read permission.
  */
          /*xwr*/
 #define __P000	PAGE_NONE
 #define __P001	PAGE_RO
 #define __P010	PAGE_RO
 #define __P011	PAGE_RO
 #define __P100	PAGE_RO
 #define __P101	PAGE_RO
 #define __P110	PAGE_RO
 #define __P111	PAGE_RO
 #define __S000	PAGE_NONE
 #define __S001	PAGE_RO
 #define __S010	PAGE_RW
 #define __S011	PAGE_RW
 #define __S100	PAGE_RO
 #define __S101	PAGE_RO
 #define __S110	PAGE_RW
 #define __S111	PAGE_RW
 static inline int mm_exclusive(struct mm_struct *mm)
 {
 	return likely(mm == current->active_mm &&
 		      atomic_read(&mm->context.attach_count) <= 1);
 }
 static inline int mm_has_pgste(struct mm_struct *mm)
 {
 #ifdef CONFIG_PGSTE
 	if (unlikely(mm->context.has_pgste))
 		return 1;
 #endif
 	return 0;
 }
 /*
  * pgd/pmd/pte query functions
  */
 #ifndef CONFIG_64BIT
 static inline int pgd_present(pgd_t pgd) { return 1; }
 static inline int pgd_none(pgd_t pgd)    { return 0; }
 static inline int pgd_bad(pgd_t pgd)     { return 0; }
 static inline int pud_present(pud_t pud) { return 1; }
 static inline int pud_none(pud_t pud)	 { return 0; }
 static inline int pud_bad(pud_t pud)	 { return 0; }
 #else /* CONFIG_64BIT */
 static inline int pgd_present(pgd_t pgd)
 {
 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
 		return 1;
 	return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
 }
 static inline int pgd_none(pgd_t pgd)
 {
 	if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
 		return 0;
 	return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL;
 }
 static inline int pgd_bad(pgd_t pgd)
 {
 	/*
 	 * With dynamic page table levels the pgd can be a region table
 	 * entry or a segment table entry. Check for the bit that are
 	 * invalid for either table entry.
 	 */
 	unsigned long mask =
 		~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
 		~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
 	return (pgd_val(pgd) & mask) != 0;
 }
 static inline int pud_present(pud_t pud)
 {
 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
 		return 1;
 	return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
 }
 static inline int pud_none(pud_t pud)
 {
 	if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
 		return 0;
 	return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL;
 }
 static inline int pud_bad(pud_t pud)
 {
 	/*
 	 * With dynamic page table levels the pud can be a region table
 	 * entry or a segment table entry. Check for the bit that are
 	 * invalid for either table entry.
 	 */
 	unsigned long mask =
 		~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
 		~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
 	return (pud_val(pud) & mask) != 0;
 }
 #endif /* CONFIG_64BIT */
 static inline int pmd_present(pmd_t pmd)
 {
 	return (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) != 0UL;
 }
 static inline int pmd_none(pmd_t pmd)
 {
 	return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) != 0UL;
 }
 static inline int pmd_bad(pmd_t pmd)
 {
 	unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV;
 	return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
 }
 #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
 extern void pmdp_splitting_flush(struct vm_area_struct *vma,
 				 unsigned long addr, pmd_t *pmdp);
+#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
+extern int pmdp_set_access_flags(struct vm_area_struct *vma,
+				 unsigned long address, pmd_t *pmdp,
+				 pmd_t entry, int dirty);
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
+				  unsigned long address, pmd_t *pmdp);
+#define __HAVE_ARCH_PMD_WRITE
+static inline int pmd_write(pmd_t pmd)
+{
+	return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
+}
+static inline int pmd_young(pmd_t pmd)
+{
+	return 0;
+}
 static inline int pte_none(pte_t pte)
 {
 	return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
 }
 static inline int pte_present(pte_t pte)
 {
 	unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT | _PAGE_SWX;
 	return (pte_val(pte) & mask) == _PAGE_TYPE_NONE ||
 		(!(pte_val(pte) & _PAGE_INVALID) &&
 		 !(pte_val(pte) & _PAGE_SWT));
 }
 static inline int pte_file(pte_t pte)
 {
 	unsigned long mask = _PAGE_RO | _PAGE_INVALID | _PAGE_SWT;
 	return (pte_val(pte) & mask) == _PAGE_TYPE_FILE;
 }
 static inline int pte_special(pte_t pte)
 {
 	return (pte_val(pte) & _PAGE_SPECIAL);
 }
 #define __HAVE_ARCH_PTE_SAME
 static inline int pte_same(pte_t a, pte_t b)
 {
 	return pte_val(a) == pte_val(b);
 }
 static inline pgste_t pgste_get_lock(pte_t *ptep)
 {
 	unsigned long new = 0;
 #ifdef CONFIG_PGSTE
 	unsigned long old;
 	preempt_disable();
 	asm(
 		"	lg	%0,%2\n"
 		"0:	lgr	%1,%0\n"
 		"	nihh	%0,0xff7f\n"	/* clear RCP_PCL_BIT in old */
 		"	oihh	%1,0x0080\n"	/* set RCP_PCL_BIT in new */
 		"	csg	%0,%1,%2\n"
 		"	jl	0b\n"
 		: "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
 		: "Q" (ptep[PTRS_PER_PTE]) : "cc");
 #endif
 	return __pgste(new);
 }
 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
 {
 #ifdef CONFIG_PGSTE
 	asm(
 		"	nihh	%1,0xff7f\n"	/* clear RCP_PCL_BIT */
 		"	stg	%1,%0\n"
 		: "=Q" (ptep[PTRS_PER_PTE])
 		: "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) : "cc");
 	preempt_enable();
 #endif
 }
 static inline pgste_t pgste_update_all(pte_t *ptep, pgste_t pgste)
 {
 #ifdef CONFIG_PGSTE
 	unsigned long address, bits;
 	unsigned char skey;
 	if (!pte_present(*ptep))
 		return pgste;
 	address = pte_val(*ptep) & PAGE_MASK;
 	skey = page_get_storage_key(address);
 	bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
 	/* Clear page changed & referenced bit in the storage key */
 	if (bits & _PAGE_CHANGED)
 		page_set_storage_key(address, skey ^ bits, 1);
 	else if (bits)
 		page_reset_referenced(address);
 	/* Transfer page changed & referenced bit to guest bits in pgste */
 	pgste_val(pgste) |= bits << 48;		/* RCP_GR_BIT & RCP_GC_BIT */
 	/* Get host changed & referenced bits from pgste */
 	bits |= (pgste_val(pgste) & (RCP_HR_BIT | RCP_HC_BIT)) >> 52;
 	/* Clear host bits in pgste. */
 	pgste_val(pgste) &= ~(RCP_HR_BIT | RCP_HC_BIT);
 	pgste_val(pgste) &= ~(RCP_ACC_BITS | RCP_FP_BIT);
 	/* Copy page access key and fetch protection bit to pgste */
 	pgste_val(pgste) |=
 		(unsigned long) (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
 	/* Transfer changed and referenced to kvm user bits */
 	pgste_val(pgste) |= bits << 45;		/* KVM_UR_BIT & KVM_UC_BIT */
 	/* Transfer changed & referenced to pte sofware bits */
 	pte_val(*ptep) |= bits << 1;		/* _PAGE_SWR & _PAGE_SWC */
 #endif
 	return pgste;
 }
 static inline pgste_t pgste_update_young(pte_t *ptep, pgste_t pgste)
 {
 #ifdef CONFIG_PGSTE
 	int young;
 	if (!pte_present(*ptep))
 		return pgste;
 	young = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
 	/* Transfer page referenced bit to pte software bit (host view) */
 	if (young || (pgste_val(pgste) & RCP_HR_BIT))
 		pte_val(*ptep) |= _PAGE_SWR;
 	/* Clear host referenced bit in pgste. */
 	pgste_val(pgste) &= ~RCP_HR_BIT;
 	/* Transfer page referenced bit to guest bit in pgste */
 	pgste_val(pgste) |= (unsigned long) young << 50; /* set RCP_GR_BIT */
 #endif
 	return pgste;
 }
 static inline void pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
 {
 #ifdef CONFIG_PGSTE
 	unsigned long address;
 	unsigned long okey, nkey;
 	if (!pte_present(entry))
 		return;
 	address = pte_val(entry) & PAGE_MASK;
 	okey = nkey = page_get_storage_key(address);
 	nkey &= ~(_PAGE_ACC_BITS | _PAGE_FP_BIT);
 	/* Set page access key and fetch protection bit from pgste */
 	nkey |= (pgste_val(pgste) & (RCP_ACC_BITS | RCP_FP_BIT)) >> 56;
 	if (okey != nkey)
 		page_set_storage_key(address, nkey, 1);
 #endif
 }
 /**
  * struct gmap_struct - guest address space
  * @mm: pointer to the parent mm_struct
  * @table: pointer to the page directory
  * @asce: address space control element for gmap page table
  * @crst_list: list of all crst tables used in the guest address space
  */
 struct gmap {
 	struct list_head list;
 	struct mm_struct *mm;
 	unsigned long *table;
 	unsigned long asce;
 	struct list_head crst_list;
 };
 /**
  * struct gmap_rmap - reverse mapping for segment table entries
  * @next: pointer to the next gmap_rmap structure in the list
  * @entry: pointer to a segment table entry
  */
 struct gmap_rmap {
 	struct list_head list;
 	unsigned long *entry;
 };
 /**
  * struct gmap_pgtable - gmap information attached to a page table
  * @vmaddr: address of the 1MB segment in the process virtual memory
  * @mapper: list of segment table entries maping a page table
  */
 struct gmap_pgtable {
 	unsigned long vmaddr;
 	struct list_head mapper;
 };
 struct gmap *gmap_alloc(struct mm_struct *mm);
 void gmap_free(struct gmap *gmap);
 void gmap_enable(struct gmap *gmap);
 void gmap_disable(struct gmap *gmap);
 int gmap_map_segment(struct gmap *gmap, unsigned long from,
 		     unsigned long to, unsigned long length);
 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len);
 unsigned long __gmap_fault(unsigned long address, struct gmap *);
 unsigned long gmap_fault(unsigned long address, struct gmap *);
 void gmap_discard(unsigned long from, unsigned long to, struct gmap *);
 /*
  * Certain architectures need to do special things when PTEs
  * within a page table are directly modified.  Thus, the following
  * hook is made available.
  */
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep, pte_t entry)
 {
 	pgste_t pgste;
 	if (mm_has_pgste(mm)) {
 		pgste = pgste_get_lock(ptep);
 		pgste_set_pte(ptep, pgste, entry);
 		*ptep = entry;
 		pgste_set_unlock(ptep, pgste);
 	} else
 		*ptep = entry;
 }
 /*
  * query functions pte_write/pte_dirty/pte_young only work if
  * pte_present() is true. Undefined behaviour if not..
  */
 static inline int pte_write(pte_t pte)
 {
 	return (pte_val(pte) & _PAGE_RO) == 0;
 }
 static inline int pte_dirty(pte_t pte)
 {
 #ifdef CONFIG_PGSTE
 	if (pte_val(pte) & _PAGE_SWC)
 		return 1;
 #endif
 	return 0;
 }
 static inline int pte_young(pte_t pte)
 {
 #ifdef CONFIG_PGSTE
 	if (pte_val(pte) & _PAGE_SWR)
 		return 1;
 #endif
 	return 0;
 }
 /*
  * pgd/pmd/pte modification functions
  */
 static inline void pgd_clear(pgd_t *pgd)
 {
 #ifdef CONFIG_64BIT
 	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
 		pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
 #endif
 }
 static inline void pud_clear(pud_t *pud)
 {
 #ifdef CONFIG_64BIT
 	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 		pud_val(*pud) = _REGION3_ENTRY_EMPTY;
 #endif
 }
 static inline void pmd_clear(pmd_t *pmdp)
 {
 	pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
 }
 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 }
 /*
  * The following pte modification functions only work if
  * pte_present() is true. Undefined behaviour if not..
  */
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 {
 	pte_val(pte) &= _PAGE_CHG_MASK;
 	pte_val(pte) |= pgprot_val(newprot);
 	return pte;
 }
 static inline pte_t pte_wrprotect(pte_t pte)
 {
 	/* Do not clobber _PAGE_TYPE_NONE pages!  */
 	if (!(pte_val(pte) & _PAGE_INVALID))
 		pte_val(pte) |= _PAGE_RO;
 	return pte;
 }
 static inline pte_t pte_mkwrite(pte_t pte)
 {
 	pte_val(pte) &= ~_PAGE_RO;
 	return pte;
 }
 static inline pte_t pte_mkclean(pte_t pte)
 {
 #ifdef CONFIG_PGSTE
 	pte_val(pte) &= ~_PAGE_SWC;
 #endif
 	return pte;
 }
 static inline pte_t pte_mkdirty(pte_t pte)
 {
 	return pte;
 }
 static inline pte_t pte_mkold(pte_t pte)
 {
 #ifdef CONFIG_PGSTE
 	pte_val(pte) &= ~_PAGE_SWR;
 #endif
 	return pte;
 }
 static inline pte_t pte_mkyoung(pte_t pte)
 {
 	return pte;
 }
 static inline pte_t pte_mkspecial(pte_t pte)
 {
 	pte_val(pte) |= _PAGE_SPECIAL;
 	return pte;
 }
 #ifdef CONFIG_HUGETLB_PAGE
 static inline pte_t pte_mkhuge(pte_t pte)
 {
 	/*
 	 * PROT_NONE needs to be remapped from the pte type to the ste type.
 	 * The HW invalid bit is also different for pte and ste. The pte
 	 * invalid bit happens to be the same as the ste _SEGMENT_ENTRY_LARGE
 	 * bit, so we don't have to clear it.
 	 */
 	if (pte_val(pte) & _PAGE_INVALID) {
 		if (pte_val(pte) & _PAGE_SWT)
 			pte_val(pte) |= _HPAGE_TYPE_NONE;
 		pte_val(pte) |= _SEGMENT_ENTRY_INV;
 	}
 	/*
 	 * Clear SW pte bits SWT and SWX, there are no SW bits in a segment
 	 * table entry.
 	 */
 	pte_val(pte) &= ~(_PAGE_SWT | _PAGE_SWX);
 	/*
 	 * Also set the change-override bit because we don't need dirty bit
 	 * tracking for hugetlbfs pages.
 	 */
 	pte_val(pte) |= (_SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_CO);
 	return pte;
 }
 #endif
 /*
  * Get (and clear) the user dirty bit for a pte.
  */
 static inline int ptep_test_and_clear_user_dirty(struct mm_struct *mm,
 						 pte_t *ptep)
 {
 	pgste_t pgste;
 	int dirty = 0;
 	if (mm_has_pgste(mm)) {
 		pgste = pgste_get_lock(ptep);
 		pgste = pgste_update_all(ptep, pgste);
 		dirty = !!(pgste_val(pgste) & KVM_UC_BIT);
 		pgste_val(pgste) &= ~KVM_UC_BIT;
 		pgste_set_unlock(ptep, pgste);
 		return dirty;
 	}
 	return dirty;
 }
 /*
  * Get (and clear) the user referenced bit for a pte.
  */
 static inline int ptep_test_and_clear_user_young(struct mm_struct *mm,
 						 pte_t *ptep)
 {
 	pgste_t pgste;
 	int young = 0;
 	if (mm_has_pgste(mm)) {
 		pgste = pgste_get_lock(ptep);
 		pgste = pgste_update_young(ptep, pgste);
 		young = !!(pgste_val(pgste) & KVM_UR_BIT);
 		pgste_val(pgste) &= ~KVM_UR_BIT;
 		pgste_set_unlock(ptep, pgste);
 	}
 	return young;
 }
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
 					    unsigned long addr, pte_t *ptep)
 {
 	pgste_t pgste;
 	pte_t pte;
 	if (mm_has_pgste(vma->vm_mm)) {
 		pgste = pgste_get_lock(ptep);
 		pgste = pgste_update_young(ptep, pgste);
 		pte = *ptep;
 		*ptep = pte_mkold(pte);
 		pgste_set_unlock(ptep, pgste);
 		return pte_young(pte);
 	}
 	return 0;
 }
 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
 static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
 					 unsigned long address, pte_t *ptep)
 {
 	/* No need to flush TLB
 	 * On s390 reference bits are in storage key and never in TLB
 	 * With virtualization we handle the reference bit, without we
 	 * we can simply return */
 	return ptep_test_and_clear_young(vma, address, ptep);
 }
 static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
 {
 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
 #ifndef CONFIG_64BIT
 		/* pto must point to the start of the segment table */
 		pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
 #else
 		/* ipte in zarch mode can do the math */
 		pte_t *pto = ptep;
 #endif
 		asm volatile(
 			"	ipte	%2,%3"
 			: "=m" (*ptep) : "m" (*ptep),
 			  "a" (pto), "a" (address));
 	}
 }
 /*
  * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
  * both clear the TLB for the unmapped pte. The reason is that
  * ptep_get_and_clear is used in common code (e.g. change_pte_range)
  * to modify an active pte. The sequence is
  *   1) ptep_get_and_clear
  *   2) set_pte_at
  *   3) flush_tlb_range
  * On s390 the tlb needs to get flushed with the modification of the pte
  * if the pte is active. The only way how this can be implemented is to
  * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
  * is a nop.
  */
 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
 				       unsigned long address, pte_t *ptep)
 {
 	pgste_t pgste;
 	pte_t pte;
 	mm->context.flush_mm = 1;
 	if (mm_has_pgste(mm))
 		pgste = pgste_get_lock(ptep);
 	pte = *ptep;
 	if (!mm_exclusive(mm))
 		__ptep_ipte(address, ptep);
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 	if (mm_has_pgste(mm)) {
 		pgste = pgste_update_all(&pte, pgste);
 		pgste_set_unlock(ptep, pgste);
 	}
 	return pte;
 }
 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm,
 					   unsigned long address,
 					   pte_t *ptep)
 {
 	pte_t pte;
 	mm->context.flush_mm = 1;
 	if (mm_has_pgste(mm))
 		pgste_get_lock(ptep);
 	pte = *ptep;
 	if (!mm_exclusive(mm))
 		__ptep_ipte(address, ptep);
 	return pte;
 }
 static inline void ptep_modify_prot_commit(struct mm_struct *mm,
 					   unsigned long address,
 					   pte_t *ptep, pte_t pte)
 {
 	*ptep = pte;
 	if (mm_has_pgste(mm))
 		pgste_set_unlock(ptep, *(pgste_t *)(ptep + PTRS_PER_PTE));
 }
 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH
 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
 				     unsigned long address, pte_t *ptep)
 {
 	pgste_t pgste;
 	pte_t pte;
 	if (mm_has_pgste(vma->vm_mm))
 		pgste = pgste_get_lock(ptep);
 	pte = *ptep;
 	__ptep_ipte(address, ptep);
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 	if (mm_has_pgste(vma->vm_mm)) {
 		pgste = pgste_update_all(&pte, pgste);
 		pgste_set_unlock(ptep, pgste);
 	}
 	return pte;
 }
 /*
  * The batched pte unmap code uses ptep_get_and_clear_full to clear the
  * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
  * tlbs of an mm if it can guarantee that the ptes of the mm_struct
  * cannot be accessed while the batched unmap is running. In this case
  * full==1 and a simple pte_clear is enough. See tlb.h.
  */
 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
 					    unsigned long address,
 					    pte_t *ptep, int full)
 {
 	pgste_t pgste;
 	pte_t pte;
 	if (mm_has_pgste(mm))
 		pgste = pgste_get_lock(ptep);
 	pte = *ptep;
 	if (!full)
 		__ptep_ipte(address, ptep);
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 	if (mm_has_pgste(mm)) {
 		pgste = pgste_update_all(&pte, pgste);
 		pgste_set_unlock(ptep, pgste);
 	}
 	return pte;
 }
 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
 static inline pte_t ptep_set_wrprotect(struct mm_struct *mm,
 				       unsigned long address, pte_t *ptep)
 {
 	pgste_t pgste;
 	pte_t pte = *ptep;
 	if (pte_write(pte)) {
 		mm->context.flush_mm = 1;
 		if (mm_has_pgste(mm))
 			pgste = pgste_get_lock(ptep);
 		if (!mm_exclusive(mm))
 			__ptep_ipte(address, ptep);
 		*ptep = pte_wrprotect(pte);
 		if (mm_has_pgste(mm))
 			pgste_set_unlock(ptep, pgste);
 	}
 	return pte;
 }
 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
 static inline int ptep_set_access_flags(struct vm_area_struct *vma,
 					unsigned long address, pte_t *ptep,
 					pte_t entry, int dirty)
 {
 	pgste_t pgste;
 	if (pte_same(*ptep, entry))
 		return 0;
 	if (mm_has_pgste(vma->vm_mm))
 		pgste = pgste_get_lock(ptep);
 	__ptep_ipte(address, ptep);
 	*ptep = entry;
 	if (mm_has_pgste(vma->vm_mm))
 		pgste_set_unlock(ptep, pgste);
 	return 1;
 }
 /*
  * Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
  */
 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot)
 {
 	pte_t __pte;
 	pte_val(__pte) = physpage + pgprot_val(pgprot);
 	return __pte;
 }
 static inline pte_t mk_pte(struct page *page, pgprot_t pgprot)
 {
 	unsigned long physpage = page_to_phys(page);
 	return mk_pte_phys(physpage, pgprot);
 }
 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
 #define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
 #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
 #define pgd_offset_k(address) pgd_offset(&init_mm, address)
 #ifndef CONFIG_64BIT
 #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
 #define pud_deref(pmd) ({ BUG(); 0UL; })
 #define pgd_deref(pmd) ({ BUG(); 0UL; })
 #define pud_offset(pgd, address) ((pud_t *) pgd)
 #define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address))
 #else /* CONFIG_64BIT */
 #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
 #define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
 #define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
 {
 	pud_t *pud = (pud_t *) pgd;
 	if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
 		pud = (pud_t *) pgd_deref(*pgd);
 	return pud  + pud_index(address);
 }
 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 {
 	pmd_t *pmd = (pmd_t *) pud;
 	if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 		pmd = (pmd_t *) pud_deref(*pud);
 	return pmd + pmd_index(address);
 }
 #endif /* CONFIG_64BIT */
 #define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
 #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
 #define pte_page(x) pfn_to_page(pte_pfn(x))
 #define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
 /* Find an entry in the lowest level page table.. */
 #define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
 #define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
 #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
 #define pte_unmap(pte) do { } while (0)
+static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
+{
+	unsigned long sto = (unsigned long) pmdp -
+			    pmd_index(address) * sizeof(pmd_t);
+	if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
+		asm volatile(
+			"	.insn	rrf,0xb98e0000,%2,%3,0,0"
+			: "=m" (*pmdp)
+			: "m" (*pmdp), "a" (sto),
+			  "a" ((address & HPAGE_MASK))
+			: "cc"
+		);
+	}
+}
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define __HAVE_ARCH_PGTABLE_DEPOSIT
 extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable);
 #define __HAVE_ARCH_PGTABLE_WITHDRAW
 extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm);
 static inline int pmd_trans_splitting(pmd_t pmd)
 {
 	return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
+}
+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+			      pmd_t *pmdp, pmd_t entry)
+{
+	*pmdp = entry;
+}
+static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
+{
+	unsigned long pgprot_pmd = 0;
+	if (pgprot_val(pgprot) & _PAGE_INVALID) {
+		if (pgprot_val(pgprot) & _PAGE_SWT)
+			pgprot_pmd |= _HPAGE_TYPE_NONE;
+		pgprot_pmd |= _SEGMENT_ENTRY_INV;
+	}
+	if (pgprot_val(pgprot) & _PAGE_RO)
+		pgprot_pmd |= _SEGMENT_ENTRY_RO;
+	return pgprot_pmd;
+}
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+	pmd_val(pmd) &= _SEGMENT_CHG_MASK;
+	pmd_val(pmd) |= massage_pgprot_pmd(newprot);
+	return pmd;
+}
+static inline pmd_t pmd_mkhuge(pmd_t pmd)
+{
+	pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
+	return pmd;
+}
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
+	return pmd;
+}
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+	pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
+	return pmd;
+}
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+	/* No dirty bit in the segment table entry. */
+	return pmd;
+}
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+	/* No referenced bit in the segment table entry. */
+	return pmd;
+}
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+	/* No referenced bit in the segment table entry. */
+	return pmd;
+}
+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
+static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+					    unsigned long address, pmd_t *pmdp)
+{
+	unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
+	long tmp, rc;
+	int counter;
+	rc = 0;
+	if (MACHINE_HAS_RRBM) {
+		counter = PTRS_PER_PTE >> 6;
+		asm volatile(
+			"0:	.insn	rre,0xb9ae0000,%0,%3\n"	/* rrbm */
+			"	ogr	%1,%0\n"
+			"	la	%3,0(%4,%3)\n"
+			"	brct	%2,0b\n"
+			: "=&d" (tmp), "+&d" (rc), "+d" (counter),
+			  "+a" (pmd_addr)
+			: "a" (64 * 4096UL) : "cc");
+		rc = !!rc;
+	} else {
+		counter = PTRS_PER_PTE;
+		asm volatile(
+			"0:	rrbe	0,%2\n"
+			"	la	%2,0(%3,%2)\n"
+			"	brc	12,1f\n"
+			"	lhi	%0,1\n"
+			"1:	brct	%1,0b\n"
+			: "+d" (rc), "+d" (counter), "+a" (pmd_addr)
+			: "a" (4096UL) : "cc");
+	}
+	return rc;
+}
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
+				       unsigned long address, pmd_t *pmdp)
+{
+	pmd_t pmd = *pmdp;
+	__pmd_idte(address, pmdp);
+	pmd_clear(pmdp);
+	return pmd;
+}
+#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
+static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
+				     unsigned long address, pmd_t *pmdp)
+{
+	return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
+}
+#define __HAVE_ARCH_PMDP_INVALIDATE
+static inline void pmdp_invalidate(struct vm_area_struct *vma,
+				   unsigned long address, pmd_t *pmdp)
+{
+	__pmd_idte(address, pmdp);
+}
+static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
+{
+	pmd_t __pmd;
+	pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
+	return __pmd;
+}
+#define pfn_pmd(pfn, pgprot)	mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
+#define mk_pmd(page, pgprot)	pfn_pmd(page_to_pfn(page), (pgprot))
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+	return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
+}
+static inline int has_transparent_hugepage(void)
+{
+	return MACHINE_HAS_HPAGE ? 1 : 0;
+}
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+	if (pmd_trans_huge(pmd))
+		return pmd_val(pmd) >> HPAGE_SHIFT;
+	else
+		return pmd_val(pmd) >> PAGE_SHIFT;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 /*
  * 31 bit swap entry format:
  * A page-table entry has some bits we have to treat in a special way.
  * Bits 0, 20 and bit 23 have to be zero, otherwise an specification
  * exception will occur instead of a page translation exception. The
  * specifiation exception has the bad habit not to store necessary
  * information in the lowcore.
  * Bit 21 and bit 22 are the page invalid bit and the page protection
  * bit. We set both to indicate a swapped page.
  * Bit 30 and 31 are used to distinguish the different page types. For
  * a swapped page these bits need to be zero.
  * This leaves the bits 1-19 and bits 24-29 to store type and offset.
  * We use the 5 bits from 25-29 for the type and the 20 bits from 1-19
  * plus 24 for the offset.
  * 0|     offset        |0110|o|type |00|
  * 0 0000000001111111111 2222 2 22222 33
  * 0 1234567890123456789 0123 4 56789 01
  *
  * 64 bit swap entry format:
  * A page-table entry has some bits we have to treat in a special way.
  * Bits 52 and bit 55 have to be zero, otherwise an specification
  * exception will occur instead of a page translation exception. The
  * specifiation exception has the bad habit not to store necessary
  * information in the lowcore.
  * Bit 53 and bit 54 are the page invalid bit and the page protection
  * bit. We set both to indicate a swapped page.
  * Bit 62 and 63 are used to distinguish the different page types. For
  * a swapped page these bits need to be zero.
  * This leaves the bits 0-51 and bits 56-61 to store type and offset.
  * We use the 5 bits from 57-61 for the type and the 53 bits from 0-51
  * plus 56 for the offset.
  * |                      offset                        |0110|o|type |00|
  *  0000000000111111111122222222223333333333444444444455 5555 5 55566 66
  *  0123456789012345678901234567890123456789012345678901 2345 6 78901 23
  */
 #ifndef CONFIG_64BIT
 #define __SWP_OFFSET_MASK (~0UL >> 12)
 #else
 #define __SWP_OFFSET_MASK (~0UL >> 11)
 #endif
 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset)
 {
 	pte_t pte;
 	offset &= __SWP_OFFSET_MASK;
 	pte_val(pte) = _PAGE_TYPE_SWAP | ((type & 0x1f) << 2) |
 		((offset & 1UL) << 7) | ((offset & ~1UL) << 11);
 	return pte;
 }
 #define __swp_type(entry)	(((entry).val >> 2) & 0x1f)
 #define __swp_offset(entry)	(((entry).val >> 11) | (((entry).val >> 7) & 1))
 #define __swp_entry(type,offset) ((swp_entry_t) { pte_val(mk_swap_pte((type),(offset))) })
 #define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
 #define __swp_entry_to_pte(x)	((pte_t) { (x).val })
 #ifndef CONFIG_64BIT
 # define PTE_FILE_MAX_BITS	26
 #else /* CONFIG_64BIT */
 # define PTE_FILE_MAX_BITS	59
 #endif /* CONFIG_64BIT */
 #define pte_to_pgoff(__pte) \
 	((((__pte).pte >> 12) << 7) + (((__pte).pte >> 1) & 0x7f))
 #define pgoff_to_pte(__off) \
 	((pte_t) { ((((__off) & 0x7f) << 1) + (((__off) >> 7) << 12)) \
 		   | _PAGE_TYPE_FILE })
 #endif /* !__ASSEMBLY__ */
 #define kern_addr_valid(addr)   (1)
 extern int vmem_add_mapping(unsigned long start, unsigned long size);
 extern int vmem_remove_mapping(unsigned long start, unsigned long size);
 extern int s390_enable_sie(void);
 /*
  * No page table caches to initialise
  */
 #define pgtable_cache_init()	do { } while (0)
 #include <asm-generic/pgtable.h>
 #endif /* _S390_PAGE_H */

arch/s390/include/asm/setup.h

Diff comments View file @ 1ae1c1d

 /*
  *  S390 version
  *    Copyright IBM Corp. 1999, 2010
  */
 #ifndef _ASM_S390_SETUP_H
 #define _ASM_S390_SETUP_H
 #define COMMAND_LINE_SIZE	4096
 #define ARCH_COMMAND_LINE_SIZE	896
 #ifdef __KERNEL__
 #define PARMAREA		0x10400
 #define MEMORY_CHUNKS		256
 #ifndef __ASSEMBLY__
 #include <asm/lowcore.h>
 #include <asm/types.h>
 #ifndef CONFIG_64BIT
 #define IPL_DEVICE        (*(unsigned long *)  (0x10404))
 #define INITRD_START      (*(unsigned long *)  (0x1040C))
 #define INITRD_SIZE       (*(unsigned long *)  (0x10414))
 #define OLDMEM_BASE	  (*(unsigned long *)  (0x1041C))
 #define OLDMEM_SIZE	  (*(unsigned long *)  (0x10424))
 #else /* CONFIG_64BIT */
 #define IPL_DEVICE        (*(unsigned long *)  (0x10400))
 #define INITRD_START      (*(unsigned long *)  (0x10408))
 #define INITRD_SIZE       (*(unsigned long *)  (0x10410))
 #define OLDMEM_BASE	  (*(unsigned long *)  (0x10418))
 #define OLDMEM_SIZE	  (*(unsigned long *)  (0x10420))
 #endif /* CONFIG_64BIT */
 #define COMMAND_LINE      ((char *)            (0x10480))
 #define CHUNK_READ_WRITE 0
 #define CHUNK_READ_ONLY  1
 #define CHUNK_OLDMEM	 4
 #define CHUNK_CRASHK	 5
 struct mem_chunk {
 	unsigned long addr;
 	unsigned long size;
 	int type;
 };
 extern struct mem_chunk memory_chunk[];
 extern unsigned long real_memory_size;
 extern int memory_end_set;
 extern unsigned long memory_end;
 void detect_memory_layout(struct mem_chunk chunk[]);
 void create_mem_hole(struct mem_chunk memory_chunk[], unsigned long addr,
 		     unsigned long size, int type);
 #define PRIMARY_SPACE_MODE	0
 #define ACCESS_REGISTER_MODE	1
 #define SECONDARY_SPACE_MODE	2
 #define HOME_SPACE_MODE		3
 extern unsigned int s390_user_mode;
 /*
  * Machine features detected in head.S
  */
 #define MACHINE_FLAG_VM		(1UL << 0)
 #define MACHINE_FLAG_IEEE	(1UL << 1)
 #define MACHINE_FLAG_CSP	(1UL << 3)
 #define MACHINE_FLAG_MVPG	(1UL << 4)
 #define MACHINE_FLAG_DIAG44	(1UL << 5)
 #define MACHINE_FLAG_IDTE	(1UL << 6)
 #define MACHINE_FLAG_DIAG9C	(1UL << 7)
 #define MACHINE_FLAG_MVCOS	(1UL << 8)
 #define MACHINE_FLAG_KVM	(1UL << 9)
 #define MACHINE_FLAG_HPAGE	(1UL << 10)
 #define MACHINE_FLAG_PFMF	(1UL << 11)
 #define MACHINE_FLAG_LPAR	(1UL << 12)
 #define MACHINE_FLAG_SPP	(1UL << 13)
 #define MACHINE_FLAG_TOPOLOGY	(1UL << 14)
 #define MACHINE_FLAG_TE		(1UL << 15)
+#define MACHINE_FLAG_RRBM	(1UL << 16)
 #define MACHINE_IS_VM		(S390_lowcore.machine_flags & MACHINE_FLAG_VM)
 #define MACHINE_IS_KVM		(S390_lowcore.machine_flags & MACHINE_FLAG_KVM)
 #define MACHINE_IS_LPAR		(S390_lowcore.machine_flags & MACHINE_FLAG_LPAR)
 #define MACHINE_HAS_DIAG9C	(S390_lowcore.machine_flags & MACHINE_FLAG_DIAG9C)
 #ifndef CONFIG_64BIT
 #define MACHINE_HAS_IEEE	(S390_lowcore.machine_flags & MACHINE_FLAG_IEEE)
 #define MACHINE_HAS_CSP		(S390_lowcore.machine_flags & MACHINE_FLAG_CSP)
 #define MACHINE_HAS_IDTE	(0)
 #define MACHINE_HAS_DIAG44	(1)
 #define MACHINE_HAS_MVPG	(S390_lowcore.machine_flags & MACHINE_FLAG_MVPG)
 #define MACHINE_HAS_MVCOS	(0)
 #define MACHINE_HAS_HPAGE	(0)
 #define MACHINE_HAS_PFMF	(0)
 #define MACHINE_HAS_SPP		(0)
 #define MACHINE_HAS_TOPOLOGY	(0)
-#define MACHINE_HAS_TE		      (0)
+#define MACHINE_HAS_TE		(0)
+#define MACHINE_HAS_RRBM	(0)
 #else /* CONFIG_64BIT */
 #define MACHINE_HAS_IEEE	(1)
 #define MACHINE_HAS_CSP		(1)
 #define MACHINE_HAS_IDTE	(S390_lowcore.machine_flags & MACHINE_FLAG_IDTE)
 #define MACHINE_HAS_DIAG44	(S390_lowcore.machine_flags & MACHINE_FLAG_DIAG44)
 #define MACHINE_HAS_MVPG	(1)
 #define MACHINE_HAS_MVCOS	(S390_lowcore.machine_flags & MACHINE_FLAG_MVCOS)
 #define MACHINE_HAS_HPAGE	(S390_lowcore.machine_flags & MACHINE_FLAG_HPAGE)
 #define MACHINE_HAS_PFMF	(S390_lowcore.machine_flags & MACHINE_FLAG_PFMF)
 #define MACHINE_HAS_SPP		(S390_lowcore.machine_flags & MACHINE_FLAG_SPP)
 #define MACHINE_HAS_TOPOLOGY	(S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY)
 #define MACHINE_HAS_TE		(S390_lowcore.machine_flags & MACHINE_FLAG_TE)
+#define MACHINE_HAS_RRBM	(S390_lowcore.machine_flags & MACHINE_FLAG_RRBM)
 #endif /* CONFIG_64BIT */
 #define ZFCPDUMP_HSA_SIZE	(32UL<<20)
 #define ZFCPDUMP_HSA_SIZE_MAX	(64UL<<20)
 /*
  * Console mode. Override with conmode=
  */
 extern unsigned int console_mode;
 extern unsigned int console_devno;
 extern unsigned int console_irq;
 extern char vmhalt_cmd[];
 extern char vmpoff_cmd[];
 #define CONSOLE_IS_UNDEFINED	(console_mode == 0)
 #define CONSOLE_IS_SCLP		(console_mode == 1)
 #define CONSOLE_IS_3215		(console_mode == 2)
 #define CONSOLE_IS_3270		(console_mode == 3)
 #define SET_CONSOLE_SCLP	do { console_mode = 1; } while (0)
 #define SET_CONSOLE_3215	do { console_mode = 2; } while (0)
 #define SET_CONSOLE_3270	do { console_mode = 3; } while (0)
 #define NSS_NAME_SIZE	8
 extern char kernel_nss_name[];
 #ifdef CONFIG_PFAULT
 extern int pfault_init(void);
 extern void pfault_fini(void);
 #else /* CONFIG_PFAULT */
 #define pfault_init()		({-1;})
 #define pfault_fini()		do { } while (0)
 #endif /* CONFIG_PFAULT */
 extern void cmma_init(void);
 extern void (*_machine_restart)(char *command);
 extern void (*_machine_halt)(void);
 extern void (*_machine_power_off)(void);
 #else /* __ASSEMBLY__ */
 #ifndef CONFIG_64BIT
 #define IPL_DEVICE        0x10404
 #define INITRD_START      0x1040C
 #define INITRD_SIZE       0x10414
 #define OLDMEM_BASE	  0x1041C
 #define OLDMEM_SIZE	  0x10424
 #else /* CONFIG_64BIT */
 #define IPL_DEVICE        0x10400
 #define INITRD_START      0x10408
 #define INITRD_SIZE       0x10410
 #define OLDMEM_BASE	  0x10418
 #define OLDMEM_SIZE	  0x10420
 #endif /* CONFIG_64BIT */
 #define COMMAND_LINE      0x10480
 #endif /* __ASSEMBLY__ */
 #endif /* __KERNEL__ */
 #endif /* _ASM_S390_SETUP_H */

arch/s390/include/asm/tlb.h

Diff comments View file @ 1ae1c1d

 #ifndef _S390_TLB_H
 #define _S390_TLB_H
 /*
  * TLB flushing on s390 is complicated. The following requirement
  * from the principles of operation is the most arduous:
  *
  * "A valid table entry must not be changed while it is attached
  * to any CPU and may be used for translation by that CPU except to
  * (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY,
  * or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page
  * table entry, or (3) make a change by means of a COMPARE AND SWAP
  * AND PURGE instruction that purges the TLB."
  *
  * The modification of a pte of an active mm struct therefore is
  * a two step process: i) invalidate the pte, ii) store the new pte.
  * This is true for the page protection bit as well.
  * The only possible optimization is to flush at the beginning of
  * a tlb_gather_mmu cycle if the mm_struct is currently not in use.
  *
  * Pages used for the page tables is a different story. FIXME: more
  */
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/swap.h>
 #include <asm/processor.h>
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 struct mmu_gather {
 	struct mm_struct *mm;
 	struct mmu_table_batch *batch;
 	unsigned int fullmm;
 };
 struct mmu_table_batch {
 	struct rcu_head		rcu;
 	unsigned int		nr;
 	void			*tables[0];
 };
 #define MAX_TABLE_BATCH		\
 	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
 extern void tlb_table_flush(struct mmu_gather *tlb);
 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 static inline void tlb_gather_mmu(struct mmu_gather *tlb,
 				  struct mm_struct *mm,
 				  unsigned int full_mm_flush)
 {
 	tlb->mm = mm;
 	tlb->fullmm = full_mm_flush;
 	tlb->batch = NULL;
 	if (tlb->fullmm)
 		__tlb_flush_mm(mm);
 }
 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
 {
 	tlb_table_flush(tlb);
 }
 static inline void tlb_finish_mmu(struct mmu_gather *tlb,
 				  unsigned long start, unsigned long end)
 {
 	tlb_table_flush(tlb);
 }
 /*
  * Release the page cache reference for a pte removed by
  * tlb_ptep_clear_flush. In both flush modes the tlb for a page cache page
  * has already been freed, so just do free_page_and_swap_cache.
  */
 static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
 	free_page_and_swap_cache(page);
 	return 1; /* avoid calling tlb_flush_mmu */
 }
 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
 {
 	free_page_and_swap_cache(page);
 }
 /*
  * pte_free_tlb frees a pte table and clears the CRSTE for the
  * page table from the tlb.
  */
 static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
 				unsigned long address)
 {
 	if (!tlb->fullmm)
 		return page_table_free_rcu(tlb, (unsigned long *) pte);
 	page_table_free(tlb->mm, (unsigned long *) pte);
 }
 /*
  * pmd_free_tlb frees a pmd table and clears the CRSTE for the
  * segment table entry from the tlb.
  * If the mm uses a two level page table the single pmd is freed
  * as the pgd. pmd_free_tlb checks the asce_limit against 2GB
  * to avoid the double free of the pmd in this case.
  */
 static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
 				unsigned long address)
 {
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 31))
 		return;
 	if (!tlb->fullmm)
 		return tlb_remove_table(tlb, pmd);
 	crst_table_free(tlb->mm, (unsigned long *) pmd);
 #endif
 }
 /*
  * pud_free_tlb frees a pud table and clears the CRSTE for the
  * region third table entry from the tlb.
  * If the mm uses a three level page table the single pud is freed
  * as the pgd. pud_free_tlb checks the asce_limit against 4TB
  * to avoid the double free of the pud in this case.
  */
 static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 				unsigned long address)
 {
 #ifdef CONFIG_64BIT
 	if (tlb->mm->context.asce_limit <= (1UL << 42))
 		return;
 	if (!tlb->fullmm)
 		return tlb_remove_table(tlb, pud);
 	crst_table_free(tlb->mm, (unsigned long *) pud);
 #endif
 }
 #define tlb_start_vma(tlb, vma)			do { } while (0)
 #define tlb_end_vma(tlb, vma)			do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)	do { } while (0)
+#define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr)	do { } while (0)
 #define tlb_migrate_finish(mm)			do { } while (0)
 #endif /* _S390_TLB_H */

arch/s390/kernel/early.c

Diff comments View file @ 1ae1c1d

 /*
  *    Copyright IBM Corp. 2007, 2009
  *    Author(s): Hongjie Yang <hongjie@us.ibm.com>,
  *		 Heiko Carstens <heiko.carstens@de.ibm.com>
  */
 #define KMSG_COMPONENT "setup"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 #include <linux/compiler.h>
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/ctype.h>
 #include <linux/ftrace.h>
 #include <linux/lockdep.h>
 #include <linux/module.h>
 #include <linux/pfn.h>
 #include <linux/uaccess.h>
 #include <linux/kernel.h>
 #include <asm/ebcdic.h>
 #include <asm/ipl.h>
 #include <asm/lowcore.h>
 #include <asm/processor.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sysinfo.h>
 #include <asm/cpcmd.h>
 #include <asm/sclp.h>
 #include <asm/facility.h>
 #include "entry.h"
 /*
  * Create a Kernel NSS if the SAVESYS= parameter is defined
  */
 #define DEFSYS_CMD_SIZE		128
 #define SAVESYS_CMD_SIZE	32
 char kernel_nss_name[NSS_NAME_SIZE + 1];
 static void __init setup_boot_command_line(void);
 /*
  * Get the TOD clock running.
  */
 static void __init reset_tod_clock(void)
 {
 	u64 time;
 	if (store_clock(&time) == 0)
 		return;
 	/* TOD clock not running. Set the clock to Unix Epoch. */
 	if (set_clock(TOD_UNIX_EPOCH) != 0 || store_clock(&time) != 0)
 		disabled_wait(0);
 	sched_clock_base_cc = TOD_UNIX_EPOCH;
 	S390_lowcore.last_update_clock = sched_clock_base_cc;
 }
 #ifdef CONFIG_SHARED_KERNEL
 int __init savesys_ipl_nss(char *cmd, const int cmdlen);
 asm(
 	"	.section .init.text,\"ax\",@progbits\n"
 	"	.align	4\n"
 	"	.type	savesys_ipl_nss, @function\n"
 	"savesys_ipl_nss:\n"
 #ifdef CONFIG_64BIT
 	"	stmg	6,15,48(15)\n"
 	"	lgr	14,3\n"
 	"	sam31\n"
 	"	diag	2,14,0x8\n"
 	"	sam64\n"
 	"	lgr	2,14\n"
 	"	lmg	6,15,48(15)\n"
 #else
 	"	stm	6,15,24(15)\n"
 	"	lr	14,3\n"
 	"	diag	2,14,0x8\n"
 	"	lr	2,14\n"
 	"	lm	6,15,24(15)\n"
 #endif
 	"	br	14\n"
 	"	.size	savesys_ipl_nss, .-savesys_ipl_nss\n"
 	"	.previous\n");
 static __initdata char upper_command_line[COMMAND_LINE_SIZE];
 static noinline __init void create_kernel_nss(void)
 {
 	unsigned int i, stext_pfn, eshared_pfn, end_pfn, min_size;
 #ifdef CONFIG_BLK_DEV_INITRD
 	unsigned int sinitrd_pfn, einitrd_pfn;
 #endif
 	int response;
 	int hlen;
 	size_t len;
 	char *savesys_ptr;
 	char defsys_cmd[DEFSYS_CMD_SIZE];
 	char savesys_cmd[SAVESYS_CMD_SIZE];
 	/* Do nothing if we are not running under VM */
 	if (!MACHINE_IS_VM)
 		return;
 	/* Convert COMMAND_LINE to upper case */
 	for (i = 0; i < strlen(boot_command_line); i++)
 		upper_command_line[i] = toupper(boot_command_line[i]);
 	savesys_ptr = strstr(upper_command_line, "SAVESYS=");
 	if (!savesys_ptr)
 		return;
 	savesys_ptr += 8;    /* Point to the beginning of the NSS name */
 	for (i = 0; i < NSS_NAME_SIZE; i++) {
 		if (savesys_ptr[i] == ' ' || savesys_ptr[i] == '\0')
 			break;
 		kernel_nss_name[i] = savesys_ptr[i];
 	}
 	stext_pfn = PFN_DOWN(__pa(&_stext));
 	eshared_pfn = PFN_DOWN(__pa(&_eshared));
 	end_pfn = PFN_UP(__pa(&_end));
 	min_size = end_pfn << 2;
 	hlen = snprintf(defsys_cmd, DEFSYS_CMD_SIZE,
 			"DEFSYS %s 00000-%.5X EW %.5X-%.5X SR %.5X-%.5X",
 			kernel_nss_name, stext_pfn - 1, stext_pfn,
 			eshared_pfn - 1, eshared_pfn, end_pfn);
 #ifdef CONFIG_BLK_DEV_INITRD
 	if (INITRD_START && INITRD_SIZE) {
 		sinitrd_pfn = PFN_DOWN(__pa(INITRD_START));
 		einitrd_pfn = PFN_UP(__pa(INITRD_START + INITRD_SIZE));
 		min_size = einitrd_pfn << 2;
 		hlen += snprintf(defsys_cmd + hlen, DEFSYS_CMD_SIZE - hlen,
 				 " EW %.5X-%.5X", sinitrd_pfn, einitrd_pfn);
 	}
 #endif
 	snprintf(defsys_cmd + hlen, DEFSYS_CMD_SIZE - hlen,
 		 " EW MINSIZE=%.7iK PARMREGS=0-13", min_size);
 	defsys_cmd[DEFSYS_CMD_SIZE - 1] = '\0';
 	snprintf(savesys_cmd, SAVESYS_CMD_SIZE, "SAVESYS %s \n IPL %s",
 		 kernel_nss_name, kernel_nss_name);
 	savesys_cmd[SAVESYS_CMD_SIZE - 1] = '\0';
 	__cpcmd(defsys_cmd, NULL, 0, &response);
 	if (response != 0) {
 		pr_err("Defining the Linux kernel NSS failed with rc=%d\n",
 			response);
 		kernel_nss_name[0] = '\0';
 		return;
 	}
 	len = strlen(savesys_cmd);
 	ASCEBC(savesys_cmd, len);
 	response = savesys_ipl_nss(savesys_cmd, len);
 	/* On success: response is equal to the command size,
 	 *	       max SAVESYS_CMD_SIZE
 	 * On error: response contains the numeric portion of cp error message.
 	 *	     for SAVESYS it will be >= 263
 	 *	     for missing privilege class, it will be 1
 	 */
 	if (response > SAVESYS_CMD_SIZE || response == 1) {
 		pr_err("Saving the Linux kernel NSS failed with rc=%d\n",
 			response);
 		kernel_nss_name[0] = '\0';
 		return;
 	}
 	/* re-initialize cputime accounting. */
 	sched_clock_base_cc = get_clock();
 	S390_lowcore.last_update_clock = sched_clock_base_cc;
 	S390_lowcore.last_update_timer = 0x7fffffffffffffffULL;
 	S390_lowcore.user_timer = 0;
 	S390_lowcore.system_timer = 0;
 	asm volatile("SPT 0(%0)" : : "a" (&S390_lowcore.last_update_timer));
 	/* re-setup boot command line with new ipl vm parms */
 	ipl_update_parameters();
 	setup_boot_command_line();
 	ipl_flags = IPL_NSS_VALID;
 }
 #else /* CONFIG_SHARED_KERNEL */
 static inline void create_kernel_nss(void) { }
 #endif /* CONFIG_SHARED_KERNEL */
 /*
  * Clear bss memory
  */
 static noinline __init void clear_bss_section(void)
 {
 	memset(__bss_start, 0, __bss_stop - __bss_start);
 }
 /*
  * Initialize storage key for kernel pages
  */
 static noinline __init void init_kernel_storage_key(void)
 {
 	unsigned long end_pfn, init_pfn;
 	end_pfn = PFN_UP(__pa(&_end));
 	for (init_pfn = 0 ; init_pfn < end_pfn; init_pfn++)
 		page_set_storage_key(init_pfn << PAGE_SHIFT,
 				     PAGE_DEFAULT_KEY, 0);
 }
 static __initdata char sysinfo_page[PAGE_SIZE] __aligned(PAGE_SIZE);
 static noinline __init void detect_machine_type(void)
 {
 	struct sysinfo_3_2_2 *vmms = (struct sysinfo_3_2_2 *)&sysinfo_page;
 	/* Check current-configuration-level */
 	if (stsi(NULL, 0, 0, 0) <= 2) {
 		S390_lowcore.machine_flags |= MACHINE_FLAG_LPAR;
 		return;
 	}
 	/* Get virtual-machine cpu information. */
 	if (stsi(vmms, 3, 2, 2) || !vmms->count)
 		return;
 	/* Running under KVM? If not we assume z/VM */
 	if (!memcmp(vmms->vm[0].cpi, "\xd2\xe5\xd4", 3))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_KVM;
 	else
 		S390_lowcore.machine_flags |= MACHINE_FLAG_VM;
 }
 static __init void setup_topology(void)
 {
 #ifdef CONFIG_64BIT
 	int max_mnest;
 	if (!test_facility(11))
 		return;
 	S390_lowcore.machine_flags |= MACHINE_FLAG_TOPOLOGY;
 	for (max_mnest = 6; max_mnest > 1; max_mnest--) {
 		if (stsi(&sysinfo_page, 15, 1, max_mnest) == 0)
 			break;
 	}
 	topology_max_mnest = max_mnest;
 #endif
 }
 static void early_pgm_check_handler(void)
 {
 	const struct exception_table_entry *fixup;
 	unsigned long addr;
 	addr = S390_lowcore.program_old_psw.addr;
 	fixup = search_exception_tables(addr & PSW_ADDR_INSN);
 	if (!fixup)
 		disabled_wait(0);
 	S390_lowcore.program_old_psw.addr = extable_fixup(fixup)|PSW_ADDR_AMODE;
 }
 static noinline __init void setup_lowcore_early(void)
 {
 	psw_t psw;
 	psw.mask = PSW_MASK_BASE | PSW_DEFAULT_KEY | PSW_MASK_EA | PSW_MASK_BA;
 	psw.addr = PSW_ADDR_AMODE | (unsigned long) s390_base_ext_handler;
 	S390_lowcore.external_new_psw = psw;
 	psw.addr = PSW_ADDR_AMODE | (unsigned long) s390_base_pgm_handler;
 	S390_lowcore.program_new_psw = psw;
 	s390_base_pgm_handler_fn = early_pgm_check_handler;
 }
 static noinline __init void setup_facility_list(void)
 {
 	stfle(S390_lowcore.stfle_fac_list,
 	      ARRAY_SIZE(S390_lowcore.stfle_fac_list));
 }
 static noinline __init void setup_hpage(void)
 {
 	if (!test_facility(2) || !test_facility(8))
 		return;
 	S390_lowcore.machine_flags |= MACHINE_FLAG_HPAGE;
 	__ctl_set_bit(0, 23);
 }
 static __init void detect_mvpg(void)
 {
 #ifndef CONFIG_64BIT
 	int rc;
 	asm volatile(
 		"	la	0,0\n"
 		"	mvpg	%2,%2\n"
 		"0:	la	%0,0\n"
 		"1:\n"
 		EX_TABLE(0b,1b)
 		: "=d" (rc) : "0" (-EOPNOTSUPP), "a" (0) : "memory", "cc", "0");
 	if (!rc)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_MVPG;
 #endif
 }
 static __init void detect_ieee(void)
 {
 #ifndef CONFIG_64BIT
 	int rc, tmp;
 	asm volatile(
 		"	efpc	%1,0\n"
 		"0:	la	%0,0\n"
 		"1:\n"
 		EX_TABLE(0b,1b)
 		: "=d" (rc), "=d" (tmp): "0" (-EOPNOTSUPP) : "cc");
 	if (!rc)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_IEEE;
 #endif
 }
 static __init void detect_csp(void)
 {
 #ifndef CONFIG_64BIT
 	int rc;
 	asm volatile(
 		"	la	0,0\n"
 		"	la	1,0\n"
 		"	la	2,4\n"
 		"	csp	0,2\n"
 		"0:	la	%0,0\n"
 		"1:\n"
 		EX_TABLE(0b,1b)
 		: "=d" (rc) : "0" (-EOPNOTSUPP) : "cc", "0", "1", "2");
 	if (!rc)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_CSP;
 #endif
 }
 static __init void detect_diag9c(void)
 {
 	unsigned int cpu_address;
 	int rc;
 	cpu_address = stap();
 	asm volatile(
 		"	diag	%2,0,0x9c\n"
 		"0:	la	%0,0\n"
 		"1:\n"
 		EX_TABLE(0b,1b)
 		: "=d" (rc) : "0" (-EOPNOTSUPP), "d" (cpu_address) : "cc");
 	if (!rc)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_DIAG9C;
 }
 static __init void detect_diag44(void)
 {
 #ifdef CONFIG_64BIT
 	int rc;
 	asm volatile(
 		"	diag	0,0,0x44\n"
 		"0:	la	%0,0\n"
 		"1:\n"
 		EX_TABLE(0b,1b)
 		: "=d" (rc) : "0" (-EOPNOTSUPP) : "cc");
 	if (!rc)
 		S390_lowcore.machine_flags |= MACHINE_FLAG_DIAG44;
 #endif
 }
 static __init void detect_machine_facilities(void)
 {
 #ifdef CONFIG_64BIT
 	if (test_facility(3))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_IDTE;
 	if (test_facility(8))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_PFMF;
 	if (test_facility(27))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_MVCOS;
 	if (test_facility(40))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_SPP;
 	if (test_facility(50) && test_facility(73))
 		S390_lowcore.machine_flags |= MACHINE_FLAG_TE;
+	if (test_facility(66))
+		S390_lowcore.machine_flags |= MACHINE_FLAG_RRBM;
 #endif
 }
 static __init void rescue_initrd(void)
 {
 #ifdef CONFIG_BLK_DEV_INITRD
 	unsigned long min_initrd_addr = (unsigned long) _end + (4UL << 20);
 	/*
 	 * Just like in case of IPL from VM reader we make sure there is a
 	 * gap of 4MB between end of kernel and start of initrd.
 	 * That way we can also be sure that saving an NSS will succeed,
 	 * which however only requires different segments.
 	 */
 	if (!INITRD_START || !INITRD_SIZE)
 		return;
 	if (INITRD_START >= min_initrd_addr)
 		return;
 	memmove((void *) min_initrd_addr, (void *) INITRD_START, INITRD_SIZE);
 	INITRD_START = min_initrd_addr;
 #endif
 }
 /* Set up boot command line */
 static void __init append_to_cmdline(size_t (*ipl_data)(char *, size_t))
 {
 	char *parm, *delim;
 	size_t rc, len;
 	len = strlen(boot_command_line);
 	delim = boot_command_line + len;	/* '\0' character position */
 	parm  = boot_command_line + len + 1;	/* append right after '\0' */
 	rc = ipl_data(parm, COMMAND_LINE_SIZE - len - 1);
 	if (rc) {
 		if (*parm == '=')
 			memmove(boot_command_line, parm + 1, rc);
 		else
 			*delim = ' ';		/* replace '\0' with space */
 	}
 }
 static inline int has_ebcdic_char(const char *str)
 {
 	int i;
 	for (i = 0; str[i]; i++)
 		if (str[i] & 0x80)
 			return 1;
 	return 0;
 }
 static void __init setup_boot_command_line(void)
 {
 	COMMAND_LINE[ARCH_COMMAND_LINE_SIZE - 1] = 0;
 	/* convert arch command line to ascii if necessary */
 	if (has_ebcdic_char(COMMAND_LINE))
 		EBCASC(COMMAND_LINE, ARCH_COMMAND_LINE_SIZE);
 	/* copy arch command line */
 	strlcpy(boot_command_line, strstrip(COMMAND_LINE),
 		ARCH_COMMAND_LINE_SIZE);
 	/* append IPL PARM data to the boot command line */
 	if (MACHINE_IS_VM)
 		append_to_cmdline(append_ipl_vmparm);
 	append_to_cmdline(append_ipl_scpdata);
 }
 /*
  * Save ipl parameters, clear bss memory, initialize storage keys
  * and create a kernel NSS at startup if the SAVESYS= parm is defined
  */
 void __init startup_init(void)
 {
 	reset_tod_clock();
 	ipl_save_parameters();
 	rescue_initrd();
 	clear_bss_section();
 	init_kernel_storage_key();
 	lockdep_init();
 	lockdep_off();
 	setup_lowcore_early();
 	setup_facility_list();
 	detect_machine_type();
 	ipl_update_parameters();
 	setup_boot_command_line();
 	create_kernel_nss();
 	detect_mvpg();
 	detect_ieee();
 	detect_csp();
 	detect_diag9c();
 	detect_diag44();
 	detect_machine_facilities();
 	setup_hpage();
 	setup_topology();
 	sclp_facilities_detect();
 	detect_memory_layout(memory_chunk);
 #ifdef CONFIG_DYNAMIC_FTRACE
 	S390_lowcore.ftrace_func = (unsigned long)ftrace_caller;
 #endif
 	lockdep_on();
 }

arch/s390/mm/pgtable.c

Diff comments View file @ 1ae1c1d

 /*
  *    Copyright IBM Corp. 2007, 2011
  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
  */
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/gfp.h>
 #include <linux/mm.h>
 #include <linux/swap.h>
 #include <linux/smp.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
 #include <linux/spinlock.h>
 #include <linux/module.h>
 #include <linux/quicklist.h>
 #include <linux/rcupdate.h>
 #include <linux/slab.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
 #ifndef CONFIG_64BIT
 #define ALLOC_ORDER	1
 #define FRAG_MASK	0x0f
 #else
 #define ALLOC_ORDER	2
 #define FRAG_MASK	0x03
 #endif
 unsigned long *crst_table_alloc(struct mm_struct *mm)
 {
 	struct page *page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
 	if (!page)
 		return NULL;
 	return (unsigned long *) page_to_phys(page);
 }
 void crst_table_free(struct mm_struct *mm, unsigned long *table)
 {
 	free_pages((unsigned long) table, ALLOC_ORDER);
 }
 #ifdef CONFIG_64BIT
 int crst_table_upgrade(struct mm_struct *mm, unsigned long limit)
 {
 	unsigned long *table, *pgd;
 	unsigned long entry;
 	BUG_ON(limit > (1UL << 53));
 repeat:
 	table = crst_table_alloc(mm);
 	if (!table)
 		return -ENOMEM;
 	spin_lock_bh(&mm->page_table_lock);
 	if (mm->context.asce_limit < limit) {
 		pgd = (unsigned long *) mm->pgd;
 		if (mm->context.asce_limit <= (1UL << 31)) {
 			entry = _REGION3_ENTRY_EMPTY;
 			mm->context.asce_limit = 1UL << 42;
 			mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 						_ASCE_USER_BITS |
 						_ASCE_TYPE_REGION3;
 		} else {
 			entry = _REGION2_ENTRY_EMPTY;
 			mm->context.asce_limit = 1UL << 53;
 			mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 						_ASCE_USER_BITS |
 						_ASCE_TYPE_REGION2;
 		}
 		crst_table_init(table, entry);
 		pgd_populate(mm, (pgd_t *) table, (pud_t *) pgd);
 		mm->pgd = (pgd_t *) table;
 		mm->task_size = mm->context.asce_limit;
 		table = NULL;
 	}
 	spin_unlock_bh(&mm->page_table_lock);
 	if (table)
 		crst_table_free(mm, table);
 	if (mm->context.asce_limit < limit)
 		goto repeat;
 	return 0;
 }
 void crst_table_downgrade(struct mm_struct *mm, unsigned long limit)
 {
 	pgd_t *pgd;
 	while (mm->context.asce_limit > limit) {
 		pgd = mm->pgd;
 		switch (pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) {
 		case _REGION_ENTRY_TYPE_R2:
 			mm->context.asce_limit = 1UL << 42;
 			mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 						_ASCE_USER_BITS |
 						_ASCE_TYPE_REGION3;
 			break;
 		case _REGION_ENTRY_TYPE_R3:
 			mm->context.asce_limit = 1UL << 31;
 			mm->context.asce_bits = _ASCE_TABLE_LENGTH |
 						_ASCE_USER_BITS |
 						_ASCE_TYPE_SEGMENT;
 			break;
 		default:
 			BUG();
 		}
 		mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN);
 		mm->task_size = mm->context.asce_limit;
 		crst_table_free(mm, (unsigned long *) pgd);
 	}
 }
 #endif
 #ifdef CONFIG_PGSTE
 /**
  * gmap_alloc - allocate a guest address space
  * @mm: pointer to the parent mm_struct
  *
  * Returns a guest address space structure.
  */
 struct gmap *gmap_alloc(struct mm_struct *mm)
 {
 	struct gmap *gmap;
 	struct page *page;
 	unsigned long *table;
 	gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL);
 	if (!gmap)
 		goto out;
 	INIT_LIST_HEAD(&gmap->crst_list);
 	gmap->mm = mm;
 	page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
 	if (!page)
 		goto out_free;
 	list_add(&page->lru, &gmap->crst_list);
 	table = (unsigned long *) page_to_phys(page);
 	crst_table_init(table, _REGION1_ENTRY_EMPTY);
 	gmap->table = table;
 	gmap->asce = _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH |
 		     _ASCE_USER_BITS | __pa(table);
 	list_add(&gmap->list, &mm->context.gmap_list);
 	return gmap;
 out_free:
 	kfree(gmap);
 out:
 	return NULL;
 }
 EXPORT_SYMBOL_GPL(gmap_alloc);
 static int gmap_unlink_segment(struct gmap *gmap, unsigned long *table)
 {
 	struct gmap_pgtable *mp;
 	struct gmap_rmap *rmap;
 	struct page *page;
 	if (*table & _SEGMENT_ENTRY_INV)
 		return 0;
 	page = pfn_to_page(*table >> PAGE_SHIFT);
 	mp = (struct gmap_pgtable *) page->index;
 	list_for_each_entry(rmap, &mp->mapper, list) {
 		if (rmap->entry != table)
 			continue;
 		list_del(&rmap->list);
 		kfree(rmap);
 		break;
 	}
 	*table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
 	return 1;
 }
 static void gmap_flush_tlb(struct gmap *gmap)
 {
 	if (MACHINE_HAS_IDTE)
 		__tlb_flush_idte((unsigned long) gmap->table |
 				 _ASCE_TYPE_REGION1);
 	else
 		__tlb_flush_global();
 }
 /**
  * gmap_free - free a guest address space
  * @gmap: pointer to the guest address space structure
  */
 void gmap_free(struct gmap *gmap)
 {
 	struct page *page, *next;
 	unsigned long *table;
 	int i;
 	/* Flush tlb. */
 	if (MACHINE_HAS_IDTE)
 		__tlb_flush_idte((unsigned long) gmap->table |
 				 _ASCE_TYPE_REGION1);
 	else
 		__tlb_flush_global();
 	/* Free all segment & region tables. */
 	down_read(&gmap->mm->mmap_sem);
 	spin_lock(&gmap->mm->page_table_lock);
 	list_for_each_entry_safe(page, next, &gmap->crst_list, lru) {
 		table = (unsigned long *) page_to_phys(page);
 		if ((*table & _REGION_ENTRY_TYPE_MASK) == 0)
 			/* Remove gmap rmap structures for segment table. */
 			for (i = 0; i < PTRS_PER_PMD; i++, table++)
 				gmap_unlink_segment(gmap, table);
 		__free_pages(page, ALLOC_ORDER);
 	}
 	spin_unlock(&gmap->mm->page_table_lock);
 	up_read(&gmap->mm->mmap_sem);
 	list_del(&gmap->list);
 	kfree(gmap);
 }
 EXPORT_SYMBOL_GPL(gmap_free);
 /**
  * gmap_enable - switch primary space to the guest address space
  * @gmap: pointer to the guest address space structure
  */
 void gmap_enable(struct gmap *gmap)
 {
 	S390_lowcore.gmap = (unsigned long) gmap;
 }
 EXPORT_SYMBOL_GPL(gmap_enable);
 /**
  * gmap_disable - switch back to the standard primary address space
  * @gmap: pointer to the guest address space structure
  */
 void gmap_disable(struct gmap *gmap)
 {
 	S390_lowcore.gmap = 0UL;
 }
 EXPORT_SYMBOL_GPL(gmap_disable);
 /*
  * gmap_alloc_table is assumed to be called with mmap_sem held
  */
 static int gmap_alloc_table(struct gmap *gmap,
 			       unsigned long *table, unsigned long init)
 {
 	struct page *page;
 	unsigned long *new;
 	/* since we dont free the gmap table until gmap_free we can unlock */
 	spin_unlock(&gmap->mm->page_table_lock);
 	page = alloc_pages(GFP_KERNEL, ALLOC_ORDER);
 	spin_lock(&gmap->mm->page_table_lock);
 	if (!page)
 		return -ENOMEM;
 	new = (unsigned long *) page_to_phys(page);
 	crst_table_init(new, init);
 	if (*table & _REGION_ENTRY_INV) {
 		list_add(&page->lru, &gmap->crst_list);
 		*table = (unsigned long) new | _REGION_ENTRY_LENGTH |
 			(*table & _REGION_ENTRY_TYPE_MASK);
 	} else
 		__free_pages(page, ALLOC_ORDER);
 	return 0;
 }
 /**
  * gmap_unmap_segment - unmap segment from the guest address space
  * @gmap: pointer to the guest address space structure
  * @addr: address in the guest address space
  * @len: length of the memory area to unmap
  *
  * Returns 0 if the unmap succeded, -EINVAL if not.
  */
 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
 {
 	unsigned long *table;
 	unsigned long off;
 	int flush;
 	if ((to | len) & (PMD_SIZE - 1))
 		return -EINVAL;
 	if (len == 0 || to + len < to)
 		return -EINVAL;
 	flush = 0;
 	down_read(&gmap->mm->mmap_sem);
 	spin_lock(&gmap->mm->page_table_lock);
 	for (off = 0; off < len; off += PMD_SIZE) {
 		/* Walk the guest addr space page table */
 		table = gmap->table + (((to + off) >> 53) & 0x7ff);
 		if (*table & _REGION_ENTRY_INV)
 			goto out;
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 42) & 0x7ff);
 		if (*table & _REGION_ENTRY_INV)
 			goto out;
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 31) & 0x7ff);
 		if (*table & _REGION_ENTRY_INV)
 			goto out;
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 20) & 0x7ff);
 		/* Clear segment table entry in guest address space. */
 		flush |= gmap_unlink_segment(gmap, table);
 		*table = _SEGMENT_ENTRY_INV;
 	}
 out:
 	spin_unlock(&gmap->mm->page_table_lock);
 	up_read(&gmap->mm->mmap_sem);
 	if (flush)
 		gmap_flush_tlb(gmap);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(gmap_unmap_segment);
 /**
  * gmap_mmap_segment - map a segment to the guest address space
  * @gmap: pointer to the guest address space structure
  * @from: source address in the parent address space
  * @to: target address in the guest address space
  *
  * Returns 0 if the mmap succeded, -EINVAL or -ENOMEM if not.
  */
 int gmap_map_segment(struct gmap *gmap, unsigned long from,
 		     unsigned long to, unsigned long len)
 {
 	unsigned long *table;
 	unsigned long off;
 	int flush;
 	if ((from | to | len) & (PMD_SIZE - 1))
 		return -EINVAL;
 	if (len == 0 || from + len > PGDIR_SIZE ||
 	    from + len < from || to + len < to)
 		return -EINVAL;
 	flush = 0;
 	down_read(&gmap->mm->mmap_sem);
 	spin_lock(&gmap->mm->page_table_lock);
 	for (off = 0; off < len; off += PMD_SIZE) {
 		/* Walk the gmap address space page table */
 		table = gmap->table + (((to + off) >> 53) & 0x7ff);
 		if ((*table & _REGION_ENTRY_INV) &&
 		    gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY))
 			goto out_unmap;
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 42) & 0x7ff);
 		if ((*table & _REGION_ENTRY_INV) &&
 		    gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY))
 			goto out_unmap;
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 31) & 0x7ff);
 		if ((*table & _REGION_ENTRY_INV) &&
 		    gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY))
 			goto out_unmap;
 		table = (unsigned long *) (*table & _REGION_ENTRY_ORIGIN);
 		table = table + (((to + off) >> 20) & 0x7ff);
 		/* Store 'from' address in an invalid segment table entry. */
 		flush |= gmap_unlink_segment(gmap, table);
 		*table = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | (from + off);
 	}
 	spin_unlock(&gmap->mm->page_table_lock);
 	up_read(&gmap->mm->mmap_sem);
 	if (flush)
 		gmap_flush_tlb(gmap);
 	return 0;
 out_unmap:
 	spin_unlock(&gmap->mm->page_table_lock);
 	up_read(&gmap->mm->mmap_sem);
 	gmap_unmap_segment(gmap, to, len);
 	return -ENOMEM;
 }
 EXPORT_SYMBOL_GPL(gmap_map_segment);
 /*
  * this function is assumed to be called with mmap_sem held
  */
 unsigned long __gmap_fault(unsigned long address, struct gmap *gmap)
 {
 	unsigned long *table, vmaddr, segment;
 	struct mm_struct *mm;
 	struct gmap_pgtable *mp;
 	struct gmap_rmap *rmap;
 	struct vm_area_struct *vma;
 	struct page *page;
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
 	current->thread.gmap_addr = address;
 	mm = gmap->mm;
 	/* Walk the gmap address space page table */
 	table = gmap->table + ((address >> 53) & 0x7ff);
 	if (unlikely(*table & _REGION_ENTRY_INV))
 		return -EFAULT;
 	table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 	table = table + ((address >> 42) & 0x7ff);
 	if (unlikely(*table & _REGION_ENTRY_INV))
 		return -EFAULT;
 	table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 	table = table + ((address >> 31) & 0x7ff);
 	if (unlikely(*table & _REGION_ENTRY_INV))
 		return -EFAULT;
 	table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 	table = table + ((address >> 20) & 0x7ff);
 	/* Convert the gmap address to an mm address. */
 	segment = *table;
 	if (likely(!(segment & _SEGMENT_ENTRY_INV))) {
 		page = pfn_to_page(segment >> PAGE_SHIFT);
 		mp = (struct gmap_pgtable *) page->index;
 		return mp->vmaddr | (address & ~PMD_MASK);
 	} else if (segment & _SEGMENT_ENTRY_RO) {
 		vmaddr = segment & _SEGMENT_ENTRY_ORIGIN;
 		vma = find_vma(mm, vmaddr);
 		if (!vma || vma->vm_start > vmaddr)
 			return -EFAULT;
 		/* Walk the parent mm page table */
 		pgd = pgd_offset(mm, vmaddr);
 		pud = pud_alloc(mm, pgd, vmaddr);
 		if (!pud)
 			return -ENOMEM;
 		pmd = pmd_alloc(mm, pud, vmaddr);
 		if (!pmd)
 			return -ENOMEM;
 		if (!pmd_present(*pmd) &&
 		    __pte_alloc(mm, vma, pmd, vmaddr))
 			return -ENOMEM;
 		/* pmd now points to a valid segment table entry. */
 		rmap = kmalloc(sizeof(*rmap), GFP_KERNEL|__GFP_REPEAT);
 		if (!rmap)
 			return -ENOMEM;
 		/* Link gmap segment table entry location to page table. */
 		page = pmd_page(*pmd);
 		mp = (struct gmap_pgtable *) page->index;
 		rmap->entry = table;
 		spin_lock(&mm->page_table_lock);
 		list_add(&rmap->list, &mp->mapper);
 		spin_unlock(&mm->page_table_lock);
 		/* Set gmap segment table entry to page table. */
 		*table = pmd_val(*pmd) & PAGE_MASK;
 		return vmaddr | (address & ~PMD_MASK);
 	}
 	return -EFAULT;
 }
 unsigned long gmap_fault(unsigned long address, struct gmap *gmap)
 {
 	unsigned long rc;
 	down_read(&gmap->mm->mmap_sem);
 	rc = __gmap_fault(address, gmap);
 	up_read(&gmap->mm->mmap_sem);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(gmap_fault);
 void gmap_discard(unsigned long from, unsigned long to, struct gmap *gmap)
 {
 	unsigned long *table, address, size;
 	struct vm_area_struct *vma;
 	struct gmap_pgtable *mp;
 	struct page *page;
 	down_read(&gmap->mm->mmap_sem);
 	address = from;
 	while (address < to) {
 		/* Walk the gmap address space page table */
 		table = gmap->table + ((address >> 53) & 0x7ff);
 		if (unlikely(*table & _REGION_ENTRY_INV)) {
 			address = (address + PMD_SIZE) & PMD_MASK;
 			continue;
 		}
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + ((address >> 42) & 0x7ff);
 		if (unlikely(*table & _REGION_ENTRY_INV)) {
 			address = (address + PMD_SIZE) & PMD_MASK;
 			continue;
 		}
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + ((address >> 31) & 0x7ff);
 		if (unlikely(*table & _REGION_ENTRY_INV)) {
 			address = (address + PMD_SIZE) & PMD_MASK;
 			continue;
 		}
 		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
 		table = table + ((address >> 20) & 0x7ff);
 		if (unlikely(*table & _SEGMENT_ENTRY_INV)) {
 			address = (address + PMD_SIZE) & PMD_MASK;
 			continue;
 		}
 		page = pfn_to_page(*table >> PAGE_SHIFT);
 		mp = (struct gmap_pgtable *) page->index;
 		vma = find_vma(gmap->mm, mp->vmaddr);
 		size = min(to - address, PMD_SIZE - (address & ~PMD_MASK));
 		zap_page_range(vma, mp->vmaddr | (address & ~PMD_MASK),
 			       size, NULL);
 		address = (address + PMD_SIZE) & PMD_MASK;
 	}
 	up_read(&gmap->mm->mmap_sem);
 }
 EXPORT_SYMBOL_GPL(gmap_discard);
 void gmap_unmap_notifier(struct mm_struct *mm, unsigned long *table)
 {
 	struct gmap_rmap *rmap, *next;
 	struct gmap_pgtable *mp;
 	struct page *page;
 	int flush;
 	flush = 0;
 	spin_lock(&mm->page_table_lock);
 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 	mp = (struct gmap_pgtable *) page->index;
 	list_for_each_entry_safe(rmap, next, &mp->mapper, list) {
 		*rmap->entry =
 			_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO | mp->vmaddr;
 		list_del(&rmap->list);
 		kfree(rmap);
 		flush = 1;
 	}
 	spin_unlock(&mm->page_table_lock);
 	if (flush)
 		__tlb_flush_global();
 }
 static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
 						    unsigned long vmaddr)
 {
 	struct page *page;
 	unsigned long *table;
 	struct gmap_pgtable *mp;
 	page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
 	if (!page)
 		return NULL;
 	mp = kmalloc(sizeof(*mp), GFP_KERNEL|__GFP_REPEAT);
 	if (!mp) {
 		__free_page(page);
 		return NULL;
 	}
 	pgtable_page_ctor(page);
 	mp->vmaddr = vmaddr & PMD_MASK;
 	INIT_LIST_HEAD(&mp->mapper);
 	page->index = (unsigned long) mp;
 	atomic_set(&page->_mapcount, 3);
 	table = (unsigned long *) page_to_phys(page);
 	clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE/2);
 	clear_table(table + PTRS_PER_PTE, 0, PAGE_SIZE/2);
 	return table;
 }
 static inline void page_table_free_pgste(unsigned long *table)
 {
 	struct page *page;
 	struct gmap_pgtable *mp;
 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 	mp = (struct gmap_pgtable *) page->index;
 	BUG_ON(!list_empty(&mp->mapper));
 	pgtable_page_dtor(page);
 	atomic_set(&page->_mapcount, -1);
 	kfree(mp);
 	__free_page(page);
 }
 #else /* CONFIG_PGSTE */
 static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
 						    unsigned long vmaddr)
 {
 	return NULL;
 }
 static inline void page_table_free_pgste(unsigned long *table)
 {
 }
 static inline void gmap_unmap_notifier(struct mm_struct *mm,
 					  unsigned long *table)
 {
 }
 #endif /* CONFIG_PGSTE */
 static inline unsigned int atomic_xor_bits(atomic_t *v, unsigned int bits)
 {
 	unsigned int old, new;
 	do {
 		old = atomic_read(v);
 		new = old ^ bits;
 	} while (atomic_cmpxchg(v, old, new) != old);
 	return new;
 }
 /*
  * page table entry allocation/free routines.
  */
 unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr)
 {
 	unsigned long *uninitialized_var(table);
 	struct page *uninitialized_var(page);
 	unsigned int mask, bit;
 	if (mm_has_pgste(mm))
 		return page_table_alloc_pgste(mm, vmaddr);
 	/* Allocate fragments of a 4K page as 1K/2K page table */
 	spin_lock_bh(&mm->context.list_lock);
 	mask = FRAG_MASK;
 	if (!list_empty(&mm->context.pgtable_list)) {
 		page = list_first_entry(&mm->context.pgtable_list,
 					struct page, lru);
 		table = (unsigned long *) page_to_phys(page);
 		mask = atomic_read(&page->_mapcount);
 		mask = mask | (mask >> 4);
 	}
 	if ((mask & FRAG_MASK) == FRAG_MASK) {
 		spin_unlock_bh(&mm->context.list_lock);
 		page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
 		if (!page)
 			return NULL;
 		pgtable_page_ctor(page);
 		atomic_set(&page->_mapcount, 1);
 		table = (unsigned long *) page_to_phys(page);
 		clear_table(table, _PAGE_TYPE_EMPTY, PAGE_SIZE);
 		spin_lock_bh(&mm->context.list_lock);
 		list_add(&page->lru, &mm->context.pgtable_list);
 	} else {
 		for (bit = 1; mask & bit; bit <<= 1)
 			table += PTRS_PER_PTE;
 		mask = atomic_xor_bits(&page->_mapcount, bit);
 		if ((mask & FRAG_MASK) == FRAG_MASK)
 			list_del(&page->lru);
 	}
 	spin_unlock_bh(&mm->context.list_lock);
 	return table;
 }
 void page_table_free(struct mm_struct *mm, unsigned long *table)
 {
 	struct page *page;
 	unsigned int bit, mask;
 	if (mm_has_pgste(mm)) {
 		gmap_unmap_notifier(mm, table);
 		return page_table_free_pgste(table);
 	}
 	/* Free 1K/2K page table fragment of a 4K page */
 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 	bit = 1 << ((__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)));
 	spin_lock_bh(&mm->context.list_lock);
 	if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
 		list_del(&page->lru);
 	mask = atomic_xor_bits(&page->_mapcount, bit);
 	if (mask & FRAG_MASK)
 		list_add(&page->lru, &mm->context.pgtable_list);
 	spin_unlock_bh(&mm->context.list_lock);
 	if (mask == 0) {
 		pgtable_page_dtor(page);
 		atomic_set(&page->_mapcount, -1);
 		__free_page(page);
 	}
 }
 static void __page_table_free_rcu(void *table, unsigned bit)
 {
 	struct page *page;
 	if (bit == FRAG_MASK)
 		return page_table_free_pgste(table);
 	/* Free 1K/2K page table fragment of a 4K page */
 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 	if (atomic_xor_bits(&page->_mapcount, bit) == 0) {
 		pgtable_page_dtor(page);
 		atomic_set(&page->_mapcount, -1);
 		__free_page(page);
 	}
 }
 void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table)
 {
 	struct mm_struct *mm;
 	struct page *page;
 	unsigned int bit, mask;
 	mm = tlb->mm;
 	if (mm_has_pgste(mm)) {
 		gmap_unmap_notifier(mm, table);
 		table = (unsigned long *) (__pa(table) | FRAG_MASK);
 		tlb_remove_table(tlb, table);
 		return;
 	}
 	bit = 1 << ((__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)));
 	page = pfn_to_page(__pa(table) >> PAGE_SHIFT);
 	spin_lock_bh(&mm->context.list_lock);
 	if ((atomic_read(&page->_mapcount) & FRAG_MASK) != FRAG_MASK)
 		list_del(&page->lru);
 	mask = atomic_xor_bits(&page->_mapcount, bit | (bit << 4));
 	if (mask & FRAG_MASK)
 		list_add_tail(&page->lru, &mm->context.pgtable_list);
 	spin_unlock_bh(&mm->context.list_lock);
 	table = (unsigned long *) (__pa(table) | (bit << 4));
 	tlb_remove_table(tlb, table);
 }
 void __tlb_remove_table(void *_table)
 {
 	const unsigned long mask = (FRAG_MASK << 4) | FRAG_MASK;
 	void *table = (void *)((unsigned long) _table & ~mask);
 	unsigned type = (unsigned long) _table & mask;
 	if (type)
 		__page_table_free_rcu(table, type);
 	else
 		free_pages((unsigned long) table, ALLOC_ORDER);
 }
 static void tlb_remove_table_smp_sync(void *arg)
 {
 	/* Simply deliver the interrupt */
 }
 static void tlb_remove_table_one(void *table)
 {
 	/*
 	 * This isn't an RCU grace period and hence the page-tables cannot be
 	 * assumed to be actually RCU-freed.
 	 *
 	 * It is however sufficient for software page-table walkers that rely
 	 * on IRQ disabling. See the comment near struct mmu_table_batch.
 	 */
 	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
 	__tlb_remove_table(table);
 }
 static void tlb_remove_table_rcu(struct rcu_head *head)
 {
 	struct mmu_table_batch *batch;
 	int i;
 	batch = container_of(head, struct mmu_table_batch, rcu);
 	for (i = 0; i < batch->nr; i++)
 		__tlb_remove_table(batch->tables[i]);
 	free_page((unsigned long)batch);
 }
 void tlb_table_flush(struct mmu_gather *tlb)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 	if (*batch) {
 		__tlb_flush_mm(tlb->mm);
 		call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
 		*batch = NULL;
 	}
 }
 void tlb_remove_table(struct mmu_gather *tlb, void *table)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 	if (*batch == NULL) {
 		*batch = (struct mmu_table_batch *)
 			__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
 		if (*batch == NULL) {
 			__tlb_flush_mm(tlb->mm);
 			tlb_remove_table_one(table);
 			return;
 		}
 		(*batch)->nr = 0;
 	}
 	(*batch)->tables[(*batch)->nr++] = table;
 	if ((*batch)->nr == MAX_TABLE_BATCH)
 		tlb_table_flush(tlb);
 }
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void thp_split_vma(struct vm_area_struct *vma)
 {
 	unsigned long addr;
 	struct page *page;
 	for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
 		page = follow_page(vma, addr, FOLL_SPLIT);
 	}
 }
 void thp_split_mm(struct mm_struct *mm)
 {
 	struct vm_area_struct *vma = mm->mmap;
 	while (vma != NULL) {
 		thp_split_vma(vma);
 		vma->vm_flags &= ~VM_HUGEPAGE;
 		vma->vm_flags |= VM_NOHUGEPAGE;
 		vma = vma->vm_next;
 	}
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 /*
  * switch on pgstes for its userspace process (for kvm)
  */
 int s390_enable_sie(void)
 {
 	struct task_struct *tsk = current;
 	struct mm_struct *mm, *old_mm;
 	/* Do we have switched amode? If no, we cannot do sie */
 	if (s390_user_mode == HOME_SPACE_MODE)
 		return -EINVAL;
 	/* Do we have pgstes? if yes, we are done */
 	if (mm_has_pgste(tsk->mm))
 		return 0;
 	/* lets check if we are allowed to replace the mm */
 	task_lock(tsk);
 	if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
 #ifdef CONFIG_AIO
 	    !hlist_empty(&tsk->mm->ioctx_list) ||
 #endif
 	    tsk->mm != tsk->active_mm) {
 		task_unlock(tsk);
 		return -EINVAL;
 	}
 	task_unlock(tsk);
 	/* we copy the mm and let dup_mm create the page tables with_pgstes */
 	tsk->mm->context.alloc_pgste = 1;
 	/* make sure that both mms have a correct rss state */
 	sync_mm_rss(tsk->mm);
 	mm = dup_mm(tsk);
 	tsk->mm->context.alloc_pgste = 0;
 	if (!mm)
 		return -ENOMEM;
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	/* split thp mappings and disable thp for future mappings */
 	thp_split_mm(mm);
 	mm->def_flags |= VM_NOHUGEPAGE;
 #endif
 	/* Now lets check again if something happened */
 	task_lock(tsk);
 	if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
 #ifdef CONFIG_AIO
 	    !hlist_empty(&tsk->mm->ioctx_list) ||
 #endif
 	    tsk->mm != tsk->active_mm) {
 		mmput(mm);
 		task_unlock(tsk);
 		return -EINVAL;
 	}
 	/* ok, we are alone. No ptrace, no threads, etc. */
 	old_mm = tsk->mm;
 	tsk->mm = tsk->active_mm = mm;
 	preempt_disable();
 	update_mm(mm, tsk);
 	atomic_inc(&mm->context.attach_count);
 	atomic_dec(&old_mm->context.attach_count);
 	cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
 	preempt_enable();
 	task_unlock(tsk);
 	mmput(old_mm);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(s390_enable_sie);
 #if defined(CONFIG_DEBUG_PAGEALLOC) && defined(CONFIG_HIBERNATION)
 bool kernel_page_present(struct page *page)
 {
 	unsigned long addr;
 	int cc;
 	addr = page_to_phys(page);
 	asm volatile(
 		"	lra	%1,0(%1)\n"
 		"	ipm	%0\n"
 		"	srl	%0,28"
 		: "=d" (cc), "+a" (addr) : : "cc");
 	return cc == 0;
 }
 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
+			   pmd_t *pmdp)
+{
+	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+	/* No need to flush TLB
+	 * On s390 reference bits are in storage key and never in TLB */
+	return pmdp_test_and_clear_young(vma, address, pmdp);
+}
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+			  unsigned long address, pmd_t *pmdp,
+			  pmd_t entry, int dirty)
+{
+	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+	if (pmd_same(*pmdp, entry))
+		return 0;
+	pmdp_invalidate(vma, address, pmdp);
+	set_pmd_at(vma->vm_mm, address, pmdp, entry);
+	return 1;
+}
 static void pmdp_splitting_flush_sync(void *arg)
 {
 	/* Simply deliver the interrupt */
 }
 void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
 			  pmd_t *pmdp)
 {
 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 	if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
 			      (unsigned long *) pmdp)) {
 		/* need to serialize against gup-fast (IRQ disabled) */
 		smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
 	}
 }
 void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
 {
 	struct list_head *lh = (struct list_head *) pgtable;
 	assert_spin_locked(&mm->page_table_lock);
 	/* FIFO */
 	if (!mm->pmd_huge_pte)
 		INIT_LIST_HEAD(lh);
 	else
 		list_add(lh, (struct list_head *) mm->pmd_huge_pte);
 	mm->pmd_huge_pte = pgtable;
 }
 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
 {
 	struct list_head *lh;
 	pgtable_t pgtable;
 	pte_t *ptep;
 	assert_spin_locked(&mm->page_table_lock);
 	/* FIFO */
 	pgtable = mm->pmd_huge_pte;
 	lh = (struct list_head *) pgtable;
 	if (list_empty(lh))
 		mm->pmd_huge_pte = NULL;
 	else {
 		mm->pmd_huge_pte = (pgtable_t) lh->next;
 		list_del(lh);
 	}
 	ptep = (pte_t *) pgtable;
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 	ptep++;
 	pte_val(*ptep) = _PAGE_TYPE_EMPTY;
 	return pgtable;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */