Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit 505569d208e61ab14f4b87957be0970ab33eb319

Authored by Linus Torvalds 2015-01-12 03:53:46 +0800

Exists in ti-lsk-linux-4.1.y and in 10 other branches

Merge branch 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 fixes from Ingo Molnar:
 "Misc fixes: two vdso fixes, two kbuild fixes and a boot failure fix
  with certain odd memory mappings"

* 'x86-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86, vdso: Use asm volatile in __getcpu
  x86/build: Clean auto-generated processor feature files
  x86: Fix mkcapflags.sh bash-ism
  x86: Fix step size adjustment during initial memory mapping
  x86_64, vdso: Fix the vdso address randomization algorithm

Showing 6 changed files Inline Diff

arch/x86/boot/Makefile
arch/x86/include/asm/vgtod.h
arch/x86/kernel/cpu/Makefile
arch/x86/kernel/cpu/mkcapflags.sh
arch/x86/mm/init.c
arch/x86/vdso/vma.c

arch/x86/boot/Makefile

Diff comments View file @ 505569d

 #
 # arch/x86/boot/Makefile
 #
 # This file is subject to the terms and conditions of the GNU General Public
 # License.  See the file "COPYING" in the main directory of this archive
 # for more details.
 #
 # Copyright (C) 1994 by Linus Torvalds
 # Changed by many, many contributors over the years.
 #
 # If you want to preset the SVGA mode, uncomment the next line and
 # set SVGA_MODE to whatever number you want.
 # Set it to -DSVGA_MODE=NORMAL_VGA if you just want the EGA/VGA mode.
 # The number is the same as you would ordinarily press at bootup.
 SVGA_MODE	:= -DSVGA_MODE=NORMAL_VGA
 targets		:= vmlinux.bin setup.bin setup.elf bzImage
 targets		+= fdimage fdimage144 fdimage288 image.iso mtools.conf
 subdir-		:= compressed
 setup-y		+= a20.o bioscall.o cmdline.o copy.o cpu.o cpuflags.o cpucheck.o
 setup-y		+= early_serial_console.o edd.o header.o main.o mca.o memory.o
 setup-y		+= pm.o pmjump.o printf.o regs.o string.o tty.o video.o
 setup-y		+= video-mode.o version.o
 setup-$(CONFIG_X86_APM_BOOT) += apm.o
 # The link order of the video-*.o modules can matter.  In particular,
 # video-vga.o *must* be listed first, followed by video-vesa.o.
 # Hardware-specific drivers should follow in the order they should be
 # probed, and video-bios.o should typically be last.
 setup-y		+= video-vga.o
 setup-y		+= video-vesa.o
 setup-y		+= video-bios.o
 targets		+= $(setup-y)
 hostprogs-y	:= tools/build
 hostprogs-$(CONFIG_X86_FEATURE_NAMES) += mkcpustr
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include \
 		    -include include/generated/autoconf.h \
 	            -D__EXPORTED_HEADERS__
 ifdef CONFIG_X86_FEATURE_NAMES
 $(obj)/cpu.o: $(obj)/cpustr.h
 quiet_cmd_cpustr = CPUSTR  $@
       cmd_cpustr = $(obj)/mkcpustr > $@
 targets += cpustr.h
 $(obj)/cpustr.h: $(obj)/mkcpustr FORCE
 	$(call if_changed,cpustr)
 endif
+clean-files += cpustr.h
 # ---------------------------------------------------------------------------
 KBUILD_CFLAGS	:= $(USERINCLUDE) $(REALMODE_CFLAGS) -D_SETUP
 KBUILD_AFLAGS	:= $(KBUILD_CFLAGS) -D__ASSEMBLY__
 GCOV_PROFILE := n
 $(obj)/bzImage: asflags-y  := $(SVGA_MODE)
 quiet_cmd_image = BUILD   $@
 cmd_image = $(obj)/tools/build $(obj)/setup.bin $(obj)/vmlinux.bin \
 			       $(obj)/zoffset.h $@
 $(obj)/bzImage: $(obj)/setup.bin $(obj)/vmlinux.bin $(obj)/tools/build FORCE
 	$(call if_changed,image)
 	@echo 'Kernel: $@ is ready' ' (#'`cat .version`')'
 OBJCOPYFLAGS_vmlinux.bin := -O binary -R .note -R .comment -S
 $(obj)/vmlinux.bin: $(obj)/compressed/vmlinux FORCE
 	$(call if_changed,objcopy)
 SETUP_OBJS = $(addprefix $(obj)/,$(setup-y))
 sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|_end\)$$/\#define VO_\2 0x\1/p'
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
 targets += voffset.h
 $(obj)/voffset.h: vmlinux FORCE
 	$(call if_changed,voffset)
 sed-zoffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(startup_32\|startup_64\|efi32_stub_entry\|efi64_stub_entry\|efi_pe_entry\|input_data\|_end\|z_.*\)$$/\#define ZO_\2 0x\1/p'
 quiet_cmd_zoffset = ZOFFSET $@
       cmd_zoffset = $(NM) $< | sed -n $(sed-zoffset) > $@
 targets += zoffset.h
 $(obj)/zoffset.h: $(obj)/compressed/vmlinux FORCE
 	$(call if_changed,zoffset)
 AFLAGS_header.o += -I$(obj)
 $(obj)/header.o: $(obj)/voffset.h $(obj)/zoffset.h
 LDFLAGS_setup.elf	:= -T
 $(obj)/setup.elf: $(src)/setup.ld $(SETUP_OBJS) FORCE
 	$(call if_changed,ld)
 OBJCOPYFLAGS_setup.bin	:= -O binary
 $(obj)/setup.bin: $(obj)/setup.elf FORCE
 	$(call if_changed,objcopy)
 $(obj)/compressed/vmlinux: FORCE
 	$(Q)$(MAKE) $(build)=$(obj)/compressed $@
 # Set this if you want to pass append arguments to the
 # bzdisk/fdimage/isoimage kernel
 FDARGS =
 # Set this if you want an initrd included with the
 # bzdisk/fdimage/isoimage kernel
 FDINITRD =
 image_cmdline = default linux $(FDARGS) $(if $(FDINITRD),initrd=initrd.img,)
 $(obj)/mtools.conf: $(src)/mtools.conf.in
 	sed -e 's|@OBJ@|$(obj)|g' < $< > $@
 # This requires write access to /dev/fd0
 bzdisk: $(obj)/bzImage $(obj)/mtools.conf
 	MTOOLSRC=$(obj)/mtools.conf mformat a:			; sync
 	syslinux /dev/fd0					; sync
 	echo '$(image_cmdline)' | \
 		MTOOLSRC=$(src)/mtools.conf mcopy - a:syslinux.cfg
 	if [ -f '$(FDINITRD)' ] ; then \
 		MTOOLSRC=$(obj)/mtools.conf mcopy '$(FDINITRD)' a:initrd.img ; \
 	fi
 	MTOOLSRC=$(obj)/mtools.conf mcopy $(obj)/bzImage a:linux	; sync
 # These require being root or having syslinux 2.02 or higher installed
 fdimage fdimage144: $(obj)/bzImage $(obj)/mtools.conf
 	dd if=/dev/zero of=$(obj)/fdimage bs=1024 count=1440
 	MTOOLSRC=$(obj)/mtools.conf mformat v:			; sync
 	syslinux $(obj)/fdimage					; sync
 	echo '$(image_cmdline)' | \
 		MTOOLSRC=$(obj)/mtools.conf mcopy - v:syslinux.cfg
 	if [ -f '$(FDINITRD)' ] ; then \
 		MTOOLSRC=$(obj)/mtools.conf mcopy '$(FDINITRD)' v:initrd.img ; \
 	fi
 	MTOOLSRC=$(obj)/mtools.conf mcopy $(obj)/bzImage v:linux	; sync
 fdimage288: $(obj)/bzImage $(obj)/mtools.conf
 	dd if=/dev/zero of=$(obj)/fdimage bs=1024 count=2880
 	MTOOLSRC=$(obj)/mtools.conf mformat w:			; sync
 	syslinux $(obj)/fdimage					; sync
 	echo '$(image_cmdline)' | \
 		MTOOLSRC=$(obj)/mtools.conf mcopy - w:syslinux.cfg
 	if [ -f '$(FDINITRD)' ] ; then \
 		MTOOLSRC=$(obj)/mtools.conf mcopy '$(FDINITRD)' w:initrd.img ; \
 	fi
 	MTOOLSRC=$(obj)/mtools.conf mcopy $(obj)/bzImage w:linux	; sync
 isoimage: $(obj)/bzImage
 	-rm -rf $(obj)/isoimage
 	mkdir $(obj)/isoimage
 	for i in lib lib64 share end ; do \
 		if [ -f /usr/$$i/syslinux/isolinux.bin ] ; then \
 			cp /usr/$$i/syslinux/isolinux.bin $(obj)/isoimage ; \
 			break ; \
 		fi ; \
 		if [ $$i = end ] ; then exit 1 ; fi ; \
 	done
 	cp $(obj)/bzImage $(obj)/isoimage/linux
 	echo '$(image_cmdline)' > $(obj)/isoimage/isolinux.cfg
 	if [ -f '$(FDINITRD)' ] ; then \
 		cp '$(FDINITRD)' $(obj)/isoimage/initrd.img ; \
 	fi
 	mkisofs -J -r -o $(obj)/image.iso -b isolinux.bin -c boot.cat \
 		-no-emul-boot -boot-load-size 4 -boot-info-table \
 		$(obj)/isoimage
 	isohybrid $(obj)/image.iso 2>/dev/null || true
 	rm -rf $(obj)/isoimage
 bzlilo: $(obj)/bzImage
 	if [ -f $(INSTALL_PATH)/vmlinuz ]; then mv $(INSTALL_PATH)/vmlinuz $(INSTALL_PATH)/vmlinuz.old; fi
 	if [ -f $(INSTALL_PATH)/System.map ]; then mv $(INSTALL_PATH)/System.map $(INSTALL_PATH)/System.old; fi
 	cat $(obj)/bzImage > $(INSTALL_PATH)/vmlinuz
 	cp System.map $(INSTALL_PATH)/
 	if [ -x /sbin/lilo ]; then /sbin/lilo; else /etc/lilo/install; fi
 install:
 	sh $(srctree)/$(src)/install.sh $(KERNELRELEASE) $(obj)/bzImage \
 		System.map "$(INSTALL_PATH)"

arch/x86/include/asm/vgtod.h

Diff comments View file @ 505569d

 #ifndef _ASM_X86_VGTOD_H
 #define _ASM_X86_VGTOD_H
 #include <linux/compiler.h>
 #include <linux/clocksource.h>
 #ifdef BUILD_VDSO32_64
 typedef u64 gtod_long_t;
 #else
 typedef unsigned long gtod_long_t;
 #endif
 /*
  * vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time
  * so be carefull by modifying this structure.
  */
 struct vsyscall_gtod_data {
 	unsigned seq;
 	int vclock_mode;
 	cycle_t	cycle_last;
 	cycle_t	mask;
 	u32	mult;
 	u32	shift;
 	/* open coded 'struct timespec' */
 	u64		wall_time_snsec;
 	gtod_long_t	wall_time_sec;
 	gtod_long_t	monotonic_time_sec;
 	u64		monotonic_time_snsec;
 	gtod_long_t	wall_time_coarse_sec;
 	gtod_long_t	wall_time_coarse_nsec;
 	gtod_long_t	monotonic_time_coarse_sec;
 	gtod_long_t	monotonic_time_coarse_nsec;
 	int		tz_minuteswest;
 	int		tz_dsttime;
 };
 extern struct vsyscall_gtod_data vsyscall_gtod_data;
 static inline unsigned gtod_read_begin(const struct vsyscall_gtod_data *s)
 {
 	unsigned ret;
 repeat:
 	ret = ACCESS_ONCE(s->seq);
 	if (unlikely(ret & 1)) {
 		cpu_relax();
 		goto repeat;
 	}
 	smp_rmb();
 	return ret;
 }
 static inline int gtod_read_retry(const struct vsyscall_gtod_data *s,
 					unsigned start)
 {
 	smp_rmb();
 	return unlikely(s->seq != start);
 }
 static inline void gtod_write_begin(struct vsyscall_gtod_data *s)
 {
 	++s->seq;
 	smp_wmb();
 }
 static inline void gtod_write_end(struct vsyscall_gtod_data *s)
 {
 	smp_wmb();
 	++s->seq;
 }
 #ifdef CONFIG_X86_64
 #define VGETCPU_CPU_MASK 0xfff
 static inline unsigned int __getcpu(void)
 {
 	unsigned int p;
 	/*
 	 * Load per CPU data from GDT.  LSL is faster than RDTSCP and
-	 * works on all CPUs.
+	 * works on all CPUs.  This is volatile so that it orders
+	 * correctly wrt barrier() and to keep gcc from cleverly
+	 * hoisting it out of the calling function.
 	 */
-	asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
+	asm volatile ("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 	return p;
 }
 #endif /* CONFIG_X86_64 */
 #endif /* _ASM_X86_VGTOD_H */

arch/x86/kernel/cpu/Makefile

Diff comments View file @ 505569d

 #
 # Makefile for x86-compatible CPU details, features and quirks
 #
 # Don't trace early stages of a secondary CPU boot
 ifdef CONFIG_FUNCTION_TRACER
 CFLAGS_REMOVE_common.o = -pg
 CFLAGS_REMOVE_perf_event.o = -pg
 endif
 # Make sure load_percpu_segment has no stackprotector
 nostackp := $(call cc-option, -fno-stack-protector)
 CFLAGS_common.o		:= $(nostackp)
 obj-y			:= intel_cacheinfo.o scattered.o topology.o
 obj-y			+= common.o
 obj-y			+= rdrand.o
 obj-y			+= match.o
 obj-$(CONFIG_PROC_FS)	+= proc.o
 obj-$(CONFIG_X86_FEATURE_NAMES) += capflags.o powerflags.o
 obj-$(CONFIG_X86_32)	+= bugs.o
 obj-$(CONFIG_X86_64)	+= bugs_64.o
 obj-$(CONFIG_CPU_SUP_INTEL)		+= intel.o
 obj-$(CONFIG_CPU_SUP_AMD)		+= amd.o
 obj-$(CONFIG_CPU_SUP_CYRIX_32)		+= cyrix.o
 obj-$(CONFIG_CPU_SUP_CENTAUR)		+= centaur.o
 obj-$(CONFIG_CPU_SUP_TRANSMETA_32)	+= transmeta.o
 obj-$(CONFIG_CPU_SUP_UMC_32)		+= umc.o
 obj-$(CONFIG_PERF_EVENTS)		+= perf_event.o
 ifdef CONFIG_PERF_EVENTS
 obj-$(CONFIG_CPU_SUP_AMD)		+= perf_event_amd.o perf_event_amd_uncore.o
 ifdef CONFIG_AMD_IOMMU
 obj-$(CONFIG_CPU_SUP_AMD)		+= perf_event_amd_iommu.o
 endif
 obj-$(CONFIG_CPU_SUP_INTEL)		+= perf_event_p6.o perf_event_knc.o perf_event_p4.o
 obj-$(CONFIG_CPU_SUP_INTEL)		+= perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o
 obj-$(CONFIG_CPU_SUP_INTEL)		+= perf_event_intel_rapl.o
 obj-$(CONFIG_PERF_EVENTS_INTEL_UNCORE)	+= perf_event_intel_uncore.o \
 					   perf_event_intel_uncore_snb.o \
 					   perf_event_intel_uncore_snbep.o \
 					   perf_event_intel_uncore_nhmex.o
 endif
 obj-$(CONFIG_X86_MCE)			+= mcheck/
 obj-$(CONFIG_MTRR)			+= mtrr/
 obj-$(CONFIG_MICROCODE)			+= microcode/
 obj-$(CONFIG_X86_LOCAL_APIC)		+= perfctr-watchdog.o perf_event_amd_ibs.o
 obj-$(CONFIG_HYPERVISOR_GUEST)		+= vmware.o hypervisor.o mshyperv.o
 ifdef CONFIG_X86_FEATURE_NAMES
 quiet_cmd_mkcapflags = MKCAP   $@
       cmd_mkcapflags = $(CONFIG_SHELL) $(srctree)/$(src)/mkcapflags.sh $< $@
 cpufeature = $(src)/../../include/asm/cpufeature.h
 targets += capflags.c
 $(obj)/capflags.c: $(cpufeature) $(src)/mkcapflags.sh FORCE
 	$(call if_changed,mkcapflags)
 endif
+clean-files += capflags.c

arch/x86/kernel/cpu/mkcapflags.sh

Diff comments View file @ 505569d

1	#!/bin/sh	1	#!/bin/sh
2	#	2	#
3	# Generate the x86_cap/bug_flags[] arrays from include/asm/cpufeature.h	3	# Generate the x86_cap/bug_flags[] arrays from include/asm/cpufeature.h
4	#	4	#
5		5
6	IN=$1	6	IN=$1
7	OUT=$2	7	OUT=$2
8		8
9	function dump_array()	9	function dump_array()
10	{	10	{
11	ARRAY=$1	11	ARRAY=$1
12	SIZE=$2	12	SIZE=$2
13	PFX=$3	13	PFX=$3
14	POSTFIX=$4	14	POSTFIX=$4
15		15
16	PFX_SZ=$(echo $PFX \| wc -c)	16	PFX_SZ=$(echo $PFX \| wc -c)
17	TABS="$(printf '\t\t\t\t\t')"	17	TABS="$(printf '\t\t\t\t\t')"
18		18
19	echo "const char * const $ARRAY[$SIZE] = {"	19	echo "const char * const $ARRAY[$SIZE] = {"
20		20
21	# Iterate through any input lines starting with #define $PFX	21	# Iterate through any input lines starting with #define $PFX
22	sed -n -e 's/\t/ /g' -e "s/^ # define *$PFX//p" $IN \|	22	sed -n -e 's/\t/ /g' -e "s/^ # define *$PFX//p" $IN \|
23	while read i	23	while read i
24	do	24	do
25	# Name is everything up to the first whitespace	25	# Name is everything up to the first whitespace
26	NAME="$(echo "$i" \| sed 's/ .*//')"	26	NAME="$(echo "$i" \| sed 's/ .*//')"
27		27
28	# If the /* comment */ starts with a quote string, grab that.	28	# If the /* comment */ starts with a quote string, grab that.
29	VALUE="$(echo "$i" \| sed -n 's@./\ $"[^"]"$.\/@\1@p')"	29	VALUE="$(echo "$i" \| sed -n 's@./\ $"[^"]"$.\/@\1@p')"
30	[ -z "$VALUE" ] && VALUE="\"$NAME\""	30	[ -z "$VALUE" ] && VALUE="\"$NAME\""
31	[ "$VALUE" == '""' ] && continue	31	[ "$VALUE" = '""' ] && continue
32		32
33	# Name is uppercase, VALUE is all lowercase	33	# Name is uppercase, VALUE is all lowercase
34	VALUE="$(echo "$VALUE" \| tr A-Z a-z)"	34	VALUE="$(echo "$VALUE" \| tr A-Z a-z)"
35		35
36	if [ -n "$POSTFIX" ]; then	36	if [ -n "$POSTFIX" ]; then
37	T=$(( $PFX_SZ + $(echo $POSTFIX \| wc -c) + 2 ))	37	T=$(( $PFX_SZ + $(echo $POSTFIX \| wc -c) + 2 ))
38	TABS="$(printf '\t\t\t\t\t\t')"	38	TABS="$(printf '\t\t\t\t\t\t')"
39	TABCOUNT=$(( ( 6*8 - ($T + 1) - $(echo "$NAME" \| wc -c) ) / 8 ))	39	TABCOUNT=$(( ( 6*8 - ($T + 1) - $(echo "$NAME" \| wc -c) ) / 8 ))
40	printf "\t[%s - %s]%.*s = %s,\n" "$PFX$NAME" "$POSTFIX" "$TABCOUNT" "$TABS" "$VALUE"	40	printf "\t[%s - %s]%.*s = %s,\n" "$PFX$NAME" "$POSTFIX" "$TABCOUNT" "$TABS" "$VALUE"
41	else	41	else
42	TABCOUNT=$(( ( 5*8 - ($PFX_SZ + 1) - $(echo "$NAME" \| wc -c) ) / 8 ))	42	TABCOUNT=$(( ( 5*8 - ($PFX_SZ + 1) - $(echo "$NAME" \| wc -c) ) / 8 ))
43	printf "\t[%s]%.*s = %s,\n" "$PFX$NAME" "$TABCOUNT" "$TABS" "$VALUE"	43	printf "\t[%s]%.*s = %s,\n" "$PFX$NAME" "$TABCOUNT" "$TABS" "$VALUE"
44	fi	44	fi
45	done	45	done
46	echo "};"	46	echo "};"
47	}	47	}
48		48
49	trap 'rm "$OUT"' EXIT	49	trap 'rm "$OUT"' EXIT
50		50
51	(	51	(
52	echo "#ifndef _ASM_X86_CPUFEATURE_H"	52	echo "#ifndef _ASM_X86_CPUFEATURE_H"
53	echo "#include <asm/cpufeature.h>"	53	echo "#include <asm/cpufeature.h>"
54	echo "#endif"	54	echo "#endif"
55	echo ""	55	echo ""
56		56
57	dump_array "x86_cap_flags" "NCAPINTS*32" "X86_FEATURE_" ""	57	dump_array "x86_cap_flags" "NCAPINTS*32" "X86_FEATURE_" ""
58	echo ""	58	echo ""
59		59
60	dump_array "x86_bug_flags" "NBUGINTS32" "X86_BUG_" "NCAPINTS32"	60	dump_array "x86_bug_flags" "NBUGINTS32" "X86_BUG_" "NCAPINTS32"
61		61
62	) > $OUT	62	) > $OUT
63		63
64	trap - EXIT	64	trap - EXIT
65		65

arch/x86/mm/init.c

Diff comments View file @ 505569d

 #include <linux/gfp.h>
 #include <linux/initrd.h>
 #include <linux/ioport.h>
 #include <linux/swap.h>
 #include <linux/memblock.h>
 #include <linux/bootmem.h>	/* for max_low_pfn */
 #include <asm/cacheflush.h>
 #include <asm/e820.h>
 #include <asm/init.h>
 #include <asm/page.h>
 #include <asm/page_types.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/tlbflush.h>
 #include <asm/tlb.h>
 #include <asm/proto.h>
 #include <asm/dma.h>		/* for MAX_DMA_PFN */
 #include <asm/microcode.h>
 /*
  * We need to define the tracepoints somewhere, and tlb.c
  * is only compied when SMP=y.
  */
 #define CREATE_TRACE_POINTS
 #include <trace/events/tlb.h>
 #include "mm_internal.h"
 /*
  * Tables translating between page_cache_type_t and pte encoding.
  * Minimal supported modes are defined statically, modified if more supported
  * cache modes are available.
  * Index into __cachemode2pte_tbl is the cachemode.
  * Index into __pte2cachemode_tbl are the caching attribute bits of the pte
  * (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
  */
 uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
 	[_PAGE_CACHE_MODE_WB]		= 0,
 	[_PAGE_CACHE_MODE_WC]		= _PAGE_PWT,
 	[_PAGE_CACHE_MODE_UC_MINUS]	= _PAGE_PCD,
 	[_PAGE_CACHE_MODE_UC]		= _PAGE_PCD | _PAGE_PWT,
 	[_PAGE_CACHE_MODE_WT]		= _PAGE_PCD,
 	[_PAGE_CACHE_MODE_WP]		= _PAGE_PCD,
 };
 EXPORT_SYMBOL_GPL(__cachemode2pte_tbl);
 uint8_t __pte2cachemode_tbl[8] = {
 	[__pte2cm_idx(0)] = _PAGE_CACHE_MODE_WB,
 	[__pte2cm_idx(_PAGE_PWT)] = _PAGE_CACHE_MODE_WC,
 	[__pte2cm_idx(_PAGE_PCD)] = _PAGE_CACHE_MODE_UC_MINUS,
 	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD)] = _PAGE_CACHE_MODE_UC,
 	[__pte2cm_idx(_PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
 	[__pte2cm_idx(_PAGE_PWT | _PAGE_PAT)] = _PAGE_CACHE_MODE_WC,
 	[__pte2cm_idx(_PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
 	[__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
 };
 EXPORT_SYMBOL_GPL(__pte2cachemode_tbl);
 static unsigned long __initdata pgt_buf_start;
 static unsigned long __initdata pgt_buf_end;
 static unsigned long __initdata pgt_buf_top;
 static unsigned long min_pfn_mapped;
 static bool __initdata can_use_brk_pgt = true;
 /*
  * Pages returned are already directly mapped.
  *
  * Changing that is likely to break Xen, see commit:
  *
  *    279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
  *
  * for detailed information.
  */
 __ref void *alloc_low_pages(unsigned int num)
 {
 	unsigned long pfn;
 	int i;
 	if (after_bootmem) {
 		unsigned int order;
 		order = get_order((unsigned long)num << PAGE_SHIFT);
 		return (void *)__get_free_pages(GFP_ATOMIC | __GFP_NOTRACK |
 						__GFP_ZERO, order);
 	}
 	if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
 		unsigned long ret;
 		if (min_pfn_mapped >= max_pfn_mapped)
 			panic("alloc_low_pages: ran out of memory");
 		ret = memblock_find_in_range(min_pfn_mapped << PAGE_SHIFT,
 					max_pfn_mapped << PAGE_SHIFT,
 					PAGE_SIZE * num , PAGE_SIZE);
 		if (!ret)
 			panic("alloc_low_pages: can not alloc memory");
 		memblock_reserve(ret, PAGE_SIZE * num);
 		pfn = ret >> PAGE_SHIFT;
 	} else {
 		pfn = pgt_buf_end;
 		pgt_buf_end += num;
 		printk(KERN_DEBUG "BRK [%#010lx, %#010lx] PGTABLE\n",
 			pfn << PAGE_SHIFT, (pgt_buf_end << PAGE_SHIFT) - 1);
 	}
 	for (i = 0; i < num; i++) {
 		void *adr;
 		adr = __va((pfn + i) << PAGE_SHIFT);
 		clear_page(adr);
 	}
 	return __va(pfn << PAGE_SHIFT);
 }
 /* need 3 4k for initial PMD_SIZE,  3 4k for 0-ISA_END_ADDRESS */
 #define INIT_PGT_BUF_SIZE	(6 * PAGE_SIZE)
 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
 void  __init early_alloc_pgt_buf(void)
 {
 	unsigned long tables = INIT_PGT_BUF_SIZE;
 	phys_addr_t base;
 	base = __pa(extend_brk(tables, PAGE_SIZE));
 	pgt_buf_start = base >> PAGE_SHIFT;
 	pgt_buf_end = pgt_buf_start;
 	pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
 }
 int after_bootmem;
 int direct_gbpages
 #ifdef CONFIG_DIRECT_GBPAGES
 				= 1
 #endif
 ;
 static void __init init_gbpages(void)
 {
 #ifdef CONFIG_X86_64
 	if (direct_gbpages && cpu_has_gbpages)
 		printk(KERN_INFO "Using GB pages for direct mapping\n");
 	else
 		direct_gbpages = 0;
 #endif
 }
 struct map_range {
 	unsigned long start;
 	unsigned long end;
 	unsigned page_size_mask;
 };
 static int page_size_mask;
 static void __init probe_page_size_mask(void)
 {
 	init_gbpages();
 #if !defined(CONFIG_DEBUG_PAGEALLOC) && !defined(CONFIG_KMEMCHECK)
 	/*
 	 * For CONFIG_DEBUG_PAGEALLOC, identity mapping will use small pages.
 	 * This will simplify cpa(), which otherwise needs to support splitting
 	 * large pages into small in interrupt context, etc.
 	 */
 	if (direct_gbpages)
 		page_size_mask |= 1 << PG_LEVEL_1G;
 	if (cpu_has_pse)
 		page_size_mask |= 1 << PG_LEVEL_2M;
 #endif
 	/* Enable PSE if available */
 	if (cpu_has_pse)
 		set_in_cr4(X86_CR4_PSE);
 	/* Enable PGE if available */
 	if (cpu_has_pge) {
 		set_in_cr4(X86_CR4_PGE);
 		__supported_pte_mask |= _PAGE_GLOBAL;
 	}
 }
 #ifdef CONFIG_X86_32
 #define NR_RANGE_MR 3
 #else /* CONFIG_X86_64 */
 #define NR_RANGE_MR 5
 #endif
 static int __meminit save_mr(struct map_range *mr, int nr_range,
 			     unsigned long start_pfn, unsigned long end_pfn,
 			     unsigned long page_size_mask)
 {
 	if (start_pfn < end_pfn) {
 		if (nr_range >= NR_RANGE_MR)
 			panic("run out of range for init_memory_mapping\n");
 		mr[nr_range].start = start_pfn<<PAGE_SHIFT;
 		mr[nr_range].end   = end_pfn<<PAGE_SHIFT;
 		mr[nr_range].page_size_mask = page_size_mask;
 		nr_range++;
 	}
 	return nr_range;
 }
 /*
  * adjust the page_size_mask for small range to go with
  *	big page size instead small one if nearby are ram too.
  */
 static void __init_refok adjust_range_page_size_mask(struct map_range *mr,
 							 int nr_range)
 {
 	int i;
 	for (i = 0; i < nr_range; i++) {
 		if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
 		    !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
 			unsigned long start = round_down(mr[i].start, PMD_SIZE);
 			unsigned long end = round_up(mr[i].end, PMD_SIZE);
 #ifdef CONFIG_X86_32
 			if ((end >> PAGE_SHIFT) > max_low_pfn)
 				continue;
 #endif
 			if (memblock_is_region_memory(start, end - start))
 				mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
 		}
 		if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
 		    !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
 			unsigned long start = round_down(mr[i].start, PUD_SIZE);
 			unsigned long end = round_up(mr[i].end, PUD_SIZE);
 			if (memblock_is_region_memory(start, end - start))
 				mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
 		}
 	}
 }
 static int __meminit split_mem_range(struct map_range *mr, int nr_range,
 				     unsigned long start,
 				     unsigned long end)
 {
 	unsigned long start_pfn, end_pfn, limit_pfn;
 	unsigned long pfn;
 	int i;
 	limit_pfn = PFN_DOWN(end);
 	/* head if not big page alignment ? */
 	pfn = start_pfn = PFN_DOWN(start);
 #ifdef CONFIG_X86_32
 	/*
 	 * Don't use a large page for the first 2/4MB of memory
 	 * because there are often fixed size MTRRs in there
 	 * and overlapping MTRRs into large pages can cause
 	 * slowdowns.
 	 */
 	if (pfn == 0)
 		end_pfn = PFN_DOWN(PMD_SIZE);
 	else
 		end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
 	end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #endif
 	if (end_pfn > limit_pfn)
 		end_pfn = limit_pfn;
 	if (start_pfn < end_pfn) {
 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 		pfn = end_pfn;
 	}
 	/* big page (2M) range */
 	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 #ifdef CONFIG_X86_32
 	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #else /* CONFIG_X86_64 */
 	end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
 	if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
 		end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 #endif
 	if (start_pfn < end_pfn) {
 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 				page_size_mask & (1<<PG_LEVEL_2M));
 		pfn = end_pfn;
 	}
 #ifdef CONFIG_X86_64
 	/* big page (1G) range */
 	start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
 	end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
 	if (start_pfn < end_pfn) {
 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 				page_size_mask &
 				 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
 		pfn = end_pfn;
 	}
 	/* tail is not big page (1G) alignment */
 	start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
 	end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
 	if (start_pfn < end_pfn) {
 		nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
 				page_size_mask & (1<<PG_LEVEL_2M));
 		pfn = end_pfn;
 	}
 #endif
 	/* tail is not big page (2M) alignment */
 	start_pfn = pfn;
 	end_pfn = limit_pfn;
 	nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
 	if (!after_bootmem)
 		adjust_range_page_size_mask(mr, nr_range);
 	/* try to merge same page size and continuous */
 	for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
 		unsigned long old_start;
 		if (mr[i].end != mr[i+1].start ||
 		    mr[i].page_size_mask != mr[i+1].page_size_mask)
 			continue;
 		/* move it */
 		old_start = mr[i].start;
 		memmove(&mr[i], &mr[i+1],
 			(nr_range - 1 - i) * sizeof(struct map_range));
 		mr[i--].start = old_start;
 		nr_range--;
 	}
 	for (i = 0; i < nr_range; i++)
 		printk(KERN_DEBUG " [mem %#010lx-%#010lx] page %s\n",
 				mr[i].start, mr[i].end - 1,
 			(mr[i].page_size_mask & (1<<PG_LEVEL_1G))?"1G":(
 			 (mr[i].page_size_mask & (1<<PG_LEVEL_2M))?"2M":"4k"));
 	return nr_range;
 }
 struct range pfn_mapped[E820_X_MAX];
 int nr_pfn_mapped;
 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
 {
 	nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_X_MAX,
 					     nr_pfn_mapped, start_pfn, end_pfn);
 	nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_X_MAX);
 	max_pfn_mapped = max(max_pfn_mapped, end_pfn);
 	if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
 		max_low_pfn_mapped = max(max_low_pfn_mapped,
 					 min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
 }
 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
 {
 	int i;
 	for (i = 0; i < nr_pfn_mapped; i++)
 		if ((start_pfn >= pfn_mapped[i].start) &&
 		    (end_pfn <= pfn_mapped[i].end))
 			return true;
 	return false;
 }
 /*
  * Setup the direct mapping of the physical memory at PAGE_OFFSET.
  * This runs before bootmem is initialized and gets pages directly from
  * the physical memory. To access them they are temporarily mapped.
  */
 unsigned long __init_refok init_memory_mapping(unsigned long start,
 					       unsigned long end)
 {
 	struct map_range mr[NR_RANGE_MR];
 	unsigned long ret = 0;
 	int nr_range, i;
 	pr_info("init_memory_mapping: [mem %#010lx-%#010lx]\n",
 	       start, end - 1);
 	memset(mr, 0, sizeof(mr));
 	nr_range = split_mem_range(mr, 0, start, end);
 	for (i = 0; i < nr_range; i++)
 		ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
 						   mr[i].page_size_mask);
 	add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
 	return ret >> PAGE_SHIFT;
 }
 /*
  * We need to iterate through the E820 memory map and create direct mappings
  * for only E820_RAM and E820_KERN_RESERVED regions. We cannot simply
  * create direct mappings for all pfns from [0 to max_low_pfn) and
  * [4GB to max_pfn) because of possible memory holes in high addresses
  * that cannot be marked as UC by fixed/variable range MTRRs.
  * Depending on the alignment of E820 ranges, this may possibly result
  * in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
  *
  * init_mem_mapping() calls init_range_memory_mapping() with big range.
  * That range would have hole in the middle or ends, and only ram parts
  * will be mapped in init_range_memory_mapping().
  */
 static unsigned long __init init_range_memory_mapping(
 					   unsigned long r_start,
 					   unsigned long r_end)
 {
 	unsigned long start_pfn, end_pfn;
 	unsigned long mapped_ram_size = 0;
 	int i;
 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
 		u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
 		u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
 		if (start >= end)
 			continue;
 		/*
 		 * if it is overlapping with brk pgt, we need to
 		 * alloc pgt buf from memblock instead.
 		 */
 		can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
 				    min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
 		init_memory_mapping(start, end);
 		mapped_ram_size += end - start;
 		can_use_brk_pgt = true;
 	}
 	return mapped_ram_size;
 }
 static unsigned long __init get_new_step_size(unsigned long step_size)
 {
 	/*
-	 * Explain why we shift by 5 and why we don't have to worry about
+	 * Initial mapped size is PMD_SIZE (2M).
-	 * 'step_size << 5' overflowing:
-	 *
-	 * initial mapped size is PMD_SIZE (2M).
 	 * We can not set step_size to be PUD_SIZE (1G) yet.
 	 * In worse case, when we cross the 1G boundary, and
 	 * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
-	 * to map 1G range with PTE. Use 5 as shift for now.
+	 * to map 1G range with PTE. Hence we use one less than the
+	 * difference of page table level shifts.
 	 *
-	 * Don't need to worry about overflow, on 32bit, when step_size
+	 * Don't need to worry about overflow in the top-down case, on 32bit,
-	 * is 0, round_down() returns 0 for start, and that turns it
+	 * when step_size is 0, round_down() returns 0 for start, and that
-	 * into 0x100000000ULL.
+	 * turns it into 0x100000000ULL.
+	 * In the bottom-up case, round_up(x, 0) returns 0 though too, which
+	 * needs to be taken into consideration by the code below.
 	 */
-	return step_size << 5;
+	return step_size << (PMD_SHIFT - PAGE_SHIFT - 1);
 }
 /**
  * memory_map_top_down - Map [map_start, map_end) top down
  * @map_start: start address of the target memory range
  * @map_end: end address of the target memory range
  *
  * This function will setup direct mapping for memory range
  * [map_start, map_end) in top-down. That said, the page tables
  * will be allocated at the end of the memory, and we map the
  * memory in top-down.
  */
 static void __init memory_map_top_down(unsigned long map_start,
 				       unsigned long map_end)
 {
 	unsigned long real_end, start, last_start;
 	unsigned long step_size;
 	unsigned long addr;
 	unsigned long mapped_ram_size = 0;
-	unsigned long new_mapped_ram_size;
 	/* xen has big range in reserved near end of ram, skip it at first.*/
 	addr = memblock_find_in_range(map_start, map_end, PMD_SIZE, PMD_SIZE);
 	real_end = addr + PMD_SIZE;
 	/* step_size need to be small so pgt_buf from BRK could cover it */
 	step_size = PMD_SIZE;
 	max_pfn_mapped = 0; /* will get exact value next */
 	min_pfn_mapped = real_end >> PAGE_SHIFT;
 	last_start = start = real_end;
 	/*
 	 * We start from the top (end of memory) and go to the bottom.
 	 * The memblock_find_in_range() gets us a block of RAM from the
 	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
 	 * for page table.
 	 */
 	while (last_start > map_start) {
 		if (last_start > step_size) {
 			start = round_down(last_start - 1, step_size);
 			if (start < map_start)
 				start = map_start;
 		} else
 			start = map_start;
-		new_mapped_ram_size = init_range_memory_mapping(start,
+		mapped_ram_size += init_range_memory_mapping(start,
 							last_start);
 		last_start = start;
 		min_pfn_mapped = last_start >> PAGE_SHIFT;
-		/* only increase step_size after big range get mapped */
+		if (mapped_ram_size >= step_size)
-		if (new_mapped_ram_size > mapped_ram_size)
 			step_size = get_new_step_size(step_size);
-		mapped_ram_size += new_mapped_ram_size;
 	}
 	if (real_end < map_end)
 		init_range_memory_mapping(real_end, map_end);
 }
 /**
  * memory_map_bottom_up - Map [map_start, map_end) bottom up
  * @map_start: start address of the target memory range
  * @map_end: end address of the target memory range
  *
  * This function will setup direct mapping for memory range
  * [map_start, map_end) in bottom-up. Since we have limited the
  * bottom-up allocation above the kernel, the page tables will
  * be allocated just above the kernel and we map the memory
  * in [map_start, map_end) in bottom-up.
  */
 static void __init memory_map_bottom_up(unsigned long map_start,
 					unsigned long map_end)
 {
-	unsigned long next, new_mapped_ram_size, start;
+	unsigned long next, start;
 	unsigned long mapped_ram_size = 0;
 	/* step_size need to be small so pgt_buf from BRK could cover it */
 	unsigned long step_size = PMD_SIZE;
 	start = map_start;
 	min_pfn_mapped = start >> PAGE_SHIFT;
 	/*
 	 * We start from the bottom (@map_start) and go to the top (@map_end).
 	 * The memblock_find_in_range() gets us a block of RAM from the
 	 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
 	 * for page table.
 	 */
 	while (start < map_end) {
-		if (map_end - start > step_size) {
+		if (step_size && map_end - start > step_size) {
 			next = round_up(start + 1, step_size);
 			if (next > map_end)
 				next = map_end;
-		} else
+		} else {
 			next = map_end;
+		}
-		new_mapped_ram_size = init_range_memory_mapping(start, next);
+		mapped_ram_size += init_range_memory_mapping(start, next);
 		start = next;
-		if (new_mapped_ram_size > mapped_ram_size)
+		if (mapped_ram_size >= step_size)
 			step_size = get_new_step_size(step_size);
-		mapped_ram_size += new_mapped_ram_size;
 	}
 }
 void __init init_mem_mapping(void)
 {
 	unsigned long end;
 	probe_page_size_mask();
 #ifdef CONFIG_X86_64
 	end = max_pfn << PAGE_SHIFT;
 #else
 	end = max_low_pfn << PAGE_SHIFT;
 #endif
 	/* the ISA range is always mapped regardless of memory holes */
 	init_memory_mapping(0, ISA_END_ADDRESS);
 	/*
 	 * If the allocation is in bottom-up direction, we setup direct mapping
 	 * in bottom-up, otherwise we setup direct mapping in top-down.
 	 */
 	if (memblock_bottom_up()) {
 		unsigned long kernel_end = __pa_symbol(_end);
 		/*
 		 * we need two separate calls here. This is because we want to
 		 * allocate page tables above the kernel. So we first map
 		 * [kernel_end, end) to make memory above the kernel be mapped
 		 * as soon as possible. And then use page tables allocated above
 		 * the kernel to map [ISA_END_ADDRESS, kernel_end).
 		 */
 		memory_map_bottom_up(kernel_end, end);
 		memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
 	} else {
 		memory_map_top_down(ISA_END_ADDRESS, end);
 	}
 #ifdef CONFIG_X86_64
 	if (max_pfn > max_low_pfn) {
 		/* can we preseve max_low_pfn ?*/
 		max_low_pfn = max_pfn;
 	}
 #else
 	early_ioremap_page_table_range_init();
 #endif
 	load_cr3(swapper_pg_dir);
 	__flush_tlb_all();
 	early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
 }
 /*
  * devmem_is_allowed() checks to see if /dev/mem access to a certain address
  * is valid. The argument is a physical page number.
  *
  *
  * On x86, access has to be given to the first megabyte of ram because that area
  * contains bios code and data regions used by X and dosemu and similar apps.
  * Access has to be given to non-kernel-ram areas as well, these contain the PCI
  * mmio resources as well as potential bios/acpi data regions.
  */
 int devmem_is_allowed(unsigned long pagenr)
 {
 	if (pagenr < 256)
 		return 1;
 	if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
 		return 0;
 	if (!page_is_ram(pagenr))
 		return 1;
 	return 0;
 }
 void free_init_pages(char *what, unsigned long begin, unsigned long end)
 {
 	unsigned long begin_aligned, end_aligned;
 	/* Make sure boundaries are page aligned */
 	begin_aligned = PAGE_ALIGN(begin);
 	end_aligned   = end & PAGE_MASK;
 	if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
 		begin = begin_aligned;
 		end   = end_aligned;
 	}
 	if (begin >= end)
 		return;
 	/*
 	 * If debugging page accesses then do not free this memory but
 	 * mark them not present - any buggy init-section access will
 	 * create a kernel page fault:
 	 */
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	printk(KERN_INFO "debug: unmapping init [mem %#010lx-%#010lx]\n",
 		begin, end - 1);
 	set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
 #else
 	/*
 	 * We just marked the kernel text read only above, now that
 	 * we are going to free part of that, we need to make that
 	 * writeable and non-executable first.
 	 */
 	set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
 	set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
 	free_reserved_area((void *)begin, (void *)end, POISON_FREE_INITMEM, what);
 #endif
 }
 void free_initmem(void)
 {
 	free_init_pages("unused kernel",
 			(unsigned long)(&__init_begin),
 			(unsigned long)(&__init_end));
 }
 #ifdef CONFIG_BLK_DEV_INITRD
 void __init free_initrd_mem(unsigned long start, unsigned long end)
 {
 #ifdef CONFIG_MICROCODE_EARLY
 	/*
 	 * Remember, initrd memory may contain microcode or other useful things.
 	 * Before we lose initrd mem, we need to find a place to hold them
 	 * now that normal virtual memory is enabled.
 	 */
 	save_microcode_in_initrd();
 #endif
 	/*
 	 * end could be not aligned, and We can not align that,
 	 * decompresser could be confused by aligned initrd_end
 	 * We already reserve the end partial page before in
 	 *   - i386_start_kernel()
 	 *   - x86_64_start_kernel()
 	 *   - relocate_initrd()
 	 * So here We can do PAGE_ALIGN() safely to get partial page to be freed
 	 */
 	free_init_pages("initrd", start, PAGE_ALIGN(end));
 }
 #endif
 void __init zone_sizes_init(void)
 {
 	unsigned long max_zone_pfns[MAX_NR_ZONES];
 	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
 #ifdef CONFIG_ZONE_DMA
 	max_zone_pfns[ZONE_DMA]		= min(MAX_DMA_PFN, max_low_pfn);
 #endif
 #ifdef CONFIG_ZONE_DMA32
 	max_zone_pfns[ZONE_DMA32]	= min(MAX_DMA32_PFN, max_low_pfn);
 #endif
 	max_zone_pfns[ZONE_NORMAL]	= max_low_pfn;
 #ifdef CONFIG_HIGHMEM
 	max_zone_pfns[ZONE_HIGHMEM]	= max_pfn;
 #endif
 	free_area_init_nodes(max_zone_pfns);
 }
 void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
 {
 	/* entry 0 MUST be WB (hardwired to speed up translations) */
 	BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB);

arch/x86/vdso/vma.c

Diff comments View file @ 505569d

1	/*	1	/*
2	* Copyright 2007 Andi Kleen, SUSE Labs.	2	* Copyright 2007 Andi Kleen, SUSE Labs.
3	* Subject to the GPL, v.2	3	* Subject to the GPL, v.2
4	*	4	*
5	* This contains most of the x86 vDSO kernel-side code.	5	* This contains most of the x86 vDSO kernel-side code.
6	*/	6	*/
7	#include <linux/mm.h>	7	#include <linux/mm.h>
8	#include <linux/err.h>	8	#include <linux/err.h>
9	#include <linux/sched.h>	9	#include <linux/sched.h>
10	#include <linux/slab.h>	10	#include <linux/slab.h>
11	#include <linux/init.h>	11	#include <linux/init.h>
12	#include <linux/random.h>	12	#include <linux/random.h>
13	#include <linux/elf.h>	13	#include <linux/elf.h>
14	#include <linux/cpu.h>	14	#include <linux/cpu.h>
15	#include <asm/vgtod.h>	15	#include <asm/vgtod.h>
16	#include <asm/proto.h>	16	#include <asm/proto.h>
17	#include <asm/vdso.h>	17	#include <asm/vdso.h>
18	#include <asm/vvar.h>	18	#include <asm/vvar.h>
19	#include <asm/page.h>	19	#include <asm/page.h>
20	#include <asm/hpet.h>	20	#include <asm/hpet.h>
21	#include <asm/desc.h>	21	#include <asm/desc.h>
22		22
23	#if defined(CONFIG_X86_64)	23	#if defined(CONFIG_X86_64)
24	unsigned int __read_mostly vdso64_enabled = 1;	24	unsigned int __read_mostly vdso64_enabled = 1;
25	#endif	25	#endif
26		26
27	void __init init_vdso_image(const struct vdso_image *image)	27	void __init init_vdso_image(const struct vdso_image *image)
28	{	28	{
29	int i;	29	int i;
30	int npages = (image->size) / PAGE_SIZE;	30	int npages = (image->size) / PAGE_SIZE;
31		31
32	BUG_ON(image->size % PAGE_SIZE != 0);	32	BUG_ON(image->size % PAGE_SIZE != 0);
33	for (i = 0; i < npages; i++)	33	for (i = 0; i < npages; i++)
34	image->text_mapping.pages[i] =	34	image->text_mapping.pages[i] =
35	virt_to_page(image->data + i*PAGE_SIZE);	35	virt_to_page(image->data + i*PAGE_SIZE);
36		36
37	apply_alternatives((struct alt_instr *)(image->data + image->alt),	37	apply_alternatives((struct alt_instr *)(image->data + image->alt),
38	(struct alt_instr *)(image->data + image->alt +	38	(struct alt_instr *)(image->data + image->alt +
39	image->alt_len));	39	image->alt_len));
40	}	40	}
41		41
42	struct linux_binprm;	42	struct linux_binprm;
43		43
44	/* Put the vdso above the (randomized) stack with another randomized offset.	44	/*
45	This way there is no hole in the middle of address space.	45	* Put the vdso above the (randomized) stack with another randomized
46	To save memory make sure it is still in the same PTE as the stack top.	46	* offset. This way there is no hole in the middle of address space.
47	This doesn't give that many random bits.	47	* To save memory make sure it is still in the same PTE as the stack
48		48	* top. This doesn't give that many random bits.
49	Only used for the 64-bit and x32 vdsos. */	49	*
		50	* Note that this algorithm is imperfect: the distribution of the vdso
		51	* start address within a PMD is biased toward the end.
		52	*
		53	* Only used for the 64-bit and x32 vdsos.
		54	*/
50	static unsigned long vdso_addr(unsigned long start, unsigned len)	55	static unsigned long vdso_addr(unsigned long start, unsigned len)
51	{	56	{
52	#ifdef CONFIG_X86_32	57	#ifdef CONFIG_X86_32
53	return 0;	58	return 0;
54	#else	59	#else
55	unsigned long addr, end;	60	unsigned long addr, end;
56	unsigned offset;	61	unsigned offset;
57	end = (start + PMD_SIZE - 1) & PMD_MASK;	62
		63	/*
		64	* Round up the start address. It can start out unaligned as a result
		65	* of stack start randomization.
		66	*/
		67	start = PAGE_ALIGN(start);
		68
		69	/* Round the lowest possible end address up to a PMD boundary. */
		70	end = (start + len + PMD_SIZE - 1) & PMD_MASK;
58	if (end >= TASK_SIZE_MAX)	71	if (end >= TASK_SIZE_MAX)
59	end = TASK_SIZE_MAX;	72	end = TASK_SIZE_MAX;
60	end -= len;	73	end -= len;
61	/* This loses some more bits than a modulo, but is cheaper */
62	offset = get_random_int() & (PTRS_PER_PTE - 1);
63	addr = start + (offset << PAGE_SHIFT);
64	if (addr >= end)
65	addr = end;
66		74
		75	if (end > start) {
		76	offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1);
		77	addr = start + (offset << PAGE_SHIFT);
		78	} else {
		79	addr = start;
		80	}
		81
67	/*	82	/*
68	* page-align it here so that get_unmapped_area doesn't	83	* Forcibly align the final address in case we have a hardware
69	* align it wrongfully again to the next page. addr can come in 4K	84	* issue that requires alignment for performance reasons.
70	* unaligned here as a result of stack start randomization.
71	*/	85	*/
72	addr = PAGE_ALIGN(addr);
73	addr = align_vdso_addr(addr);	86	addr = align_vdso_addr(addr);
74		87
75	return addr;	88	return addr;
76	#endif	89	#endif
77	}	90	}
78		91
79	static int map_vdso(const struct vdso_image *image, bool calculate_addr)	92	static int map_vdso(const struct vdso_image *image, bool calculate_addr)
80	{	93	{
81	struct mm_struct *mm = current->mm;	94	struct mm_struct *mm = current->mm;
82	struct vm_area_struct *vma;	95	struct vm_area_struct *vma;
83	unsigned long addr, text_start;	96	unsigned long addr, text_start;
84	int ret = 0;	97	int ret = 0;
85	static struct page *no_pages[] = {NULL};	98	static struct page *no_pages[] = {NULL};
86	static struct vm_special_mapping vvar_mapping = {	99	static struct vm_special_mapping vvar_mapping = {
87	.name = "[vvar]",	100	.name = "[vvar]",
88	.pages = no_pages,	101	.pages = no_pages,
89	};	102	};
90		103
91	if (calculate_addr) {	104	if (calculate_addr) {
92	addr = vdso_addr(current->mm->start_stack,	105	addr = vdso_addr(current->mm->start_stack,
93	image->size - image->sym_vvar_start);	106	image->size - image->sym_vvar_start);
94	} else {	107	} else {
95	addr = 0;	108	addr = 0;
96	}	109	}
97		110
98	down_write(&mm->mmap_sem);	111	down_write(&mm->mmap_sem);
99		112
100	addr = get_unmapped_area(NULL, addr,	113	addr = get_unmapped_area(NULL, addr,
101	image->size - image->sym_vvar_start, 0, 0);	114	image->size - image->sym_vvar_start, 0, 0);
102	if (IS_ERR_VALUE(addr)) {	115	if (IS_ERR_VALUE(addr)) {
103	ret = addr;	116	ret = addr;
104	goto up_fail;	117	goto up_fail;
105	}	118	}
106		119
107	text_start = addr - image->sym_vvar_start;	120	text_start = addr - image->sym_vvar_start;
108	current->mm->context.vdso = (void __user *)text_start;	121	current->mm->context.vdso = (void __user *)text_start;
109		122
110	/*	123	/*
111	* MAYWRITE to allow gdb to COW and set breakpoints	124	* MAYWRITE to allow gdb to COW and set breakpoints
112	*/	125	*/
113	vma = _install_special_mapping(mm,	126	vma = _install_special_mapping(mm,
114	text_start,	127	text_start,
115	image->size,	128	image->size,
116	VM_READ\|VM_EXEC\|	129	VM_READ\|VM_EXEC\|
117	VM_MAYREAD\|VM_MAYWRITE\|VM_MAYEXEC,	130	VM_MAYREAD\|VM_MAYWRITE\|VM_MAYEXEC,
118	&image->text_mapping);	131	&image->text_mapping);
119		132
120	if (IS_ERR(vma)) {	133	if (IS_ERR(vma)) {
121	ret = PTR_ERR(vma);	134	ret = PTR_ERR(vma);
122	goto up_fail;	135	goto up_fail;
123	}	136	}
124		137
125	vma = _install_special_mapping(mm,	138	vma = _install_special_mapping(mm,
126	addr,	139	addr,
127	-image->sym_vvar_start,	140	-image->sym_vvar_start,
128	VM_READ\|VM_MAYREAD,	141	VM_READ\|VM_MAYREAD,
129	&vvar_mapping);	142	&vvar_mapping);
130		143
131	if (IS_ERR(vma)) {	144	if (IS_ERR(vma)) {
132	ret = PTR_ERR(vma);	145	ret = PTR_ERR(vma);
133	goto up_fail;	146	goto up_fail;
134	}	147	}
135		148
136	if (image->sym_vvar_page)	149	if (image->sym_vvar_page)
137	ret = remap_pfn_range(vma,	150	ret = remap_pfn_range(vma,
138	text_start + image->sym_vvar_page,	151	text_start + image->sym_vvar_page,
139	__pa_symbol(&__vvar_page) >> PAGE_SHIFT,	152	__pa_symbol(&__vvar_page) >> PAGE_SHIFT,
140	PAGE_SIZE,	153	PAGE_SIZE,
141	PAGE_READONLY);	154	PAGE_READONLY);
142		155
143	if (ret)	156	if (ret)
144	goto up_fail;	157	goto up_fail;
145		158
146	#ifdef CONFIG_HPET_TIMER	159	#ifdef CONFIG_HPET_TIMER
147	if (hpet_address && image->sym_hpet_page) {	160	if (hpet_address && image->sym_hpet_page) {
148	ret = io_remap_pfn_range(vma,	161	ret = io_remap_pfn_range(vma,
149	text_start + image->sym_hpet_page,	162	text_start + image->sym_hpet_page,
150	hpet_address >> PAGE_SHIFT,	163	hpet_address >> PAGE_SHIFT,
151	PAGE_SIZE,	164	PAGE_SIZE,
152	pgprot_noncached(PAGE_READONLY));	165	pgprot_noncached(PAGE_READONLY));
153		166
154	if (ret)	167	if (ret)
155	goto up_fail;	168	goto up_fail;
156	}	169	}
157	#endif	170	#endif
158		171
159	up_fail:	172	up_fail:
160	if (ret)	173	if (ret)
161	current->mm->context.vdso = NULL;	174	current->mm->context.vdso = NULL;
162		175
163	up_write(&mm->mmap_sem);	176	up_write(&mm->mmap_sem);
164	return ret;	177	return ret;
165	}	178	}
166		179
167	#if defined(CONFIG_X86_32) \|\| defined(CONFIG_COMPAT)	180	#if defined(CONFIG_X86_32) \|\| defined(CONFIG_COMPAT)
168	static int load_vdso32(void)	181	static int load_vdso32(void)
169	{	182	{
170	int ret;	183	int ret;
171		184
172	if (vdso32_enabled != 1) /* Other values all mean "disabled" */	185	if (vdso32_enabled != 1) /* Other values all mean "disabled" */
173	return 0;	186	return 0;
174		187
175	ret = map_vdso(selected_vdso32, false);	188	ret = map_vdso(selected_vdso32, false);
176	if (ret)	189	if (ret)
177	return ret;	190	return ret;
178		191
179	if (selected_vdso32->sym_VDSO32_SYSENTER_RETURN)	192	if (selected_vdso32->sym_VDSO32_SYSENTER_RETURN)
180	current_thread_info()->sysenter_return =	193	current_thread_info()->sysenter_return =
181	current->mm->context.vdso +	194	current->mm->context.vdso +
182	selected_vdso32->sym_VDSO32_SYSENTER_RETURN;	195	selected_vdso32->sym_VDSO32_SYSENTER_RETURN;
183		196
184	return 0;	197	return 0;
185	}	198	}
186	#endif	199	#endif
187		200
188	#ifdef CONFIG_X86_64	201	#ifdef CONFIG_X86_64
189	int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)	202	int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
190	{	203	{
191	if (!vdso64_enabled)	204	if (!vdso64_enabled)
192	return 0;	205	return 0;
193		206
194	return map_vdso(&vdso_image_64, true);	207	return map_vdso(&vdso_image_64, true);
195	}	208	}
196		209
197	#ifdef CONFIG_COMPAT	210	#ifdef CONFIG_COMPAT
198	int compat_arch_setup_additional_pages(struct linux_binprm *bprm,	211	int compat_arch_setup_additional_pages(struct linux_binprm *bprm,
199	int uses_interp)	212	int uses_interp)
200	{	213	{
201	#ifdef CONFIG_X86_X32_ABI	214	#ifdef CONFIG_X86_X32_ABI
202	if (test_thread_flag(TIF_X32)) {	215	if (test_thread_flag(TIF_X32)) {
203	if (!vdso64_enabled)	216	if (!vdso64_enabled)
204	return 0;	217	return 0;
205		218
206	return map_vdso(&vdso_image_x32, true);	219	return map_vdso(&vdso_image_x32, true);
207	}	220	}
208	#endif	221	#endif
209		222
210	return load_vdso32();	223	return load_vdso32();
211	}	224	}
212	#endif	225	#endif
213	#else	226	#else
214	int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)	227	int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
215	{	228	{
216	return load_vdso32();	229	return load_vdso32();
217	}	230	}
218	#endif	231	#endif
219		232
220	#ifdef CONFIG_X86_64	233	#ifdef CONFIG_X86_64
221	static __init int vdso_setup(char *s)	234	static __init int vdso_setup(char *s)
222	{	235	{
223	vdso64_enabled = simple_strtoul(s, NULL, 0);	236	vdso64_enabled = simple_strtoul(s, NULL, 0);
224	return 0;	237	return 0;
225	}	238	}
226	__setup("vdso=", vdso_setup);	239	__setup("vdso=", vdso_setup);
227	#endif	240	#endif
228		241
229	#ifdef CONFIG_X86_64	242	#ifdef CONFIG_X86_64
230	static void vgetcpu_cpu_init(void *arg)	243	static void vgetcpu_cpu_init(void *arg)
231	{	244	{
232	int cpu = smp_processor_id();	245	int cpu = smp_processor_id();
233	struct desc_struct d = { };	246	struct desc_struct d = { };
234	unsigned long node = 0;	247	unsigned long node = 0;
235	#ifdef CONFIG_NUMA	248	#ifdef CONFIG_NUMA
236	node = cpu_to_node(cpu);	249	node = cpu_to_node(cpu);
237	#endif	250	#endif
238	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))	251	if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
239	write_rdtscp_aux((node << 12) \| cpu);	252	write_rdtscp_aux((node << 12) \| cpu);
240		253
241	/*	254	/*
242	* Store cpu number in limit so that it can be loaded	255	* Store cpu number in limit so that it can be loaded
243	* quickly in user space in vgetcpu. (12 bits for the CPU	256	* quickly in user space in vgetcpu. (12 bits for the CPU
244	* and 8 bits for the node)	257	* and 8 bits for the node)
245	*/	258	*/
246	d.limit0 = cpu \| ((node & 0xf) << 12);	259	d.limit0 = cpu \| ((node & 0xf) << 12);
247	d.limit = node >> 4;	260	d.limit = node >> 4;
248	d.type = 5; /* RO data, expand down, accessed */	261	d.type = 5; /* RO data, expand down, accessed */
249	d.dpl = 3; /* Visible to user code */	262	d.dpl = 3; /* Visible to user code */
250	d.s = 1; /* Not a system segment */	263	d.s = 1; /* Not a system segment */
251	d.p = 1; /* Present */	264	d.p = 1; /* Present */
252	d.d = 1; /* 32-bit */	265	d.d = 1; /* 32-bit */
253		266
254	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);	267	write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
255	}	268	}
256		269
257	static int	270	static int
258	vgetcpu_cpu_notifier(struct notifier_block n, unsigned long action, void arg)	271	vgetcpu_cpu_notifier(struct notifier_block n, unsigned long action, void arg)
259	{	272	{
260	long cpu = (long)arg;	273	long cpu = (long)arg;
261		274
262	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN)	275	if (action == CPU_ONLINE \|\| action == CPU_ONLINE_FROZEN)
263	smp_call_function_single(cpu, vgetcpu_cpu_init, NULL, 1);	276	smp_call_function_single(cpu, vgetcpu_cpu_init, NULL, 1);
264		277
265	return NOTIFY_DONE;	278	return NOTIFY_DONE;
266	}	279	}
267		280
268	static int __init init_vdso(void)	281	static int __init init_vdso(void)
269	{	282	{
270	init_vdso_image(&vdso_image_64);	283	init_vdso_image(&vdso_image_64);
271		284
272	#ifdef CONFIG_X86_X32_ABI	285	#ifdef CONFIG_X86_X32_ABI
273	init_vdso_image(&vdso_image_x32);	286	init_vdso_image(&vdso_image_x32);
274	#endif	287	#endif
275		288
276	cpu_notifier_register_begin();	289	cpu_notifier_register_begin();
277		290
278	on_each_cpu(vgetcpu_cpu_init, NULL, 1);	291	on_each_cpu(vgetcpu_cpu_init, NULL, 1);
279	/* notifier priority > KVM */	292	/* notifier priority > KVM */
280	__hotcpu_notifier(vgetcpu_cpu_notifier, 30);	293	__hotcpu_notifier(vgetcpu_cpu_notifier, 30);
281		294