Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 79a69d342d71b2b4eafdf51e2451606cfe380a44

Authored by Linus Torvalds 2013-02-21 06:30:33 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

Merge tag 'arm64-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64

Pull arm64 patches from Catalin Marinas:

 - SMP support for the PSCI booting protocol (power state coordination
   interface).

 - Simple earlyprintk support.

 - Platform devices populated by default from the DT (SoC-agnostic).

 - CONTEXTIDR support (used by external trace tools).

* tag 'arm64-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux-aarch64:
  arm64: mm: update CONTEXTIDR register to contain PID of current process
  arm64: atomics: fix grossly inconsistent asm constraints for exclusives
  arm64: compat: use compat_uptr_t type for compat_ucontext.uc_link
  arm64: Select ARCH_WANT_FRAME_POINTERS
  arm64: Add kvm_para.h and xor.h generic headers
  arm64: SMP: enable PSCI boot method
  arm64: psci: add support for PSCI invocations from the kernel
  arm64: SMP: rework the SMP code to be enabling method agnostic
  arm64: perf: add guest vs host discrimination
  arm64: add COMPAT_PSR_*_BIT flags
  arm64: Add simple earlyprintk support
  arm64: Populate the platform devices

Showing 29 changed files Inline Diff

Documentation/arm64/memory.txt
arch/arm64/Kconfig
arch/arm64/Kconfig.debug
arch/arm64/include/asm/Kbuild
arch/arm64/include/asm/atomic.h
arch/arm64/include/asm/cmpxchg.h
arch/arm64/include/asm/futex.h
arch/arm64/include/asm/io.h
arch/arm64/include/asm/memory.h
arch/arm64/include/asm/mmu.h
arch/arm64/include/asm/mmu_context.h
arch/arm64/include/asm/perf_event.h
arch/arm64/include/asm/psci.h
arch/arm64/include/asm/ptrace.h
arch/arm64/include/asm/smp.h
arch/arm64/include/asm/spinlock.h
arch/arm64/include/uapi/asm/Kbuild
arch/arm64/kernel/Makefile
arch/arm64/kernel/early_printk.c
arch/arm64/kernel/head.S
arch/arm64/kernel/perf_event.c
arch/arm64/kernel/process.c
arch/arm64/kernel/psci.c
arch/arm64/kernel/setup.c
arch/arm64/kernel/signal32.c
arch/arm64/kernel/smp.c
arch/arm64/kernel/smp_psci.c
arch/arm64/kernel/smp_spin_table.c
arch/arm64/mm/mmu.c

Documentation/arm64/memory.txt

Diff comments View file @ 79a69d3

 		     Memory Layout on AArch64 Linux
 		     ==============================
 Author: Catalin Marinas <catalin.marinas@arm.com>
 Date  : 20 February 2012
 This document describes the virtual memory layout used by the AArch64
 Linux kernel. The architecture allows up to 4 levels of translation
 tables with a 4KB page size and up to 3 levels with a 64KB page size.
 AArch64 Linux uses 3 levels of translation tables with the 4KB page
 configuration, allowing 39-bit (512GB) virtual addresses for both user
 and kernel. With 64KB pages, only 2 levels of translation tables are
 used but the memory layout is the same.
 User addresses have bits 63:39 set to 0 while the kernel addresses have
 the same bits set to 1. TTBRx selection is given by bit 63 of the
 virtual address. The swapper_pg_dir contains only kernel (global)
 mappings while the user pgd contains only user (non-global) mappings.
 The swapper_pgd_dir address is written to TTBR1 and never written to
 TTBR0.
 AArch64 Linux memory layout:
 Start			End			Size		Use
 -----------------------------------------------------------------------
 0000000000000000	0000007fffffffff	 512GB		user
 ffffff8000000000	ffffffbbfffeffff	~240GB		vmalloc
 ffffffbbffff0000	ffffffbbffffffff	  64KB		[guard page]
 ffffffbc00000000	ffffffbdffffffff	   8GB		vmemmap
 ffffffbe00000000	ffffffbffbbfffff	  ~8GB		[guard, future vmmemap]
+ffffffbffbc00000	ffffffbffbdfffff	   2MB		earlyprintk device
 ffffffbffbe00000	ffffffbffbe0ffff	  64KB		PCI I/O space
 ffffffbbffff0000	ffffffbcffffffff	  ~2MB		[guard]
 ffffffbffc000000	ffffffbfffffffff	  64MB		modules
 ffffffc000000000	ffffffffffffffff	 256GB		kernel logical memory map
 Translation table lookup with 4KB pages:
 +--------+--------+--------+--------+--------+--------+--------+--------+
 |63    56|55    48|47    40|39    32|31    24|23    16|15     8|7      0|
 +--------+--------+--------+--------+--------+--------+--------+--------+
  |                 |         |         |         |         |
  |                 |         |         |         |         v
  |                 |         |         |         |   [11:0]  in-page offset
  |                 |         |         |         +-> [20:12] L3 index
  |                 |         |         +-----------> [29:21] L2 index
  |                 |         +---------------------> [38:30] L1 index
  |                 +-------------------------------> [47:39] L0 index (not used)
  +-------------------------------------------------> [63] TTBR0/1
 Translation table lookup with 64KB pages:
 +--------+--------+--------+--------+--------+--------+--------+--------+
 |63    56|55    48|47    40|39    32|31    24|23    16|15     8|7      0|
 +--------+--------+--------+--------+--------+--------+--------+--------+
  |                 |    |               |              |
  |                 |    |               |              v
  |                 |    |               |            [15:0]  in-page offset
  |                 |    |               +----------> [28:16] L3 index
  |                 |    +--------------------------> [41:29] L2 index (only 38:29 used)
  |                 +-------------------------------> [47:42] L1 index (not used)
  +-------------------------------------------------> [63] TTBR0/1

arch/arm64/Kconfig

Diff comments View file @ 79a69d3

 config ARM64
 	def_bool y
 	select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
 	select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
+	select ARCH_WANT_FRAME_POINTERS
 	select ARM_AMBA
 	select CLONE_BACKWARDS
 	select COMMON_CLK
 	select GENERIC_CLOCKEVENTS
 	select GENERIC_HARDIRQS_NO_DEPRECATED
 	select GENERIC_IOMAP
 	select GENERIC_IRQ_PROBE
 	select GENERIC_IRQ_SHOW
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select HARDIRQS_SW_RESEND
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_DEBUG_BUGVERBOSE
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_ATTRS
 	select HAVE_GENERIC_DMA_COHERENT
 	select HAVE_GENERIC_HARDIRQS
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_MEMBLOCK
 	select HAVE_PERF_EVENTS
 	select IRQ_DOMAIN
 	select MODULES_USE_ELF_RELA
 	select NO_BOOTMEM
 	select OF
 	select OF_EARLY_FLATTREE
 	select PERF_USE_VMALLOC
 	select RTC_LIB
 	select SPARSE_IRQ
 	select SYSCTL_EXCEPTION_TRACE
 	help
 	  ARM 64-bit (AArch64) Linux support.
 config 64BIT
 	def_bool y
 config ARCH_PHYS_ADDR_T_64BIT
 	def_bool y
 config MMU
 	def_bool y
 config NO_IOPORT
 	def_bool y
 config STACKTRACE_SUPPORT
 	def_bool y
 config LOCKDEP_SUPPORT
 	def_bool y
 config TRACE_IRQFLAGS_SUPPORT
 	def_bool y
 config GENERIC_LOCKBREAK
 	def_bool y
 	depends on SMP && PREEMPT
 config RWSEM_GENERIC_SPINLOCK
 	def_bool y
 config GENERIC_HWEIGHT
 	def_bool y
 config GENERIC_CSUM
         def_bool y
 config GENERIC_CALIBRATE_DELAY
 	def_bool y
 config ZONE_DMA32
 	def_bool y
 config ARCH_DMA_ADDR_T_64BIT
 	def_bool y
 config NEED_DMA_MAP_STATE
 	def_bool y
 config NEED_SG_DMA_LENGTH
 	def_bool y
 config SWIOTLB
 	def_bool y
 config IOMMU_HELPER
 	def_bool SWIOTLB
 config GENERIC_GPIO
 	def_bool y
 source "init/Kconfig"
 source "kernel/Kconfig.freezer"
 menu "System Type"
 endmenu
 menu "Bus support"
 config ARM_AMBA
 	bool
 endmenu
 menu "Kernel Features"
 source "kernel/time/Kconfig"
 config ARM64_64K_PAGES
 	bool "Enable 64KB pages support"
 	help
 	  This feature enables 64KB pages support (4KB by default)
 	  allowing only two levels of page tables and faster TLB
 	  look-up. AArch32 emulation is not available when this feature
 	  is enabled.
 config SMP
 	bool "Symmetric Multi-Processing"
 	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU.  If
 	  you say N here, the kernel will run on single and
 	  multiprocessor machines, but will use only one CPU of a
 	  multiprocessor machine. If you say Y here, the kernel will run
 	  on many, but not all, single processor machines. On a single
 	  processor machine, the kernel will run faster if you say N
 	  here.
 	  If you don't know what to do here, say N.
 config NR_CPUS
 	int "Maximum number of CPUs (2-32)"
 	range 2 32
 	depends on SMP
 	default "4"
 source kernel/Kconfig.preempt
 config HZ
 	int
 	default 100
 config ARCH_HAS_HOLES_MEMORYMODEL
 	def_bool y if SPARSEMEM
 config ARCH_SPARSEMEM_ENABLE
 	def_bool y
 	select SPARSEMEM_VMEMMAP_ENABLE
 config ARCH_SPARSEMEM_DEFAULT
 	def_bool ARCH_SPARSEMEM_ENABLE
 config ARCH_SELECT_MEMORY_MODEL
 	def_bool ARCH_SPARSEMEM_ENABLE
 config HAVE_ARCH_PFN_VALID
 	def_bool ARCH_HAS_HOLES_MEMORYMODEL || !SPARSEMEM
 config HW_PERF_EVENTS
 	bool "Enable hardware performance counter support for perf events"
 	depends on PERF_EVENTS
 	default y
 	help
 	  Enable hardware performance counter support for perf events. If
 	  disabled, perf events will use software events only.
 source "mm/Kconfig"
 endmenu
 menu "Boot options"
 config CMDLINE
 	string "Default kernel command string"
 	default ""
 	help
 	  Provide a set of default command-line options at build time by
 	  entering them here. As a minimum, you should specify the the
 	  root device (e.g. root=/dev/nfs).
 config CMDLINE_FORCE
 	bool "Always use the default kernel command string"
 	help
 	  Always use the default kernel command string, even if the boot
 	  loader passes other arguments to the kernel.
 	  This is useful if you cannot or don't want to change the
 	  command-line options your boot loader passes to the kernel.
 endmenu
 menu "Userspace binary formats"
 source "fs/Kconfig.binfmt"
 config COMPAT
 	bool "Kernel support for 32-bit EL0"
 	depends on !ARM64_64K_PAGES
 	select COMPAT_BINFMT_ELF
 	select HAVE_UID16
 	help
 	  This option enables support for a 32-bit EL0 running under a 64-bit
 	  kernel at EL1. AArch32-specific components such as system calls,
 	  the user helper functions, VFP support and the ptrace interface are
 	  handled appropriately by the kernel.
 	  If you want to execute 32-bit userspace applications, say Y.
 config SYSVIPC_COMPAT
 	def_bool y
 	depends on COMPAT && SYSVIPC
 endmenu
 source "net/Kconfig"
 source "drivers/Kconfig"
 source "fs/Kconfig"
 source "arch/arm64/Kconfig.debug"
 source "security/Kconfig"
 source "crypto/Kconfig"
 source "lib/Kconfig"

arch/arm64/Kconfig.debug

Diff comments View file @ 79a69d3

 menu "Kernel hacking"
 source "lib/Kconfig.debug"
 config FRAME_POINTER
 	bool
 	default y
 config DEBUG_ERRORS
 	bool "Verbose kernel error messages"
 	depends on DEBUG_KERNEL
 	help
 	  This option controls verbose debugging information which can be
 	  printed when the kernel detects an internal error. This debugging
 	  information is useful to kernel hackers when tracking down problems,
 	  but mostly meaningless to other people. It's safe to say Y unless
 	  you are concerned with the code size or don't want to see these
 	  messages.
 config DEBUG_STACK_USAGE
 	bool "Enable stack utilization instrumentation"
 	depends on DEBUG_KERNEL
 	help
 	  Enables the display of the minimum amount of free stack which each
 	  task has ever had available in the sysrq-T output.
+config EARLY_PRINTK
+	bool "Early printk support"
+	default y
+	help
+	  Say Y here if you want to have an early console using the
+	  earlyprintk=<name>[,<addr>][,<options>] kernel parameter. It
+	  is assumed that the early console device has been initialised
+	  by the boot loader prior to starting the Linux kernel.
+config PID_IN_CONTEXTIDR
+	bool "Write the current PID to the CONTEXTIDR register"
+	help
+	  Enabling this option causes the kernel to write the current PID to
+	  the CONTEXTIDR register, at the expense of some additional
+	  instructions during context switch. Say Y here only if you are
+	  planning to use hardware trace tools with this kernel.
 endmenu

arch/arm64/include/asm/Kbuild

Diff comments View file @ 79a69d3

 generic-y += bug.h
 generic-y += bugs.h
 generic-y += checksum.h
 generic-y += clkdev.h
 generic-y += cputime.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += div64.h
 generic-y += dma.h
 generic-y += emergency-restart.h
 generic-y += errno.h
 generic-y += ftrace.h
 generic-y += hw_irq.h
 generic-y += ioctl.h
 generic-y += ioctls.h
 generic-y += ipcbuf.h
 generic-y += irq_regs.h
 generic-y += kdebug.h
 generic-y += kmap_types.h
+generic-y += kvm_para.h
 generic-y += local.h
 generic-y += local64.h
 generic-y += mman.h
 generic-y += msgbuf.h
 generic-y += mutex.h
 generic-y += pci.h
 generic-y += percpu.h
 generic-y += poll.h
 generic-y += posix_types.h
 generic-y += resource.h
 generic-y += scatterlist.h
 generic-y += sections.h
 generic-y += segment.h
 generic-y += sembuf.h
 generic-y += serial.h
 generic-y += shmbuf.h
 generic-y += sizes.h
 generic-y += socket.h
 generic-y += sockios.h
 generic-y += string.h
 generic-y += switch_to.h
 generic-y += swab.h
 generic-y += termbits.h
 generic-y += termios.h
 generic-y += topology.h
 generic-y += trace_clock.h
 generic-y += types.h
 generic-y += unaligned.h
 generic-y += user.h
+generic-y += xor.h

arch/arm64/include/asm/atomic.h

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* Based on arch/arm/include/asm/atomic.h	2	* Based on arch/arm/include/asm/atomic.h
3	*	3	*
4	* Copyright (C) 1996 Russell King.	4	* Copyright (C) 1996 Russell King.
5	* Copyright (C) 2002 Deep Blue Solutions Ltd.	5	* Copyright (C) 2002 Deep Blue Solutions Ltd.
6	* Copyright (C) 2012 ARM Ltd.	6	* Copyright (C) 2012 ARM Ltd.
7	*	7	*
8	* This program is free software; you can redistribute it and/or modify	8	* This program is free software; you can redistribute it and/or modify
9	* it under the terms of the GNU General Public License version 2 as	9	* it under the terms of the GNU General Public License version 2 as
10	* published by the Free Software Foundation.	10	* published by the Free Software Foundation.
11	*	11	*
12	* This program is distributed in the hope that it will be useful,	12	* This program is distributed in the hope that it will be useful,
13	* but WITHOUT ANY WARRANTY; without even the implied warranty of	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	* GNU General Public License for more details.	15	* GNU General Public License for more details.
16	*	16	*
17	* You should have received a copy of the GNU General Public License	17	* You should have received a copy of the GNU General Public License
18	* along with this program. If not, see <http://www.gnu.org/licenses/>.	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
19	*/	19	*/
20	#ifndef __ASM_ATOMIC_H	20	#ifndef __ASM_ATOMIC_H
21	#define __ASM_ATOMIC_H	21	#define __ASM_ATOMIC_H
22		22
23	#include <linux/compiler.h>	23	#include <linux/compiler.h>
24	#include <linux/types.h>	24	#include <linux/types.h>
25		25
26	#include <asm/barrier.h>	26	#include <asm/barrier.h>
27	#include <asm/cmpxchg.h>	27	#include <asm/cmpxchg.h>
28		28
29	#define ATOMIC_INIT(i) { (i) }	29	#define ATOMIC_INIT(i) { (i) }
30		30
31	#ifdef __KERNEL__	31	#ifdef __KERNEL__
32		32
33	/*	33	/*
34	* On ARM, ordinary assignment (str instruction) doesn't clear the local	34	* On ARM, ordinary assignment (str instruction) doesn't clear the local
35	* strex/ldrex monitor on some implementations. The reason we can use it for	35	* strex/ldrex monitor on some implementations. The reason we can use it for
36	* atomic_set() is the clrex or dummy strex done on every exception return.	36	* atomic_set() is the clrex or dummy strex done on every exception return.
37	*/	37	*/
38	#define atomic_read(v) ((volatile int )&(v)->counter)	38	#define atomic_read(v) ((volatile int )&(v)->counter)
39	#define atomic_set(v,i) (((v)->counter) = (i))	39	#define atomic_set(v,i) (((v)->counter) = (i))
40		40
41	/*	41	/*
42	* AArch64 UP and SMP safe atomic ops. We use load exclusive and	42	* AArch64 UP and SMP safe atomic ops. We use load exclusive and
43	* store exclusive to ensure that these are atomic. We may loop	43	* store exclusive to ensure that these are atomic. We may loop
44	* to ensure that the update happens.	44	* to ensure that the update happens.
45	*/	45	*/
46	static inline void atomic_add(int i, atomic_t *v)	46	static inline void atomic_add(int i, atomic_t *v)
47	{	47	{
48	unsigned long tmp;	48	unsigned long tmp;
49	int result;	49	int result;
50		50
51	asm volatile("// atomic_add\n"	51	asm volatile("// atomic_add\n"
52	"1: ldxr %w0, [%3]\n"	52	"1: ldxr %w0, %2\n"
53	" add %w0, %w0, %w4\n"	53	" add %w0, %w0, %w3\n"
54	" stxr %w1, %w0, [%3]\n"	54	" stxr %w1, %w0, %2\n"
55	" cbnz %w1, 1b"	55	" cbnz %w1, 1b"
56	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	56	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
57	: "r" (&v->counter), "Ir" (i)	57	: "Ir" (i)
58	: "cc");	58	: "cc");
59	}	59	}
60		60
61	static inline int atomic_add_return(int i, atomic_t *v)	61	static inline int atomic_add_return(int i, atomic_t *v)
62	{	62	{
63	unsigned long tmp;	63	unsigned long tmp;
64	int result;	64	int result;
65		65
66	asm volatile("// atomic_add_return\n"	66	asm volatile("// atomic_add_return\n"
67	"1: ldaxr %w0, [%3]\n"	67	"1: ldaxr %w0, %2\n"
68	" add %w0, %w0, %w4\n"	68	" add %w0, %w0, %w3\n"
69	" stlxr %w1, %w0, [%3]\n"	69	" stlxr %w1, %w0, %2\n"
70	" cbnz %w1, 1b"	70	" cbnz %w1, 1b"
71	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	71	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
72	: "r" (&v->counter), "Ir" (i)	72	: "Ir" (i)
73	: "cc");	73	: "cc", "memory");
74		74
75	return result;	75	return result;
76	}	76	}
77		77
78	static inline void atomic_sub(int i, atomic_t *v)	78	static inline void atomic_sub(int i, atomic_t *v)
79	{	79	{
80	unsigned long tmp;	80	unsigned long tmp;
81	int result;	81	int result;
82		82
83	asm volatile("// atomic_sub\n"	83	asm volatile("// atomic_sub\n"
84	"1: ldxr %w0, [%3]\n"	84	"1: ldxr %w0, %2\n"
85	" sub %w0, %w0, %w4\n"	85	" sub %w0, %w0, %w3\n"
86	" stxr %w1, %w0, [%3]\n"	86	" stxr %w1, %w0, %2\n"
87	" cbnz %w1, 1b"	87	" cbnz %w1, 1b"
88	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	88	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
89	: "r" (&v->counter), "Ir" (i)	89	: "Ir" (i)
90	: "cc");	90	: "cc");
91	}	91	}
92		92
93	static inline int atomic_sub_return(int i, atomic_t *v)	93	static inline int atomic_sub_return(int i, atomic_t *v)
94	{	94	{
95	unsigned long tmp;	95	unsigned long tmp;
96	int result;	96	int result;
97		97
98	asm volatile("// atomic_sub_return\n"	98	asm volatile("// atomic_sub_return\n"
99	"1: ldaxr %w0, [%3]\n"	99	"1: ldaxr %w0, %2\n"
100	" sub %w0, %w0, %w4\n"	100	" sub %w0, %w0, %w3\n"
101	" stlxr %w1, %w0, [%3]\n"	101	" stlxr %w1, %w0, %2\n"
102	" cbnz %w1, 1b"	102	" cbnz %w1, 1b"
103	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	103	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
104	: "r" (&v->counter), "Ir" (i)	104	: "Ir" (i)
105	: "cc");	105	: "cc", "memory");
106		106
107	return result;	107	return result;
108	}	108	}
109		109
110	static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)	110	static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
111	{	111	{
112	unsigned long tmp;	112	unsigned long tmp;
113	int oldval;	113	int oldval;
114		114
115	asm volatile("// atomic_cmpxchg\n"	115	asm volatile("// atomic_cmpxchg\n"
116	"1: ldaxr %w1, [%3]\n"	116	"1: ldaxr %w1, %2\n"
117	" cmp %w1, %w4\n"	117	" cmp %w1, %w3\n"
118	" b.ne 2f\n"	118	" b.ne 2f\n"
119	" stlxr %w0, %w5, [%3]\n"	119	" stlxr %w0, %w4, %2\n"
120	" cbnz %w0, 1b\n"	120	" cbnz %w0, 1b\n"
121	"2:"	121	"2:"
122	: "=&r" (tmp), "=&r" (oldval), "+o" (ptr->counter)	122	: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
123	: "r" (&ptr->counter), "Ir" (old), "r" (new)	123	: "Ir" (old), "r" (new)
124	: "cc");	124	: "cc", "memory");
125		125
126	return oldval;	126	return oldval;
127	}	127	}
128		128
129	static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)	129	static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
130	{	130	{
131	unsigned long tmp, tmp2;	131	unsigned long tmp, tmp2;
132		132
133	asm volatile("// atomic_clear_mask\n"	133	asm volatile("// atomic_clear_mask\n"
134	"1: ldxr %0, [%3]\n"	134	"1: ldxr %0, %2\n"
135	" bic %0, %0, %4\n"	135	" bic %0, %0, %3\n"
136	" stxr %w1, %0, [%3]\n"	136	" stxr %w1, %0, %2\n"
137	" cbnz %w1, 1b"	137	" cbnz %w1, 1b"
138	: "=&r" (tmp), "=&r" (tmp2), "+o" (*addr)	138	: "=&r" (tmp), "=&r" (tmp2), "+Q" (*addr)
139	: "r" (addr), "Ir" (mask)	139	: "Ir" (mask)
140	: "cc");	140	: "cc");
141	}	141	}
142		142
143	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))	143	#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
144		144
145	static inline int __atomic_add_unless(atomic_t *v, int a, int u)	145	static inline int __atomic_add_unless(atomic_t *v, int a, int u)
146	{	146	{
147	int c, old;	147	int c, old;
148		148
149	c = atomic_read(v);	149	c = atomic_read(v);
150	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)	150	while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c)
151	c = old;	151	c = old;
152	return c;	152	return c;
153	}	153	}
154		154
155	#define atomic_inc(v) atomic_add(1, v)	155	#define atomic_inc(v) atomic_add(1, v)
156	#define atomic_dec(v) atomic_sub(1, v)	156	#define atomic_dec(v) atomic_sub(1, v)
157		157
158	#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)	158	#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0)
159	#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)	159	#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
160	#define atomic_inc_return(v) (atomic_add_return(1, v))	160	#define atomic_inc_return(v) (atomic_add_return(1, v))
161	#define atomic_dec_return(v) (atomic_sub_return(1, v))	161	#define atomic_dec_return(v) (atomic_sub_return(1, v))
162	#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)	162	#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
163		163
164	#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)	164	#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0)
165		165
166	#define smp_mb__before_atomic_dec() smp_mb()	166	#define smp_mb__before_atomic_dec() smp_mb()
167	#define smp_mb__after_atomic_dec() smp_mb()	167	#define smp_mb__after_atomic_dec() smp_mb()
168	#define smp_mb__before_atomic_inc() smp_mb()	168	#define smp_mb__before_atomic_inc() smp_mb()
169	#define smp_mb__after_atomic_inc() smp_mb()	169	#define smp_mb__after_atomic_inc() smp_mb()
170		170
171	/*	171	/*
172	* 64-bit atomic operations.	172	* 64-bit atomic operations.
173	*/	173	*/
174	#define ATOMIC64_INIT(i) { (i) }	174	#define ATOMIC64_INIT(i) { (i) }
175		175
176	#define atomic64_read(v) ((volatile long long )&(v)->counter)	176	#define atomic64_read(v) ((volatile long long )&(v)->counter)
177	#define atomic64_set(v,i) (((v)->counter) = (i))	177	#define atomic64_set(v,i) (((v)->counter) = (i))
178		178
179	static inline void atomic64_add(u64 i, atomic64_t *v)	179	static inline void atomic64_add(u64 i, atomic64_t *v)
180	{	180	{
181	long result;	181	long result;
182	unsigned long tmp;	182	unsigned long tmp;
183		183
184	asm volatile("// atomic64_add\n"	184	asm volatile("// atomic64_add\n"
185	"1: ldxr %0, [%3]\n"	185	"1: ldxr %0, %2\n"
186	" add %0, %0, %4\n"	186	" add %0, %0, %3\n"
187	" stxr %w1, %0, [%3]\n"	187	" stxr %w1, %0, %2\n"
188	" cbnz %w1, 1b"	188	" cbnz %w1, 1b"
189	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	189	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
190	: "r" (&v->counter), "Ir" (i)	190	: "Ir" (i)
191	: "cc");	191	: "cc");
192	}	192	}
193		193
194	static inline long atomic64_add_return(long i, atomic64_t *v)	194	static inline long atomic64_add_return(long i, atomic64_t *v)
195	{	195	{
196	long result;	196	long result;
197	unsigned long tmp;	197	unsigned long tmp;
198		198
199	asm volatile("// atomic64_add_return\n"	199	asm volatile("// atomic64_add_return\n"
200	"1: ldaxr %0, [%3]\n"	200	"1: ldaxr %0, %2\n"
201	" add %0, %0, %4\n"	201	" add %0, %0, %3\n"
202	" stlxr %w1, %0, [%3]\n"	202	" stlxr %w1, %0, %2\n"
203	" cbnz %w1, 1b"	203	" cbnz %w1, 1b"
204	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	204	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
205	: "r" (&v->counter), "Ir" (i)	205	: "Ir" (i)
206	: "cc");	206	: "cc", "memory");
207		207
208	return result;	208	return result;
209	}	209	}
210		210
211	static inline void atomic64_sub(u64 i, atomic64_t *v)	211	static inline void atomic64_sub(u64 i, atomic64_t *v)
212	{	212	{
213	long result;	213	long result;
214	unsigned long tmp;	214	unsigned long tmp;
215		215
216	asm volatile("// atomic64_sub\n"	216	asm volatile("// atomic64_sub\n"
217	"1: ldxr %0, [%3]\n"	217	"1: ldxr %0, %2\n"
218	" sub %0, %0, %4\n"	218	" sub %0, %0, %3\n"
219	" stxr %w1, %0, [%3]\n"	219	" stxr %w1, %0, %2\n"
220	" cbnz %w1, 1b"	220	" cbnz %w1, 1b"
221	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	221	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
222	: "r" (&v->counter), "Ir" (i)	222	: "Ir" (i)
223	: "cc");	223	: "cc");
224	}	224	}
225		225
226	static inline long atomic64_sub_return(long i, atomic64_t *v)	226	static inline long atomic64_sub_return(long i, atomic64_t *v)
227	{	227	{
228	long result;	228	long result;
229	unsigned long tmp;	229	unsigned long tmp;
230		230
231	asm volatile("// atomic64_sub_return\n"	231	asm volatile("// atomic64_sub_return\n"
232	"1: ldaxr %0, [%3]\n"	232	"1: ldaxr %0, %2\n"
233	" sub %0, %0, %4\n"	233	" sub %0, %0, %3\n"
234	" stlxr %w1, %0, [%3]\n"	234	" stlxr %w1, %0, %2\n"
235	" cbnz %w1, 1b"	235	" cbnz %w1, 1b"
236	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	236	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
237	: "r" (&v->counter), "Ir" (i)	237	: "Ir" (i)
238	: "cc");	238	: "cc", "memory");
239		239
240	return result;	240	return result;
241	}	241	}
242		242
243	static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)	243	static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
244	{	244	{
245	long oldval;	245	long oldval;
246	unsigned long res;	246	unsigned long res;
247		247
248	asm volatile("// atomic64_cmpxchg\n"	248	asm volatile("// atomic64_cmpxchg\n"
249	"1: ldaxr %1, [%3]\n"	249	"1: ldaxr %1, %2\n"
250	" cmp %1, %4\n"	250	" cmp %1, %3\n"
251	" b.ne 2f\n"	251	" b.ne 2f\n"
252	" stlxr %w0, %5, [%3]\n"	252	" stlxr %w0, %4, %2\n"
253	" cbnz %w0, 1b\n"	253	" cbnz %w0, 1b\n"
254	"2:"	254	"2:"
255	: "=&r" (res), "=&r" (oldval), "+o" (ptr->counter)	255	: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
256	: "r" (&ptr->counter), "Ir" (old), "r" (new)	256	: "Ir" (old), "r" (new)
257	: "cc");	257	: "cc", "memory");
258		258
259	return oldval;	259	return oldval;
260	}	260	}
261		261
262	#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))	262	#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
263		263
264	static inline long atomic64_dec_if_positive(atomic64_t *v)	264	static inline long atomic64_dec_if_positive(atomic64_t *v)
265	{	265	{
266	long result;	266	long result;
267	unsigned long tmp;	267	unsigned long tmp;
268		268
269	asm volatile("// atomic64_dec_if_positive\n"	269	asm volatile("// atomic64_dec_if_positive\n"
270	"1: ldaxr %0, [%3]\n"	270	"1: ldaxr %0, %2\n"
271	" subs %0, %0, #1\n"	271	" subs %0, %0, #1\n"
272	" b.mi 2f\n"	272	" b.mi 2f\n"
273	" stlxr %w1, %0, [%3]\n"	273	" stlxr %w1, %0, %2\n"
274	" cbnz %w1, 1b\n"	274	" cbnz %w1, 1b\n"
275	"2:"	275	"2:"
276	: "=&r" (result), "=&r" (tmp), "+o" (v->counter)	276	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
277	: "r" (&v->counter)	277	:
278	: "cc");	278	: "cc", "memory");
279		279
280	return result;	280	return result;
281	}	281	}
282		282
283	static inline int atomic64_add_unless(atomic64_t *v, long a, long u)	283	static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
284	{	284	{
285	long c, old;	285	long c, old;
286		286
287	c = atomic64_read(v);	287	c = atomic64_read(v);
288	while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)	288	while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c)
289	c = old;	289	c = old;
290		290
291	return c != u;	291	return c != u;
292	}	292	}
293		293
294	#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)	294	#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0)
295	#define atomic64_inc(v) atomic64_add(1LL, (v))	295	#define atomic64_inc(v) atomic64_add(1LL, (v))
296	#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))	296	#define atomic64_inc_return(v) atomic64_add_return(1LL, (v))
297	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)	297	#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
298	#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)	298	#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0)
299	#define atomic64_dec(v) atomic64_sub(1LL, (v))	299	#define atomic64_dec(v) atomic64_sub(1LL, (v))
300	#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))	300	#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v))
301	#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)	301	#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0)
302	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)	302	#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL)
303		303
304	#endif	304	#endif
305	#endif	305	#endif
306		306

arch/arm64/include/asm/cmpxchg.h

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* Based on arch/arm/include/asm/cmpxchg.h	2	* Based on arch/arm/include/asm/cmpxchg.h
3	*	3	*
4	* Copyright (C) 2012 ARM Ltd.	4	* Copyright (C) 2012 ARM Ltd.
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify	6	* This program is free software; you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License version 2 as	7	* it under the terms of the GNU General Public License version 2 as
8	* published by the Free Software Foundation.	8	* published by the Free Software Foundation.
9	*	9	*
10	* This program is distributed in the hope that it will be useful,	10	* This program is distributed in the hope that it will be useful,
11	* but WITHOUT ANY WARRANTY; without even the implied warranty of	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	* GNU General Public License for more details.	13	* GNU General Public License for more details.
14	*	14	*
15	* You should have received a copy of the GNU General Public License	15	* You should have received a copy of the GNU General Public License
16	* along with this program. If not, see <http://www.gnu.org/licenses/>.	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
17	*/	17	*/
18	#ifndef __ASM_CMPXCHG_H	18	#ifndef __ASM_CMPXCHG_H
19	#define __ASM_CMPXCHG_H	19	#define __ASM_CMPXCHG_H
20		20
21	#include <linux/bug.h>	21	#include <linux/bug.h>
22		22
23	#include <asm/barrier.h>	23	#include <asm/barrier.h>
24		24
25	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)	25	static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
26	{	26	{
27	unsigned long ret, tmp;	27	unsigned long ret, tmp;
28		28
29	switch (size) {	29	switch (size) {
30	case 1:	30	case 1:
31	asm volatile("// __xchg1\n"	31	asm volatile("// __xchg1\n"
32	"1: ldaxrb %w0, [%3]\n"	32	"1: ldaxrb %w0, %2\n"
33	" stlxrb %w1, %w2, [%3]\n"	33	" stlxrb %w1, %w3, %2\n"
34	" cbnz %w1, 1b\n"	34	" cbnz %w1, 1b\n"
35	: "=&r" (ret), "=&r" (tmp)	35	: "=&r" (ret), "=&r" (tmp), "+Q" ((u8 )ptr)
36	: "r" (x), "r" (ptr)	36	: "r" (x)
37	: "memory", "cc");	37	: "cc", "memory");
38	break;	38	break;
39	case 2:	39	case 2:
40	asm volatile("// __xchg2\n"	40	asm volatile("// __xchg2\n"
41	"1: ldaxrh %w0, [%3]\n"	41	"1: ldaxrh %w0, %2\n"
42	" stlxrh %w1, %w2, [%3]\n"	42	" stlxrh %w1, %w3, %2\n"
43	" cbnz %w1, 1b\n"	43	" cbnz %w1, 1b\n"
44	: "=&r" (ret), "=&r" (tmp)	44	: "=&r" (ret), "=&r" (tmp), "+Q" ((u16 )ptr)
45	: "r" (x), "r" (ptr)	45	: "r" (x)
46	: "memory", "cc");	46	: "cc", "memory");
47	break;	47	break;
48	case 4:	48	case 4:
49	asm volatile("// __xchg4\n"	49	asm volatile("// __xchg4\n"
50	"1: ldaxr %w0, [%3]\n"	50	"1: ldaxr %w0, %2\n"
51	" stlxr %w1, %w2, [%3]\n"	51	" stlxr %w1, %w3, %2\n"
52	" cbnz %w1, 1b\n"	52	" cbnz %w1, 1b\n"
53	: "=&r" (ret), "=&r" (tmp)	53	: "=&r" (ret), "=&r" (tmp), "+Q" ((u32 )ptr)
54	: "r" (x), "r" (ptr)	54	: "r" (x)
55	: "memory", "cc");	55	: "cc", "memory");
56	break;	56	break;
57	case 8:	57	case 8:
58	asm volatile("// __xchg8\n"	58	asm volatile("// __xchg8\n"
59	"1: ldaxr %0, [%3]\n"	59	"1: ldaxr %0, %2\n"
60	" stlxr %w1, %2, [%3]\n"	60	" stlxr %w1, %3, %2\n"
61	" cbnz %w1, 1b\n"	61	" cbnz %w1, 1b\n"
62	: "=&r" (ret), "=&r" (tmp)	62	: "=&r" (ret), "=&r" (tmp), "+Q" ((u64 )ptr)
63	: "r" (x), "r" (ptr)	63	: "r" (x)
64	: "memory", "cc");	64	: "cc", "memory");
65	break;	65	break;
66	default:	66	default:
67	BUILD_BUG();	67	BUILD_BUG();
68	}	68	}
69		69
70	return ret;	70	return ret;
71	}	71	}
72		72
73	#define xchg(ptr,x) \	73	#define xchg(ptr,x) \
74	((__typeof__((ptr)))__xchg((unsigned long)(x),(ptr),sizeof((ptr))))	74	((__typeof__((ptr)))__xchg((unsigned long)(x),(ptr),sizeof((ptr))))
75		75
76	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,	76	static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
77	unsigned long new, int size)	77	unsigned long new, int size)
78	{	78	{
79	unsigned long oldval = 0, res;	79	unsigned long oldval = 0, res;
80		80
81	switch (size) {	81	switch (size) {
82	case 1:	82	case 1:
83	do {	83	do {
84	asm volatile("// __cmpxchg1\n"	84	asm volatile("// __cmpxchg1\n"
85	" ldxrb %w1, [%2]\n"	85	" ldxrb %w1, %2\n"
86	" mov %w0, #0\n"	86	" mov %w0, #0\n"
87	" cmp %w1, %w3\n"	87	" cmp %w1, %w3\n"
88	" b.ne 1f\n"	88	" b.ne 1f\n"
89	" stxrb %w0, %w4, [%2]\n"	89	" stxrb %w0, %w4, %2\n"
90	"1:\n"	90	"1:\n"
91	: "=&r" (res), "=&r" (oldval)	91	: "=&r" (res), "=&r" (oldval), "+Q" ((u8 )ptr)
92	: "r" (ptr), "Ir" (old), "r" (new)	92	: "Ir" (old), "r" (new)
93	: "cc");	93	: "cc");
94	} while (res);	94	} while (res);
95	break;	95	break;
96		96
97	case 2:	97	case 2:
98	do {	98	do {
99	asm volatile("// __cmpxchg2\n"	99	asm volatile("// __cmpxchg2\n"
100	" ldxrh %w1, [%2]\n"	100	" ldxrh %w1, %2\n"
101	" mov %w0, #0\n"	101	" mov %w0, #0\n"
102	" cmp %w1, %w3\n"	102	" cmp %w1, %w3\n"
103	" b.ne 1f\n"	103	" b.ne 1f\n"
104	" stxrh %w0, %w4, [%2]\n"	104	" stxrh %w0, %w4, %2\n"
105	"1:\n"	105	"1:\n"
106	: "=&r" (res), "=&r" (oldval)	106	: "=&r" (res), "=&r" (oldval), "+Q" ((u16 )ptr)
107	: "r" (ptr), "Ir" (old), "r" (new)	107	: "Ir" (old), "r" (new)
108	: "memory", "cc");	108	: "cc");
109	} while (res);	109	} while (res);
110	break;	110	break;
111		111
112	case 4:	112	case 4:
113	do {	113	do {
114	asm volatile("// __cmpxchg4\n"	114	asm volatile("// __cmpxchg4\n"
115	" ldxr %w1, [%2]\n"	115	" ldxr %w1, %2\n"
116	" mov %w0, #0\n"	116	" mov %w0, #0\n"
117	" cmp %w1, %w3\n"	117	" cmp %w1, %w3\n"
118	" b.ne 1f\n"	118	" b.ne 1f\n"
119	" stxr %w0, %w4, [%2]\n"	119	" stxr %w0, %w4, %2\n"
120	"1:\n"	120	"1:\n"
121	: "=&r" (res), "=&r" (oldval)	121	: "=&r" (res), "=&r" (oldval), "+Q" ((u32 )ptr)
122	: "r" (ptr), "Ir" (old), "r" (new)	122	: "Ir" (old), "r" (new)
123	: "cc");	123	: "cc");
124	} while (res);	124	} while (res);
125	break;	125	break;
126		126
127	case 8:	127	case 8:
128	do {	128	do {
129	asm volatile("// __cmpxchg8\n"	129	asm volatile("// __cmpxchg8\n"
130	" ldxr %1, [%2]\n"	130	" ldxr %1, %2\n"
131	" mov %w0, #0\n"	131	" mov %w0, #0\n"
132	" cmp %1, %3\n"	132	" cmp %1, %3\n"
133	" b.ne 1f\n"	133	" b.ne 1f\n"
134	" stxr %w0, %4, [%2]\n"	134	" stxr %w0, %4, %2\n"
135	"1:\n"	135	"1:\n"
136	: "=&r" (res), "=&r" (oldval)	136	: "=&r" (res), "=&r" (oldval), "+Q" ((u64 )ptr)
137	: "r" (ptr), "Ir" (old), "r" (new)	137	: "Ir" (old), "r" (new)
138	: "cc");	138	: "cc");
139	} while (res);	139	} while (res);
140	break;	140	break;
141		141
142	default:	142	default:
143	BUILD_BUG();	143	BUILD_BUG();
144	}	144	}
145		145
146	return oldval;	146	return oldval;
147	}	147	}
148		148
149	static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,	149	static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
150	unsigned long new, int size)	150	unsigned long new, int size)
151	{	151	{
152	unsigned long ret;	152	unsigned long ret;
153		153
154	smp_mb();	154	smp_mb();
155	ret = __cmpxchg(ptr, old, new, size);	155	ret = __cmpxchg(ptr, old, new, size);
156	smp_mb();	156	smp_mb();
157		157
158	return ret;	158	return ret;
159	}	159	}
160		160
161	#define cmpxchg(ptr,o,n) \	161	#define cmpxchg(ptr,o,n) \
162	((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \	162	((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \
163	(unsigned long)(o), \	163	(unsigned long)(o), \
164	(unsigned long)(n), \	164	(unsigned long)(n), \
165	sizeof(*(ptr))))	165	sizeof(*(ptr))))
166		166
167	#define cmpxchg_local(ptr,o,n) \	167	#define cmpxchg_local(ptr,o,n) \
168	((__typeof__(*(ptr)))__cmpxchg((ptr), \	168	((__typeof__(*(ptr)))__cmpxchg((ptr), \
169	(unsigned long)(o), \	169	(unsigned long)(o), \
170	(unsigned long)(n), \	170	(unsigned long)(n), \
171	sizeof(*(ptr))))	171	sizeof(*(ptr))))
172		172
173	#endif /* __ASM_CMPXCHG_H */	173	#endif /* __ASM_CMPXCHG_H */
174		174

arch/arm64/include/asm/futex.h

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* Copyright (C) 2012 ARM Ltd.	2	* Copyright (C) 2012 ARM Ltd.
3	*	3	*
4	* This program is free software; you can redistribute it and/or modify	4	* This program is free software; you can redistribute it and/or modify
5	* it under the terms of the GNU General Public License version 2 as	5	* it under the terms of the GNU General Public License version 2 as
6	* published by the Free Software Foundation.	6	* published by the Free Software Foundation.
7	*	7	*
8	* This program is distributed in the hope that it will be useful,	8	* This program is distributed in the hope that it will be useful,
9	* but WITHOUT ANY WARRANTY; without even the implied warranty of	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	* GNU General Public License for more details.	11	* GNU General Public License for more details.
12	*	12	*
13	* You should have received a copy of the GNU General Public License	13	* You should have received a copy of the GNU General Public License
14	* along with this program. If not, see <http://www.gnu.org/licenses/>.	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
15	*/	15	*/
16	#ifndef __ASM_FUTEX_H	16	#ifndef __ASM_FUTEX_H
17	#define __ASM_FUTEX_H	17	#define __ASM_FUTEX_H
18		18
19	#ifdef __KERNEL__	19	#ifdef __KERNEL__
20		20
21	#include <linux/futex.h>	21	#include <linux/futex.h>
22	#include <linux/uaccess.h>	22	#include <linux/uaccess.h>
23	#include <asm/errno.h>	23	#include <asm/errno.h>
24		24
25	#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \	25	#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
26	asm volatile( \	26	asm volatile( \
27	"1: ldaxr %w1, %2\n" \	27	"1: ldaxr %w1, %2\n" \
28	insn "\n" \	28	insn "\n" \
29	"2: stlxr %w3, %w0, %2\n" \	29	"2: stlxr %w3, %w0, %2\n" \
30	" cbnz %w3, 1b\n" \	30	" cbnz %w3, 1b\n" \
31	"3:\n" \	31	"3:\n" \
32	" .pushsection .fixup,\"ax\"\n" \	32	" .pushsection .fixup,\"ax\"\n" \
33	"4: mov %w0, %w5\n" \	33	"4: mov %w0, %w5\n" \
34	" b 3b\n" \	34	" b 3b\n" \
35	" .popsection\n" \	35	" .popsection\n" \
36	" .pushsection __ex_table,\"a\"\n" \	36	" .pushsection __ex_table,\"a\"\n" \
37	" .align 3\n" \	37	" .align 3\n" \
38	" .quad 1b, 4b, 2b, 4b\n" \	38	" .quad 1b, 4b, 2b, 4b\n" \
39	" .popsection\n" \	39	" .popsection\n" \
40	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \	40	: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
41	: "r" (oparg), "Ir" (-EFAULT) \	41	: "r" (oparg), "Ir" (-EFAULT) \
42	: "cc")	42	: "cc", "memory")
43		43
44	static inline int	44	static inline int
45	futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)	45	futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
46	{	46	{
47	int op = (encoded_op >> 28) & 7;	47	int op = (encoded_op >> 28) & 7;
48	int cmp = (encoded_op >> 24) & 15;	48	int cmp = (encoded_op >> 24) & 15;
49	int oparg = (encoded_op << 8) >> 20;	49	int oparg = (encoded_op << 8) >> 20;
50	int cmparg = (encoded_op << 20) >> 20;	50	int cmparg = (encoded_op << 20) >> 20;
51	int oldval = 0, ret, tmp;	51	int oldval = 0, ret, tmp;
52		52
53	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))	53	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
54	oparg = 1 << oparg;	54	oparg = 1 << oparg;
55		55
56	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))	56	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
57	return -EFAULT;	57	return -EFAULT;
58		58
59	pagefault_disable(); /* implies preempt_disable() */	59	pagefault_disable(); /* implies preempt_disable() */
60		60
61	switch (op) {	61	switch (op) {
62	case FUTEX_OP_SET:	62	case FUTEX_OP_SET:
63	__futex_atomic_op("mov %w0, %w4",	63	__futex_atomic_op("mov %w0, %w4",
64	ret, oldval, uaddr, tmp, oparg);	64	ret, oldval, uaddr, tmp, oparg);
65	break;	65	break;
66	case FUTEX_OP_ADD:	66	case FUTEX_OP_ADD:
67	__futex_atomic_op("add %w0, %w1, %w4",	67	__futex_atomic_op("add %w0, %w1, %w4",
68	ret, oldval, uaddr, tmp, oparg);	68	ret, oldval, uaddr, tmp, oparg);
69	break;	69	break;
70	case FUTEX_OP_OR:	70	case FUTEX_OP_OR:
71	__futex_atomic_op("orr %w0, %w1, %w4",	71	__futex_atomic_op("orr %w0, %w1, %w4",
72	ret, oldval, uaddr, tmp, oparg);	72	ret, oldval, uaddr, tmp, oparg);
73	break;	73	break;
74	case FUTEX_OP_ANDN:	74	case FUTEX_OP_ANDN:
75	__futex_atomic_op("and %w0, %w1, %w4",	75	__futex_atomic_op("and %w0, %w1, %w4",
76	ret, oldval, uaddr, tmp, ~oparg);	76	ret, oldval, uaddr, tmp, ~oparg);
77	break;	77	break;
78	case FUTEX_OP_XOR:	78	case FUTEX_OP_XOR:
79	__futex_atomic_op("eor %w0, %w1, %w4",	79	__futex_atomic_op("eor %w0, %w1, %w4",
80	ret, oldval, uaddr, tmp, oparg);	80	ret, oldval, uaddr, tmp, oparg);
81	break;	81	break;
82	default:	82	default:
83	ret = -ENOSYS;	83	ret = -ENOSYS;
84	}	84	}
85		85
86	pagefault_enable(); /* subsumes preempt_enable() */	86	pagefault_enable(); /* subsumes preempt_enable() */
87		87
88	if (!ret) {	88	if (!ret) {
89	switch (cmp) {	89	switch (cmp) {
90	case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break;	90	case FUTEX_OP_CMP_EQ: ret = (oldval == cmparg); break;
91	case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break;	91	case FUTEX_OP_CMP_NE: ret = (oldval != cmparg); break;
92	case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break;	92	case FUTEX_OP_CMP_LT: ret = (oldval < cmparg); break;
93	case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break;	93	case FUTEX_OP_CMP_GE: ret = (oldval >= cmparg); break;
94	case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break;	94	case FUTEX_OP_CMP_LE: ret = (oldval <= cmparg); break;
95	case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break;	95	case FUTEX_OP_CMP_GT: ret = (oldval > cmparg); break;
96	default: ret = -ENOSYS;	96	default: ret = -ENOSYS;
97	}	97	}
98	}	98	}
99	return ret;	99	return ret;
100	}	100	}
101		101
102	static inline int	102	static inline int
103	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,	103	futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
104	u32 oldval, u32 newval)	104	u32 oldval, u32 newval)
105	{	105	{
106	int ret = 0;	106	int ret = 0;
107	u32 val, tmp;	107	u32 val, tmp;
108		108
109	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))	109	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
110	return -EFAULT;	110	return -EFAULT;
111		111
112	asm volatile("// futex_atomic_cmpxchg_inatomic\n"	112	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
113	"1: ldaxr %w1, %2\n"	113	"1: ldaxr %w1, %2\n"
114	" sub %w3, %w1, %w4\n"	114	" sub %w3, %w1, %w4\n"
115	" cbnz %w3, 3f\n"	115	" cbnz %w3, 3f\n"
116	"2: stlxr %w3, %w5, %2\n"	116	"2: stlxr %w3, %w5, %2\n"
117	" cbnz %w3, 1b\n"	117	" cbnz %w3, 1b\n"
118	"3:\n"	118	"3:\n"
119	" .pushsection .fixup,\"ax\"\n"	119	" .pushsection .fixup,\"ax\"\n"
120	"4: mov %w0, %w6\n"	120	"4: mov %w0, %w6\n"
121	" b 3b\n"	121	" b 3b\n"
122	" .popsection\n"	122	" .popsection\n"
123	" .pushsection __ex_table,\"a\"\n"	123	" .pushsection __ex_table,\"a\"\n"
124	" .align 3\n"	124	" .align 3\n"
125	" .quad 1b, 4b, 2b, 4b\n"	125	" .quad 1b, 4b, 2b, 4b\n"
126	" .popsection\n"	126	" .popsection\n"
127	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)	127	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp)
128	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)	128	: "r" (oldval), "r" (newval), "Ir" (-EFAULT)
129	: "cc", "memory");	129	: "cc", "memory");
130		130
131	*uval = val;	131	*uval = val;
132	return ret;	132	return ret;
133	}	133	}
134		134
135	#endif /* __KERNEL__ */	135	#endif /* __KERNEL__ */
136	#endif /* __ASM_FUTEX_H */	136	#endif /* __ASM_FUTEX_H */
137		137

arch/arm64/include/asm/io.h

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/include/asm/io.h
  *
  * Copyright (C) 1996-2000 Russell King
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_IO_H
 #define __ASM_IO_H
 #ifdef __KERNEL__
 #include <linux/types.h>
 #include <asm/byteorder.h>
 #include <asm/barrier.h>
 #include <asm/pgtable.h>
 /*
  * Generic IO read/write.  These perform native-endian accesses.
  */
 static inline void __raw_writeb(u8 val, volatile void __iomem *addr)
 {
 	asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr));
 }
 static inline void __raw_writew(u16 val, volatile void __iomem *addr)
 {
 	asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr));
 }
 static inline void __raw_writel(u32 val, volatile void __iomem *addr)
 {
 	asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr));
 }
 static inline void __raw_writeq(u64 val, volatile void __iomem *addr)
 {
 	asm volatile("str %0, [%1]" : : "r" (val), "r" (addr));
 }
 static inline u8 __raw_readb(const volatile void __iomem *addr)
 {
 	u8 val;
 	asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr));
 	return val;
 }
 static inline u16 __raw_readw(const volatile void __iomem *addr)
 {
 	u16 val;
 	asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr));
 	return val;
 }
 static inline u32 __raw_readl(const volatile void __iomem *addr)
 {
 	u32 val;
 	asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr));
 	return val;
 }
 static inline u64 __raw_readq(const volatile void __iomem *addr)
 {
 	u64 val;
 	asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr));
 	return val;
 }
 /* IO barriers */
 #define __iormb()		rmb()
 #define __iowmb()		wmb()
 #define mmiowb()		do { } while (0)
 /*
  * Relaxed I/O memory access primitives. These follow the Device memory
  * ordering rules but do not guarantee any ordering relative to Normal memory
  * accesses.
  */
 #define readb_relaxed(c)	({ u8  __v = __raw_readb(c); __v; })
 #define readw_relaxed(c)	({ u16 __v = le16_to_cpu((__force __le16)__raw_readw(c)); __v; })
 #define readl_relaxed(c)	({ u32 __v = le32_to_cpu((__force __le32)__raw_readl(c)); __v; })
 #define writeb_relaxed(v,c)	((void)__raw_writeb((v),(c)))
 #define writew_relaxed(v,c)	((void)__raw_writew((__force u16)cpu_to_le16(v),(c)))
 #define writel_relaxed(v,c)	((void)__raw_writel((__force u32)cpu_to_le32(v),(c)))
 /*
  * I/O memory access primitives. Reads are ordered relative to any
  * following Normal memory access. Writes are ordered relative to any prior
  * Normal memory access.
  */
 #define readb(c)		({ u8  __v = readb_relaxed(c); __iormb(); __v; })
 #define readw(c)		({ u16 __v = readw_relaxed(c); __iormb(); __v; })
 #define readl(c)		({ u32 __v = readl_relaxed(c); __iormb(); __v; })
 #define writeb(v,c)		({ __iowmb(); writeb_relaxed((v),(c)); })
 #define writew(v,c)		({ __iowmb(); writew_relaxed((v),(c)); })
 #define writel(v,c)		({ __iowmb(); writel_relaxed((v),(c)); })
 /*
  *  I/O port access primitives.
  */
 #define IO_SPACE_LIMIT		0xffff
 #define PCI_IOBASE		((void __iomem *)(MODULES_VADDR - SZ_2M))
 static inline u8 inb(unsigned long addr)
 {
 	return readb(addr + PCI_IOBASE);
 }
 static inline u16 inw(unsigned long addr)
 {
 	return readw(addr + PCI_IOBASE);
 }
 static inline u32 inl(unsigned long addr)
 {
 	return readl(addr + PCI_IOBASE);
 }
 static inline void outb(u8 b, unsigned long addr)
 {
 	writeb(b, addr + PCI_IOBASE);
 }
 static inline void outw(u16 b, unsigned long addr)
 {
 	writew(b, addr + PCI_IOBASE);
 }
 static inline void outl(u32 b, unsigned long addr)
 {
 	writel(b, addr + PCI_IOBASE);
 }
 #define inb_p(addr)	inb(addr)
 #define inw_p(addr)	inw(addr)
 #define inl_p(addr)	inl(addr)
 #define outb_p(x, addr)	outb((x), (addr))
 #define outw_p(x, addr)	outw((x), (addr))
 #define outl_p(x, addr)	outl((x), (addr))
 static inline void insb(unsigned long addr, void *buffer, int count)
 {
 	u8 *buf = buffer;
 	while (count--)
 		*buf++ = __raw_readb(addr + PCI_IOBASE);
 }
 static inline void insw(unsigned long addr, void *buffer, int count)
 {
 	u16 *buf = buffer;
 	while (count--)
 		*buf++ = __raw_readw(addr + PCI_IOBASE);
 }
 static inline void insl(unsigned long addr, void *buffer, int count)
 {
 	u32 *buf = buffer;
 	while (count--)
 		*buf++ = __raw_readl(addr + PCI_IOBASE);
 }
 static inline void outsb(unsigned long addr, const void *buffer, int count)
 {
 	const u8 *buf = buffer;
 	while (count--)
 		__raw_writeb(*buf++, addr + PCI_IOBASE);
 }
 static inline void outsw(unsigned long addr, const void *buffer, int count)
 {
 	const u16 *buf = buffer;
 	while (count--)
 		__raw_writew(*buf++, addr + PCI_IOBASE);
 }
 static inline void outsl(unsigned long addr, const void *buffer, int count)
 {
 	const u32 *buf = buffer;
 	while (count--)
 		__raw_writel(*buf++, addr + PCI_IOBASE);
 }
 #define insb_p(port,to,len)	insb(port,to,len)
 #define insw_p(port,to,len)	insw(port,to,len)
 #define insl_p(port,to,len)	insl(port,to,len)
 #define outsb_p(port,from,len)	outsb(port,from,len)
 #define outsw_p(port,from,len)	outsw(port,from,len)
 #define outsl_p(port,from,len)	outsl(port,from,len)
 /*
  * String version of I/O memory access operations.
  */
 extern void __memcpy_fromio(void *, const volatile void __iomem *, size_t);
 extern void __memcpy_toio(volatile void __iomem *, const void *, size_t);
 extern void __memset_io(volatile void __iomem *, int, size_t);
 #define memset_io(c,v,l)	__memset_io((c),(v),(l))
 #define memcpy_fromio(a,c,l)	__memcpy_fromio((a),(c),(l))
 #define memcpy_toio(c,a,l)	__memcpy_toio((c),(a),(l))
 /*
  * I/O memory mapping functions.
  */
 extern void __iomem *__ioremap(phys_addr_t phys_addr, size_t size, pgprot_t prot);
 extern void __iounmap(volatile void __iomem *addr);
 #define PROT_DEFAULT		(PTE_TYPE_PAGE | PTE_AF | PTE_DIRTY)
 #define PROT_DEVICE_nGnRE	(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE))
 #define PROT_NORMAL_NC		(PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL_NC))
 #define ioremap(addr, size)		__ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
 #define ioremap_nocache(addr, size)	__ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE))
 #define ioremap_wc(addr, size)		__ioremap((addr), (size), __pgprot(PROT_NORMAL_NC))
 #define iounmap				__iounmap
+#define PROT_SECT_DEFAULT	(PMD_TYPE_SECT | PMD_SECT_AF)
+#define PROT_SECT_DEVICE_nGnRE	(PROT_SECT_DEFAULT | PTE_PXN | PTE_UXN | PMD_ATTRINDX(MT_DEVICE_nGnRE))
 #define ARCH_HAS_IOREMAP_WC
 #include <asm-generic/iomap.h>
 /*
  * More restrictive address range checking than the default implementation
  * (PHYS_OFFSET and PHYS_MASK taken into account).
  */
 #define ARCH_HAS_VALID_PHYS_ADDR_RANGE
 extern int valid_phys_addr_range(unsigned long addr, size_t size);
 extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size);
 extern int devmem_is_allowed(unsigned long pfn);
 /*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
  * access
  */
 #define xlate_dev_mem_ptr(p)	__va(p)
 /*
  * Convert a virtual cached pointer to an uncached pointer
  */
 #define xlate_dev_kmem_ptr(p)	p
 #endif	/* __KERNEL__ */
 #endif	/* __ASM_IO_H */

arch/arm64/include/asm/memory.h

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/include/asm/memory.h
  *
  * Copyright (C) 2000-2002 Russell King
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  * Note: this file should not be included by non-asm/.h files
  */
 #ifndef __ASM_MEMORY_H
 #define __ASM_MEMORY_H
 #include <linux/compiler.h>
 #include <linux/const.h>
 #include <linux/types.h>
 #include <asm/sizes.h>
 /*
  * Allow for constants defined here to be used from assembly code
  * by prepending the UL suffix only with actual C code compilation.
  */
 #define UL(x) _AC(x, UL)
 /*
  * PAGE_OFFSET - the virtual address of the start of the kernel image.
  * VA_BITS - the maximum number of bits for virtual addresses.
  * TASK_SIZE - the maximum size of a user space task.
  * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
  * The module space lives between the addresses given by TASK_SIZE
  * and PAGE_OFFSET - it must be within 128MB of the kernel text.
  */
 #define PAGE_OFFSET		UL(0xffffffc000000000)
 #define MODULES_END		(PAGE_OFFSET)
 #define MODULES_VADDR		(MODULES_END - SZ_64M)
+#define EARLYCON_IOBASE		(MODULES_VADDR - SZ_4M)
 #define VA_BITS			(39)
 #define TASK_SIZE_64		(UL(1) << VA_BITS)
 #ifdef CONFIG_COMPAT
 #define TASK_SIZE_32		UL(0x100000000)
 #define TASK_SIZE		(test_thread_flag(TIF_32BIT) ? \
 				TASK_SIZE_32 : TASK_SIZE_64)
 #else
 #define TASK_SIZE		TASK_SIZE_64
 #endif /* CONFIG_COMPAT */
 #define TASK_UNMAPPED_BASE	(PAGE_ALIGN(TASK_SIZE / 4))
 #if TASK_SIZE_64 > MODULES_VADDR
 #error Top of 64-bit user space clashes with start of module space
 #endif
 /*
  * Physical vs virtual RAM address space conversion.  These are
  * private definitions which should NOT be used outside memory.h
  * files.  Use virt_to_phys/phys_to_virt/__pa/__va instead.
  */
 #define __virt_to_phys(x)	(((phys_addr_t)(x) - PAGE_OFFSET + PHYS_OFFSET))
 #define __phys_to_virt(x)	((unsigned long)((x) - PHYS_OFFSET + PAGE_OFFSET))
 /*
  * Convert a physical address to a Page Frame Number and back
  */
 #define	__phys_to_pfn(paddr)	((unsigned long)((paddr) >> PAGE_SHIFT))
 #define	__pfn_to_phys(pfn)	((phys_addr_t)(pfn) << PAGE_SHIFT)
 /*
  * Convert a page to/from a physical address
  */
 #define page_to_phys(page)	(__pfn_to_phys(page_to_pfn(page)))
 #define phys_to_page(phys)	(pfn_to_page(__phys_to_pfn(phys)))
 /*
  * Memory types available.
  */
 #define MT_DEVICE_nGnRnE	0
 #define MT_DEVICE_nGnRE		1
 #define MT_DEVICE_GRE		2
 #define MT_NORMAL_NC		3
 #define MT_NORMAL		4
 #ifndef __ASSEMBLY__
 extern phys_addr_t		memstart_addr;
 /* PHYS_OFFSET - the physical address of the start of memory. */
 #define PHYS_OFFSET		({ memstart_addr; })
 /*
  * PFNs are used to describe any physical page; this means
  * PFN 0 == physical address 0.
  *
  * This is the PFN of the first RAM page in the kernel
  * direct-mapped view.  We assume this is the first page
  * of RAM in the mem_map as well.
  */
 #define PHYS_PFN_OFFSET	(PHYS_OFFSET >> PAGE_SHIFT)
 /*
  * Note: Drivers should NOT use these.  They are the wrong
  * translation for translating DMA addresses.  Use the driver
  * DMA support - see dma-mapping.h.
  */
 static inline phys_addr_t virt_to_phys(const volatile void *x)
 {
 	return __virt_to_phys((unsigned long)(x));
 }
 static inline void *phys_to_virt(phys_addr_t x)
 {
 	return (void *)(__phys_to_virt(x));
 }
 /*
  * Drivers should NOT use these either.
  */
 #define __pa(x)			__virt_to_phys((unsigned long)(x))
 #define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
 #define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
 /*
  *  virt_to_page(k)	convert a _valid_ virtual address to struct page *
  *  virt_addr_valid(k)	indicates whether a virtual address is valid
  */
 #define ARCH_PFN_OFFSET		PHYS_PFN_OFFSET
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 #define	virt_addr_valid(kaddr)	(((void *)(kaddr) >= (void *)PAGE_OFFSET) && \
 				 ((void *)(kaddr) < (void *)high_memory))
 #endif
 #include <asm-generic/memory_model.h>
 #endif

arch/arm64/include/asm/mmu.h

Diff comments View file @ 79a69d3

 /*
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_MMU_H
 #define __ASM_MMU_H
 typedef struct {
 	unsigned int id;
 	raw_spinlock_t id_lock;
 	void *vdso;
 } mm_context_t;
 #define ASID(mm)	((mm)->context.id & 0xffff)
 extern void paging_init(void);
 extern void setup_mm_for_reboot(void);
+extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
 #endif

arch/arm64/include/asm/mmu_context.h

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/include/asm/mmu_context.h
  *
  * Copyright (C) 1996 Russell King.
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_MMU_CONTEXT_H
 #define __ASM_MMU_CONTEXT_H
 #include <linux/compiler.h>
 #include <linux/sched.h>
 #include <asm/cacheflush.h>
 #include <asm/proc-fns.h>
 #include <asm-generic/mm_hooks.h>
 #include <asm/cputype.h>
 #include <asm/pgtable.h>
 #define MAX_ASID_BITS	16
 extern unsigned int cpu_last_asid;
 void __init_new_context(struct task_struct *tsk, struct mm_struct *mm);
 void __new_context(struct mm_struct *mm);
+#ifdef CONFIG_PID_IN_CONTEXTIDR
+static inline void contextidr_thread_switch(struct task_struct *next)
+{
+	asm(
+	"	msr	contextidr_el1, %0\n"
+	"	isb"
+	:
+	: "r" (task_pid_nr(next)));
+}
+#else
+static inline void contextidr_thread_switch(struct task_struct *next)
+{
+}
+#endif
 /*
  * Set TTBR0 to empty_zero_page. No translations will be possible via TTBR0.
  */
 static inline void cpu_set_reserved_ttbr0(void)
 {
 	unsigned long ttbr = page_to_phys(empty_zero_page);
 	asm(
 	"	msr	ttbr0_el1, %0			// set TTBR0\n"
 	"	isb"
 	:
 	: "r" (ttbr));
 }
 static inline void switch_new_context(struct mm_struct *mm)
 {
 	unsigned long flags;
 	__new_context(mm);
 	local_irq_save(flags);
 	cpu_switch_mm(mm->pgd, mm);
 	local_irq_restore(flags);
 }
 static inline void check_and_switch_context(struct mm_struct *mm,
 					    struct task_struct *tsk)
 {
 	/*
 	 * Required during context switch to avoid speculative page table
 	 * walking with the wrong TTBR.
 	 */
 	cpu_set_reserved_ttbr0();
 	if (!((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS))
 		/*
 		 * The ASID is from the current generation, just switch to the
 		 * new pgd. This condition is only true for calls from
 		 * context_switch() and interrupts are already disabled.
 		 */
 		cpu_switch_mm(mm->pgd, mm);
 	else if (irqs_disabled())
 		/*
 		 * Defer the new ASID allocation until after the context
 		 * switch critical region since __new_context() cannot be
 		 * called with interrupts disabled.
 		 */
 		set_ti_thread_flag(task_thread_info(tsk), TIF_SWITCH_MM);
 	else
 		/*
 		 * That is a direct call to switch_mm() or activate_mm() with
 		 * interrupts enabled and a new context.
 		 */
 		switch_new_context(mm);
 }
 #define init_new_context(tsk,mm)	(__init_new_context(tsk,mm),0)
 #define destroy_context(mm)		do { } while(0)
 #define finish_arch_post_lock_switch \
 	finish_arch_post_lock_switch
 static inline void finish_arch_post_lock_switch(void)
 {
 	if (test_and_clear_thread_flag(TIF_SWITCH_MM)) {
 		struct mm_struct *mm = current->mm;
 		unsigned long flags;
 		__new_context(mm);
 		local_irq_save(flags);
 		cpu_switch_mm(mm->pgd, mm);
 		local_irq_restore(flags);
 	}
 }
 /*
  * This is called when "tsk" is about to enter lazy TLB mode.
  *
  * mm:  describes the currently active mm context
  * tsk: task which is entering lazy tlb
  * cpu: cpu number which is entering lazy tlb
  *
  * tsk->mm will be NULL
  */
 static inline void
 enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 }
 /*
  * This is the actual mm switch as far as the scheduler
  * is concerned.  No registers are touched.  We avoid
  * calling the CPU specific function when the mm hasn't
  * actually changed.
  */
 static inline void
 switch_mm(struct mm_struct *prev, struct mm_struct *next,
 	  struct task_struct *tsk)
 {
 	unsigned int cpu = smp_processor_id();
 #ifdef CONFIG_SMP
 	/* check for possible thread migration */
 	if (!cpumask_empty(mm_cpumask(next)) &&
 	    !cpumask_test_cpu(cpu, mm_cpumask(next)))
 		__flush_icache_all();
 #endif
 	if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next)
 		check_and_switch_context(next, tsk);
 }
 #define deactivate_mm(tsk,mm)	do { } while (0)
 #define activate_mm(prev,next)	switch_mm(prev, next, NULL)
 #endif

arch/arm64/include/asm/perf_event.h

Diff comments View file @ 79a69d3

 /*
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_PERF_EVENT_H
 #define __ASM_PERF_EVENT_H
-/* It's quiet around here... */
+#ifdef CONFIG_HW_PERF_EVENTS
+struct pt_regs;
+extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
+extern unsigned long perf_misc_flags(struct pt_regs *regs);
+#define perf_misc_flags(regs)	perf_misc_flags(regs)
+#endif
 #endif

arch/arm64/include/asm/psci.h

Diff comments View file @ 79a69d3

File was created	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License version 2 as
	4	* published by the Free Software Foundation.
	5	*
	6	* This program is distributed in the hope that it will be useful,
	7	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	8	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	9	* GNU General Public License for more details.
	10	*
	11	* Copyright (C) 2013 ARM Limited
	12	*/
	13
	14	#ifndef __ASM_PSCI_H
	15	#define __ASM_PSCI_H
	16
	17	#define PSCI_POWER_STATE_TYPE_STANDBY 0
	18	#define PSCI_POWER_STATE_TYPE_POWER_DOWN 1
	19
	20	struct psci_power_state {
	21	u16 id;
	22	u8 type;
	23	u8 affinity_level;
	24	};
	25
	26	struct psci_operations {
	27	int (*cpu_suspend)(struct psci_power_state state,
	28	unsigned long entry_point);
	29	int (*cpu_off)(struct psci_power_state state);
	30	int (*cpu_on)(unsigned long cpuid, unsigned long entry_point);
	31	int (*migrate)(unsigned long cpuid);
	32	};
	33
	34	extern struct psci_operations psci_ops;
	35
	36	int psci_init(void);
	37
	38	#endif /* __ASM_PSCI_H */
	39

arch/arm64/include/asm/ptrace.h

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/include/asm/ptrace.h
  *
  * Copyright (C) 1996-2003 Russell King
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_PTRACE_H
 #define __ASM_PTRACE_H
 #include <uapi/asm/ptrace.h>
 /* AArch32-specific ptrace requests */
 #define COMPAT_PTRACE_GETREGS		12
 #define COMPAT_PTRACE_SETREGS		13
 #define COMPAT_PTRACE_GET_THREAD_AREA	22
 #define COMPAT_PTRACE_SET_SYSCALL	23
 #define COMPAT_PTRACE_GETVFPREGS	27
 #define COMPAT_PTRACE_SETVFPREGS	28
 #define COMPAT_PTRACE_GETHBPREGS	29
 #define COMPAT_PTRACE_SETHBPREGS	30
 /* AArch32 CPSR bits */
 #define COMPAT_PSR_MODE_MASK	0x0000001f
 #define COMPAT_PSR_MODE_USR	0x00000010
 #define COMPAT_PSR_MODE_FIQ	0x00000011
 #define COMPAT_PSR_MODE_IRQ	0x00000012
 #define COMPAT_PSR_MODE_SVC	0x00000013
 #define COMPAT_PSR_MODE_ABT	0x00000017
 #define COMPAT_PSR_MODE_HYP	0x0000001a
 #define COMPAT_PSR_MODE_UND	0x0000001b
 #define COMPAT_PSR_MODE_SYS	0x0000001f
 #define COMPAT_PSR_T_BIT	0x00000020
+#define COMPAT_PSR_F_BIT	0x00000040
+#define COMPAT_PSR_I_BIT	0x00000080
+#define COMPAT_PSR_A_BIT	0x00000100
+#define COMPAT_PSR_E_BIT	0x00000200
+#define COMPAT_PSR_J_BIT	0x01000000
+#define COMPAT_PSR_Q_BIT	0x08000000
+#define COMPAT_PSR_V_BIT	0x10000000
+#define COMPAT_PSR_C_BIT	0x20000000
+#define COMPAT_PSR_Z_BIT	0x40000000
+#define COMPAT_PSR_N_BIT	0x80000000
 #define COMPAT_PSR_IT_MASK	0x0600fc00	/* If-Then execution state mask */
 /*
  * These are 'magic' values for PTRACE_PEEKUSR that return info about where a
  * process is located in memory.
  */
 #define COMPAT_PT_TEXT_ADDR		0x10000
 #define COMPAT_PT_DATA_ADDR		0x10004
 #define COMPAT_PT_TEXT_END_ADDR		0x10008
 #ifndef __ASSEMBLY__
 /* sizeof(struct user) for AArch32 */
 #define COMPAT_USER_SZ	296
 /* Architecturally defined mapping between AArch32 and AArch64 registers */
 #define compat_usr(x)	regs[(x)]
 #define compat_sp	regs[13]
 #define compat_lr	regs[14]
 #define compat_sp_hyp	regs[15]
 #define compat_sp_irq	regs[16]
 #define compat_lr_irq	regs[17]
 #define compat_sp_svc	regs[18]
 #define compat_lr_svc	regs[19]
 #define compat_sp_abt	regs[20]
 #define compat_lr_abt	regs[21]
 #define compat_sp_und	regs[22]
 #define compat_lr_und	regs[23]
 #define compat_r8_fiq	regs[24]
 #define compat_r9_fiq	regs[25]
 #define compat_r10_fiq	regs[26]
 #define compat_r11_fiq	regs[27]
 #define compat_r12_fiq	regs[28]
 #define compat_sp_fiq	regs[29]
 #define compat_lr_fiq	regs[30]
 /*
  * This struct defines the way the registers are stored on the stack during an
  * exception. Note that sizeof(struct pt_regs) has to be a multiple of 16 (for
  * stack alignment). struct user_pt_regs must form a prefix of struct pt_regs.
  */
 struct pt_regs {
 	union {
 		struct user_pt_regs user_regs;
 		struct {
 			u64 regs[31];
 			u64 sp;
 			u64 pc;
 			u64 pstate;
 		};
 	};
 	u64 orig_x0;
 	u64 syscallno;
 };
 #define arch_has_single_step()	(1)
 #ifdef CONFIG_COMPAT
 #define compat_thumb_mode(regs) \
 	(((regs)->pstate & COMPAT_PSR_T_BIT))
 #else
 #define compat_thumb_mode(regs) (0)
 #endif
 #define user_mode(regs)	\
 	(((regs)->pstate & PSR_MODE_MASK) == PSR_MODE_EL0t)
 #define compat_user_mode(regs)	\
 	(((regs)->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) == \
 	 (PSR_MODE32_BIT | PSR_MODE_EL0t))
 #define processor_mode(regs) \
 	((regs)->pstate & PSR_MODE_MASK)
 #define interrupts_enabled(regs) \
 	(!((regs)->pstate & PSR_I_BIT))
 #define fast_interrupts_enabled(regs) \
 	(!((regs)->pstate & PSR_F_BIT))
 #define user_stack_pointer(regs) \
 	((regs)->sp)
 /*
  * Are the current registers suitable for user mode? (used to maintain
  * security in signal handlers)
  */
 static inline int valid_user_regs(struct user_pt_regs *regs)
 {
 	if (user_mode(regs) && (regs->pstate & PSR_I_BIT) == 0) {
 		regs->pstate &= ~(PSR_F_BIT | PSR_A_BIT);
 		/* The T bit is reserved for AArch64 */
 		if (!(regs->pstate & PSR_MODE32_BIT))
 			regs->pstate &= ~COMPAT_PSR_T_BIT;
 		return 1;
 	}
 	/*
 	 * Force PSR to something logical...
 	 */
 	regs->pstate &= PSR_f | PSR_s | (PSR_x & ~PSR_A_BIT) | \
 			COMPAT_PSR_T_BIT | PSR_MODE32_BIT;
 	if (!(regs->pstate & PSR_MODE32_BIT)) {
 		regs->pstate &= ~COMPAT_PSR_T_BIT;
 		regs->pstate |= PSR_MODE_EL0t;
 	}
 	return 0;
 }
 #define instruction_pointer(regs)	(regs)->pc
 #ifdef CONFIG_SMP
 extern unsigned long profile_pc(struct pt_regs *regs);
 #else
 #define profile_pc(regs) instruction_pointer(regs)
 #endif
 extern int aarch32_break_trap(struct pt_regs *regs);
 #endif /* __ASSEMBLY__ */
 #endif

arch/arm64/include/asm/smp.h

Diff comments View file @ 79a69d3

 /*
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #ifndef __ASM_SMP_H
 #define __ASM_SMP_H
 #include <linux/threads.h>
 #include <linux/cpumask.h>
 #include <linux/thread_info.h>
 #ifndef CONFIG_SMP
 # error "<asm/smp.h> included in non-SMP build"
 #endif
 #define raw_smp_processor_id() (current_thread_info()->cpu)
 struct seq_file;
 /*
  * generate IPI list text
  */
 extern void show_ipi_list(struct seq_file *p, int prec);
 /*
  * Called from C code, this handles an IPI.
  */
 extern void handle_IPI(int ipinr, struct pt_regs *regs);
 /*
  * Setup the set of possible CPUs (via set_cpu_possible)
  */
 extern void smp_init_cpus(void);
 /*
  * Provide a function to raise an IPI cross call on CPUs in callmap.
  */
 extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
 /*
  * Called from the secondary holding pen, this is the secondary CPU entry point.
  */
 asmlinkage void secondary_start_kernel(void);
 /*
  * Initial data for bringing up a secondary CPU.
  */
 struct secondary_data {
 	void *stack;
 };
 extern struct secondary_data secondary_data;
 extern void secondary_holding_pen(void);
 extern volatile unsigned long secondary_holding_pen_release;
 extern void arch_send_call_function_single_ipi(int cpu);
 extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
+struct device_node;
+struct smp_enable_ops {
+	const char	*name;
+	int		(*init_cpu)(struct device_node *, int);
+	int		(*prepare_cpu)(int);
+};
+extern const struct smp_enable_ops smp_spin_table_ops;
+extern const struct smp_enable_ops smp_psci_ops;
 #endif /* ifndef __ASM_SMP_H */

arch/arm64/include/asm/spinlock.h

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* Copyright (C) 2012 ARM Ltd.	2	* Copyright (C) 2012 ARM Ltd.
3	*	3	*
4	* This program is free software; you can redistribute it and/or modify	4	* This program is free software; you can redistribute it and/or modify
5	* it under the terms of the GNU General Public License version 2 as	5	* it under the terms of the GNU General Public License version 2 as
6	* published by the Free Software Foundation.	6	* published by the Free Software Foundation.
7	*	7	*
8	* This program is distributed in the hope that it will be useful,	8	* This program is distributed in the hope that it will be useful,
9	* but WITHOUT ANY WARRANTY; without even the implied warranty of	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11	* GNU General Public License for more details.	11	* GNU General Public License for more details.
12	*	12	*
13	* You should have received a copy of the GNU General Public License	13	* You should have received a copy of the GNU General Public License
14	* along with this program. If not, see <http://www.gnu.org/licenses/>.	14	* along with this program. If not, see <http://www.gnu.org/licenses/>.
15	*/	15	*/
16	#ifndef __ASM_SPINLOCK_H	16	#ifndef __ASM_SPINLOCK_H
17	#define __ASM_SPINLOCK_H	17	#define __ASM_SPINLOCK_H
18		18
19	#include <asm/spinlock_types.h>	19	#include <asm/spinlock_types.h>
20	#include <asm/processor.h>	20	#include <asm/processor.h>
21		21
22	/*	22	/*
23	* Spinlock implementation.	23	* Spinlock implementation.
24	*	24	*
25	* The old value is read exclusively and the new one, if unlocked, is written	25	* The old value is read exclusively and the new one, if unlocked, is written
26	* exclusively. In case of failure, the loop is restarted.	26	* exclusively. In case of failure, the loop is restarted.
27	*	27	*
28	* The memory barriers are implicit with the load-acquire and store-release	28	* The memory barriers are implicit with the load-acquire and store-release
29	* instructions.	29	* instructions.
30	*	30	*
31	* Unlocked value: 0	31	* Unlocked value: 0
32	* Locked value: 1	32	* Locked value: 1
33	*/	33	*/
34		34
35	#define arch_spin_is_locked(x) ((x)->lock != 0)	35	#define arch_spin_is_locked(x) ((x)->lock != 0)
36	#define arch_spin_unlock_wait(lock) \	36	#define arch_spin_unlock_wait(lock) \
37	do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)	37	do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0)
38		38
39	#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)	39	#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
40		40
41	static inline void arch_spin_lock(arch_spinlock_t *lock)	41	static inline void arch_spin_lock(arch_spinlock_t *lock)
42	{	42	{
43	unsigned int tmp;	43	unsigned int tmp;
44		44
45	asm volatile(	45	asm volatile(
46	" sevl\n"	46	" sevl\n"
47	"1: wfe\n"	47	"1: wfe\n"
48	"2: ldaxr %w0, [%1]\n"	48	"2: ldaxr %w0, %1\n"
49	" cbnz %w0, 1b\n"	49	" cbnz %w0, 1b\n"
50	" stxr %w0, %w2, [%1]\n"	50	" stxr %w0, %w2, %1\n"
51	" cbnz %w0, 2b\n"	51	" cbnz %w0, 2b\n"
52	: "=&r" (tmp)	52	: "=&r" (tmp), "+Q" (lock->lock)
53	: "r" (&lock->lock), "r" (1)	53	: "r" (1)
54	: "memory");	54	: "cc", "memory");
55	}	55	}
56		56
57	static inline int arch_spin_trylock(arch_spinlock_t *lock)	57	static inline int arch_spin_trylock(arch_spinlock_t *lock)
58	{	58	{
59	unsigned int tmp;	59	unsigned int tmp;
60		60
61	asm volatile(	61	asm volatile(
62	" ldaxr %w0, [%1]\n"	62	" ldaxr %w0, %1\n"
63	" cbnz %w0, 1f\n"	63	" cbnz %w0, 1f\n"
64	" stxr %w0, %w2, [%1]\n"	64	" stxr %w0, %w2, %1\n"
65	"1:\n"	65	"1:\n"
66	: "=&r" (tmp)	66	: "=&r" (tmp), "+Q" (lock->lock)
67	: "r" (&lock->lock), "r" (1)	67	: "r" (1)
68	: "memory");	68	: "cc", "memory");
69		69
70	return !tmp;	70	return !tmp;
71	}	71	}
72		72
73	static inline void arch_spin_unlock(arch_spinlock_t *lock)	73	static inline void arch_spin_unlock(arch_spinlock_t *lock)
74	{	74	{
75	asm volatile(	75	asm volatile(
76	" stlr %w1, [%0]\n"	76	" stlr %w1, %0\n"
77	: : "r" (&lock->lock), "r" (0) : "memory");	77	: "=Q" (lock->lock) : "r" (0) : "memory");
78	}	78	}
79		79
80	/*	80	/*
81	* Write lock implementation.	81	* Write lock implementation.
82	*	82	*
83	* Write locks set bit 31. Unlocking, is done by writing 0 since the lock is	83	* Write locks set bit 31. Unlocking, is done by writing 0 since the lock is
84	* exclusively held.	84	* exclusively held.
85	*	85	*
86	* The memory barriers are implicit with the load-acquire and store-release	86	* The memory barriers are implicit with the load-acquire and store-release
87	* instructions.	87	* instructions.
88	*/	88	*/
89		89
90	static inline void arch_write_lock(arch_rwlock_t *rw)	90	static inline void arch_write_lock(arch_rwlock_t *rw)
91	{	91	{
92	unsigned int tmp;	92	unsigned int tmp;
93		93
94	asm volatile(	94	asm volatile(
95	" sevl\n"	95	" sevl\n"
96	"1: wfe\n"	96	"1: wfe\n"
97	"2: ldaxr %w0, [%1]\n"	97	"2: ldaxr %w0, %1\n"
98	" cbnz %w0, 1b\n"	98	" cbnz %w0, 1b\n"
99	" stxr %w0, %w2, [%1]\n"	99	" stxr %w0, %w2, %1\n"
100	" cbnz %w0, 2b\n"	100	" cbnz %w0, 2b\n"
101	: "=&r" (tmp)	101	: "=&r" (tmp), "+Q" (rw->lock)
102	: "r" (&rw->lock), "r" (0x80000000)	102	: "r" (0x80000000)
103	: "memory");	103	: "cc", "memory");
104	}	104	}
105		105
106	static inline int arch_write_trylock(arch_rwlock_t *rw)	106	static inline int arch_write_trylock(arch_rwlock_t *rw)
107	{	107	{
108	unsigned int tmp;	108	unsigned int tmp;
109		109
110	asm volatile(	110	asm volatile(
111	" ldaxr %w0, [%1]\n"	111	" ldaxr %w0, %1\n"
112	" cbnz %w0, 1f\n"	112	" cbnz %w0, 1f\n"
113	" stxr %w0, %w2, [%1]\n"	113	" stxr %w0, %w2, %1\n"
114	"1:\n"	114	"1:\n"
115	: "=&r" (tmp)	115	: "=&r" (tmp), "+Q" (rw->lock)
116	: "r" (&rw->lock), "r" (0x80000000)	116	: "r" (0x80000000)
117	: "memory");	117	: "cc", "memory");
118		118
119	return !tmp;	119	return !tmp;
120	}	120	}
121		121
122	static inline void arch_write_unlock(arch_rwlock_t *rw)	122	static inline void arch_write_unlock(arch_rwlock_t *rw)
123	{	123	{
124	asm volatile(	124	asm volatile(
125	" stlr %w1, [%0]\n"	125	" stlr %w1, %0\n"
126	: : "r" (&rw->lock), "r" (0) : "memory");	126	: "=Q" (rw->lock) : "r" (0) : "memory");
127	}	127	}
128		128
129	/* write_can_lock - would write_trylock() succeed? */	129	/* write_can_lock - would write_trylock() succeed? */
130	#define arch_write_can_lock(x) ((x)->lock == 0)	130	#define arch_write_can_lock(x) ((x)->lock == 0)
131		131
132	/*	132	/*
133	* Read lock implementation.	133	* Read lock implementation.
134	*	134	*
135	* It exclusively loads the lock value, increments it and stores the new value	135	* It exclusively loads the lock value, increments it and stores the new value
136	* back if positive and the CPU still exclusively owns the location. If the	136	* back if positive and the CPU still exclusively owns the location. If the
137	* value is negative, the lock is already held.	137	* value is negative, the lock is already held.
138	*	138	*
139	* During unlocking there may be multiple active read locks but no write lock.	139	* During unlocking there may be multiple active read locks but no write lock.
140	*	140	*
141	* The memory barriers are implicit with the load-acquire and store-release	141	* The memory barriers are implicit with the load-acquire and store-release
142	* instructions.	142	* instructions.
143	*/	143	*/
144	static inline void arch_read_lock(arch_rwlock_t *rw)	144	static inline void arch_read_lock(arch_rwlock_t *rw)
145	{	145	{
146	unsigned int tmp, tmp2;	146	unsigned int tmp, tmp2;
147		147
148	asm volatile(	148	asm volatile(
149	" sevl\n"	149	" sevl\n"
150	"1: wfe\n"	150	"1: wfe\n"
151	"2: ldaxr %w0, [%2]\n"	151	"2: ldaxr %w0, %2\n"
152	" add %w0, %w0, #1\n"	152	" add %w0, %w0, #1\n"
153	" tbnz %w0, #31, 1b\n"	153	" tbnz %w0, #31, 1b\n"
154	" stxr %w1, %w0, [%2]\n"	154	" stxr %w1, %w0, %2\n"
155	" cbnz %w1, 2b\n"	155	" cbnz %w1, 2b\n"
156	: "=&r" (tmp), "=&r" (tmp2)	156	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
157	: "r" (&rw->lock)	157	:
158	: "memory");	158	: "cc", "memory");
159	}	159	}
160		160
161	static inline void arch_read_unlock(arch_rwlock_t *rw)	161	static inline void arch_read_unlock(arch_rwlock_t *rw)
162	{	162	{
163	unsigned int tmp, tmp2;	163	unsigned int tmp, tmp2;
164		164
165	asm volatile(	165	asm volatile(
166	"1: ldxr %w0, [%2]\n"	166	"1: ldxr %w0, %2\n"
167	" sub %w0, %w0, #1\n"	167	" sub %w0, %w0, #1\n"
168	" stlxr %w1, %w0, [%2]\n"	168	" stlxr %w1, %w0, %2\n"
169	" cbnz %w1, 1b\n"	169	" cbnz %w1, 1b\n"
170	: "=&r" (tmp), "=&r" (tmp2)	170	: "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock)
171	: "r" (&rw->lock)	171	:
172	: "memory");	172	: "cc", "memory");
173	}	173	}
174		174
175	static inline int arch_read_trylock(arch_rwlock_t *rw)	175	static inline int arch_read_trylock(arch_rwlock_t *rw)
176	{	176	{
177	unsigned int tmp, tmp2 = 1;	177	unsigned int tmp, tmp2 = 1;
178		178
179	asm volatile(	179	asm volatile(
180	" ldaxr %w0, [%2]\n"	180	" ldaxr %w0, %2\n"
181	" add %w0, %w0, #1\n"	181	" add %w0, %w0, #1\n"
182	" tbnz %w0, #31, 1f\n"	182	" tbnz %w0, #31, 1f\n"
183	" stxr %w1, %w0, [%2]\n"	183	" stxr %w1, %w0, %2\n"
184	"1:\n"	184	"1:\n"
185	: "=&r" (tmp), "+r" (tmp2)	185	: "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock)
186	: "r" (&rw->lock)	186	:
187	: "memory");	187	: "cc", "memory");
188		188
189	return !tmp2;	189	return !tmp2;
190	}	190	}
191		191
192	/* read_can_lock - would read_trylock() succeed? */	192	/* read_can_lock - would read_trylock() succeed? */
193	#define arch_read_can_lock(x) ((x)->lock < 0x80000000)	193	#define arch_read_can_lock(x) ((x)->lock < 0x80000000)
194		194
195	#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)	195	#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
196	#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)	196	#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
197		197
198	#define arch_spin_relax(lock) cpu_relax()	198	#define arch_spin_relax(lock) cpu_relax()
199	#define arch_read_relax(lock) cpu_relax()	199	#define arch_read_relax(lock) cpu_relax()
200	#define arch_write_relax(lock) cpu_relax()	200	#define arch_write_relax(lock) cpu_relax()
201		201
202	#endif /* __ASM_SPINLOCK_H */	202	#endif /* __ASM_SPINLOCK_H */
203		203

arch/arm64/include/uapi/asm/Kbuild

Diff comments View file @ 79a69d3

 # UAPI Header export list
 include include/uapi/asm-generic/Kbuild.asm
+generic-y += kvm_para.h
 header-y += auxvec.h
 header-y += bitsperlong.h
 header-y += byteorder.h
 header-y += fcntl.h
 header-y += hwcap.h
+header-y += kvm_para.h
 header-y += param.h
 header-y += ptrace.h
 header-y += setup.h
 header-y += sigcontext.h
 header-y += siginfo.h
 header-y += signal.h
 header-y += stat.h
 header-y += statfs.h
 header-y += unistd.h

arch/arm64/kernel/Makefile

Diff comments View file @ 79a69d3

 #
 # Makefile for the linux kernel.
 #
 CPPFLAGS_vmlinux.lds	:= -DTEXT_OFFSET=$(TEXT_OFFSET)
 AFLAGS_head.o		:= -DTEXT_OFFSET=$(TEXT_OFFSET)
 # Object file lists.
 arm64-obj-y		:= cputable.o debug-monitors.o entry.o irq.o fpsimd.o	\
 			   entry-fpsimd.o process.o ptrace.o setup.o signal.o	\
 			   sys.o stacktrace.o time.o traps.o io.o vdso.o	\
-			   hyp-stub.o
+			   hyp-stub.o psci.o
 arm64-obj-$(CONFIG_COMPAT)		+= sys32.o kuser32.o signal32.o 	\
 					   sys_compat.o
 arm64-obj-$(CONFIG_MODULES)		+= arm64ksyms.o module.o
-arm64-obj-$(CONFIG_SMP)			+= smp.o
+arm64-obj-$(CONFIG_SMP)			+= smp.o smp_spin_table.o smp_psci.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)	+= perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
+arm64-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
 head-y					:= head.o
 extra-y					:= $(head-y) vmlinux.lds
 # vDSO - this must be built first to generate the symbol offsets
 $(call objectify,$(arm64-obj-y)): $(obj)/vdso/vdso-offsets.h
 $(obj)/vdso/vdso-offsets.h: $(obj)/vdso

arch/arm64/kernel/early_printk.c

Diff comments View file @ 79a69d3

File was created	1	/*
	2	* Earlyprintk support.
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	* Author: Catalin Marinas <catalin.marinas@arm.com>
	6	*
	7	* This program is free software: you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	* GNU General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	18	*/
	19	#include <linux/kernel.h>
	20	#include <linux/console.h>
	21	#include <linux/init.h>
	22	#include <linux/string.h>
	23	#include <linux/mm.h>
	24	#include <linux/io.h>
	25
	26	#include <linux/amba/serial.h>
	27
	28	static void __iomem *early_base;
	29	static void (*printch)(char ch);
	30
	31	/*
	32	* PL011 single character TX.
	33	*/
	34	static void pl011_printch(char ch)
	35	{
	36	while (readl_relaxed(early_base + UART01x_FR) & UART01x_FR_TXFF)
	37	;
	38	writeb_relaxed(ch, early_base + UART01x_DR);
	39	while (readl_relaxed(early_base + UART01x_FR) & UART01x_FR_BUSY)
	40	;
	41	}
	42
	43	struct earlycon_match {
	44	const char *name;
	45	void (*printch)(char ch);
	46	};
	47
	48	static const struct earlycon_match earlycon_match[] __initconst = {
	49	{ .name = "pl011", .printch = pl011_printch, },
	50	{}
	51	};
	52
	53	static void early_write(struct console con, const char s, unsigned n)
	54	{
	55	while (n-- > 0) {
	56	if (*s == '\n')
	57	printch('\r');
	58	printch(*s);
	59	s++;
	60	}
	61	}
	62
	63	static struct console early_console = {
	64	.name = "earlycon",
	65	.write = early_write,
	66	.flags = CON_PRINTBUFFER \| CON_BOOT,
	67	.index = -1,
	68	};
	69
	70	/*
	71	* Parse earlyprintk=... parameter in the format:
	72	*
	73	* <name>[,<addr>][,<options>]
	74	*
	75	* and register the early console. It is assumed that the UART has been
	76	* initialised by the bootloader already.
	77	*/
	78	static int __init setup_early_printk(char *buf)
	79	{
	80	const struct earlycon_match *match = earlycon_match;
	81	phys_addr_t paddr = 0;
	82
	83	if (!buf) {
	84	pr_warning("No earlyprintk arguments passed.\n");
	85	return 0;
	86	}
	87
	88	while (match->name) {
	89	size_t len = strlen(match->name);
	90	if (!strncmp(buf, match->name, len)) {
	91	buf += len;
	92	break;
	93	}
	94	match++;
	95	}
	96	if (!match->name) {
	97	pr_warning("Unknown earlyprintk arguments: %s\n", buf);
	98	return 0;
	99	}
	100
	101	/* I/O address */
	102	if (!strncmp(buf, ",0x", 3)) {
	103	char *e;
	104	paddr = simple_strtoul(buf + 1, &e, 16);
	105	buf = e;
	106	}
	107	/* no options parsing yet */
	108
	109	if (paddr)
	110	early_base = early_io_map(paddr, EARLYCON_IOBASE);
	111
	112	printch = match->printch;
	113	register_console(&early_console);
	114
	115	return 0;
	116	}
	117
	118	early_param("earlyprintk", setup_early_printk);
	119

arch/arm64/kernel/head.S

Diff comments View file @ 79a69d3

 /*
  * Low-level CPU initialisation
  * Based on arch/arm/kernel/head.S
  *
  * Copyright (C) 1994-2002 Russell King
  * Copyright (C) 2003-2012 ARM Ltd.
  * Authors:	Catalin Marinas <catalin.marinas@arm.com>
  *		Will Deacon <will.deacon@arm.com>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
 #include <asm/assembler.h>
 #include <asm/ptrace.h>
 #include <asm/asm-offsets.h>
 #include <asm/memory.h>
 #include <asm/thread_info.h>
 #include <asm/pgtable-hwdef.h>
 #include <asm/pgtable.h>
 #include <asm/page.h>
 #include <asm/virt.h>
 /*
  * swapper_pg_dir is the virtual address of the initial page table. We place
  * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has
  * 2 pages and is placed below swapper_pg_dir.
  */
 #define KERNEL_RAM_VADDR	(PAGE_OFFSET + TEXT_OFFSET)
 #if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000
 #error KERNEL_RAM_VADDR must start at 0xXXX80000
 #endif
 #define SWAPPER_DIR_SIZE	(3 * PAGE_SIZE)
 #define IDMAP_DIR_SIZE		(2 * PAGE_SIZE)
 	.globl	swapper_pg_dir
 	.equ	swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE
 	.globl	idmap_pg_dir
 	.equ	idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE
 	.macro	pgtbl, ttb0, ttb1, phys
 	add	\ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE
 	sub	\ttb0, \ttb1, #IDMAP_DIR_SIZE
 	.endm
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT	PAGE_SHIFT
 #define BLOCK_SIZE	PAGE_SIZE
 #else
 #define BLOCK_SHIFT	SECTION_SHIFT
 #define BLOCK_SIZE	SECTION_SIZE
 #endif
 #define KERNEL_START	KERNEL_RAM_VADDR
 #define KERNEL_END	_end
 /*
  * Initial memory map attributes.
  */
 #ifndef CONFIG_SMP
 #define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF
 #define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF
 #else
 #define PTE_FLAGS	PTE_TYPE_PAGE | PTE_AF | PTE_SHARED
 #define PMD_FLAGS	PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S
 #endif
 #ifdef CONFIG_ARM64_64K_PAGES
 #define MM_MMUFLAGS	PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS
-#define IO_MMUFLAGS	PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_XN | PTE_FLAGS
 #else
 #define MM_MMUFLAGS	PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS
-#define IO_MMUFLAGS	PMD_ATTRINDX(MT_DEVICE_nGnRE) | PMD_SECT_XN | PMD_FLAGS
 #endif
 /*
  * Kernel startup entry point.
  * ---------------------------
  *
  * The requirements are:
  *   MMU = off, D-cache = off, I-cache = on or off,
  *   x0 = physical address to the FDT blob.
  *
  * This code is mostly position independent so you call this at
  * __pa(PAGE_OFFSET + TEXT_OFFSET).
  *
  * Note that the callee-saved registers are used for storing variables
  * that are useful before the MMU is enabled. The allocations are described
  * in the entry routines.
  */
 	__HEAD
 	/*
 	 * DO NOT MODIFY. Image header expected by Linux boot-loaders.
 	 */
 	b	stext				// branch to kernel start, magic
 	.long	0				// reserved
 	.quad	TEXT_OFFSET			// Image load offset from start of RAM
 	.quad	0				// reserved
 	.quad	0				// reserved
 ENTRY(stext)
 	mov	x21, x0				// x21=FDT
 	bl	__calc_phys_offset		// x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET
 	bl	el2_setup			// Drop to EL1
 	mrs	x22, midr_el1			// x22=cpuid
 	mov	x0, x22
 	bl	lookup_processor_type
 	mov	x23, x0				// x23=current cpu_table
 	cbz	x23, __error_p			// invalid processor (x23=0)?
 	bl	__vet_fdt
 	bl	__create_page_tables		// x25=TTBR0, x26=TTBR1
 	/*
 	 * The following calls CPU specific code in a position independent
 	 * manner. See arch/arm64/mm/proc.S for details. x23 = base of
 	 * cpu_info structure selected by lookup_processor_type above.
 	 * On return, the CPU will be ready for the MMU to be turned on and
 	 * the TCR will have been set.
 	 */
 	ldr	x27, __switch_data		// address to jump to after
 						// MMU has been enabled
 	adr	lr, __enable_mmu		// return (PIC) address
 	ldr	x12, [x23, #CPU_INFO_SETUP]
 	add	x12, x12, x28			// __virt_to_phys
 	br	x12				// initialise processor
 ENDPROC(stext)
 /*
  * If we're fortunate enough to boot at EL2, ensure that the world is
  * sane before dropping to EL1.
  */
 ENTRY(el2_setup)
 	mrs	x0, CurrentEL
 	cmp	x0, #PSR_MODE_EL2t
 	ccmp	x0, #PSR_MODE_EL2h, #0x4, ne
 	ldr	x0, =__boot_cpu_mode		// Compute __boot_cpu_mode
 	add	x0, x0, x28
 	b.eq	1f
 	str	wzr, [x0]			// Remember we don't have EL2...
 	ret
 	/* Hyp configuration. */
 1:	ldr	w1, =BOOT_CPU_MODE_EL2
 	str	w1, [x0, #4]			// This CPU has EL2
 	mov	x0, #(1 << 31)			// 64-bit EL1
 	msr	hcr_el2, x0
 	/* Generic timers. */
 	mrs	x0, cnthctl_el2
 	orr	x0, x0, #3			// Enable EL1 physical timers
 	msr	cnthctl_el2, x0
 	msr	cntvoff_el2, xzr		// Clear virtual offset
 	/* Populate ID registers. */
 	mrs	x0, midr_el1
 	mrs	x1, mpidr_el1
 	msr	vpidr_el2, x0
 	msr	vmpidr_el2, x1
 	/* sctlr_el1 */
 	mov	x0, #0x0800			// Set/clear RES{1,0} bits
 	movk	x0, #0x30d0, lsl #16
 	msr	sctlr_el1, x0
 	/* Coprocessor traps. */
 	mov	x0, #0x33ff
 	msr	cptr_el2, x0			// Disable copro. traps to EL2
 #ifdef CONFIG_COMPAT
 	msr	hstr_el2, xzr			// Disable CP15 traps to EL2
 #endif
 	/* Stage-2 translation */
 	msr	vttbr_el2, xzr
 	/* Hypervisor stub */
 	adr	x0, __hyp_stub_vectors
 	msr	vbar_el2, x0
 	/* spsr */
 	mov	x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
 		      PSR_MODE_EL1h)
 	msr	spsr_el2, x0
 	msr	elr_el2, lr
 	eret
 ENDPROC(el2_setup)
 /*
  * We need to find out the CPU boot mode long after boot, so we need to
  * store it in a writable variable.
  *
  * This is not in .bss, because we set it sufficiently early that the boot-time
  * zeroing of .bss would clobber it.
  */
 	.pushsection	.data
 ENTRY(__boot_cpu_mode)
 	.long	BOOT_CPU_MODE_EL2
 	.long	0
 	.popsection
 	.align	3
 2:	.quad	.
 	.quad	PAGE_OFFSET
 #ifdef CONFIG_SMP
 	.pushsection    .smp.pen.text, "ax"
 	.align	3
 1:	.quad	.
 	.quad	secondary_holding_pen_release
 	/*
 	 * This provides a "holding pen" for platforms to hold all secondary
 	 * cores are held until we're ready for them to initialise.
 	 */
 ENTRY(secondary_holding_pen)
 	bl	__calc_phys_offset		// x24=phys offset
 	bl	el2_setup			// Drop to EL1
 	mrs	x0, mpidr_el1
 	and	x0, x0, #15			// CPU number
 	adr	x1, 1b
 	ldp	x2, x3, [x1]
 	sub	x1, x1, x2
 	add	x3, x3, x1
 pen:	ldr	x4, [x3]
 	cmp	x4, x0
 	b.eq	secondary_startup
 	wfe
 	b	pen
 ENDPROC(secondary_holding_pen)
 	.popsection
 ENTRY(secondary_startup)
 	/*
 	 * Common entry point for secondary CPUs.
 	 */
 	mrs	x22, midr_el1			// x22=cpuid
 	mov	x0, x22
 	bl	lookup_processor_type
 	mov	x23, x0				// x23=current cpu_table
 	cbz	x23, __error_p			// invalid processor (x23=0)?
 	pgtbl	x25, x26, x24			// x25=TTBR0, x26=TTBR1
 	ldr	x12, [x23, #CPU_INFO_SETUP]
 	add	x12, x12, x28			// __virt_to_phys
 	blr	x12				// initialise processor
 	ldr	x21, =secondary_data
 	ldr	x27, =__secondary_switched	// address to jump to after enabling the MMU
 	b	__enable_mmu
 ENDPROC(secondary_startup)
 ENTRY(__secondary_switched)
 	ldr	x0, [x21]			// get secondary_data.stack
 	mov	sp, x0
 	mov	x29, #0
 	b	secondary_start_kernel
 ENDPROC(__secondary_switched)
 #endif	/* CONFIG_SMP */
 /*
  * Setup common bits before finally enabling the MMU. Essentially this is just
  * loading the page table pointer and vector base registers.
  *
  * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on
  * the MMU.
  */
 __enable_mmu:
 	ldr	x5, =vectors
 	msr	vbar_el1, x5
 	msr	ttbr0_el1, x25			// load TTBR0
 	msr	ttbr1_el1, x26			// load TTBR1
 	isb
 	b	__turn_mmu_on
 ENDPROC(__enable_mmu)
 /*
  * Enable the MMU. This completely changes the structure of the visible memory
  * space. You will not be able to trace execution through this.
  *
  *  x0  = system control register
  *  x27 = *virtual* address to jump to upon completion
  *
  * other registers depend on the function called upon completion
  */
 	.align	6
 __turn_mmu_on:
 	msr	sctlr_el1, x0
 	isb
 	br	x27
 ENDPROC(__turn_mmu_on)
 /*
  * Calculate the start of physical memory.
  */
 __calc_phys_offset:
 	adr	x0, 1f
 	ldp	x1, x2, [x0]
 	sub	x28, x0, x1			// x28 = PHYS_OFFSET - PAGE_OFFSET
 	add	x24, x2, x28			// x24 = PHYS_OFFSET
 	ret
 ENDPROC(__calc_phys_offset)
 	.align 3
 1:	.quad	.
 	.quad	PAGE_OFFSET
 /*
  * Macro to populate the PGD for the corresponding block entry in the next
  * level (tbl) for the given virtual address.
  *
  * Preserves:	pgd, tbl, virt
  * Corrupts:	tmp1, tmp2
  */
 	.macro	create_pgd_entry, pgd, tbl, virt, tmp1, tmp2
 	lsr	\tmp1, \virt, #PGDIR_SHIFT
 	and	\tmp1, \tmp1, #PTRS_PER_PGD - 1	// PGD index
 	orr	\tmp2, \tbl, #3			// PGD entry table type
 	str	\tmp2, [\pgd, \tmp1, lsl #3]
 	.endm
 /*
  * Macro to populate block entries in the page table for the start..end
  * virtual range (inclusive).
  *
  * Preserves:	tbl, flags
  * Corrupts:	phys, start, end, pstate
  */
 	.macro	create_block_map, tbl, flags, phys, start, end, idmap=0
 	lsr	\phys, \phys, #BLOCK_SHIFT
 	.if	\idmap
 	and	\start, \phys, #PTRS_PER_PTE - 1	// table index
 	.else
 	lsr	\start, \start, #BLOCK_SHIFT
 	and	\start, \start, #PTRS_PER_PTE - 1	// table index
 	.endif
 	orr	\phys, \flags, \phys, lsl #BLOCK_SHIFT	// table entry
 	.ifnc	\start,\end
 	lsr	\end, \end, #BLOCK_SHIFT
 	and	\end, \end, #PTRS_PER_PTE - 1		// table end index
 	.endif
 9999:	str	\phys, [\tbl, \start, lsl #3]		// store the entry
 	.ifnc	\start,\end
 	add	\start, \start, #1			// next entry
 	add	\phys, \phys, #BLOCK_SIZE		// next block
 	cmp	\start, \end
 	b.ls	9999b
 	.endif
 	.endm
 /*
  * Setup the initial page tables. We only setup the barest amount which is
  * required to get the kernel running. The following sections are required:
  *   - identity mapping to enable the MMU (low address, TTBR0)
  *   - first few MB of the kernel linear mapping to jump to once the MMU has
  *     been enabled, including the FDT blob (TTBR1)
+ *   - UART mapping if CONFIG_EARLY_PRINTK is enabled (TTBR1)
  */
 __create_page_tables:
 	pgtbl	x25, x26, x24			// idmap_pg_dir and swapper_pg_dir addresses
 	/*
 	 * Clear the idmap and swapper page tables.
 	 */
 	mov	x0, x25
 	add	x6, x26, #SWAPPER_DIR_SIZE
 1:	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
 	cmp	x0, x6
 	b.lo	1b
 	ldr	x7, =MM_MMUFLAGS
 	/*
 	 * Create the identity mapping.
 	 */
 	add	x0, x25, #PAGE_SIZE		// section table address
 	adr	x3, __turn_mmu_on		// virtual/physical address
 	create_pgd_entry x25, x0, x3, x5, x6
 	create_block_map x0, x7, x3, x5, x5, idmap=1
 	/*
 	 * Map the kernel image (starting with PHYS_OFFSET).
 	 */
 	add	x0, x26, #PAGE_SIZE		// section table address
 	mov	x5, #PAGE_OFFSET
 	create_pgd_entry x26, x0, x5, x3, x6
 	ldr	x6, =KERNEL_END - 1
 	mov	x3, x24				// phys offset
 	create_block_map x0, x7, x3, x5, x6
 	/*
 	 * Map the FDT blob (maximum 2MB; must be within 512MB of
 	 * PHYS_OFFSET).
 	 */
 	mov	x3, x21				// FDT phys address
 	and	x3, x3, #~((1 << 21) - 1)	// 2MB aligned
 	mov	x6, #PAGE_OFFSET
 	sub	x5, x3, x24			// subtract PHYS_OFFSET
 	tst	x5, #~((1 << 29) - 1)		// within 512MB?
 	csel	x21, xzr, x21, ne		// zero the FDT pointer
 	b.ne	1f
 	add	x5, x5, x6			// __va(FDT blob)
 	add	x6, x5, #1 << 21		// 2MB for the FDT blob
 	sub	x6, x6, #1			// inclusive range
 	create_block_map x0, x7, x3, x5, x6
 1:
+#ifdef CONFIG_EARLY_PRINTK
+	/*
+	 * Create the pgd entry for the UART mapping. The full mapping is done
+	 * later based earlyprintk kernel parameter.
+	 */
+	ldr	x5, =EARLYCON_IOBASE		// UART virtual address
+	add	x0, x26, #2 * PAGE_SIZE		// section table address
+	create_pgd_entry x26, x0, x5, x6, x7
+#endif
 	ret
 ENDPROC(__create_page_tables)
 	.ltorg
 	.align	3
 	.type	__switch_data, %object
 __switch_data:
 	.quad	__mmap_switched
 	.quad	__data_loc			// x4
 	.quad	_data				// x5
 	.quad	__bss_start			// x6
 	.quad	_end				// x7
 	.quad	processor_id			// x4
 	.quad	__fdt_pointer			// x5
 	.quad	memstart_addr			// x6
 	.quad	init_thread_union + THREAD_START_SP // sp
 /*
  * The following fragment of code is executed with the MMU on in MMU mode, and
  * uses absolute addresses; this is not position independent.
  */
 __mmap_switched:
 	adr	x3, __switch_data + 8
 	ldp	x4, x5, [x3], #16
 	ldp	x6, x7, [x3], #16
 	cmp	x4, x5				// Copy data segment if needed
 1:	ccmp	x5, x6, #4, ne
 	b.eq	2f
 	ldr	x16, [x4], #8
 	str	x16, [x5], #8
 	b	1b
 2:
 1:	cmp	x6, x7
 	b.hs	2f
 	str	xzr, [x6], #8			// Clear BSS
 	b	1b
 2:
 	ldp	x4, x5, [x3], #16
 	ldr	x6, [x3], #8
 	ldr	x16, [x3]
 	mov	sp, x16
 	str	x22, [x4]			// Save processor ID
 	str	x21, [x5]			// Save FDT pointer
 	str	x24, [x6]			// Save PHYS_OFFSET
 	mov	x29, #0
 	b	start_kernel
 ENDPROC(__mmap_switched)
 /*
  * Exception handling. Something went wrong and we can't proceed. We ought to
  * tell the user, but since we don't have any guarantee that we're even
  * running on the right architecture, we do virtually nothing.
  */
 __error_p:
 ENDPROC(__error_p)
 __error:
 1:	nop
 	b	1b
 ENDPROC(__error)
 /*
  * This function gets the processor ID in w0 and searches the cpu_table[] for
  * a match. It returns a pointer to the struct cpu_info it found. The
  * cpu_table[] must end with an empty (all zeros) structure.
  *
  * This routine can be called via C code and it needs to work with the MMU
  * both disabled and enabled (the offset is calculated automatically).
  */
 ENTRY(lookup_processor_type)
 	adr	x1, __lookup_processor_type_data
 	ldp	x2, x3, [x1]
 	sub	x1, x1, x2			// get offset between VA and PA
 	add	x3, x3, x1			// convert VA to PA
 1:
 	ldp	w5, w6, [x3]			// load cpu_id_val and cpu_id_mask
 	cbz	w5, 2f				// end of list?
 	and	w6, w6, w0
 	cmp	w5, w6
 	b.eq	3f
 	add	x3, x3, #CPU_INFO_SZ
 	b	1b
 2:
 	mov	x3, #0				// unknown processor
 3:
 	mov	x0, x3
 	ret
 ENDPROC(lookup_processor_type)
 	.align	3
 	.type	__lookup_processor_type_data, %object
 __lookup_processor_type_data:
 	.quad	.
 	.quad	cpu_table
 	.size	__lookup_processor_type_data, . - __lookup_processor_type_data
 /*
  * Determine validity of the x21 FDT pointer.
  * The dtb must be 8-byte aligned and live in the first 512M of memory.
  */
 __vet_fdt:
 	tst	x21, #0x7
 	b.ne	1f
 	cmp	x21, x24
 	b.lt	1f
 	mov	x0, #(1 << 29)
 	add	x0, x0, x24
 	cmp	x21, x0
 	b.ge	1f
 	ret
 1:
 	mov	x21, #0
 	ret

arch/arm64/kernel/perf_event.c

Diff comments View file @ 79a69d3

 /*
  * PMU support
  *
  * Copyright (C) 2012 ARM Limited
  * Author: Will Deacon <will.deacon@arm.com>
  *
  * This code is based heavily on the ARMv7 perf event code.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #define pr_fmt(fmt) "hw perfevents: " fmt
 #include <linux/bitmap.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/export.h>
 #include <linux/perf_event.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/uaccess.h>
 #include <asm/cputype.h>
 #include <asm/irq.h>
 #include <asm/irq_regs.h>
 #include <asm/pmu.h>
 #include <asm/stacktrace.h>
 /*
  * ARMv8 supports a maximum of 32 events.
  * The cycle counter is included in this total.
  */
 #define ARMPMU_MAX_HWEVENTS		32
 static DEFINE_PER_CPU(struct perf_event * [ARMPMU_MAX_HWEVENTS], hw_events);
 static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)], used_mask);
 static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events);
 #define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
 /* Set at runtime when we know what CPU type we are. */
 static struct arm_pmu *cpu_pmu;
 int
 armpmu_get_max_events(void)
 {
 	int max_events = 0;
 	if (cpu_pmu != NULL)
 		max_events = cpu_pmu->num_events;
 	return max_events;
 }
 EXPORT_SYMBOL_GPL(armpmu_get_max_events);
 int perf_num_counters(void)
 {
 	return armpmu_get_max_events();
 }
 EXPORT_SYMBOL_GPL(perf_num_counters);
 #define HW_OP_UNSUPPORTED		0xFFFF
 #define C(_x) \
 	PERF_COUNT_HW_CACHE_##_x
 #define CACHE_OP_UNSUPPORTED		0xFFFF
 static int
 armpmu_map_cache_event(const unsigned (*cache_map)
 				      [PERF_COUNT_HW_CACHE_MAX]
 				      [PERF_COUNT_HW_CACHE_OP_MAX]
 				      [PERF_COUNT_HW_CACHE_RESULT_MAX],
 		       u64 config)
 {
 	unsigned int cache_type, cache_op, cache_result, ret;
 	cache_type = (config >>  0) & 0xff;
 	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 		return -EINVAL;
 	cache_op = (config >>  8) & 0xff;
 	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 		return -EINVAL;
 	cache_result = (config >> 16) & 0xff;
 	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 		return -EINVAL;
 	ret = (int)(*cache_map)[cache_type][cache_op][cache_result];
 	if (ret == CACHE_OP_UNSUPPORTED)
 		return -ENOENT;
 	return ret;
 }
 static int
 armpmu_map_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config)
 {
 	int mapping = (*event_map)[config];
 	return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
 }
 static int
 armpmu_map_raw_event(u32 raw_event_mask, u64 config)
 {
 	return (int)(config & raw_event_mask);
 }
 static int map_cpu_event(struct perf_event *event,
 			 const unsigned (*event_map)[PERF_COUNT_HW_MAX],
 			 const unsigned (*cache_map)
 					[PERF_COUNT_HW_CACHE_MAX]
 					[PERF_COUNT_HW_CACHE_OP_MAX]
 					[PERF_COUNT_HW_CACHE_RESULT_MAX],
 			 u32 raw_event_mask)
 {
 	u64 config = event->attr.config;
 	switch (event->attr.type) {
 	case PERF_TYPE_HARDWARE:
 		return armpmu_map_event(event_map, config);
 	case PERF_TYPE_HW_CACHE:
 		return armpmu_map_cache_event(cache_map, config);
 	case PERF_TYPE_RAW:
 		return armpmu_map_raw_event(raw_event_mask, config);
 	}
 	return -ENOENT;
 }
 int
 armpmu_event_set_period(struct perf_event *event,
 			struct hw_perf_event *hwc,
 			int idx)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	s64 left = local64_read(&hwc->period_left);
 	s64 period = hwc->sample_period;
 	int ret = 0;
 	if (unlikely(left <= -period)) {
 		left = period;
 		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 	if (unlikely(left <= 0)) {
 		left += period;
 		local64_set(&hwc->period_left, left);
 		hwc->last_period = period;
 		ret = 1;
 	}
 	if (left > (s64)armpmu->max_period)
 		left = armpmu->max_period;
 	local64_set(&hwc->prev_count, (u64)-left);
 	armpmu->write_counter(idx, (u64)(-left) & 0xffffffff);
 	perf_event_update_userpage(event);
 	return ret;
 }
 u64
 armpmu_event_update(struct perf_event *event,
 		    struct hw_perf_event *hwc,
 		    int idx)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	u64 delta, prev_raw_count, new_raw_count;
 again:
 	prev_raw_count = local64_read(&hwc->prev_count);
 	new_raw_count = armpmu->read_counter(idx);
 	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 			     new_raw_count) != prev_raw_count)
 		goto again;
 	delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
 	local64_add(delta, &event->count);
 	local64_sub(delta, &hwc->period_left);
 	return new_raw_count;
 }
 static void
 armpmu_read(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
 	/* Don't read disabled counters! */
 	if (hwc->idx < 0)
 		return;
 	armpmu_event_update(event, hwc, hwc->idx);
 }
 static void
 armpmu_stop(struct perf_event *event, int flags)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct hw_perf_event *hwc = &event->hw;
 	/*
 	 * ARM pmu always has to update the counter, so ignore
 	 * PERF_EF_UPDATE, see comments in armpmu_start().
 	 */
 	if (!(hwc->state & PERF_HES_STOPPED)) {
 		armpmu->disable(hwc, hwc->idx);
 		barrier(); /* why? */
 		armpmu_event_update(event, hwc, hwc->idx);
 		hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 	}
 }
 static void
 armpmu_start(struct perf_event *event, int flags)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct hw_perf_event *hwc = &event->hw;
 	/*
 	 * ARM pmu always has to reprogram the period, so ignore
 	 * PERF_EF_RELOAD, see the comment below.
 	 */
 	if (flags & PERF_EF_RELOAD)
 		WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 	hwc->state = 0;
 	/*
 	 * Set the period again. Some counters can't be stopped, so when we
 	 * were stopped we simply disabled the IRQ source and the counter
 	 * may have been left counting. If we don't do this step then we may
 	 * get an interrupt too soon or *way* too late if the overflow has
 	 * happened since disabling.
 	 */
 	armpmu_event_set_period(event, hwc, hwc->idx);
 	armpmu->enable(hwc, hwc->idx);
 }
 static void
 armpmu_del(struct perf_event *event, int flags)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 	struct hw_perf_event *hwc = &event->hw;
 	int idx = hwc->idx;
 	WARN_ON(idx < 0);
 	armpmu_stop(event, PERF_EF_UPDATE);
 	hw_events->events[idx] = NULL;
 	clear_bit(idx, hw_events->used_mask);
 	perf_event_update_userpage(event);
 }
 static int
 armpmu_add(struct perf_event *event, int flags)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 	struct hw_perf_event *hwc = &event->hw;
 	int idx;
 	int err = 0;
 	perf_pmu_disable(event->pmu);
 	/* If we don't have a space for the counter then finish early. */
 	idx = armpmu->get_event_idx(hw_events, hwc);
 	if (idx < 0) {
 		err = idx;
 		goto out;
 	}
 	/*
 	 * If there is an event in the counter we are going to use then make
 	 * sure it is disabled.
 	 */
 	event->hw.idx = idx;
 	armpmu->disable(hwc, idx);
 	hw_events->events[idx] = event;
 	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 	if (flags & PERF_EF_START)
 		armpmu_start(event, PERF_EF_RELOAD);
 	/* Propagate our changes to the userspace mapping. */
 	perf_event_update_userpage(event);
 out:
 	perf_pmu_enable(event->pmu);
 	return err;
 }
 static int
 validate_event(struct pmu_hw_events *hw_events,
 	       struct perf_event *event)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct hw_perf_event fake_event = event->hw;
 	struct pmu *leader_pmu = event->group_leader->pmu;
 	if (event->pmu != leader_pmu || event->state <= PERF_EVENT_STATE_OFF)
 		return 1;
 	return armpmu->get_event_idx(hw_events, &fake_event) >= 0;
 }
 static int
 validate_group(struct perf_event *event)
 {
 	struct perf_event *sibling, *leader = event->group_leader;
 	struct pmu_hw_events fake_pmu;
 	DECLARE_BITMAP(fake_used_mask, ARMPMU_MAX_HWEVENTS);
 	/*
 	 * Initialise the fake PMU. We only need to populate the
 	 * used_mask for the purposes of validation.
 	 */
 	memset(fake_used_mask, 0, sizeof(fake_used_mask));
 	fake_pmu.used_mask = fake_used_mask;
 	if (!validate_event(&fake_pmu, leader))
 		return -EINVAL;
 	list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 		if (!validate_event(&fake_pmu, sibling))
 			return -EINVAL;
 	}
 	if (!validate_event(&fake_pmu, event))
 		return -EINVAL;
 	return 0;
 }
 static void
 armpmu_release_hardware(struct arm_pmu *armpmu)
 {
 	int i, irq, irqs;
 	struct platform_device *pmu_device = armpmu->plat_device;
 	irqs = min(pmu_device->num_resources, num_possible_cpus());
 	for (i = 0; i < irqs; ++i) {
 		if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs))
 			continue;
 		irq = platform_get_irq(pmu_device, i);
 		if (irq >= 0)
 			free_irq(irq, armpmu);
 	}
 }
 static int
 armpmu_reserve_hardware(struct arm_pmu *armpmu)
 {
 	int i, err, irq, irqs;
 	struct platform_device *pmu_device = armpmu->plat_device;
 	if (!pmu_device) {
 		pr_err("no PMU device registered\n");
 		return -ENODEV;
 	}
 	irqs = min(pmu_device->num_resources, num_possible_cpus());
 	if (irqs < 1) {
 		pr_err("no irqs for PMUs defined\n");
 		return -ENODEV;
 	}
 	for (i = 0; i < irqs; ++i) {
 		err = 0;
 		irq = platform_get_irq(pmu_device, i);
 		if (irq < 0)
 			continue;
 		/*
 		 * If we have a single PMU interrupt that we can't shift,
 		 * assume that we're running on a uniprocessor machine and
 		 * continue. Otherwise, continue without this interrupt.
 		 */
 		if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) {
 			pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
 				    irq, i);
 			continue;
 		}
 		err = request_irq(irq, armpmu->handle_irq,
 				  IRQF_NOBALANCING,
 				  "arm-pmu", armpmu);
 		if (err) {
 			pr_err("unable to request IRQ%d for ARM PMU counters\n",
 				irq);
 			armpmu_release_hardware(armpmu);
 			return err;
 		}
 		cpumask_set_cpu(i, &armpmu->active_irqs);
 	}
 	return 0;
 }
 static void
 hw_perf_event_destroy(struct perf_event *event)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	atomic_t *active_events	 = &armpmu->active_events;
 	struct mutex *pmu_reserve_mutex = &armpmu->reserve_mutex;
 	if (atomic_dec_and_mutex_lock(active_events, pmu_reserve_mutex)) {
 		armpmu_release_hardware(armpmu);
 		mutex_unlock(pmu_reserve_mutex);
 	}
 }
 static int
 event_requires_mode_exclusion(struct perf_event_attr *attr)
 {
 	return attr->exclude_idle || attr->exclude_user ||
 	       attr->exclude_kernel || attr->exclude_hv;
 }
 static int
 __hw_perf_event_init(struct perf_event *event)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	struct hw_perf_event *hwc = &event->hw;
 	int mapping, err;
 	mapping = armpmu->map_event(event);
 	if (mapping < 0) {
 		pr_debug("event %x:%llx not supported\n", event->attr.type,
 			 event->attr.config);
 		return mapping;
 	}
 	/*
 	 * We don't assign an index until we actually place the event onto
 	 * hardware. Use -1 to signify that we haven't decided where to put it
 	 * yet. For SMP systems, each core has it's own PMU so we can't do any
 	 * clever allocation or constraints checking at this point.
 	 */
 	hwc->idx		= -1;
 	hwc->config_base	= 0;
 	hwc->config		= 0;
 	hwc->event_base		= 0;
 	/*
 	 * Check whether we need to exclude the counter from certain modes.
 	 */
 	if ((!armpmu->set_event_filter ||
 	     armpmu->set_event_filter(hwc, &event->attr)) &&
 	     event_requires_mode_exclusion(&event->attr)) {
 		pr_debug("ARM performance counters do not support mode exclusion\n");
 		return -EPERM;
 	}
 	/*
 	 * Store the event encoding into the config_base field.
 	 */
 	hwc->config_base	    |= (unsigned long)mapping;
 	if (!hwc->sample_period) {
 		/*
 		 * For non-sampling runs, limit the sample_period to half
 		 * of the counter width. That way, the new counter value
 		 * is far less likely to overtake the previous one unless
 		 * you have some serious IRQ latency issues.
 		 */
 		hwc->sample_period  = armpmu->max_period >> 1;
 		hwc->last_period    = hwc->sample_period;
 		local64_set(&hwc->period_left, hwc->sample_period);
 	}
 	err = 0;
 	if (event->group_leader != event) {
 		err = validate_group(event);
 		if (err)
 			return -EINVAL;
 	}
 	return err;
 }
 static int armpmu_event_init(struct perf_event *event)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
 	int err = 0;
 	atomic_t *active_events = &armpmu->active_events;
 	if (armpmu->map_event(event) == -ENOENT)
 		return -ENOENT;
 	event->destroy = hw_perf_event_destroy;
 	if (!atomic_inc_not_zero(active_events)) {
 		mutex_lock(&armpmu->reserve_mutex);
 		if (atomic_read(active_events) == 0)
 			err = armpmu_reserve_hardware(armpmu);
 		if (!err)
 			atomic_inc(active_events);
 		mutex_unlock(&armpmu->reserve_mutex);
 	}
 	if (err)
 		return err;
 	err = __hw_perf_event_init(event);
 	if (err)
 		hw_perf_event_destroy(event);
 	return err;
 }
 static void armpmu_enable(struct pmu *pmu)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(pmu);
 	struct pmu_hw_events *hw_events = armpmu->get_hw_events();
 	int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events);
 	if (enabled)
 		armpmu->start();
 }
 static void armpmu_disable(struct pmu *pmu)
 {
 	struct arm_pmu *armpmu = to_arm_pmu(pmu);
 	armpmu->stop();
 }
 static void __init armpmu_init(struct arm_pmu *armpmu)
 {
 	atomic_set(&armpmu->active_events, 0);
 	mutex_init(&armpmu->reserve_mutex);
 	armpmu->pmu = (struct pmu) {
 		.pmu_enable	= armpmu_enable,
 		.pmu_disable	= armpmu_disable,
 		.event_init	= armpmu_event_init,
 		.add		= armpmu_add,
 		.del		= armpmu_del,
 		.start		= armpmu_start,
 		.stop		= armpmu_stop,
 		.read		= armpmu_read,
 	};
 }
 int __init armpmu_register(struct arm_pmu *armpmu, char *name, int type)
 {
 	armpmu_init(armpmu);
 	return perf_pmu_register(&armpmu->pmu, name, type);
 }
 /*
  * ARMv8 PMUv3 Performance Events handling code.
  * Common event types.
  */
 enum armv8_pmuv3_perf_types {
 	/* Required events. */
 	ARMV8_PMUV3_PERFCTR_PMNC_SW_INCR			= 0x00,
 	ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL			= 0x03,
 	ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS			= 0x04,
 	ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED			= 0x10,
 	ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES			= 0x11,
 	ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED			= 0x12,
 	/* At least one of the following is required. */
 	ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED			= 0x08,
 	ARMV8_PMUV3_PERFCTR_OP_SPEC				= 0x1B,
 	/* Common architectural events. */
 	ARMV8_PMUV3_PERFCTR_MEM_READ				= 0x06,
 	ARMV8_PMUV3_PERFCTR_MEM_WRITE				= 0x07,
 	ARMV8_PMUV3_PERFCTR_EXC_TAKEN				= 0x09,
 	ARMV8_PMUV3_PERFCTR_EXC_EXECUTED			= 0x0A,
 	ARMV8_PMUV3_PERFCTR_CID_WRITE				= 0x0B,
 	ARMV8_PMUV3_PERFCTR_PC_WRITE				= 0x0C,
 	ARMV8_PMUV3_PERFCTR_PC_IMM_BRANCH			= 0x0D,
 	ARMV8_PMUV3_PERFCTR_PC_PROC_RETURN			= 0x0E,
 	ARMV8_PMUV3_PERFCTR_MEM_UNALIGNED_ACCESS		= 0x0F,
 	ARMV8_PMUV3_PERFCTR_TTBR_WRITE				= 0x1C,
 	/* Common microarchitectural events. */
 	ARMV8_PMUV3_PERFCTR_L1_ICACHE_REFILL			= 0x01,
 	ARMV8_PMUV3_PERFCTR_ITLB_REFILL				= 0x02,
 	ARMV8_PMUV3_PERFCTR_DTLB_REFILL				= 0x05,
 	ARMV8_PMUV3_PERFCTR_MEM_ACCESS				= 0x13,
 	ARMV8_PMUV3_PERFCTR_L1_ICACHE_ACCESS			= 0x14,
 	ARMV8_PMUV3_PERFCTR_L1_DCACHE_WB			= 0x15,
 	ARMV8_PMUV3_PERFCTR_L2_CACHE_ACCESS			= 0x16,
 	ARMV8_PMUV3_PERFCTR_L2_CACHE_REFILL			= 0x17,
 	ARMV8_PMUV3_PERFCTR_L2_CACHE_WB				= 0x18,
 	ARMV8_PMUV3_PERFCTR_BUS_ACCESS				= 0x19,
 	ARMV8_PMUV3_PERFCTR_MEM_ERROR				= 0x1A,
 	ARMV8_PMUV3_PERFCTR_BUS_CYCLES				= 0x1D,
 };
 /* PMUv3 HW events mapping. */
 static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
 	[PERF_COUNT_HW_CPU_CYCLES]		= ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES,
 	[PERF_COUNT_HW_INSTRUCTIONS]		= ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED,
 	[PERF_COUNT_HW_CACHE_REFERENCES]	= ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 	[PERF_COUNT_HW_CACHE_MISSES]		= ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= HW_OP_UNSUPPORTED,
 	[PERF_COUNT_HW_BRANCH_MISSES]		= ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 	[PERF_COUNT_HW_BUS_CYCLES]		= HW_OP_UNSUPPORTED,
 	[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND]	= HW_OP_UNSUPPORTED,
 	[PERF_COUNT_HW_STALLED_CYCLES_BACKEND]	= HW_OP_UNSUPPORTED,
 };
 static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 						[PERF_COUNT_HW_CACHE_OP_MAX]
 						[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 	[C(L1D)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 			[C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS,
 			[C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(L1I)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(LL)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(DTLB)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(ITLB)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(BPU)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 			[C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED,
 			[C(RESULT_MISS)]	= ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 	[C(NODE)] = {
 		[C(OP_READ)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_WRITE)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 		[C(OP_PREFETCH)] = {
 			[C(RESULT_ACCESS)]	= CACHE_OP_UNSUPPORTED,
 			[C(RESULT_MISS)]	= CACHE_OP_UNSUPPORTED,
 		},
 	},
 };
 /*
  * Perf Events' indices
  */
 #define	ARMV8_IDX_CYCLE_COUNTER	0
 #define	ARMV8_IDX_COUNTER0	1
 #define	ARMV8_IDX_COUNTER_LAST	(ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 #define	ARMV8_MAX_COUNTERS	32
 #define	ARMV8_COUNTER_MASK	(ARMV8_MAX_COUNTERS - 1)
 /*
  * ARMv8 low level PMU access
  */
 /*
  * Perf Event to low level counters mapping
  */
 #define	ARMV8_IDX_TO_COUNTER(x)	\
 	(((x) - ARMV8_IDX_COUNTER0) & ARMV8_COUNTER_MASK)
 /*
  * Per-CPU PMCR: config reg
  */
 #define ARMV8_PMCR_E		(1 << 0) /* Enable all counters */
 #define ARMV8_PMCR_P		(1 << 1) /* Reset all counters */
 #define ARMV8_PMCR_C		(1 << 2) /* Cycle counter reset */
 #define ARMV8_PMCR_D		(1 << 3) /* CCNT counts every 64th cpu cycle */
 #define ARMV8_PMCR_X		(1 << 4) /* Export to ETM */
 #define ARMV8_PMCR_DP		(1 << 5) /* Disable CCNT if non-invasive debug*/
 #define	ARMV8_PMCR_N_SHIFT	11	 /* Number of counters supported */
 #define	ARMV8_PMCR_N_MASK	0x1f
 #define	ARMV8_PMCR_MASK		0x3f	 /* Mask for writable bits */
 /*
  * PMOVSR: counters overflow flag status reg
  */
 #define	ARMV8_OVSR_MASK		0xffffffff	/* Mask for writable bits */
 #define	ARMV8_OVERFLOWED_MASK	ARMV8_OVSR_MASK
 /*
  * PMXEVTYPER: Event selection reg
  */
 #define	ARMV8_EVTYPE_MASK	0xc00000ff	/* Mask for writable bits */
 #define	ARMV8_EVTYPE_EVENT	0xff		/* Mask for EVENT bits */
 /*
  * Event filters for PMUv3
  */
 #define	ARMV8_EXCLUDE_EL1	(1 << 31)
 #define	ARMV8_EXCLUDE_EL0	(1 << 30)
 #define	ARMV8_INCLUDE_EL2	(1 << 27)
 static inline u32 armv8pmu_pmcr_read(void)
 {
 	u32 val;
 	asm volatile("mrs %0, pmcr_el0" : "=r" (val));
 	return val;
 }
 static inline void armv8pmu_pmcr_write(u32 val)
 {
 	val &= ARMV8_PMCR_MASK;
 	isb();
 	asm volatile("msr pmcr_el0, %0" :: "r" (val));
 }
 static inline int armv8pmu_has_overflowed(u32 pmovsr)
 {
 	return pmovsr & ARMV8_OVERFLOWED_MASK;
 }
 static inline int armv8pmu_counter_valid(int idx)
 {
 	return idx >= ARMV8_IDX_CYCLE_COUNTER && idx <= ARMV8_IDX_COUNTER_LAST;
 }
 static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx)
 {
 	int ret = 0;
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u checking wrong counter %d overflow status\n",
 			smp_processor_id(), idx);
 	} else {
 		counter = ARMV8_IDX_TO_COUNTER(idx);
 		ret = pmnc & BIT(counter);
 	}
 	return ret;
 }
 static inline int armv8pmu_select_counter(int idx)
 {
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u selecting wrong PMNC counter %d\n",
 			smp_processor_id(), idx);
 		return -EINVAL;
 	}
 	counter = ARMV8_IDX_TO_COUNTER(idx);
 	asm volatile("msr pmselr_el0, %0" :: "r" (counter));
 	isb();
 	return idx;
 }
 static inline u32 armv8pmu_read_counter(int idx)
 {
 	u32 value = 0;
 	if (!armv8pmu_counter_valid(idx))
 		pr_err("CPU%u reading wrong counter %d\n",
 			smp_processor_id(), idx);
 	else if (idx == ARMV8_IDX_CYCLE_COUNTER)
 		asm volatile("mrs %0, pmccntr_el0" : "=r" (value));
 	else if (armv8pmu_select_counter(idx) == idx)
 		asm volatile("mrs %0, pmxevcntr_el0" : "=r" (value));
 	return value;
 }
 static inline void armv8pmu_write_counter(int idx, u32 value)
 {
 	if (!armv8pmu_counter_valid(idx))
 		pr_err("CPU%u writing wrong counter %d\n",
 			smp_processor_id(), idx);
 	else if (idx == ARMV8_IDX_CYCLE_COUNTER)
 		asm volatile("msr pmccntr_el0, %0" :: "r" (value));
 	else if (armv8pmu_select_counter(idx) == idx)
 		asm volatile("msr pmxevcntr_el0, %0" :: "r" (value));
 }
 static inline void armv8pmu_write_evtype(int idx, u32 val)
 {
 	if (armv8pmu_select_counter(idx) == idx) {
 		val &= ARMV8_EVTYPE_MASK;
 		asm volatile("msr pmxevtyper_el0, %0" :: "r" (val));
 	}
 }
 static inline int armv8pmu_enable_counter(int idx)
 {
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u enabling wrong PMNC counter %d\n",
 			smp_processor_id(), idx);
 		return -EINVAL;
 	}
 	counter = ARMV8_IDX_TO_COUNTER(idx);
 	asm volatile("msr pmcntenset_el0, %0" :: "r" (BIT(counter)));
 	return idx;
 }
 static inline int armv8pmu_disable_counter(int idx)
 {
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u disabling wrong PMNC counter %d\n",
 			smp_processor_id(), idx);
 		return -EINVAL;
 	}
 	counter = ARMV8_IDX_TO_COUNTER(idx);
 	asm volatile("msr pmcntenclr_el0, %0" :: "r" (BIT(counter)));
 	return idx;
 }
 static inline int armv8pmu_enable_intens(int idx)
 {
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 			smp_processor_id(), idx);
 		return -EINVAL;
 	}
 	counter = ARMV8_IDX_TO_COUNTER(idx);
 	asm volatile("msr pmintenset_el1, %0" :: "r" (BIT(counter)));
 	return idx;
 }
 static inline int armv8pmu_disable_intens(int idx)
 {
 	u32 counter;
 	if (!armv8pmu_counter_valid(idx)) {
 		pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 			smp_processor_id(), idx);
 		return -EINVAL;
 	}
 	counter = ARMV8_IDX_TO_COUNTER(idx);
 	asm volatile("msr pmintenclr_el1, %0" :: "r" (BIT(counter)));
 	isb();
 	/* Clear the overflow flag in case an interrupt is pending. */
 	asm volatile("msr pmovsclr_el0, %0" :: "r" (BIT(counter)));
 	isb();
 	return idx;
 }
 static inline u32 armv8pmu_getreset_flags(void)
 {
 	u32 value;
 	/* Read */
 	asm volatile("mrs %0, pmovsclr_el0" : "=r" (value));
 	/* Write to clear flags */
 	value &= ARMV8_OVSR_MASK;
 	asm volatile("msr pmovsclr_el0, %0" :: "r" (value));
 	return value;
 }
 static void armv8pmu_enable_event(struct hw_perf_event *hwc, int idx)
 {
 	unsigned long flags;
 	struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 	/*
 	 * Enable counter and interrupt, and set the counter to count
 	 * the event that we're interested in.
 	 */
 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 	/*
 	 * Disable counter
 	 */
 	armv8pmu_disable_counter(idx);
 	/*
 	 * Set event (if destined for PMNx counters).
 	 */
 	armv8pmu_write_evtype(idx, hwc->config_base);
 	/*
 	 * Enable interrupt for this counter
 	 */
 	armv8pmu_enable_intens(idx);
 	/*
 	 * Enable counter
 	 */
 	armv8pmu_enable_counter(idx);
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 static void armv8pmu_disable_event(struct hw_perf_event *hwc, int idx)
 {
 	unsigned long flags;
 	struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 	/*
 	 * Disable counter and interrupt
 	 */
 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 	/*
 	 * Disable counter
 	 */
 	armv8pmu_disable_counter(idx);
 	/*
 	 * Disable interrupt for this counter
 	 */
 	armv8pmu_disable_intens(idx);
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev)
 {
 	u32 pmovsr;
 	struct perf_sample_data data;
 	struct pmu_hw_events *cpuc;
 	struct pt_regs *regs;
 	int idx;
 	/*
 	 * Get and reset the IRQ flags
 	 */
 	pmovsr = armv8pmu_getreset_flags();
 	/*
 	 * Did an overflow occur?
 	 */
 	if (!armv8pmu_has_overflowed(pmovsr))
 		return IRQ_NONE;
 	/*
 	 * Handle the counter(s) overflow(s)
 	 */
 	regs = get_irq_regs();
 	cpuc = &__get_cpu_var(cpu_hw_events);
 	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 		struct perf_event *event = cpuc->events[idx];
 		struct hw_perf_event *hwc;
 		/* Ignore if we don't have an event. */
 		if (!event)
 			continue;
 		/*
 		 * We have a single interrupt for all counters. Check that
 		 * each counter has overflowed before we process it.
 		 */
 		if (!armv8pmu_counter_has_overflowed(pmovsr, idx))
 			continue;
 		hwc = &event->hw;
 		armpmu_event_update(event, hwc, idx);
 		perf_sample_data_init(&data, 0, hwc->last_period);
 		if (!armpmu_event_set_period(event, hwc, idx))
 			continue;
 		if (perf_event_overflow(event, &data, regs))
 			cpu_pmu->disable(hwc, idx);
 	}
 	/*
 	 * Handle the pending perf events.
 	 *
 	 * Note: this call *must* be run with interrupts disabled. For
 	 * platforms that can have the PMU interrupts raised as an NMI, this
 	 * will not work.
 	 */
 	irq_work_run();
 	return IRQ_HANDLED;
 }
 static void armv8pmu_start(void)
 {
 	unsigned long flags;
 	struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 	/* Enable all counters */
 	armv8pmu_pmcr_write(armv8pmu_pmcr_read() | ARMV8_PMCR_E);
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 static void armv8pmu_stop(void)
 {
 	unsigned long flags;
 	struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 	raw_spin_lock_irqsave(&events->pmu_lock, flags);
 	/* Disable all counters */
 	armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMCR_E);
 	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 }
 static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
 				  struct hw_perf_event *event)
 {
 	int idx;
 	unsigned long evtype = event->config_base & ARMV8_EVTYPE_EVENT;
 	/* Always place a cycle counter into the cycle counter. */
 	if (evtype == ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES) {
 		if (test_and_set_bit(ARMV8_IDX_CYCLE_COUNTER, cpuc->used_mask))
 			return -EAGAIN;
 		return ARMV8_IDX_CYCLE_COUNTER;
 	}
 	/*
 	 * For anything other than a cycle counter, try and use
 	 * the events counters
 	 */
 	for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
 		if (!test_and_set_bit(idx, cpuc->used_mask))
 			return idx;
 	}
 	/* The counters are all in use. */
 	return -EAGAIN;
 }
 /*
  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
  */
 static int armv8pmu_set_event_filter(struct hw_perf_event *event,
 				     struct perf_event_attr *attr)
 {
 	unsigned long config_base = 0;
 	if (attr->exclude_idle)
 		return -EPERM;
 	if (attr->exclude_user)
 		config_base |= ARMV8_EXCLUDE_EL0;
 	if (attr->exclude_kernel)
 		config_base |= ARMV8_EXCLUDE_EL1;
 	if (!attr->exclude_hv)
 		config_base |= ARMV8_INCLUDE_EL2;
 	/*
 	 * Install the filter into config_base as this is used to
 	 * construct the event type.
 	 */
 	event->config_base = config_base;
 	return 0;
 }
 static void armv8pmu_reset(void *info)
 {
 	u32 idx, nb_cnt = cpu_pmu->num_events;
 	/* The counter and interrupt enable registers are unknown at reset. */
 	for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
 		armv8pmu_disable_event(NULL, idx);
 	/* Initialize & Reset PMNC: C and P bits. */
 	armv8pmu_pmcr_write(ARMV8_PMCR_P | ARMV8_PMCR_C);
 	/* Disable access from userspace. */
 	asm volatile("msr pmuserenr_el0, %0" :: "r" (0));
 }
 static int armv8_pmuv3_map_event(struct perf_event *event)
 {
 	return map_cpu_event(event, &armv8_pmuv3_perf_map,
 				&armv8_pmuv3_perf_cache_map, 0xFF);
 }
 static struct arm_pmu armv8pmu = {
 	.handle_irq		= armv8pmu_handle_irq,
 	.enable			= armv8pmu_enable_event,
 	.disable		= armv8pmu_disable_event,
 	.read_counter		= armv8pmu_read_counter,
 	.write_counter		= armv8pmu_write_counter,
 	.get_event_idx		= armv8pmu_get_event_idx,
 	.start			= armv8pmu_start,
 	.stop			= armv8pmu_stop,
 	.reset			= armv8pmu_reset,
 	.max_period		= (1LLU << 32) - 1,
 };
 static u32 __init armv8pmu_read_num_pmnc_events(void)
 {
 	u32 nb_cnt;
 	/* Read the nb of CNTx counters supported from PMNC */
 	nb_cnt = (armv8pmu_pmcr_read() >> ARMV8_PMCR_N_SHIFT) & ARMV8_PMCR_N_MASK;
 	/* Add the CPU cycles counter and return */
 	return nb_cnt + 1;
 }
 static struct arm_pmu *__init armv8_pmuv3_pmu_init(void)
 {
 	armv8pmu.name			= "arm/armv8-pmuv3";
 	armv8pmu.map_event		= armv8_pmuv3_map_event;
 	armv8pmu.num_events		= armv8pmu_read_num_pmnc_events();
 	armv8pmu.set_event_filter	= armv8pmu_set_event_filter;
 	return &armv8pmu;
 }
 /*
  * Ensure the PMU has sane values out of reset.
  * This requires SMP to be available, so exists as a separate initcall.
  */
 static int __init
 cpu_pmu_reset(void)
 {
 	if (cpu_pmu && cpu_pmu->reset)
 		return on_each_cpu(cpu_pmu->reset, NULL, 1);
 	return 0;
 }
 arch_initcall(cpu_pmu_reset);
 /*
  * PMU platform driver and devicetree bindings.
  */
 static struct of_device_id armpmu_of_device_ids[] = {
 	{.compatible = "arm,armv8-pmuv3"},
 	{},
 };
 static int armpmu_device_probe(struct platform_device *pdev)
 {
 	if (!cpu_pmu)
 		return -ENODEV;
 	cpu_pmu->plat_device = pdev;
 	return 0;
 }
 static struct platform_driver armpmu_driver = {
 	.driver		= {
 		.name	= "arm-pmu",
 		.of_match_table = armpmu_of_device_ids,
 	},
 	.probe		= armpmu_device_probe,
 };
 static int __init register_pmu_driver(void)
 {
 	return platform_driver_register(&armpmu_driver);
 }
 device_initcall(register_pmu_driver);
 static struct pmu_hw_events *armpmu_get_cpu_events(void)
 {
 	return &__get_cpu_var(cpu_hw_events);
 }
 static void __init cpu_pmu_init(struct arm_pmu *armpmu)
 {
 	int cpu;
 	for_each_possible_cpu(cpu) {
 		struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu);
 		events->events = per_cpu(hw_events, cpu);
 		events->used_mask = per_cpu(used_mask, cpu);
 		raw_spin_lock_init(&events->pmu_lock);
 	}
 	armpmu->get_hw_events = armpmu_get_cpu_events;
 }
 static int __init init_hw_perf_events(void)
 {
 	u64 dfr = read_cpuid(ID_AA64DFR0_EL1);
 	switch ((dfr >> 8) & 0xf) {
 	case 0x1:	/* PMUv3 */
 		cpu_pmu = armv8_pmuv3_pmu_init();
 		break;
 	}
 	if (cpu_pmu) {
 		pr_info("enabled with %s PMU driver, %d counters available\n",
 			cpu_pmu->name, cpu_pmu->num_events);
 		cpu_pmu_init(cpu_pmu);
 		armpmu_register(cpu_pmu, "cpu", PERF_TYPE_RAW);
 	} else {
 		pr_info("no hardware support available\n");
 	}
 	return 0;
 }
 early_initcall(init_hw_perf_events);
 /*
  * Callchain handling code.
  */
 struct frame_tail {
 	struct frame_tail   __user *fp;
 	unsigned long	    lr;
 } __attribute__((packed));
 /*
  * Get the return address for a single stackframe and return a pointer to the
  * next frame tail.
  */
 static struct frame_tail __user *
 user_backtrace(struct frame_tail __user *tail,
 	       struct perf_callchain_entry *entry)
 {
 	struct frame_tail buftail;
 	unsigned long err;
 	/* Also check accessibility of one struct frame_tail beyond */
 	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
 		return NULL;
 	pagefault_disable();
 	err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail));
 	pagefault_enable();
 	if (err)
 		return NULL;
 	perf_callchain_store(entry, buftail.lr);
 	/*
 	 * Frame pointers should strictly progress back up the stack
 	 * (towards higher addresses).
 	 */
 	if (tail >= buftail.fp)
 		return NULL;
 	return buftail.fp;
 }
 void perf_callchain_user(struct perf_callchain_entry *entry,
 			 struct pt_regs *regs)
 {
 	struct frame_tail __user *tail;
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+		/* We don't support guest os callchain now */
+		return;
+	}
 	tail = (struct frame_tail __user *)regs->regs[29];
 	while (entry->nr < PERF_MAX_STACK_DEPTH &&
 	       tail && !((unsigned long)tail & 0xf))
 		tail = user_backtrace(tail, entry);
 }
 /*
  * Gets called by walk_stackframe() for every stackframe. This will be called
  * whist unwinding the stackframe and is like a subroutine return so we use
  * the PC.
  */
 static int callchain_trace(struct stackframe *frame, void *data)
 {
 	struct perf_callchain_entry *entry = data;
 	perf_callchain_store(entry, frame->pc);
 	return 0;
 }
 void perf_callchain_kernel(struct perf_callchain_entry *entry,
 			   struct pt_regs *regs)
 {
 	struct stackframe frame;
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+		/* We don't support guest os callchain now */
+		return;
+	}
 	frame.fp = regs->regs[29];
 	frame.sp = regs->sp;
 	frame.pc = regs->pc;
 	walk_stackframe(&frame, callchain_trace, entry);
+}
+unsigned long perf_instruction_pointer(struct pt_regs *regs)
+{
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+		return perf_guest_cbs->get_guest_ip();
+	return instruction_pointer(regs);
+}
+unsigned long perf_misc_flags(struct pt_regs *regs)
+{
+	int misc = 0;
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+		if (perf_guest_cbs->is_user_mode())
+			misc |= PERF_RECORD_MISC_GUEST_USER;
+		else
+			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
+	} else {
+		if (user_mode(regs))
+			misc |= PERF_RECORD_MISC_USER;
+		else
+			misc |= PERF_RECORD_MISC_KERNEL;
+	}
+	return misc;
 }

arch/arm64/kernel/process.c

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/kernel/process.c
  *
  * Original Copyright (C) 1995  Linus Torvalds
  * Copyright (C) 1996-2000 Russell King - Converted to ARM.
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <stdarg.h>
 #include <linux/export.h>
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/stddef.h>
 #include <linux/unistd.h>
 #include <linux/user.h>
 #include <linux/delay.h>
 #include <linux/reboot.h>
 #include <linux/interrupt.h>
 #include <linux/kallsyms.h>
 #include <linux/init.h>
 #include <linux/cpu.h>
 #include <linux/elfcore.h>
 #include <linux/pm.h>
 #include <linux/tick.h>
 #include <linux/utsname.h>
 #include <linux/uaccess.h>
 #include <linux/random.h>
 #include <linux/hw_breakpoint.h>
 #include <linux/personality.h>
 #include <linux/notifier.h>
 #include <asm/compat.h>
 #include <asm/cacheflush.h>
+#include <asm/fpsimd.h>
+#include <asm/mmu_context.h>
 #include <asm/processor.h>
 #include <asm/stacktrace.h>
-#include <asm/fpsimd.h>
 static void setup_restart(void)
 {
 	/*
 	 * Tell the mm system that we are going to reboot -
 	 * we may need it to insert some 1:1 mappings so that
 	 * soft boot works.
 	 */
 	setup_mm_for_reboot();
 	/* Clean and invalidate caches */
 	flush_cache_all();
 	/* Turn D-cache off */
 	cpu_cache_off();
 	/* Push out any further dirty data, and ensure cache is empty */
 	flush_cache_all();
 }
 void soft_restart(unsigned long addr)
 {
 	setup_restart();
 	cpu_reset(addr);
 }
 /*
  * Function pointers to optional machine specific functions
  */
 void (*pm_power_off)(void);
 EXPORT_SYMBOL_GPL(pm_power_off);
 void (*pm_restart)(const char *cmd);
 EXPORT_SYMBOL_GPL(pm_restart);
 /*
  * This is our default idle handler.
  */
 static void default_idle(void)
 {
 	/*
 	 * This should do all the clock switching and wait for interrupt
 	 * tricks
 	 */
 	cpu_do_idle();
 	local_irq_enable();
 }
 /*
  * The idle thread.
  * We always respect 'hlt_counter' to prevent low power idle.
  */
 void cpu_idle(void)
 {
 	local_fiq_enable();
 	/* endless idle loop with no priority at all */
 	while (1) {
 		tick_nohz_idle_enter();
 		rcu_idle_enter();
 		while (!need_resched()) {
 			/*
 			 * We need to disable interrupts here to ensure
 			 * we don't miss a wakeup call.
 			 */
 			local_irq_disable();
 			if (!need_resched()) {
 				stop_critical_timings();
 				default_idle();
 				start_critical_timings();
 				/*
 				 * default_idle functions should always return
 				 * with IRQs enabled.
 				 */
 				WARN_ON(irqs_disabled());
 			} else {
 				local_irq_enable();
 			}
 		}
 		rcu_idle_exit();
 		tick_nohz_idle_exit();
 		schedule_preempt_disabled();
 	}
 }
 void machine_shutdown(void)
 {
 #ifdef CONFIG_SMP
 	smp_send_stop();
 #endif
 }
 void machine_halt(void)
 {
 	machine_shutdown();
 	while (1);
 }
 void machine_power_off(void)
 {
 	machine_shutdown();
 	if (pm_power_off)
 		pm_power_off();
 }
 void machine_restart(char *cmd)
 {
 	machine_shutdown();
 	/* Disable interrupts first */
 	local_irq_disable();
 	local_fiq_disable();
 	/* Now call the architecture specific reboot code. */
 	if (pm_restart)
 		pm_restart(cmd);
 	/*
 	 * Whoops - the architecture was unable to reboot.
 	 */
 	printk("Reboot failed -- System halted\n");
 	while (1);
 }
 void __show_regs(struct pt_regs *regs)
 {
 	int i;
 	printk("CPU: %d    %s  (%s %.*s)\n",
 		raw_smp_processor_id(), print_tainted(),
 		init_utsname()->release,
 		(int)strcspn(init_utsname()->version, " "),
 		init_utsname()->version);
 	print_symbol("PC is at %s\n", instruction_pointer(regs));
 	print_symbol("LR is at %s\n", regs->regs[30]);
 	printk("pc : [<%016llx>] lr : [<%016llx>] pstate: %08llx\n",
 	       regs->pc, regs->regs[30], regs->pstate);
 	printk("sp : %016llx\n", regs->sp);
 	for (i = 29; i >= 0; i--) {
 		printk("x%-2d: %016llx ", i, regs->regs[i]);
 		if (i % 2 == 0)
 			printk("\n");
 	}
 	printk("\n");
 }
 void show_regs(struct pt_regs * regs)
 {
 	printk("\n");
 	printk("Pid: %d, comm: %20s\n", task_pid_nr(current), current->comm);
 	__show_regs(regs);
 }
 /*
  * Free current thread data structures etc..
  */
 void exit_thread(void)
 {
 }
 void flush_thread(void)
 {
 	fpsimd_flush_thread();
 	flush_ptrace_hw_breakpoint(current);
 }
 void release_thread(struct task_struct *dead_task)
 {
 }
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 {
 	fpsimd_save_state(&current->thread.fpsimd_state);
 	*dst = *src;
 	return 0;
 }
 asmlinkage void ret_from_fork(void) asm("ret_from_fork");
 int copy_thread(unsigned long clone_flags, unsigned long stack_start,
 		unsigned long stk_sz, struct task_struct *p)
 {
 	struct pt_regs *childregs = task_pt_regs(p);
 	unsigned long tls = p->thread.tp_value;
 	memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context));
 	if (likely(!(p->flags & PF_KTHREAD))) {
 		*childregs = *current_pt_regs();
 		childregs->regs[0] = 0;
 		if (is_compat_thread(task_thread_info(p))) {
 			if (stack_start)
 				childregs->compat_sp = stack_start;
 		} else {
 			/*
 			 * Read the current TLS pointer from tpidr_el0 as it may be
 			 * out-of-sync with the saved value.
 			 */
 			asm("mrs %0, tpidr_el0" : "=r" (tls));
 			if (stack_start) {
 				/* 16-byte aligned stack mandatory on AArch64 */
 				if (stack_start & 15)
 					return -EINVAL;
 				childregs->sp = stack_start;
 			}
 		}
 		/*
 		 * If a TLS pointer was passed to clone (4th argument), use it
 		 * for the new thread.
 		 */
 		if (clone_flags & CLONE_SETTLS)
 			tls = childregs->regs[3];
 	} else {
 		memset(childregs, 0, sizeof(struct pt_regs));
 		childregs->pstate = PSR_MODE_EL1h;
 		p->thread.cpu_context.x19 = stack_start;
 		p->thread.cpu_context.x20 = stk_sz;
 	}
 	p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
 	p->thread.cpu_context.sp = (unsigned long)childregs;
 	p->thread.tp_value = tls;
 	ptrace_hw_copy_thread(p);
 	return 0;
 }
 static void tls_thread_switch(struct task_struct *next)
 {
 	unsigned long tpidr, tpidrro;
 	if (!is_compat_task()) {
 		asm("mrs %0, tpidr_el0" : "=r" (tpidr));
 		current->thread.tp_value = tpidr;
 	}
 	if (is_compat_thread(task_thread_info(next))) {
 		tpidr = 0;
 		tpidrro = next->thread.tp_value;
 	} else {
 		tpidr = next->thread.tp_value;
 		tpidrro = 0;
 	}
 	asm(
 	"	msr	tpidr_el0, %0\n"
 	"	msr	tpidrro_el0, %1"
 	: : "r" (tpidr), "r" (tpidrro));
 }
 /*
  * Thread switching.
  */
 struct task_struct *__switch_to(struct task_struct *prev,
 				struct task_struct *next)
 {
 	struct task_struct *last;
 	fpsimd_thread_switch(next);
 	tls_thread_switch(next);
 	hw_breakpoint_thread_switch(next);
 	/* the actual thread switch */
 	last = cpu_switch_to(prev, next);
+	contextidr_thread_switch(next);
 	return last;
 }
 unsigned long get_wchan(struct task_struct *p)
 {
 	struct stackframe frame;
 	int count = 0;
 	if (!p || p == current || p->state == TASK_RUNNING)
 		return 0;
 	frame.fp = thread_saved_fp(p);
 	frame.sp = thread_saved_sp(p);
 	frame.pc = thread_saved_pc(p);
 	do {
 		int ret = unwind_frame(&frame);
 		if (ret < 0)
 			return 0;
 		if (!in_sched_functions(frame.pc))
 			return frame.pc;
 	} while (count ++ < 16);
 	return 0;
 }
 unsigned long arch_align_stack(unsigned long sp)
 {
 	if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
 		sp -= get_random_int() & ~PAGE_MASK;
 	return sp & ~0xf;
 }
 static unsigned long randomize_base(unsigned long base)
 {
 	unsigned long range_end = base + (STACK_RND_MASK << PAGE_SHIFT) + 1;
 	return randomize_range(base, range_end, 0) ? : base;
 }
 unsigned long arch_randomize_brk(struct mm_struct *mm)
 {
 	return randomize_base(mm->brk);
 }
 unsigned long randomize_et_dyn(unsigned long base)
 {
 	return randomize_base(base);
 }

arch/arm64/kernel/psci.c

Diff comments View file @ 79a69d3

File was created	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License version 2 as
	4	* published by the Free Software Foundation.
	5	*
	6	* This program is distributed in the hope that it will be useful,
	7	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	8	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	9	* GNU General Public License for more details.
	10	*
	11	* Copyright (C) 2013 ARM Limited
	12	*
	13	* Author: Will Deacon <will.deacon@arm.com>
	14	*/
	15
	16	#define pr_fmt(fmt) "psci: " fmt
	17
	18	#include <linux/init.h>
	19	#include <linux/of.h>
	20
	21	#include <asm/compiler.h>
	22	#include <asm/errno.h>
	23	#include <asm/psci.h>
	24
	25	struct psci_operations psci_ops;
	26
	27	static int (*invoke_psci_fn)(u64, u64, u64, u64);
	28
	29	enum psci_function {
	30	PSCI_FN_CPU_SUSPEND,
	31	PSCI_FN_CPU_ON,
	32	PSCI_FN_CPU_OFF,
	33	PSCI_FN_MIGRATE,
	34	PSCI_FN_MAX,
	35	};
	36
	37	static u32 psci_function_id[PSCI_FN_MAX];
	38
	39	#define PSCI_RET_SUCCESS 0
	40	#define PSCI_RET_EOPNOTSUPP -1
	41	#define PSCI_RET_EINVAL -2
	42	#define PSCI_RET_EPERM -3
	43
	44	static int psci_to_linux_errno(int errno)
	45	{
	46	switch (errno) {
	47	case PSCI_RET_SUCCESS:
	48	return 0;
	49	case PSCI_RET_EOPNOTSUPP:
	50	return -EOPNOTSUPP;
	51	case PSCI_RET_EINVAL:
	52	return -EINVAL;
	53	case PSCI_RET_EPERM:
	54	return -EPERM;
	55	};
	56
	57	return -EINVAL;
	58	}
	59
	60	#define PSCI_POWER_STATE_ID_MASK 0xffff
	61	#define PSCI_POWER_STATE_ID_SHIFT 0
	62	#define PSCI_POWER_STATE_TYPE_MASK 0x1
	63	#define PSCI_POWER_STATE_TYPE_SHIFT 16
	64	#define PSCI_POWER_STATE_AFFL_MASK 0x3
	65	#define PSCI_POWER_STATE_AFFL_SHIFT 24
	66
	67	static u32 psci_power_state_pack(struct psci_power_state state)
	68	{
	69	return ((state.id & PSCI_POWER_STATE_ID_MASK)
	70	<< PSCI_POWER_STATE_ID_SHIFT) \|
	71	((state.type & PSCI_POWER_STATE_TYPE_MASK)
	72	<< PSCI_POWER_STATE_TYPE_SHIFT) \|
	73	((state.affinity_level & PSCI_POWER_STATE_AFFL_MASK)
	74	<< PSCI_POWER_STATE_AFFL_SHIFT);
	75	}
	76
	77	/*
	78	* The following two functions are invoked via the invoke_psci_fn pointer
	79	* and will not be inlined, allowing us to piggyback on the AAPCS.
	80	*/
	81	static noinline int __invoke_psci_fn_hvc(u64 function_id, u64 arg0, u64 arg1,
	82	u64 arg2)
	83	{
	84	asm volatile(
	85	__asmeq("%0", "x0")
	86	__asmeq("%1", "x1")
	87	__asmeq("%2", "x2")
	88	__asmeq("%3", "x3")
	89	"hvc #0\n"
	90	: "+r" (function_id)
	91	: "r" (arg0), "r" (arg1), "r" (arg2));
	92
	93	return function_id;
	94	}
	95
	96	static noinline int __invoke_psci_fn_smc(u64 function_id, u64 arg0, u64 arg1,
	97	u64 arg2)
	98	{
	99	asm volatile(
	100	__asmeq("%0", "x0")
	101	__asmeq("%1", "x1")
	102	__asmeq("%2", "x2")
	103	__asmeq("%3", "x3")
	104	"smc #0\n"
	105	: "+r" (function_id)
	106	: "r" (arg0), "r" (arg1), "r" (arg2));
	107
	108	return function_id;
	109	}
	110
	111	static int psci_cpu_suspend(struct psci_power_state state,
	112	unsigned long entry_point)
	113	{
	114	int err;
	115	u32 fn, power_state;
	116
	117	fn = psci_function_id[PSCI_FN_CPU_SUSPEND];
	118	power_state = psci_power_state_pack(state);
	119	err = invoke_psci_fn(fn, power_state, entry_point, 0);
	120	return psci_to_linux_errno(err);
	121	}
	122
	123	static int psci_cpu_off(struct psci_power_state state)
	124	{
	125	int err;
	126	u32 fn, power_state;
	127
	128	fn = psci_function_id[PSCI_FN_CPU_OFF];
	129	power_state = psci_power_state_pack(state);
	130	err = invoke_psci_fn(fn, power_state, 0, 0);
	131	return psci_to_linux_errno(err);
	132	}
	133
	134	static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point)
	135	{
	136	int err;
	137	u32 fn;
	138
	139	fn = psci_function_id[PSCI_FN_CPU_ON];
	140	err = invoke_psci_fn(fn, cpuid, entry_point, 0);
	141	return psci_to_linux_errno(err);
	142	}
	143
	144	static int psci_migrate(unsigned long cpuid)
	145	{
	146	int err;
	147	u32 fn;
	148
	149	fn = psci_function_id[PSCI_FN_MIGRATE];
	150	err = invoke_psci_fn(fn, cpuid, 0, 0);
	151	return psci_to_linux_errno(err);
	152	}
	153
	154	static const struct of_device_id psci_of_match[] __initconst = {
	155	{ .compatible = "arm,psci", },
	156	{},
	157	};
	158
	159	int __init psci_init(void)
	160	{
	161	struct device_node *np;
	162	const char *method;
	163	u32 id;
	164	int err = 0;
	165
	166	np = of_find_matching_node(NULL, psci_of_match);
	167	if (!np)
	168	return -ENODEV;
	169
	170	pr_info("probing function IDs from device-tree\n");
	171
	172	if (of_property_read_string(np, "method", &method)) {
	173	pr_warning("missing \"method\" property\n");
	174	err = -ENXIO;
	175	goto out_put_node;
	176	}
	177
	178	if (!strcmp("hvc", method)) {
	179	invoke_psci_fn = __invoke_psci_fn_hvc;
	180	} else if (!strcmp("smc", method)) {
	181	invoke_psci_fn = __invoke_psci_fn_smc;
	182	} else {
	183	pr_warning("invalid \"method\" property: %s\n", method);
	184	err = -EINVAL;
	185	goto out_put_node;
	186	}
	187
	188	if (!of_property_read_u32(np, "cpu_suspend", &id)) {
	189	psci_function_id[PSCI_FN_CPU_SUSPEND] = id;
	190	psci_ops.cpu_suspend = psci_cpu_suspend;
	191	}
	192
	193	if (!of_property_read_u32(np, "cpu_off", &id)) {
	194	psci_function_id[PSCI_FN_CPU_OFF] = id;
	195	psci_ops.cpu_off = psci_cpu_off;
	196	}
	197
	198	if (!of_property_read_u32(np, "cpu_on", &id)) {
	199	psci_function_id[PSCI_FN_CPU_ON] = id;
	200	psci_ops.cpu_on = psci_cpu_on;
	201	}
	202
	203	if (!of_property_read_u32(np, "migrate", &id)) {
	204	psci_function_id[PSCI_FN_MIGRATE] = id;
	205	psci_ops.migrate = psci_migrate;
	206	}
	207
	208	out_put_node:
	209	of_node_put(np);
	210	return err;
	211	}
	212

arch/arm64/kernel/setup.c

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/kernel/setup.c
  *
  * Copyright (C) 1995-2001 Russell King
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/stddef.h>
 #include <linux/ioport.h>
 #include <linux/delay.h>
 #include <linux/utsname.h>
 #include <linux/initrd.h>
 #include <linux/console.h>
 #include <linux/bootmem.h>
 #include <linux/seq_file.h>
 #include <linux/screen_info.h>
 #include <linux/init.h>
 #include <linux/kexec.h>
 #include <linux/crash_dump.h>
 #include <linux/root_dev.h>
 #include <linux/cpu.h>
 #include <linux/interrupt.h>
 #include <linux/smp.h>
 #include <linux/fs.h>
 #include <linux/proc_fs.h>
 #include <linux/memblock.h>
 #include <linux/of_fdt.h>
+#include <linux/of_platform.h>
 #include <asm/cputype.h>
 #include <asm/elf.h>
 #include <asm/cputable.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
 #include <asm/traps.h>
 #include <asm/memblock.h>
+#include <asm/psci.h>
 unsigned int processor_id;
 EXPORT_SYMBOL(processor_id);
 unsigned int elf_hwcap __read_mostly;
 EXPORT_SYMBOL_GPL(elf_hwcap);
 static const char *cpu_name;
 static const char *machine_name;
 phys_addr_t __fdt_pointer __initdata;
 /*
  * Standard memory resources
  */
 static struct resource mem_res[] = {
 	{
 		.name = "Kernel code",
 		.start = 0,
 		.end = 0,
 		.flags = IORESOURCE_MEM
 	},
 	{
 		.name = "Kernel data",
 		.start = 0,
 		.end = 0,
 		.flags = IORESOURCE_MEM
 	}
 };
 #define kernel_code mem_res[0]
 #define kernel_data mem_res[1]
 void __init early_print(const char *str, ...)
 {
 	char buf[256];
 	va_list ap;
 	va_start(ap, str);
 	vsnprintf(buf, sizeof(buf), str, ap);
 	va_end(ap);
 	printk("%s", buf);
 }
 static void __init setup_processor(void)
 {
 	struct cpu_info *cpu_info;
 	/*
 	 * locate processor in the list of supported processor
 	 * types.  The linker builds this table for us from the
 	 * entries in arch/arm/mm/proc.S
 	 */
 	cpu_info = lookup_processor_type(read_cpuid_id());
 	if (!cpu_info) {
 		printk("CPU configuration botched (ID %08x), unable to continue.\n",
 		       read_cpuid_id());
 		while (1);
 	}
 	cpu_name = cpu_info->cpu_name;
 	printk("CPU: %s [%08x] revision %d\n",
 	       cpu_name, read_cpuid_id(), read_cpuid_id() & 15);
 	sprintf(init_utsname()->machine, "aarch64");
 	elf_hwcap = 0;
 }
 static void __init setup_machine_fdt(phys_addr_t dt_phys)
 {
 	struct boot_param_header *devtree;
 	unsigned long dt_root;
 	/* Check we have a non-NULL DT pointer */
 	if (!dt_phys) {
 		early_print("\n"
 			"Error: NULL or invalid device tree blob\n"
 			"The dtb must be 8-byte aligned and passed in the first 512MB of memory\n"
 			"\nPlease check your bootloader.\n");
 		while (true)
 			cpu_relax();
 	}
 	devtree = phys_to_virt(dt_phys);
 	/* Check device tree validity */
 	if (be32_to_cpu(devtree->magic) != OF_DT_HEADER) {
 		early_print("\n"
 			"Error: invalid device tree blob at physical address 0x%p (virtual address 0x%p)\n"
 			"Expected 0x%x, found 0x%x\n"
 			"\nPlease check your bootloader.\n",
 			dt_phys, devtree, OF_DT_HEADER,
 			be32_to_cpu(devtree->magic));
 		while (true)
 			cpu_relax();
 	}
 	initial_boot_params = devtree;
 	dt_root = of_get_flat_dt_root();
 	machine_name = of_get_flat_dt_prop(dt_root, "model", NULL);
 	if (!machine_name)
 		machine_name = of_get_flat_dt_prop(dt_root, "compatible", NULL);
 	if (!machine_name)
 		machine_name = "<unknown>";
 	pr_info("Machine: %s\n", machine_name);
 	/* Retrieve various information from the /chosen node */
 	of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line);
 	/* Initialize {size,address}-cells info */
 	of_scan_flat_dt(early_init_dt_scan_root, NULL);
 	/* Setup memory, calling early_init_dt_add_memory_arch */
 	of_scan_flat_dt(early_init_dt_scan_memory, NULL);
 }
 void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 {
 	base &= PAGE_MASK;
 	size &= PAGE_MASK;
 	if (base + size < PHYS_OFFSET) {
 		pr_warning("Ignoring memory block 0x%llx - 0x%llx\n",
 			   base, base + size);
 		return;
 	}
 	if (base < PHYS_OFFSET) {
 		pr_warning("Ignoring memory range 0x%llx - 0x%llx\n",
 			   base, PHYS_OFFSET);
 		size -= PHYS_OFFSET - base;
 		base = PHYS_OFFSET;
 	}
 	memblock_add(base, size);
 }
 void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
 {
 	return __va(memblock_alloc(size, align));
 }
 /*
  * Limit the memory size that was specified via FDT.
  */
 static int __init early_mem(char *p)
 {
 	phys_addr_t limit;
 	if (!p)
 		return 1;
 	limit = memparse(p, &p) & PAGE_MASK;
 	pr_notice("Memory limited to %lldMB\n", limit >> 20);
 	memblock_enforce_memory_limit(limit);
 	return 0;
 }
 early_param("mem", early_mem);
 static void __init request_standard_resources(void)
 {
 	struct memblock_region *region;
 	struct resource *res;
 	kernel_code.start   = virt_to_phys(_text);
 	kernel_code.end     = virt_to_phys(_etext - 1);
 	kernel_data.start   = virt_to_phys(_sdata);
 	kernel_data.end     = virt_to_phys(_end - 1);
 	for_each_memblock(memory, region) {
 		res = alloc_bootmem_low(sizeof(*res));
 		res->name  = "System RAM";
 		res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region));
 		res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1;
 		res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
 		request_resource(&iomem_resource, res);
 		if (kernel_code.start >= res->start &&
 		    kernel_code.end <= res->end)
 			request_resource(res, &kernel_code);
 		if (kernel_data.start >= res->start &&
 		    kernel_data.end <= res->end)
 			request_resource(res, &kernel_data);
 	}
 }
 void __init setup_arch(char **cmdline_p)
 {
 	setup_processor();
 	setup_machine_fdt(__fdt_pointer);
 	init_mm.start_code = (unsigned long) _text;
 	init_mm.end_code   = (unsigned long) _etext;
 	init_mm.end_data   = (unsigned long) _edata;
 	init_mm.brk	   = (unsigned long) _end;
 	*cmdline_p = boot_command_line;
 	parse_early_param();
 	arm64_memblock_init();
 	paging_init();
 	request_standard_resources();
 	unflatten_device_tree();
+	psci_init();
 #ifdef CONFIG_SMP
 	smp_init_cpus();
 #endif
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)
 	conswitchp = &vga_con;
 #elif defined(CONFIG_DUMMY_CONSOLE)
 	conswitchp = &dummy_con;
 #endif
 #endif
 }
 static DEFINE_PER_CPU(struct cpu, cpu_data);
 static int __init topology_init(void)
 {
 	int i;
 	for_each_possible_cpu(i) {
 		struct cpu *cpu = &per_cpu(cpu_data, i);
 		cpu->hotpluggable = 1;
 		register_cpu(cpu, i);
 	}
 	return 0;
 }
 subsys_initcall(topology_init);
+static int __init arm64_device_probe(void)
+{
+	of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
+	return 0;
+}
+device_initcall(arm64_device_probe);
 static const char *hwcap_str[] = {
 	"fp",
 	"asimd",
 	NULL
 };
 static int c_show(struct seq_file *m, void *v)
 {
 	int i;
 	seq_printf(m, "Processor\t: %s rev %d (%s)\n",
 		   cpu_name, read_cpuid_id() & 15, ELF_PLATFORM);
 	for_each_online_cpu(i) {
 		/*
 		 * glibc reads /proc/cpuinfo to determine the number of
 		 * online processors, looking for lines beginning with
 		 * "processor".  Give glibc what it expects.
 		 */
 #ifdef CONFIG_SMP
 		seq_printf(m, "processor\t: %d\n", i);
 #endif
 		seq_printf(m, "BogoMIPS\t: %lu.%02lu\n\n",
 			   loops_per_jiffy / (500000UL/HZ),
 			   loops_per_jiffy / (5000UL/HZ) % 100);
 	}
 	/* dump out the processor features */
 	seq_puts(m, "Features\t: ");
 	for (i = 0; hwcap_str[i]; i++)
 		if (elf_hwcap & (1 << i))
 			seq_printf(m, "%s ", hwcap_str[i]);
 	seq_printf(m, "\nCPU implementer\t: 0x%02x\n", read_cpuid_id() >> 24);
 	seq_printf(m, "CPU architecture: AArch64\n");
 	seq_printf(m, "CPU variant\t: 0x%x\n", (read_cpuid_id() >> 20) & 15);
 	seq_printf(m, "CPU part\t: 0x%03x\n", (read_cpuid_id() >> 4) & 0xfff);
 	seq_printf(m, "CPU revision\t: %d\n", read_cpuid_id() & 15);
 	seq_puts(m, "\n");
 	seq_printf(m, "Hardware\t: %s\n", machine_name);
 	return 0;
 }
 static void *c_start(struct seq_file *m, loff_t *pos)
 {
 	return *pos < 1 ? (void *)1 : NULL;
 }
 static void *c_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	++*pos;
 	return NULL;
 }
 static void c_stop(struct seq_file *m, void *v)
 {
 }
 const struct seq_operations cpuinfo_op = {
 	.start	= c_start,
 	.next	= c_next,
 	.stop	= c_stop,
 	.show	= c_show
 };

arch/arm64/kernel/signal32.c

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* Based on arch/arm/kernel/signal.c	2	* Based on arch/arm/kernel/signal.c
3	*	3	*
4	* Copyright (C) 1995-2009 Russell King	4	* Copyright (C) 1995-2009 Russell King
5	* Copyright (C) 2012 ARM Ltd.	5	* Copyright (C) 2012 ARM Ltd.
6	* Modified by Will Deacon <will.deacon@arm.com>	6	* Modified by Will Deacon <will.deacon@arm.com>
7	*	7	*
8	* This program is free software; you can redistribute it and/or modify	8	* This program is free software; you can redistribute it and/or modify
9	* it under the terms of the GNU General Public License version 2 as	9	* it under the terms of the GNU General Public License version 2 as
10	* published by the Free Software Foundation.	10	* published by the Free Software Foundation.
11	*	11	*
12	* This program is distributed in the hope that it will be useful,	12	* This program is distributed in the hope that it will be useful,
13	* but WITHOUT ANY WARRANTY; without even the implied warranty of	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15	* GNU General Public License for more details.	15	* GNU General Public License for more details.
16	*	16	*
17	* You should have received a copy of the GNU General Public License	17	* You should have received a copy of the GNU General Public License
18	* along with this program. If not, see <http://www.gnu.org/licenses/>.	18	* along with this program. If not, see <http://www.gnu.org/licenses/>.
19	*/	19	*/
20		20
21	#include <linux/compat.h>	21	#include <linux/compat.h>
22	#include <linux/signal.h>	22	#include <linux/signal.h>
23	#include <linux/syscalls.h>	23	#include <linux/syscalls.h>
24	#include <linux/ratelimit.h>	24	#include <linux/ratelimit.h>
25		25
26	#include <asm/fpsimd.h>	26	#include <asm/fpsimd.h>
27	#include <asm/signal32.h>	27	#include <asm/signal32.h>
28	#include <asm/uaccess.h>	28	#include <asm/uaccess.h>
29	#include <asm/unistd32.h>	29	#include <asm/unistd32.h>
30		30
31	struct compat_sigaction {	31	struct compat_sigaction {
32	compat_uptr_t sa_handler;	32	compat_uptr_t sa_handler;
33	compat_ulong_t sa_flags;	33	compat_ulong_t sa_flags;
34	compat_uptr_t sa_restorer;	34	compat_uptr_t sa_restorer;
35	compat_sigset_t sa_mask;	35	compat_sigset_t sa_mask;
36	};	36	};
37		37
38	struct compat_old_sigaction {	38	struct compat_old_sigaction {
39	compat_uptr_t sa_handler;	39	compat_uptr_t sa_handler;
40	compat_old_sigset_t sa_mask;	40	compat_old_sigset_t sa_mask;
41	compat_ulong_t sa_flags;	41	compat_ulong_t sa_flags;
42	compat_uptr_t sa_restorer;	42	compat_uptr_t sa_restorer;
43	};	43	};
44		44
45	typedef struct compat_sigaltstack {	45	typedef struct compat_sigaltstack {
46	compat_uptr_t ss_sp;	46	compat_uptr_t ss_sp;
47	int ss_flags;	47	int ss_flags;
48	compat_size_t ss_size;	48	compat_size_t ss_size;
49	} compat_stack_t;	49	} compat_stack_t;
50		50
51	struct compat_sigcontext {	51	struct compat_sigcontext {
52	/* We always set these two fields to 0 */	52	/* We always set these two fields to 0 */
53	compat_ulong_t trap_no;	53	compat_ulong_t trap_no;
54	compat_ulong_t error_code;	54	compat_ulong_t error_code;
55		55
56	compat_ulong_t oldmask;	56	compat_ulong_t oldmask;
57	compat_ulong_t arm_r0;	57	compat_ulong_t arm_r0;
58	compat_ulong_t arm_r1;	58	compat_ulong_t arm_r1;
59	compat_ulong_t arm_r2;	59	compat_ulong_t arm_r2;
60	compat_ulong_t arm_r3;	60	compat_ulong_t arm_r3;
61	compat_ulong_t arm_r4;	61	compat_ulong_t arm_r4;
62	compat_ulong_t arm_r5;	62	compat_ulong_t arm_r5;
63	compat_ulong_t arm_r6;	63	compat_ulong_t arm_r6;
64	compat_ulong_t arm_r7;	64	compat_ulong_t arm_r7;
65	compat_ulong_t arm_r8;	65	compat_ulong_t arm_r8;
66	compat_ulong_t arm_r9;	66	compat_ulong_t arm_r9;
67	compat_ulong_t arm_r10;	67	compat_ulong_t arm_r10;
68	compat_ulong_t arm_fp;	68	compat_ulong_t arm_fp;
69	compat_ulong_t arm_ip;	69	compat_ulong_t arm_ip;
70	compat_ulong_t arm_sp;	70	compat_ulong_t arm_sp;
71	compat_ulong_t arm_lr;	71	compat_ulong_t arm_lr;
72	compat_ulong_t arm_pc;	72	compat_ulong_t arm_pc;
73	compat_ulong_t arm_cpsr;	73	compat_ulong_t arm_cpsr;
74	compat_ulong_t fault_address;	74	compat_ulong_t fault_address;
75	};	75	};
76		76
77	struct compat_ucontext {	77	struct compat_ucontext {
78	compat_ulong_t uc_flags;	78	compat_ulong_t uc_flags;
79	struct compat_ucontext *uc_link;	79	compat_uptr_t uc_link;
80	compat_stack_t uc_stack;	80	compat_stack_t uc_stack;
81	struct compat_sigcontext uc_mcontext;	81	struct compat_sigcontext uc_mcontext;
82	compat_sigset_t uc_sigmask;	82	compat_sigset_t uc_sigmask;
83	int __unused[32 - (sizeof (compat_sigset_t) / sizeof (int))];	83	int __unused[32 - (sizeof (compat_sigset_t) / sizeof (int))];
84	compat_ulong_t uc_regspace[128] __attribute__((__aligned__(8)));	84	compat_ulong_t uc_regspace[128] __attribute__((__aligned__(8)));
85	};	85	};
86		86
87	struct compat_vfp_sigframe {	87	struct compat_vfp_sigframe {
88	compat_ulong_t magic;	88	compat_ulong_t magic;
89	compat_ulong_t size;	89	compat_ulong_t size;
90	struct compat_user_vfp {	90	struct compat_user_vfp {
91	compat_u64 fpregs[32];	91	compat_u64 fpregs[32];
92	compat_ulong_t fpscr;	92	compat_ulong_t fpscr;
93	} ufp;	93	} ufp;
94	struct compat_user_vfp_exc {	94	struct compat_user_vfp_exc {
95	compat_ulong_t fpexc;	95	compat_ulong_t fpexc;
96	compat_ulong_t fpinst;	96	compat_ulong_t fpinst;
97	compat_ulong_t fpinst2;	97	compat_ulong_t fpinst2;
98	} ufp_exc;	98	} ufp_exc;
99	} __attribute__((__aligned__(8)));	99	} __attribute__((__aligned__(8)));
100		100
101	#define VFP_MAGIC 0x56465001	101	#define VFP_MAGIC 0x56465001
102	#define VFP_STORAGE_SIZE sizeof(struct compat_vfp_sigframe)	102	#define VFP_STORAGE_SIZE sizeof(struct compat_vfp_sigframe)
103		103
104	struct compat_aux_sigframe {	104	struct compat_aux_sigframe {
105	struct compat_vfp_sigframe vfp;	105	struct compat_vfp_sigframe vfp;
106		106
107	/* Something that isn't a valid magic number for any coprocessor. */	107	/* Something that isn't a valid magic number for any coprocessor. */
108	unsigned long end_magic;	108	unsigned long end_magic;
109	} __attribute__((__aligned__(8)));	109	} __attribute__((__aligned__(8)));
110		110
111	struct compat_sigframe {	111	struct compat_sigframe {
112	struct compat_ucontext uc;	112	struct compat_ucontext uc;
113	compat_ulong_t retcode[2];	113	compat_ulong_t retcode[2];
114	};	114	};
115		115
116	struct compat_rt_sigframe {	116	struct compat_rt_sigframe {
117	struct compat_siginfo info;	117	struct compat_siginfo info;
118	struct compat_sigframe sig;	118	struct compat_sigframe sig;
119	};	119	};
120		120
121	#define _BLOCKABLE (~(sigmask(SIGKILL) \| sigmask(SIGSTOP)))	121	#define _BLOCKABLE (~(sigmask(SIGKILL) \| sigmask(SIGSTOP)))
122		122
123	/*	123	/*
124	* For ARM syscalls, the syscall number has to be loaded into r7.	124	* For ARM syscalls, the syscall number has to be loaded into r7.
125	* We do not support an OABI userspace.	125	* We do not support an OABI userspace.
126	*/	126	*/
127	#define MOV_R7_NR_SIGRETURN (0xe3a07000 \| __NR_compat_sigreturn)	127	#define MOV_R7_NR_SIGRETURN (0xe3a07000 \| __NR_compat_sigreturn)
128	#define SVC_SYS_SIGRETURN (0xef000000 \| __NR_compat_sigreturn)	128	#define SVC_SYS_SIGRETURN (0xef000000 \| __NR_compat_sigreturn)
129	#define MOV_R7_NR_RT_SIGRETURN (0xe3a07000 \| __NR_compat_rt_sigreturn)	129	#define MOV_R7_NR_RT_SIGRETURN (0xe3a07000 \| __NR_compat_rt_sigreturn)
130	#define SVC_SYS_RT_SIGRETURN (0xef000000 \| __NR_compat_rt_sigreturn)	130	#define SVC_SYS_RT_SIGRETURN (0xef000000 \| __NR_compat_rt_sigreturn)
131		131
132	/*	132	/*
133	* For Thumb syscalls, we also pass the syscall number via r7. We therefore	133	* For Thumb syscalls, we also pass the syscall number via r7. We therefore
134	* need two 16-bit instructions.	134	* need two 16-bit instructions.
135	*/	135	*/
136	#define SVC_THUMB_SIGRETURN (((0xdf00 \| __NR_compat_sigreturn) << 16) \| \	136	#define SVC_THUMB_SIGRETURN (((0xdf00 \| __NR_compat_sigreturn) << 16) \| \
137	0x2700 \| __NR_compat_sigreturn)	137	0x2700 \| __NR_compat_sigreturn)
138	#define SVC_THUMB_RT_SIGRETURN (((0xdf00 \| __NR_compat_rt_sigreturn) << 16) \| \	138	#define SVC_THUMB_RT_SIGRETURN (((0xdf00 \| __NR_compat_rt_sigreturn) << 16) \| \
139	0x2700 \| __NR_compat_rt_sigreturn)	139	0x2700 \| __NR_compat_rt_sigreturn)
140		140
141	const compat_ulong_t aarch32_sigret_code[6] = {	141	const compat_ulong_t aarch32_sigret_code[6] = {
142	/*	142	/*
143	* AArch32 sigreturn code.	143	* AArch32 sigreturn code.
144	* We don't construct an OABI SWI - instead we just set the imm24 field	144	* We don't construct an OABI SWI - instead we just set the imm24 field
145	* to the EABI syscall number so that we create a sane disassembly.	145	* to the EABI syscall number so that we create a sane disassembly.
146	*/	146	*/
147	MOV_R7_NR_SIGRETURN, SVC_SYS_SIGRETURN, SVC_THUMB_SIGRETURN,	147	MOV_R7_NR_SIGRETURN, SVC_SYS_SIGRETURN, SVC_THUMB_SIGRETURN,
148	MOV_R7_NR_RT_SIGRETURN, SVC_SYS_RT_SIGRETURN, SVC_THUMB_RT_SIGRETURN,	148	MOV_R7_NR_RT_SIGRETURN, SVC_SYS_RT_SIGRETURN, SVC_THUMB_RT_SIGRETURN,
149	};	149	};
150		150
151	static inline int put_sigset_t(compat_sigset_t __user uset, sigset_t set)	151	static inline int put_sigset_t(compat_sigset_t __user uset, sigset_t set)
152	{	152	{
153	compat_sigset_t cset;	153	compat_sigset_t cset;
154		154
155	cset.sig[0] = set->sig[0] & 0xffffffffull;	155	cset.sig[0] = set->sig[0] & 0xffffffffull;
156	cset.sig[1] = set->sig[0] >> 32;	156	cset.sig[1] = set->sig[0] >> 32;
157		157
158	return copy_to_user(uset, &cset, sizeof(*uset));	158	return copy_to_user(uset, &cset, sizeof(*uset));
159	}	159	}
160		160
161	static inline int get_sigset_t(sigset_t *set,	161	static inline int get_sigset_t(sigset_t *set,
162	const compat_sigset_t __user *uset)	162	const compat_sigset_t __user *uset)
163	{	163	{
164	compat_sigset_t s32;	164	compat_sigset_t s32;
165		165
166	if (copy_from_user(&s32, uset, sizeof(*uset)))	166	if (copy_from_user(&s32, uset, sizeof(*uset)))
167	return -EFAULT;	167	return -EFAULT;
168		168
169	set->sig[0] = s32.sig[0] \| (((long)s32.sig[1]) << 32);	169	set->sig[0] = s32.sig[0] \| (((long)s32.sig[1]) << 32);
170	return 0;	170	return 0;
171	}	171	}
172		172
173	int copy_siginfo_to_user32(compat_siginfo_t __user to, siginfo_t from)	173	int copy_siginfo_to_user32(compat_siginfo_t __user to, siginfo_t from)
174	{	174	{
175	int err;	175	int err;
176		176
177	if (!access_ok(VERIFY_WRITE, to, sizeof(*to)))	177	if (!access_ok(VERIFY_WRITE, to, sizeof(*to)))
178	return -EFAULT;	178	return -EFAULT;
179		179
180	/* If you change siginfo_t structure, please be sure	180	/* If you change siginfo_t structure, please be sure
181	* this code is fixed accordingly.	181	* this code is fixed accordingly.
182	* It should never copy any pad contained in the structure	182	* It should never copy any pad contained in the structure
183	* to avoid security leaks, but must copy the generic	183	* to avoid security leaks, but must copy the generic
184	* 3 ints plus the relevant union member.	184	* 3 ints plus the relevant union member.
185	* This routine must convert siginfo from 64bit to 32bit as well	185	* This routine must convert siginfo from 64bit to 32bit as well
186	* at the same time.	186	* at the same time.
187	*/	187	*/
188	err = __put_user(from->si_signo, &to->si_signo);	188	err = __put_user(from->si_signo, &to->si_signo);
189	err \|= __put_user(from->si_errno, &to->si_errno);	189	err \|= __put_user(from->si_errno, &to->si_errno);
190	err \|= __put_user((short)from->si_code, &to->si_code);	190	err \|= __put_user((short)from->si_code, &to->si_code);
191	if (from->si_code < 0)	191	if (from->si_code < 0)
192	err \|= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad,	192	err \|= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad,
193	SI_PAD_SIZE);	193	SI_PAD_SIZE);
194	else switch (from->si_code & __SI_MASK) {	194	else switch (from->si_code & __SI_MASK) {
195	case __SI_KILL:	195	case __SI_KILL:
196	err \|= __put_user(from->si_pid, &to->si_pid);	196	err \|= __put_user(from->si_pid, &to->si_pid);
197	err \|= __put_user(from->si_uid, &to->si_uid);	197	err \|= __put_user(from->si_uid, &to->si_uid);
198	break;	198	break;
199	case __SI_TIMER:	199	case __SI_TIMER:
200	err \|= __put_user(from->si_tid, &to->si_tid);	200	err \|= __put_user(from->si_tid, &to->si_tid);
201	err \|= __put_user(from->si_overrun, &to->si_overrun);	201	err \|= __put_user(from->si_overrun, &to->si_overrun);
202	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_ptr,	202	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_ptr,
203	&to->si_ptr);	203	&to->si_ptr);
204	break;	204	break;
205	case __SI_POLL:	205	case __SI_POLL:
206	err \|= __put_user(from->si_band, &to->si_band);	206	err \|= __put_user(from->si_band, &to->si_band);
207	err \|= __put_user(from->si_fd, &to->si_fd);	207	err \|= __put_user(from->si_fd, &to->si_fd);
208	break;	208	break;
209	case __SI_FAULT:	209	case __SI_FAULT:
210	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_addr,	210	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_addr,
211	&to->si_addr);	211	&to->si_addr);
212	#ifdef BUS_MCEERR_AO	212	#ifdef BUS_MCEERR_AO
213	/*	213	/*
214	* Other callers might not initialize the si_lsb field,	214	* Other callers might not initialize the si_lsb field,
215	* so check explicitely for the right codes here.	215	* so check explicitely for the right codes here.
216	*/	216	*/
217	if (from->si_code == BUS_MCEERR_AR \|\| from->si_code == BUS_MCEERR_AO)	217	if (from->si_code == BUS_MCEERR_AR \|\| from->si_code == BUS_MCEERR_AO)
218	err \|= __put_user(from->si_addr_lsb, &to->si_addr_lsb);	218	err \|= __put_user(from->si_addr_lsb, &to->si_addr_lsb);
219	#endif	219	#endif
220	break;	220	break;
221	case __SI_CHLD:	221	case __SI_CHLD:
222	err \|= __put_user(from->si_pid, &to->si_pid);	222	err \|= __put_user(from->si_pid, &to->si_pid);
223	err \|= __put_user(from->si_uid, &to->si_uid);	223	err \|= __put_user(from->si_uid, &to->si_uid);
224	err \|= __put_user(from->si_status, &to->si_status);	224	err \|= __put_user(from->si_status, &to->si_status);
225	err \|= __put_user(from->si_utime, &to->si_utime);	225	err \|= __put_user(from->si_utime, &to->si_utime);
226	err \|= __put_user(from->si_stime, &to->si_stime);	226	err \|= __put_user(from->si_stime, &to->si_stime);
227	break;	227	break;
228	case __SI_RT: /* This is not generated by the kernel as of now. */	228	case __SI_RT: /* This is not generated by the kernel as of now. */
229	case __SI_MESGQ: /* But this is */	229	case __SI_MESGQ: /* But this is */
230	err \|= __put_user(from->si_pid, &to->si_pid);	230	err \|= __put_user(from->si_pid, &to->si_pid);
231	err \|= __put_user(from->si_uid, &to->si_uid);	231	err \|= __put_user(from->si_uid, &to->si_uid);
232	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_ptr, &to->si_ptr);	232	err \|= __put_user((compat_uptr_t)(unsigned long)from->si_ptr, &to->si_ptr);
233	break;	233	break;
234	default: /* this is just in case for now ... */	234	default: /* this is just in case for now ... */
235	err \|= __put_user(from->si_pid, &to->si_pid);	235	err \|= __put_user(from->si_pid, &to->si_pid);
236	err \|= __put_user(from->si_uid, &to->si_uid);	236	err \|= __put_user(from->si_uid, &to->si_uid);
237	break;	237	break;
238	}	238	}
239	return err;	239	return err;
240	}	240	}
241		241
242	int copy_siginfo_from_user32(siginfo_t to, compat_siginfo_t __user from)	242	int copy_siginfo_from_user32(siginfo_t to, compat_siginfo_t __user from)
243	{	243	{
244	memset(to, 0, sizeof *to);	244	memset(to, 0, sizeof *to);
245		245
246	if (copy_from_user(to, from, __ARCH_SI_PREAMBLE_SIZE) \|\|	246	if (copy_from_user(to, from, __ARCH_SI_PREAMBLE_SIZE) \|\|
247	copy_from_user(to->_sifields._pad,	247	copy_from_user(to->_sifields._pad,
248	from->_sifields._pad, SI_PAD_SIZE))	248	from->_sifields._pad, SI_PAD_SIZE))
249	return -EFAULT;	249	return -EFAULT;
250		250
251	return 0;	251	return 0;
252	}	252	}
253		253
254	/*	254	/*
255	* VFP save/restore code.	255	* VFP save/restore code.
256	*/	256	*/
257	static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)	257	static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
258	{	258	{
259	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;	259	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;
260	compat_ulong_t magic = VFP_MAGIC;	260	compat_ulong_t magic = VFP_MAGIC;
261	compat_ulong_t size = VFP_STORAGE_SIZE;	261	compat_ulong_t size = VFP_STORAGE_SIZE;
262	compat_ulong_t fpscr, fpexc;	262	compat_ulong_t fpscr, fpexc;
263	int err = 0;	263	int err = 0;
264		264
265	/*	265	/*
266	* Save the hardware registers to the fpsimd_state structure.	266	* Save the hardware registers to the fpsimd_state structure.
267	* Note that this also saves V16-31, which aren't visible	267	* Note that this also saves V16-31, which aren't visible
268	* in AArch32.	268	* in AArch32.
269	*/	269	*/
270	fpsimd_save_state(fpsimd);	270	fpsimd_save_state(fpsimd);
271		271
272	/* Place structure header on the stack */	272	/* Place structure header on the stack */
273	__put_user_error(magic, &frame->magic, err);	273	__put_user_error(magic, &frame->magic, err);
274	__put_user_error(size, &frame->size, err);	274	__put_user_error(size, &frame->size, err);
275		275
276	/*	276	/*
277	* Now copy the FP registers. Since the registers are packed,	277	* Now copy the FP registers. Since the registers are packed,
278	* we can copy the prefix we want (V0-V15) as it is.	278	* we can copy the prefix we want (V0-V15) as it is.
279	* FIXME: Won't work if big endian.	279	* FIXME: Won't work if big endian.
280	*/	280	*/
281	err \|= __copy_to_user(&frame->ufp.fpregs, fpsimd->vregs,	281	err \|= __copy_to_user(&frame->ufp.fpregs, fpsimd->vregs,
282	sizeof(frame->ufp.fpregs));	282	sizeof(frame->ufp.fpregs));
283		283
284	/* Create an AArch32 fpscr from the fpsr and the fpcr. */	284	/* Create an AArch32 fpscr from the fpsr and the fpcr. */
285	fpscr = (fpsimd->fpsr & VFP_FPSCR_STAT_MASK) \|	285	fpscr = (fpsimd->fpsr & VFP_FPSCR_STAT_MASK) \|
286	(fpsimd->fpcr & VFP_FPSCR_CTRL_MASK);	286	(fpsimd->fpcr & VFP_FPSCR_CTRL_MASK);
287	__put_user_error(fpscr, &frame->ufp.fpscr, err);	287	__put_user_error(fpscr, &frame->ufp.fpscr, err);
288		288
289	/*	289	/*
290	* The exception register aren't available so we fake up a	290	* The exception register aren't available so we fake up a
291	* basic FPEXC and zero everything else.	291	* basic FPEXC and zero everything else.
292	*/	292	*/
293	fpexc = (1 << 30);	293	fpexc = (1 << 30);
294	__put_user_error(fpexc, &frame->ufp_exc.fpexc, err);	294	__put_user_error(fpexc, &frame->ufp_exc.fpexc, err);
295	__put_user_error(0, &frame->ufp_exc.fpinst, err);	295	__put_user_error(0, &frame->ufp_exc.fpinst, err);
296	__put_user_error(0, &frame->ufp_exc.fpinst2, err);	296	__put_user_error(0, &frame->ufp_exc.fpinst2, err);
297		297
298	return err ? -EFAULT : 0;	298	return err ? -EFAULT : 0;
299	}	299	}
300		300
301	static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame)	301	static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame)
302	{	302	{
303	struct fpsimd_state fpsimd;	303	struct fpsimd_state fpsimd;
304	compat_ulong_t magic = VFP_MAGIC;	304	compat_ulong_t magic = VFP_MAGIC;
305	compat_ulong_t size = VFP_STORAGE_SIZE;	305	compat_ulong_t size = VFP_STORAGE_SIZE;
306	compat_ulong_t fpscr;	306	compat_ulong_t fpscr;
307	int err = 0;	307	int err = 0;
308		308
309	__get_user_error(magic, &frame->magic, err);	309	__get_user_error(magic, &frame->magic, err);
310	__get_user_error(size, &frame->size, err);	310	__get_user_error(size, &frame->size, err);
311		311
312	if (err)	312	if (err)
313	return -EFAULT;	313	return -EFAULT;
314	if (magic != VFP_MAGIC \|\| size != VFP_STORAGE_SIZE)	314	if (magic != VFP_MAGIC \|\| size != VFP_STORAGE_SIZE)
315	return -EINVAL;	315	return -EINVAL;
316		316
317	/*	317	/*
318	* Copy the FP registers into the start of the fpsimd_state.	318	* Copy the FP registers into the start of the fpsimd_state.
319	* FIXME: Won't work if big endian.	319	* FIXME: Won't work if big endian.
320	*/	320	*/
321	err \|= __copy_from_user(fpsimd.vregs, frame->ufp.fpregs,	321	err \|= __copy_from_user(fpsimd.vregs, frame->ufp.fpregs,
322	sizeof(frame->ufp.fpregs));	322	sizeof(frame->ufp.fpregs));
323		323
324	/* Extract the fpsr and the fpcr from the fpscr */	324	/* Extract the fpsr and the fpcr from the fpscr */
325	__get_user_error(fpscr, &frame->ufp.fpscr, err);	325	__get_user_error(fpscr, &frame->ufp.fpscr, err);
326	fpsimd.fpsr = fpscr & VFP_FPSCR_STAT_MASK;	326	fpsimd.fpsr = fpscr & VFP_FPSCR_STAT_MASK;
327	fpsimd.fpcr = fpscr & VFP_FPSCR_CTRL_MASK;	327	fpsimd.fpcr = fpscr & VFP_FPSCR_CTRL_MASK;
328		328
329	/*	329	/*
330	* We don't need to touch the exception register, so	330	* We don't need to touch the exception register, so
331	* reload the hardware state.	331	* reload the hardware state.
332	*/	332	*/
333	if (!err) {	333	if (!err) {
334	preempt_disable();	334	preempt_disable();
335	fpsimd_load_state(&fpsimd);	335	fpsimd_load_state(&fpsimd);
336	preempt_enable();	336	preempt_enable();
337	}	337	}
338		338
339	return err ? -EFAULT : 0;	339	return err ? -EFAULT : 0;
340	}	340	}
341		341
342	/*	342	/*
343	* atomically swap in the new signal mask, and wait for a signal.	343	* atomically swap in the new signal mask, and wait for a signal.
344	*/	344	*/
345	asmlinkage int compat_sys_sigsuspend(int restart, compat_ulong_t oldmask,	345	asmlinkage int compat_sys_sigsuspend(int restart, compat_ulong_t oldmask,
346	compat_old_sigset_t mask)	346	compat_old_sigset_t mask)
347	{	347	{
348	sigset_t blocked;	348	sigset_t blocked;
349		349
350	siginitset(&current->blocked, mask);	350	siginitset(&current->blocked, mask);
351	return sigsuspend(&blocked);	351	return sigsuspend(&blocked);
352	}	352	}
353		353
354	asmlinkage int compat_sys_sigaction(int sig,	354	asmlinkage int compat_sys_sigaction(int sig,
355	const struct compat_old_sigaction __user *act,	355	const struct compat_old_sigaction __user *act,
356	struct compat_old_sigaction __user *oact)	356	struct compat_old_sigaction __user *oact)
357	{	357	{
358	struct k_sigaction new_ka, old_ka;	358	struct k_sigaction new_ka, old_ka;
359	int ret;	359	int ret;
360	compat_old_sigset_t mask;	360	compat_old_sigset_t mask;
361	compat_uptr_t handler, restorer;	361	compat_uptr_t handler, restorer;
362		362
363	if (act) {	363	if (act) {
364	if (!access_ok(VERIFY_READ, act, sizeof(*act)) \|\|	364	if (!access_ok(VERIFY_READ, act, sizeof(*act)) \|\|
365	__get_user(handler, &act->sa_handler) \|\|	365	__get_user(handler, &act->sa_handler) \|\|
366	__get_user(restorer, &act->sa_restorer) \|\|	366	__get_user(restorer, &act->sa_restorer) \|\|
367	__get_user(new_ka.sa.sa_flags, &act->sa_flags) \|\|	367	__get_user(new_ka.sa.sa_flags, &act->sa_flags) \|\|
368	__get_user(mask, &act->sa_mask))	368	__get_user(mask, &act->sa_mask))
369	return -EFAULT;	369	return -EFAULT;
370		370
371	new_ka.sa.sa_handler = compat_ptr(handler);	371	new_ka.sa.sa_handler = compat_ptr(handler);
372	new_ka.sa.sa_restorer = compat_ptr(restorer);	372	new_ka.sa.sa_restorer = compat_ptr(restorer);
373	siginitset(&new_ka.sa.sa_mask, mask);	373	siginitset(&new_ka.sa.sa_mask, mask);
374	}	374	}
375		375
376	ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);	376	ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
377		377
378	if (!ret && oact) {	378	if (!ret && oact) {
379	if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) \|\|	379	if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) \|\|
380	__put_user(ptr_to_compat(old_ka.sa.sa_handler),	380	__put_user(ptr_to_compat(old_ka.sa.sa_handler),
381	&oact->sa_handler) \|\|	381	&oact->sa_handler) \|\|
382	__put_user(ptr_to_compat(old_ka.sa.sa_restorer),	382	__put_user(ptr_to_compat(old_ka.sa.sa_restorer),
383	&oact->sa_restorer) \|\|	383	&oact->sa_restorer) \|\|
384	__put_user(old_ka.sa.sa_flags, &oact->sa_flags) \|\|	384	__put_user(old_ka.sa.sa_flags, &oact->sa_flags) \|\|
385	__put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))	385	__put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
386	return -EFAULT;	386	return -EFAULT;
387	}	387	}
388		388
389	return ret;	389	return ret;
390	}	390	}
391		391
392	asmlinkage int compat_sys_rt_sigaction(int sig,	392	asmlinkage int compat_sys_rt_sigaction(int sig,
393	const struct compat_sigaction __user *act,	393	const struct compat_sigaction __user *act,
394	struct compat_sigaction __user *oact,	394	struct compat_sigaction __user *oact,
395	compat_size_t sigsetsize)	395	compat_size_t sigsetsize)
396	{	396	{
397	struct k_sigaction new_ka, old_ka;	397	struct k_sigaction new_ka, old_ka;
398	int ret;	398	int ret;
399		399
400	/* XXX: Don't preclude handling different sized sigset_t's. */	400	/* XXX: Don't preclude handling different sized sigset_t's. */
401	if (sigsetsize != sizeof(compat_sigset_t))	401	if (sigsetsize != sizeof(compat_sigset_t))
402	return -EINVAL;	402	return -EINVAL;
403		403
404	if (act) {	404	if (act) {
405	compat_uptr_t handler, restorer;	405	compat_uptr_t handler, restorer;
406		406
407	ret = get_user(handler, &act->sa_handler);	407	ret = get_user(handler, &act->sa_handler);
408	new_ka.sa.sa_handler = compat_ptr(handler);	408	new_ka.sa.sa_handler = compat_ptr(handler);
409	ret \|= get_user(restorer, &act->sa_restorer);	409	ret \|= get_user(restorer, &act->sa_restorer);
410	new_ka.sa.sa_restorer = compat_ptr(restorer);	410	new_ka.sa.sa_restorer = compat_ptr(restorer);
411	ret \|= get_sigset_t(&new_ka.sa.sa_mask, &act->sa_mask);	411	ret \|= get_sigset_t(&new_ka.sa.sa_mask, &act->sa_mask);
412	ret \|= __get_user(new_ka.sa.sa_flags, &act->sa_flags);	412	ret \|= __get_user(new_ka.sa.sa_flags, &act->sa_flags);
413	if (ret)	413	if (ret)
414	return -EFAULT;	414	return -EFAULT;
415	}	415	}
416		416
417	ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);	417	ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
418	if (!ret && oact) {	418	if (!ret && oact) {
419	ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler);	419	ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler);
420	ret \|= put_sigset_t(&oact->sa_mask, &old_ka.sa.sa_mask);	420	ret \|= put_sigset_t(&oact->sa_mask, &old_ka.sa.sa_mask);
421	ret \|= __put_user(old_ka.sa.sa_flags, &oact->sa_flags);	421	ret \|= __put_user(old_ka.sa.sa_flags, &oact->sa_flags);
422	}	422	}
423	return ret;	423	return ret;
424	}	424	}
425		425
426	int compat_do_sigaltstack(compat_uptr_t compat_uss, compat_uptr_t compat_uoss,	426	int compat_do_sigaltstack(compat_uptr_t compat_uss, compat_uptr_t compat_uoss,
427	compat_ulong_t sp)	427	compat_ulong_t sp)
428	{	428	{
429	compat_stack_t __user *newstack = compat_ptr(compat_uss);	429	compat_stack_t __user *newstack = compat_ptr(compat_uss);
430	compat_stack_t __user *oldstack = compat_ptr(compat_uoss);	430	compat_stack_t __user *oldstack = compat_ptr(compat_uoss);
431	compat_uptr_t ss_sp;	431	compat_uptr_t ss_sp;
432	int ret;	432	int ret;
433	mm_segment_t old_fs;	433	mm_segment_t old_fs;
434	stack_t uss, uoss;	434	stack_t uss, uoss;
435		435
436	/* Marshall the compat new stack into a stack_t */	436	/* Marshall the compat new stack into a stack_t */
437	if (newstack) {	437	if (newstack) {
438	if (get_user(ss_sp, &newstack->ss_sp) \|\|	438	if (get_user(ss_sp, &newstack->ss_sp) \|\|
439	__get_user(uss.ss_flags, &newstack->ss_flags) \|\|	439	__get_user(uss.ss_flags, &newstack->ss_flags) \|\|
440	__get_user(uss.ss_size, &newstack->ss_size))	440	__get_user(uss.ss_size, &newstack->ss_size))
441	return -EFAULT;	441	return -EFAULT;
442	uss.ss_sp = compat_ptr(ss_sp);	442	uss.ss_sp = compat_ptr(ss_sp);
443	}	443	}
444		444
445	old_fs = get_fs();	445	old_fs = get_fs();
446	set_fs(KERNEL_DS);	446	set_fs(KERNEL_DS);
447	/* The __user pointer casts are valid because of the set_fs() */	447	/* The __user pointer casts are valid because of the set_fs() */
448	ret = do_sigaltstack(	448	ret = do_sigaltstack(
449	newstack ? (stack_t __user *) &uss : NULL,	449	newstack ? (stack_t __user *) &uss : NULL,
450	oldstack ? (stack_t __user *) &uoss : NULL,	450	oldstack ? (stack_t __user *) &uoss : NULL,
451	(unsigned long)sp);	451	(unsigned long)sp);
452	set_fs(old_fs);	452	set_fs(old_fs);
453		453
454	/* Convert the old stack_t into a compat stack. */	454	/* Convert the old stack_t into a compat stack. */
455	if (!ret && oldstack &&	455	if (!ret && oldstack &&
456	(put_user(ptr_to_compat(uoss.ss_sp), &oldstack->ss_sp) \|\|	456	(put_user(ptr_to_compat(uoss.ss_sp), &oldstack->ss_sp) \|\|
457	__put_user(uoss.ss_flags, &oldstack->ss_flags) \|\|	457	__put_user(uoss.ss_flags, &oldstack->ss_flags) \|\|
458	__put_user(uoss.ss_size, &oldstack->ss_size)))	458	__put_user(uoss.ss_size, &oldstack->ss_size)))
459	return -EFAULT;	459	return -EFAULT;
460	return ret;	460	return ret;
461	}	461	}
462		462
463	static int compat_restore_sigframe(struct pt_regs *regs,	463	static int compat_restore_sigframe(struct pt_regs *regs,
464	struct compat_sigframe __user *sf)	464	struct compat_sigframe __user *sf)
465	{	465	{
466	int err;	466	int err;
467	sigset_t set;	467	sigset_t set;
468	struct compat_aux_sigframe __user *aux;	468	struct compat_aux_sigframe __user *aux;
469		469
470	err = get_sigset_t(&set, &sf->uc.uc_sigmask);	470	err = get_sigset_t(&set, &sf->uc.uc_sigmask);
471	if (err == 0) {	471	if (err == 0) {
472	sigdelsetmask(&set, ~_BLOCKABLE);	472	sigdelsetmask(&set, ~_BLOCKABLE);
473	set_current_blocked(&set);	473	set_current_blocked(&set);
474	}	474	}
475		475
476	__get_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err);	476	__get_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err);
477	__get_user_error(regs->regs[1], &sf->uc.uc_mcontext.arm_r1, err);	477	__get_user_error(regs->regs[1], &sf->uc.uc_mcontext.arm_r1, err);
478	__get_user_error(regs->regs[2], &sf->uc.uc_mcontext.arm_r2, err);	478	__get_user_error(regs->regs[2], &sf->uc.uc_mcontext.arm_r2, err);
479	__get_user_error(regs->regs[3], &sf->uc.uc_mcontext.arm_r3, err);	479	__get_user_error(regs->regs[3], &sf->uc.uc_mcontext.arm_r3, err);
480	__get_user_error(regs->regs[4], &sf->uc.uc_mcontext.arm_r4, err);	480	__get_user_error(regs->regs[4], &sf->uc.uc_mcontext.arm_r4, err);
481	__get_user_error(regs->regs[5], &sf->uc.uc_mcontext.arm_r5, err);	481	__get_user_error(regs->regs[5], &sf->uc.uc_mcontext.arm_r5, err);
482	__get_user_error(regs->regs[6], &sf->uc.uc_mcontext.arm_r6, err);	482	__get_user_error(regs->regs[6], &sf->uc.uc_mcontext.arm_r6, err);
483	__get_user_error(regs->regs[7], &sf->uc.uc_mcontext.arm_r7, err);	483	__get_user_error(regs->regs[7], &sf->uc.uc_mcontext.arm_r7, err);
484	__get_user_error(regs->regs[8], &sf->uc.uc_mcontext.arm_r8, err);	484	__get_user_error(regs->regs[8], &sf->uc.uc_mcontext.arm_r8, err);
485	__get_user_error(regs->regs[9], &sf->uc.uc_mcontext.arm_r9, err);	485	__get_user_error(regs->regs[9], &sf->uc.uc_mcontext.arm_r9, err);
486	__get_user_error(regs->regs[10], &sf->uc.uc_mcontext.arm_r10, err);	486	__get_user_error(regs->regs[10], &sf->uc.uc_mcontext.arm_r10, err);
487	__get_user_error(regs->regs[11], &sf->uc.uc_mcontext.arm_fp, err);	487	__get_user_error(regs->regs[11], &sf->uc.uc_mcontext.arm_fp, err);
488	__get_user_error(regs->regs[12], &sf->uc.uc_mcontext.arm_ip, err);	488	__get_user_error(regs->regs[12], &sf->uc.uc_mcontext.arm_ip, err);
489	__get_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);	489	__get_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);
490	__get_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);	490	__get_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);
491	__get_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);	491	__get_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);
492	__get_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);	492	__get_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);
493		493
494	/*	494	/*
495	* Avoid compat_sys_sigreturn() restarting.	495	* Avoid compat_sys_sigreturn() restarting.
496	*/	496	*/
497	regs->syscallno = ~0UL;	497	regs->syscallno = ~0UL;
498		498
499	err \|= !valid_user_regs(&regs->user_regs);	499	err \|= !valid_user_regs(&regs->user_regs);
500		500
501	aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace;	501	aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace;
502	if (err == 0)	502	if (err == 0)
503	err \|= compat_restore_vfp_context(&aux->vfp);	503	err \|= compat_restore_vfp_context(&aux->vfp);
504		504
505	return err;	505	return err;
506	}	506	}
507		507
508	asmlinkage int compat_sys_sigreturn(struct pt_regs *regs)	508	asmlinkage int compat_sys_sigreturn(struct pt_regs *regs)
509	{	509	{
510	struct compat_sigframe __user *frame;	510	struct compat_sigframe __user *frame;
511		511
512	/* Always make any pending restarted system calls return -EINTR */	512	/* Always make any pending restarted system calls return -EINTR */
513	current_thread_info()->restart_block.fn = do_no_restart_syscall;	513	current_thread_info()->restart_block.fn = do_no_restart_syscall;
514		514
515	/*	515	/*
516	* Since we stacked the signal on a 64-bit boundary,	516	* Since we stacked the signal on a 64-bit boundary,
517	* then 'sp' should be word aligned here. If it's	517	* then 'sp' should be word aligned here. If it's
518	* not, then the user is trying to mess with us.	518	* not, then the user is trying to mess with us.
519	*/	519	*/
520	if (regs->compat_sp & 7)	520	if (regs->compat_sp & 7)
521	goto badframe;	521	goto badframe;
522		522
523	frame = (struct compat_sigframe __user *)regs->compat_sp;	523	frame = (struct compat_sigframe __user *)regs->compat_sp;
524		524
525	if (!access_ok(VERIFY_READ, frame, sizeof (*frame)))	525	if (!access_ok(VERIFY_READ, frame, sizeof (*frame)))
526	goto badframe;	526	goto badframe;
527		527
528	if (compat_restore_sigframe(regs, frame))	528	if (compat_restore_sigframe(regs, frame))
529	goto badframe;	529	goto badframe;
530		530
531	return regs->regs[0];	531	return regs->regs[0];
532		532
533	badframe:	533	badframe:
534	if (show_unhandled_signals)	534	if (show_unhandled_signals)
535	pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n",	535	pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n",
536	current->comm, task_pid_nr(current), __func__,	536	current->comm, task_pid_nr(current), __func__,
537	regs->pc, regs->sp);	537	regs->pc, regs->sp);
538	force_sig(SIGSEGV, current);	538	force_sig(SIGSEGV, current);
539	return 0;	539	return 0;
540	}	540	}
541		541
542	asmlinkage int compat_sys_rt_sigreturn(struct pt_regs *regs)	542	asmlinkage int compat_sys_rt_sigreturn(struct pt_regs *regs)
543	{	543	{
544	struct compat_rt_sigframe __user *frame;	544	struct compat_rt_sigframe __user *frame;
545		545
546	/* Always make any pending restarted system calls return -EINTR */	546	/* Always make any pending restarted system calls return -EINTR */
547	current_thread_info()->restart_block.fn = do_no_restart_syscall;	547	current_thread_info()->restart_block.fn = do_no_restart_syscall;
548		548
549	/*	549	/*
550	* Since we stacked the signal on a 64-bit boundary,	550	* Since we stacked the signal on a 64-bit boundary,
551	* then 'sp' should be word aligned here. If it's	551	* then 'sp' should be word aligned here. If it's
552	* not, then the user is trying to mess with us.	552	* not, then the user is trying to mess with us.
553	*/	553	*/
554	if (regs->compat_sp & 7)	554	if (regs->compat_sp & 7)
555	goto badframe;	555	goto badframe;
556		556
557	frame = (struct compat_rt_sigframe __user *)regs->compat_sp;	557	frame = (struct compat_rt_sigframe __user *)regs->compat_sp;
558		558
559	if (!access_ok(VERIFY_READ, frame, sizeof (*frame)))	559	if (!access_ok(VERIFY_READ, frame, sizeof (*frame)))
560	goto badframe;	560	goto badframe;
561		561
562	if (compat_restore_sigframe(regs, &frame->sig))	562	if (compat_restore_sigframe(regs, &frame->sig))
563	goto badframe;	563	goto badframe;
564		564
565	if (compat_do_sigaltstack(ptr_to_compat(&frame->sig.uc.uc_stack),	565	if (compat_do_sigaltstack(ptr_to_compat(&frame->sig.uc.uc_stack),
566	ptr_to_compat((void __user *)NULL),	566	ptr_to_compat((void __user *)NULL),
567	regs->compat_sp) == -EFAULT)	567	regs->compat_sp) == -EFAULT)
568	goto badframe;	568	goto badframe;
569		569
570	return regs->regs[0];	570	return regs->regs[0];
571		571
572	badframe:	572	badframe:
573	if (show_unhandled_signals)	573	if (show_unhandled_signals)
574	pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n",	574	pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n",
575	current->comm, task_pid_nr(current), __func__,	575	current->comm, task_pid_nr(current), __func__,
576	regs->pc, regs->sp);	576	regs->pc, regs->sp);
577	force_sig(SIGSEGV, current);	577	force_sig(SIGSEGV, current);
578	return 0;	578	return 0;
579	}	579	}
580		580
581	static void __user compat_get_sigframe(struct k_sigaction ka,	581	static void __user compat_get_sigframe(struct k_sigaction ka,
582	struct pt_regs *regs,	582	struct pt_regs *regs,
583	int framesize)	583	int framesize)
584	{	584	{
585	compat_ulong_t sp = regs->compat_sp;	585	compat_ulong_t sp = regs->compat_sp;
586	void __user *frame;	586	void __user *frame;
587		587
588	/*	588	/*
589	* This is the X/Open sanctioned signal stack switching.	589	* This is the X/Open sanctioned signal stack switching.
590	*/	590	*/
591	if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))	591	if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp))
592	sp = current->sas_ss_sp + current->sas_ss_size;	592	sp = current->sas_ss_sp + current->sas_ss_size;
593		593
594	/*	594	/*
595	* ATPCS B01 mandates 8-byte alignment	595	* ATPCS B01 mandates 8-byte alignment
596	*/	596	*/
597	frame = compat_ptr((compat_uptr_t)((sp - framesize) & ~7));	597	frame = compat_ptr((compat_uptr_t)((sp - framesize) & ~7));
598		598
599	/*	599	/*
600	* Check that we can actually write to the signal frame.	600	* Check that we can actually write to the signal frame.
601	*/	601	*/
602	if (!access_ok(VERIFY_WRITE, frame, framesize))	602	if (!access_ok(VERIFY_WRITE, frame, framesize))
603	frame = NULL;	603	frame = NULL;
604		604
605	return frame;	605	return frame;
606	}	606	}
607		607
608	static void compat_setup_return(struct pt_regs regs, struct k_sigaction ka,	608	static void compat_setup_return(struct pt_regs regs, struct k_sigaction ka,
609	compat_ulong_t __user rc, void __user frame,	609	compat_ulong_t __user rc, void __user frame,
610	int usig)	610	int usig)
611	{	611	{
612	compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);	612	compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
613	compat_ulong_t retcode;	613	compat_ulong_t retcode;
614	compat_ulong_t spsr = regs->pstate & ~PSR_f;	614	compat_ulong_t spsr = regs->pstate & ~PSR_f;
615	int thumb;	615	int thumb;
616		616
617	/* Check if the handler is written for ARM or Thumb */	617	/* Check if the handler is written for ARM or Thumb */
618	thumb = handler & 1;	618	thumb = handler & 1;
619		619
620	if (thumb) {	620	if (thumb) {
621	spsr \|= COMPAT_PSR_T_BIT;	621	spsr \|= COMPAT_PSR_T_BIT;
622	spsr &= ~COMPAT_PSR_IT_MASK;	622	spsr &= ~COMPAT_PSR_IT_MASK;
623	} else {	623	} else {
624	spsr &= ~COMPAT_PSR_T_BIT;	624	spsr &= ~COMPAT_PSR_T_BIT;
625	}	625	}
626		626
627	if (ka->sa.sa_flags & SA_RESTORER) {	627	if (ka->sa.sa_flags & SA_RESTORER) {
628	retcode = ptr_to_compat(ka->sa.sa_restorer);	628	retcode = ptr_to_compat(ka->sa.sa_restorer);
629	} else {	629	} else {
630	/* Set up sigreturn pointer */	630	/* Set up sigreturn pointer */
631	unsigned int idx = thumb << 1;	631	unsigned int idx = thumb << 1;
632		632
633	if (ka->sa.sa_flags & SA_SIGINFO)	633	if (ka->sa.sa_flags & SA_SIGINFO)
634	idx += 3;	634	idx += 3;
635		635
636	retcode = AARCH32_VECTORS_BASE +	636	retcode = AARCH32_VECTORS_BASE +
637	AARCH32_KERN_SIGRET_CODE_OFFSET +	637	AARCH32_KERN_SIGRET_CODE_OFFSET +
638	(idx << 2) + thumb;	638	(idx << 2) + thumb;
639	}	639	}
640		640
641	regs->regs[0] = usig;	641	regs->regs[0] = usig;
642	regs->compat_sp = ptr_to_compat(frame);	642	regs->compat_sp = ptr_to_compat(frame);
643	regs->compat_lr = retcode;	643	regs->compat_lr = retcode;
644	regs->pc = handler;	644	regs->pc = handler;
645	regs->pstate = spsr;	645	regs->pstate = spsr;
646	}	646	}
647		647
648	static int compat_setup_sigframe(struct compat_sigframe __user *sf,	648	static int compat_setup_sigframe(struct compat_sigframe __user *sf,
649	struct pt_regs regs, sigset_t set)	649	struct pt_regs regs, sigset_t set)
650	{	650	{
651	struct compat_aux_sigframe __user *aux;	651	struct compat_aux_sigframe __user *aux;
652	int err = 0;	652	int err = 0;
653		653
654	__put_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err);	654	__put_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err);
655	__put_user_error(regs->regs[1], &sf->uc.uc_mcontext.arm_r1, err);	655	__put_user_error(regs->regs[1], &sf->uc.uc_mcontext.arm_r1, err);
656	__put_user_error(regs->regs[2], &sf->uc.uc_mcontext.arm_r2, err);	656	__put_user_error(regs->regs[2], &sf->uc.uc_mcontext.arm_r2, err);
657	__put_user_error(regs->regs[3], &sf->uc.uc_mcontext.arm_r3, err);	657	__put_user_error(regs->regs[3], &sf->uc.uc_mcontext.arm_r3, err);
658	__put_user_error(regs->regs[4], &sf->uc.uc_mcontext.arm_r4, err);	658	__put_user_error(regs->regs[4], &sf->uc.uc_mcontext.arm_r4, err);
659	__put_user_error(regs->regs[5], &sf->uc.uc_mcontext.arm_r5, err);	659	__put_user_error(regs->regs[5], &sf->uc.uc_mcontext.arm_r5, err);
660	__put_user_error(regs->regs[6], &sf->uc.uc_mcontext.arm_r6, err);	660	__put_user_error(regs->regs[6], &sf->uc.uc_mcontext.arm_r6, err);
661	__put_user_error(regs->regs[7], &sf->uc.uc_mcontext.arm_r7, err);	661	__put_user_error(regs->regs[7], &sf->uc.uc_mcontext.arm_r7, err);
662	__put_user_error(regs->regs[8], &sf->uc.uc_mcontext.arm_r8, err);	662	__put_user_error(regs->regs[8], &sf->uc.uc_mcontext.arm_r8, err);
663	__put_user_error(regs->regs[9], &sf->uc.uc_mcontext.arm_r9, err);	663	__put_user_error(regs->regs[9], &sf->uc.uc_mcontext.arm_r9, err);
664	__put_user_error(regs->regs[10], &sf->uc.uc_mcontext.arm_r10, err);	664	__put_user_error(regs->regs[10], &sf->uc.uc_mcontext.arm_r10, err);
665	__put_user_error(regs->regs[11], &sf->uc.uc_mcontext.arm_fp, err);	665	__put_user_error(regs->regs[11], &sf->uc.uc_mcontext.arm_fp, err);
666	__put_user_error(regs->regs[12], &sf->uc.uc_mcontext.arm_ip, err);	666	__put_user_error(regs->regs[12], &sf->uc.uc_mcontext.arm_ip, err);
667	__put_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);	667	__put_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);
668	__put_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);	668	__put_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);
669	__put_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);	669	__put_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);
670	__put_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);	670	__put_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);
671		671
672	__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);	672	__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
673	__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.error_code, err);	673	__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.error_code, err);
674	__put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);	674	__put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);
675	__put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);	675	__put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);
676		676
677	err \|= put_sigset_t(&sf->uc.uc_sigmask, set);	677	err \|= put_sigset_t(&sf->uc.uc_sigmask, set);
678		678
679	aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace;	679	aux = (struct compat_aux_sigframe __user *) sf->uc.uc_regspace;
680		680
681	if (err == 0)	681	if (err == 0)
682	err \|= compat_preserve_vfp_context(&aux->vfp);	682	err \|= compat_preserve_vfp_context(&aux->vfp);
683	__put_user_error(0, &aux->end_magic, err);	683	__put_user_error(0, &aux->end_magic, err);
684		684
685	return err;	685	return err;
686	}	686	}
687		687
688	/*	688	/*
689	* 32-bit signal handling routines called from signal.c	689	* 32-bit signal handling routines called from signal.c
690	*/	690	*/
691	int compat_setup_rt_frame(int usig, struct k_sigaction ka, siginfo_t info,	691	int compat_setup_rt_frame(int usig, struct k_sigaction ka, siginfo_t info,
692	sigset_t set, struct pt_regs regs)	692	sigset_t set, struct pt_regs regs)
693	{	693	{
694	struct compat_rt_sigframe __user *frame;	694	struct compat_rt_sigframe __user *frame;
695	compat_stack_t stack;	695	compat_stack_t stack;
696	int err = 0;	696	int err = 0;
697		697
698	frame = compat_get_sigframe(ka, regs, sizeof(*frame));	698	frame = compat_get_sigframe(ka, regs, sizeof(*frame));
699		699
700	if (!frame)	700	if (!frame)
701	return 1;	701	return 1;
702		702
703	err \|= copy_siginfo_to_user32(&frame->info, info);	703	err \|= copy_siginfo_to_user32(&frame->info, info);
704		704
705	__put_user_error(0, &frame->sig.uc.uc_flags, err);	705	__put_user_error(0, &frame->sig.uc.uc_flags, err);
706	__put_user_error(NULL, &frame->sig.uc.uc_link, err);	706	__put_user_error(0, &frame->sig.uc.uc_link, err);
707		707
708	memset(&stack, 0, sizeof(stack));	708	memset(&stack, 0, sizeof(stack));
709	stack.ss_sp = (compat_uptr_t)current->sas_ss_sp;	709	stack.ss_sp = (compat_uptr_t)current->sas_ss_sp;
710	stack.ss_flags = sas_ss_flags(regs->compat_sp);	710	stack.ss_flags = sas_ss_flags(regs->compat_sp);
711	stack.ss_size = current->sas_ss_size;	711	stack.ss_size = current->sas_ss_size;
712	err \|= __copy_to_user(&frame->sig.uc.uc_stack, &stack, sizeof(stack));	712	err \|= __copy_to_user(&frame->sig.uc.uc_stack, &stack, sizeof(stack));
713		713
714	err \|= compat_setup_sigframe(&frame->sig, regs, set);	714	err \|= compat_setup_sigframe(&frame->sig, regs, set);
715		715
716	if (err == 0) {	716	if (err == 0) {
717	compat_setup_return(regs, ka, frame->sig.retcode, frame, usig);	717	compat_setup_return(regs, ka, frame->sig.retcode, frame, usig);
718	regs->regs[1] = (compat_ulong_t)(unsigned long)&frame->info;	718	regs->regs[1] = (compat_ulong_t)(unsigned long)&frame->info;
719	regs->regs[2] = (compat_ulong_t)(unsigned long)&frame->sig.uc;	719	regs->regs[2] = (compat_ulong_t)(unsigned long)&frame->sig.uc;
720	}	720	}
721		721
722	return err;	722	return err;
723	}	723	}
724		724
725	int compat_setup_frame(int usig, struct k_sigaction ka, sigset_t set,	725	int compat_setup_frame(int usig, struct k_sigaction ka, sigset_t set,
726	struct pt_regs *regs)	726	struct pt_regs *regs)
727	{	727	{
728	struct compat_sigframe __user *frame;	728	struct compat_sigframe __user *frame;
729	int err = 0;	729	int err = 0;
730		730
731	frame = compat_get_sigframe(ka, regs, sizeof(*frame));	731	frame = compat_get_sigframe(ka, regs, sizeof(*frame));
732		732
733	if (!frame)	733	if (!frame)
734	return 1;	734	return 1;
735		735
736	__put_user_error(0x5ac3c35a, &frame->uc.uc_flags, err);	736	__put_user_error(0x5ac3c35a, &frame->uc.uc_flags, err);
737		737
738	err \|= compat_setup_sigframe(frame, regs, set);	738	err \|= compat_setup_sigframe(frame, regs, set);
739	if (err == 0)	739	if (err == 0)
740	compat_setup_return(regs, ka, frame->retcode, frame, usig);	740	compat_setup_return(regs, ka, frame->retcode, frame, usig);
741		741
742	return err;	742	return err;
743	}	743	}
744		744
745	/*	745	/*
746	* RT signals don't have generic compat wrappers.	746	* RT signals don't have generic compat wrappers.
747	* See arch/powerpc/kernel/signal_32.c	747	* See arch/powerpc/kernel/signal_32.c
748	*/	748	*/
749	asmlinkage int compat_sys_rt_sigprocmask(int how, compat_sigset_t __user *set,	749	asmlinkage int compat_sys_rt_sigprocmask(int how, compat_sigset_t __user *set,
750	compat_sigset_t __user *oset,	750	compat_sigset_t __user *oset,
751	compat_size_t sigsetsize)	751	compat_size_t sigsetsize)
752	{	752	{
753	sigset_t s;	753	sigset_t s;
754	sigset_t __user *up;	754	sigset_t __user *up;
755	int ret;	755	int ret;
756	mm_segment_t old_fs = get_fs();	756	mm_segment_t old_fs = get_fs();
757		757
758	if (set) {	758	if (set) {
759	if (get_sigset_t(&s, set))	759	if (get_sigset_t(&s, set))
760	return -EFAULT;	760	return -EFAULT;
761	}	761	}
762		762
763	set_fs(KERNEL_DS);	763	set_fs(KERNEL_DS);
764	/* This is valid because of the set_fs() */	764	/* This is valid because of the set_fs() */
765	up = (sigset_t __user *) &s;	765	up = (sigset_t __user *) &s;
766	ret = sys_rt_sigprocmask(how, set ? up : NULL, oset ? up : NULL,	766	ret = sys_rt_sigprocmask(how, set ? up : NULL, oset ? up : NULL,
767	sigsetsize);	767	sigsetsize);
768	set_fs(old_fs);	768	set_fs(old_fs);
769	if (ret)	769	if (ret)
770	return ret;	770	return ret;
771	if (oset) {	771	if (oset) {
772	if (put_sigset_t(oset, &s))	772	if (put_sigset_t(oset, &s))
773	return -EFAULT;	773	return -EFAULT;
774	}	774	}
775	return 0;	775	return 0;
776	}	776	}
777		777
778	asmlinkage int compat_sys_rt_sigpending(compat_sigset_t __user *set,	778	asmlinkage int compat_sys_rt_sigpending(compat_sigset_t __user *set,
779	compat_size_t sigsetsize)	779	compat_size_t sigsetsize)
780	{	780	{
781	sigset_t s;	781	sigset_t s;
782	int ret;	782	int ret;
783	mm_segment_t old_fs = get_fs();	783	mm_segment_t old_fs = get_fs();
784		784
785	set_fs(KERNEL_DS);	785	set_fs(KERNEL_DS);
786	/* The __user pointer cast is valid because of the set_fs() */	786	/* The __user pointer cast is valid because of the set_fs() */
787	ret = sys_rt_sigpending((sigset_t __user *) &s, sigsetsize);	787	ret = sys_rt_sigpending((sigset_t __user *) &s, sigsetsize);
788	set_fs(old_fs);	788	set_fs(old_fs);
789	if (!ret) {	789	if (!ret) {
790	if (put_sigset_t(set, &s))	790	if (put_sigset_t(set, &s))
791	return -EFAULT;	791	return -EFAULT;
792	}	792	}
793	return ret;	793	return ret;
794	}	794	}
795		795
796	asmlinkage int compat_sys_rt_sigqueueinfo(int pid, int sig,	796	asmlinkage int compat_sys_rt_sigqueueinfo(int pid, int sig,
797	compat_siginfo_t __user *uinfo)	797	compat_siginfo_t __user *uinfo)
798	{	798	{
799	siginfo_t info;	799	siginfo_t info;
800	int ret;	800	int ret;
801	mm_segment_t old_fs = get_fs();	801	mm_segment_t old_fs = get_fs();
802		802
803	ret = copy_siginfo_from_user32(&info, uinfo);	803	ret = copy_siginfo_from_user32(&info, uinfo);
804	if (unlikely(ret))	804	if (unlikely(ret))
805	return ret;	805	return ret;
806		806
807	set_fs (KERNEL_DS);	807	set_fs (KERNEL_DS);
808	/* The __user pointer cast is valid because of the set_fs() */	808	/* The __user pointer cast is valid because of the set_fs() */
809	ret = sys_rt_sigqueueinfo(pid, sig, (siginfo_t __user *) &info);	809	ret = sys_rt_sigqueueinfo(pid, sig, (siginfo_t __user *) &info);
810	set_fs (old_fs);	810	set_fs (old_fs);
811	return ret;	811	return ret;
812	}	812	}
813		813
814	void compat_setup_restart_syscall(struct pt_regs *regs)	814	void compat_setup_restart_syscall(struct pt_regs *regs)
815	{	815	{
816	regs->regs[7] = __NR_compat_restart_syscall;	816	regs->regs[7] = __NR_compat_restart_syscall;
817	}	817	}
818		818

arch/arm64/kernel/smp.c

Diff comments View file @ 79a69d3

1	/*	1	/*
2	* SMP initialisation and IPI support	2	* SMP initialisation and IPI support
3	* Based on arch/arm/kernel/smp.c	3	* Based on arch/arm/kernel/smp.c
4	*	4	*
5	* Copyright (C) 2012 ARM Ltd.	5	* Copyright (C) 2012 ARM Ltd.
6	*	6	*
7	* This program is free software; you can redistribute it and/or modify	7	* This program is free software; you can redistribute it and/or modify
8	* it under the terms of the GNU General Public License version 2 as	8	* it under the terms of the GNU General Public License version 2 as
9	* published by the Free Software Foundation.	9	* published by the Free Software Foundation.
10	*	10	*
11	* This program is distributed in the hope that it will be useful,	11	* This program is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	* GNU General Public License for more details.	14	* GNU General Public License for more details.
15	*	15	*
16	* You should have received a copy of the GNU General Public License	16	* You should have received a copy of the GNU General Public License
17	* along with this program. If not, see <http://www.gnu.org/licenses/>.	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
18	*/	18	*/
19		19
20	#include <linux/delay.h>	20	#include <linux/delay.h>
21	#include <linux/init.h>	21	#include <linux/init.h>
22	#include <linux/spinlock.h>	22	#include <linux/spinlock.h>
23	#include <linux/sched.h>	23	#include <linux/sched.h>
24	#include <linux/interrupt.h>	24	#include <linux/interrupt.h>
25	#include <linux/cache.h>	25	#include <linux/cache.h>
26	#include <linux/profile.h>	26	#include <linux/profile.h>
27	#include <linux/errno.h>	27	#include <linux/errno.h>
28	#include <linux/mm.h>	28	#include <linux/mm.h>
29	#include <linux/err.h>	29	#include <linux/err.h>
30	#include <linux/cpu.h>	30	#include <linux/cpu.h>
31	#include <linux/smp.h>	31	#include <linux/smp.h>
32	#include <linux/seq_file.h>	32	#include <linux/seq_file.h>
33	#include <linux/irq.h>	33	#include <linux/irq.h>
34	#include <linux/percpu.h>	34	#include <linux/percpu.h>
35	#include <linux/clockchips.h>	35	#include <linux/clockchips.h>
36	#include <linux/completion.h>	36	#include <linux/completion.h>
37	#include <linux/of.h>	37	#include <linux/of.h>
38		38
39	#include <asm/atomic.h>	39	#include <asm/atomic.h>
40	#include <asm/cacheflush.h>	40	#include <asm/cacheflush.h>
41	#include <asm/cputype.h>	41	#include <asm/cputype.h>
42	#include <asm/mmu_context.h>	42	#include <asm/mmu_context.h>
43	#include <asm/pgtable.h>	43	#include <asm/pgtable.h>
44	#include <asm/pgalloc.h>	44	#include <asm/pgalloc.h>
45	#include <asm/processor.h>	45	#include <asm/processor.h>
46	#include <asm/sections.h>	46	#include <asm/sections.h>
47	#include <asm/tlbflush.h>	47	#include <asm/tlbflush.h>
48	#include <asm/ptrace.h>	48	#include <asm/ptrace.h>
49		49
50	/*	50	/*
51	* as from 2.5, kernels no longer have an init_tasks structure	51	* as from 2.5, kernels no longer have an init_tasks structure
52	* so we need some other way of telling a new secondary core	52	* so we need some other way of telling a new secondary core
53	* where to place its SVC stack	53	* where to place its SVC stack
54	*/	54	*/
55	struct secondary_data secondary_data;	55	struct secondary_data secondary_data;
56	volatile unsigned long secondary_holding_pen_release = -1;	56	volatile unsigned long secondary_holding_pen_release = -1;
57		57
58	enum ipi_msg_type {	58	enum ipi_msg_type {
59	IPI_RESCHEDULE,	59	IPI_RESCHEDULE,
60	IPI_CALL_FUNC,	60	IPI_CALL_FUNC,
61	IPI_CALL_FUNC_SINGLE,	61	IPI_CALL_FUNC_SINGLE,
62	IPI_CPU_STOP,	62	IPI_CPU_STOP,
63	};	63	};
64		64
65	static DEFINE_RAW_SPINLOCK(boot_lock);	65	static DEFINE_RAW_SPINLOCK(boot_lock);
66		66
67	/*	67	/*
68	* Write secondary_holding_pen_release in a way that is guaranteed to be	68	* Write secondary_holding_pen_release in a way that is guaranteed to be
69	* visible to all observers, irrespective of whether they're taking part	69	* visible to all observers, irrespective of whether they're taking part
70	* in coherency or not. This is necessary for the hotplug code to work	70	* in coherency or not. This is necessary for the hotplug code to work
71	* reliably.	71	* reliably.
72	*/	72	*/
73	static void __cpuinit write_pen_release(int val)	73	static void __cpuinit write_pen_release(int val)
74	{	74	{
75	void start = (void )&secondary_holding_pen_release;	75	void start = (void )&secondary_holding_pen_release;
76	unsigned long size = sizeof(secondary_holding_pen_release);	76	unsigned long size = sizeof(secondary_holding_pen_release);
77		77
78	secondary_holding_pen_release = val;	78	secondary_holding_pen_release = val;
79	__flush_dcache_area(start, size);	79	__flush_dcache_area(start, size);
80	}	80	}
81		81
82	/*	82	/*
83	* Boot a secondary CPU, and assign it the specified idle task.	83	* Boot a secondary CPU, and assign it the specified idle task.
84	* This also gives us the initial stack to use for this CPU.	84	* This also gives us the initial stack to use for this CPU.
85	*/	85	*/
86	static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle)	86	static int __cpuinit boot_secondary(unsigned int cpu, struct task_struct *idle)
87	{	87	{
88	unsigned long timeout;	88	unsigned long timeout;
89		89
90	/*	90	/*
91	* Set synchronisation state between this boot processor	91	* Set synchronisation state between this boot processor
92	* and the secondary one	92	* and the secondary one
93	*/	93	*/
94	raw_spin_lock(&boot_lock);	94	raw_spin_lock(&boot_lock);
95		95
96	/*	96	/*
97	* Update the pen release flag.	97	* Update the pen release flag.
98	*/	98	*/
99	write_pen_release(cpu);	99	write_pen_release(cpu);
100		100
101	/*	101	/*
102	* Send an event, causing the secondaries to read pen_release.	102	* Send an event, causing the secondaries to read pen_release.
103	*/	103	*/
104	sev();	104	sev();
105		105
106	timeout = jiffies + (1 * HZ);	106	timeout = jiffies + (1 * HZ);
107	while (time_before(jiffies, timeout)) {	107	while (time_before(jiffies, timeout)) {
108	if (secondary_holding_pen_release == -1UL)	108	if (secondary_holding_pen_release == -1UL)
109	break;	109	break;
110	udelay(10);	110	udelay(10);
111	}	111	}
112		112
113	/*	113	/*
114	* Now the secondary core is starting up let it run its	114	* Now the secondary core is starting up let it run its
115	* calibrations, then wait for it to finish	115	* calibrations, then wait for it to finish
116	*/	116	*/
117	raw_spin_unlock(&boot_lock);	117	raw_spin_unlock(&boot_lock);
118		118
119	return secondary_holding_pen_release != -1 ? -ENOSYS : 0;	119	return secondary_holding_pen_release != -1 ? -ENOSYS : 0;
120	}	120	}
121		121
122	static DECLARE_COMPLETION(cpu_running);	122	static DECLARE_COMPLETION(cpu_running);
123		123
124	int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle)	124	int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle)
125	{	125	{
126	int ret;	126	int ret;
127		127
128	/*	128	/*
129	* We need to tell the secondary core where to find its stack and the	129	* We need to tell the secondary core where to find its stack and the
130	* page tables.	130	* page tables.
131	*/	131	*/
132	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;	132	secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
133	__flush_dcache_area(&secondary_data, sizeof(secondary_data));	133	__flush_dcache_area(&secondary_data, sizeof(secondary_data));
134		134
135	/*	135	/*
136	* Now bring the CPU into our world.	136	* Now bring the CPU into our world.
137	*/	137	*/
138	ret = boot_secondary(cpu, idle);	138	ret = boot_secondary(cpu, idle);
139	if (ret == 0) {	139	if (ret == 0) {
140	/*	140	/*
141	* CPU was successfully started, wait for it to come online or	141	* CPU was successfully started, wait for it to come online or
142	* time out.	142	* time out.
143	*/	143	*/
144	wait_for_completion_timeout(&cpu_running,	144	wait_for_completion_timeout(&cpu_running,
145	msecs_to_jiffies(1000));	145	msecs_to_jiffies(1000));
146		146
147	if (!cpu_online(cpu)) {	147	if (!cpu_online(cpu)) {
148	pr_crit("CPU%u: failed to come online\n", cpu);	148	pr_crit("CPU%u: failed to come online\n", cpu);
149	ret = -EIO;	149	ret = -EIO;
150	}	150	}
151	} else {	151	} else {
152	pr_err("CPU%u: failed to boot: %d\n", cpu, ret);	152	pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
153	}	153	}
154		154
155	secondary_data.stack = NULL;	155	secondary_data.stack = NULL;
156		156
157	return ret;	157	return ret;
158	}	158	}
159		159
160	/*	160	/*
161	* This is the secondary CPU boot entry. We're using this CPUs	161	* This is the secondary CPU boot entry. We're using this CPUs
162	* idle thread stack, but a set of temporary page tables.	162	* idle thread stack, but a set of temporary page tables.
163	*/	163	*/
164	asmlinkage void __cpuinit secondary_start_kernel(void)	164	asmlinkage void __cpuinit secondary_start_kernel(void)
165	{	165	{
166	struct mm_struct *mm = &init_mm;	166	struct mm_struct *mm = &init_mm;
167	unsigned int cpu = smp_processor_id();	167	unsigned int cpu = smp_processor_id();
168		168
169	printk("CPU%u: Booted secondary processor\n", cpu);	169	printk("CPU%u: Booted secondary processor\n", cpu);
170		170
171	/*	171	/*
172	* All kernel threads share the same mm context; grab a	172	* All kernel threads share the same mm context; grab a
173	* reference and switch to it.	173	* reference and switch to it.
174	*/	174	*/
175	atomic_inc(&mm->mm_count);	175	atomic_inc(&mm->mm_count);
176	current->active_mm = mm;	176	current->active_mm = mm;
177	cpumask_set_cpu(cpu, mm_cpumask(mm));	177	cpumask_set_cpu(cpu, mm_cpumask(mm));
178		178
179	/*	179	/*
180	* TTBR0 is only used for the identity mapping at this stage. Make it	180	* TTBR0 is only used for the identity mapping at this stage. Make it
181	* point to zero page to avoid speculatively fetching new entries.	181	* point to zero page to avoid speculatively fetching new entries.
182	*/	182	*/
183	cpu_set_reserved_ttbr0();	183	cpu_set_reserved_ttbr0();
184	flush_tlb_all();	184	flush_tlb_all();
185		185
186	preempt_disable();	186	preempt_disable();
187	trace_hardirqs_off();	187	trace_hardirqs_off();
188		188
189	/*	189	/*
190	* Let the primary processor know we're out of the	190	* Let the primary processor know we're out of the
191	* pen, then head off into the C entry point	191	* pen, then head off into the C entry point
192	*/	192	*/
193	write_pen_release(-1);	193	write_pen_release(-1);
194		194
195	/*	195	/*
196	* Synchronise with the boot thread.	196	* Synchronise with the boot thread.
197	*/	197	*/
198	raw_spin_lock(&boot_lock);	198	raw_spin_lock(&boot_lock);
199	raw_spin_unlock(&boot_lock);	199	raw_spin_unlock(&boot_lock);
200		200
201	/*	201	/*
202	* Enable local interrupts.	202	* Enable local interrupts.
203	*/	203	*/
204	notify_cpu_starting(cpu);	204	notify_cpu_starting(cpu);
205	local_irq_enable();	205	local_irq_enable();
206	local_fiq_enable();	206	local_fiq_enable();
207		207
208	/*	208	/*
209	* OK, now it's safe to let the boot CPU continue. Wait for	209	* OK, now it's safe to let the boot CPU continue. Wait for
210	* the CPU migration code to notice that the CPU is online	210	* the CPU migration code to notice that the CPU is online
211	* before we continue.	211	* before we continue.
212	*/	212	*/
213	set_cpu_online(cpu, true);	213	set_cpu_online(cpu, true);
214	complete(&cpu_running);	214	complete(&cpu_running);
215		215
216	/*	216	/*
217	* OK, it's off to the idle thread for us	217	* OK, it's off to the idle thread for us
218	*/	218	*/
219	cpu_idle();	219	cpu_idle();
220	}	220	}
221		221
222	void __init smp_cpus_done(unsigned int max_cpus)	222	void __init smp_cpus_done(unsigned int max_cpus)
223	{	223	{
224	unsigned long bogosum = loops_per_jiffy * num_online_cpus();	224	unsigned long bogosum = loops_per_jiffy * num_online_cpus();
225		225
226	pr_info("SMP: Total of %d processors activated (%lu.%02lu BogoMIPS).\n",	226	pr_info("SMP: Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
227	num_online_cpus(), bogosum / (500000/HZ),	227	num_online_cpus(), bogosum / (500000/HZ),
228	(bogosum / (5000/HZ)) % 100);	228	(bogosum / (5000/HZ)) % 100);
229	}	229	}
230		230
231	void __init smp_prepare_boot_cpu(void)	231	void __init smp_prepare_boot_cpu(void)
232	{	232	{
233	}	233	}
234		234
235	static void (smp_cross_call)(const struct cpumask , unsigned int);	235	static void (smp_cross_call)(const struct cpumask , unsigned int);
236	static phys_addr_t cpu_release_addr[NR_CPUS];
237		236
		237	static const struct smp_enable_ops *enable_ops[] __initconst = {
		238	&smp_spin_table_ops,
		239	&smp_psci_ops,
		240	NULL,
		241	};
		242
		243	static const struct smp_enable_ops *smp_enable_ops[NR_CPUS];
		244
		245	static const struct smp_enable_ops * __init smp_get_enable_ops(const char *name)
		246	{
		247	const struct smp_enable_ops *ops = enable_ops[0];
		248
		249	while (ops) {
		250	if (!strcmp(name, ops->name))
		251	return ops;
		252
		253	ops++;
		254	}
		255
		256	return NULL;
		257	}
		258
238	/*	259	/*
239	* Enumerate the possible CPU set from the device tree.	260	* Enumerate the possible CPU set from the device tree.
240	*/	261	*/
241	void __init smp_init_cpus(void)	262	void __init smp_init_cpus(void)
242	{	263	{
243	const char *enable_method;	264	const char *enable_method;
244	struct device_node *dn = NULL;	265	struct device_node *dn = NULL;
245	int cpu = 0;	266	int cpu = 0;
246		267
247	while ((dn = of_find_node_by_type(dn, "cpu"))) {	268	while ((dn = of_find_node_by_type(dn, "cpu"))) {
248	if (cpu >= NR_CPUS)	269	if (cpu >= NR_CPUS)
249	goto next;	270	goto next;
250		271
251	/*	272	/*
252	* We currently support only the "spin-table" enable-method.	273	* We currently support only the "spin-table" enable-method.
253	*/	274	*/
254	enable_method = of_get_property(dn, "enable-method", NULL);	275	enable_method = of_get_property(dn, "enable-method", NULL);
255	if (!enable_method \|\| strcmp(enable_method, "spin-table")) {	276	if (!enable_method) {
256	pr_err("CPU %d: missing or invalid enable-method property: %s\n",	277	pr_err("CPU %d: missing enable-method property\n", cpu);
257	cpu, enable_method);
258	goto next;	278	goto next;
259	}	279	}
260		280
261	/*	281	smp_enable_ops[cpu] = smp_get_enable_ops(enable_method);
262	* Determine the address from which the CPU is polling.	282
263	*/	283	if (!smp_enable_ops[cpu]) {
264	if (of_property_read_u64(dn, "cpu-release-addr",	284	pr_err("CPU %d: invalid enable-method property: %s\n",
265	&cpu_release_addr[cpu])) {	285	cpu, enable_method);
266	pr_err("CPU %d: missing or invalid cpu-release-addr property\n",
267	cpu);
268	goto next;	286	goto next;
269	}	287	}
270		288
		289	if (smp_enable_ops[cpu]->init_cpu(dn, cpu))
		290	goto next;
		291
271	set_cpu_possible(cpu, true);	292	set_cpu_possible(cpu, true);
272	next:	293	next:
273	cpu++;	294	cpu++;
274	}	295	}
275		296
276	/* sanity check */	297	/* sanity check */
277	if (cpu > NR_CPUS)	298	if (cpu > NR_CPUS)
278	pr_warning("no. of cores (%d) greater than configured maximum of %d - clipping\n",	299	pr_warning("no. of cores (%d) greater than configured maximum of %d - clipping\n",
279	cpu, NR_CPUS);	300	cpu, NR_CPUS);
280	}	301	}
281		302
282	void __init smp_prepare_cpus(unsigned int max_cpus)	303	void __init smp_prepare_cpus(unsigned int max_cpus)
283	{	304	{
284	int cpu;	305	int cpu, err;
285	void **release_addr;
286	unsigned int ncores = num_possible_cpus();	306	unsigned int ncores = num_possible_cpus();
287		307
288	/*	308	/*
289	* are we trying to boot more cores than exist?	309	* are we trying to boot more cores than exist?
290	*/	310	*/
291	if (max_cpus > ncores)	311	if (max_cpus > ncores)
292	max_cpus = ncores;	312	max_cpus = ncores;
293		313
		314	/* Don't bother if we're effectively UP */
		315	if (max_cpus <= 1)
		316	return;
		317
294	/*	318	/*
295	* Initialise the present map (which describes the set of CPUs	319	* Initialise the present map (which describes the set of CPUs
296	* actually populated at the present time) and release the	320	* actually populated at the present time) and release the
297	* secondaries from the bootloader.	321	* secondaries from the bootloader.
		322	*
		323	* Make sure we online at most (max_cpus - 1) additional CPUs.
298	*/	324	*/
		325	max_cpus--;
299	for_each_possible_cpu(cpu) {	326	for_each_possible_cpu(cpu) {
300	if (max_cpus == 0)	327	if (max_cpus == 0)
301	break;	328	break;
302		329
303	if (!cpu_release_addr[cpu])	330	if (cpu == smp_processor_id())
304	continue;	331	continue;
305		332
306	release_addr = __va(cpu_release_addr[cpu]);	333	if (!smp_enable_ops[cpu])
307	release_addr[0] = (void *)__pa(secondary_holding_pen);	334	continue;
308	__flush_dcache_area(release_addr, sizeof(release_addr[0]));
309		335
		336	err = smp_enable_ops[cpu]->prepare_cpu(cpu);
		337	if (err)
		338	continue;
		339
310	set_cpu_present(cpu, true);	340	set_cpu_present(cpu, true);
311	max_cpus--;	341	max_cpus--;
312	}	342	}
313
314	/*
315	* Send an event to wake up the secondaries.
316	*/
317	sev();
318	}	343	}
319		344
320		345
321	void __init set_smp_cross_call(void (fn)(const struct cpumask , unsigned int))	346	void __init set_smp_cross_call(void (fn)(const struct cpumask , unsigned int))
322	{	347	{
323	smp_cross_call = fn;	348	smp_cross_call = fn;
324	}	349	}
325		350
326	void arch_send_call_function_ipi_mask(const struct cpumask *mask)	351	void arch_send_call_function_ipi_mask(const struct cpumask *mask)
327	{	352	{
328	smp_cross_call(mask, IPI_CALL_FUNC);	353	smp_cross_call(mask, IPI_CALL_FUNC);
329	}	354	}
330		355
331	void arch_send_call_function_single_ipi(int cpu)	356	void arch_send_call_function_single_ipi(int cpu)
332	{	357	{
333	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);	358	smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
334	}	359	}
335		360
336	static const char *ipi_types[NR_IPI] = {	361	static const char *ipi_types[NR_IPI] = {
337	#define S(x,s) [x - IPI_RESCHEDULE] = s	362	#define S(x,s) [x - IPI_RESCHEDULE] = s
338	S(IPI_RESCHEDULE, "Rescheduling interrupts"),	363	S(IPI_RESCHEDULE, "Rescheduling interrupts"),
339	S(IPI_CALL_FUNC, "Function call interrupts"),	364	S(IPI_CALL_FUNC, "Function call interrupts"),
340	S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),	365	S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
341	S(IPI_CPU_STOP, "CPU stop interrupts"),	366	S(IPI_CPU_STOP, "CPU stop interrupts"),
342	};	367	};
343		368
344	void show_ipi_list(struct seq_file *p, int prec)	369	void show_ipi_list(struct seq_file *p, int prec)
345	{	370	{
346	unsigned int cpu, i;	371	unsigned int cpu, i;
347		372
348	for (i = 0; i < NR_IPI; i++) {	373	for (i = 0; i < NR_IPI; i++) {
349	seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE,	374	seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i + IPI_RESCHEDULE,
350	prec >= 4 ? " " : "");	375	prec >= 4 ? " " : "");
351	for_each_present_cpu(cpu)	376	for_each_present_cpu(cpu)
352	seq_printf(p, "%10u ",	377	seq_printf(p, "%10u ",
353	__get_irq_stat(cpu, ipi_irqs[i]));	378	__get_irq_stat(cpu, ipi_irqs[i]));
354	seq_printf(p, " %s\n", ipi_types[i]);	379	seq_printf(p, " %s\n", ipi_types[i]);
355	}	380	}
356	}	381	}
357		382
358	u64 smp_irq_stat_cpu(unsigned int cpu)	383	u64 smp_irq_stat_cpu(unsigned int cpu)
359	{	384	{
360	u64 sum = 0;	385	u64 sum = 0;
361	int i;	386	int i;
362		387
363	for (i = 0; i < NR_IPI; i++)	388	for (i = 0; i < NR_IPI; i++)
364	sum += __get_irq_stat(cpu, ipi_irqs[i]);	389	sum += __get_irq_stat(cpu, ipi_irqs[i]);
365		390
366	return sum;	391	return sum;
367	}	392	}
368		393
369	static DEFINE_RAW_SPINLOCK(stop_lock);	394	static DEFINE_RAW_SPINLOCK(stop_lock);
370		395
371	/*	396	/*
372	* ipi_cpu_stop - handle IPI from smp_send_stop()	397	* ipi_cpu_stop - handle IPI from smp_send_stop()
373	*/	398	*/
374	static void ipi_cpu_stop(unsigned int cpu)	399	static void ipi_cpu_stop(unsigned int cpu)
375	{	400	{
376	if (system_state == SYSTEM_BOOTING \|\|	401	if (system_state == SYSTEM_BOOTING \|\|
377	system_state == SYSTEM_RUNNING) {	402	system_state == SYSTEM_RUNNING) {
378	raw_spin_lock(&stop_lock);	403	raw_spin_lock(&stop_lock);
379	pr_crit("CPU%u: stopping\n", cpu);	404	pr_crit("CPU%u: stopping\n", cpu);
380	dump_stack();	405	dump_stack();
381	raw_spin_unlock(&stop_lock);	406	raw_spin_unlock(&stop_lock);
382	}	407	}
383		408
384	set_cpu_online(cpu, false);	409	set_cpu_online(cpu, false);
385		410
386	local_fiq_disable();	411	local_fiq_disable();
387	local_irq_disable();	412	local_irq_disable();
388		413
389	while (1)	414	while (1)
390	cpu_relax();	415	cpu_relax();
391	}	416	}
392		417
393	/*	418	/*
394	* Main handler for inter-processor interrupts	419	* Main handler for inter-processor interrupts
395	*/	420	*/
396	void handle_IPI(int ipinr, struct pt_regs *regs)	421	void handle_IPI(int ipinr, struct pt_regs *regs)
397	{	422	{
398	unsigned int cpu = smp_processor_id();	423	unsigned int cpu = smp_processor_id();
399	struct pt_regs *old_regs = set_irq_regs(regs);	424	struct pt_regs *old_regs = set_irq_regs(regs);
400		425
401	if (ipinr >= IPI_RESCHEDULE && ipinr < IPI_RESCHEDULE + NR_IPI)	426	if (ipinr >= IPI_RESCHEDULE && ipinr < IPI_RESCHEDULE + NR_IPI)
402	__inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_RESCHEDULE]);	427	__inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_RESCHEDULE]);
403		428
404	switch (ipinr) {	429	switch (ipinr) {
405	case IPI_RESCHEDULE:	430	case IPI_RESCHEDULE:
406	scheduler_ipi();	431	scheduler_ipi();
407	break;	432	break;
408		433
409	case IPI_CALL_FUNC:	434	case IPI_CALL_FUNC:
410	irq_enter();	435	irq_enter();
411	generic_smp_call_function_interrupt();	436	generic_smp_call_function_interrupt();
412	irq_exit();	437	irq_exit();
413	break;	438	break;
414		439
415	case IPI_CALL_FUNC_SINGLE:	440	case IPI_CALL_FUNC_SINGLE:
416	irq_enter();	441	irq_enter();
417	generic_smp_call_function_single_interrupt();	442	generic_smp_call_function_single_interrupt();
418	irq_exit();	443	irq_exit();
419	break;	444	break;
420		445
421	case IPI_CPU_STOP:	446	case IPI_CPU_STOP:
422	irq_enter();	447	irq_enter();
423	ipi_cpu_stop(cpu);	448	ipi_cpu_stop(cpu);
424	irq_exit();	449	irq_exit();
425	break;	450	break;
426		451
427	default:	452	default:
428	pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);	453	pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
429	break;	454	break;
430	}	455	}
431	set_irq_regs(old_regs);	456	set_irq_regs(old_regs);
432	}	457	}
433		458
434	void smp_send_reschedule(int cpu)	459	void smp_send_reschedule(int cpu)
435	{	460	{
436	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);	461	smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
437	}	462	}
438		463
439	void smp_send_stop(void)	464	void smp_send_stop(void)
440	{	465	{
441	unsigned long timeout;	466	unsigned long timeout;
442		467
443	if (num_online_cpus() > 1) {	468	if (num_online_cpus() > 1) {
444	cpumask_t mask;	469	cpumask_t mask;
445		470
446	cpumask_copy(&mask, cpu_online_mask);	471	cpumask_copy(&mask, cpu_online_mask);
447	cpu_clear(smp_processor_id(), mask);	472	cpu_clear(smp_processor_id(), mask);
448		473
449	smp_cross_call(&mask, IPI_CPU_STOP);	474	smp_cross_call(&mask, IPI_CPU_STOP);
450	}	475	}
451		476
452	/* Wait up to one second for other CPUs to stop */	477	/* Wait up to one second for other CPUs to stop */
453	timeout = USEC_PER_SEC;	478	timeout = USEC_PER_SEC;
454	while (num_online_cpus() > 1 && timeout--)	479	while (num_online_cpus() > 1 && timeout--)
455	udelay(1);	480	udelay(1);
456		481
457	if (num_online_cpus() > 1)	482	if (num_online_cpus() > 1)

arch/arm64/kernel/smp_psci.c

Diff comments View file @ 79a69d3

File was created	1	/*
	2	* PSCI SMP initialisation
	3	*
	4	* Copyright (C) 2013 ARM Ltd.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18
	19	#include <linux/init.h>
	20	#include <linux/of.h>
	21	#include <linux/smp.h>
	22
	23	#include <asm/psci.h>
	24
	25	static int __init smp_psci_init_cpu(struct device_node *dn, int cpu)
	26	{
	27	return 0;
	28	}
	29
	30	static int __init smp_psci_prepare_cpu(int cpu)
	31	{
	32	int err;
	33
	34	if (!psci_ops.cpu_on) {
	35	pr_err("psci: no cpu_on method, not booting CPU%d\n", cpu);
	36	return -ENODEV;
	37	}
	38
	39	err = psci_ops.cpu_on(cpu, __pa(secondary_holding_pen));
	40	if (err) {
	41	pr_err("psci: failed to boot CPU%d (%d)\n", cpu, err);
	42	return err;
	43	}
	44
	45	return 0;
	46	}
	47
	48	const struct smp_enable_ops smp_psci_ops __initconst = {
	49	.name = "psci",
	50	.init_cpu = smp_psci_init_cpu,
	51	.prepare_cpu = smp_psci_prepare_cpu,
	52	};
	53

arch/arm64/kernel/smp_spin_table.c

Diff comments View file @ 79a69d3

File was created	1	/*
	2	* Spin Table SMP initialisation
	3	*
	4	* Copyright (C) 2013 ARM Ltd.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	* You should have received a copy of the GNU General Public License
	16	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	17	*/
	18
	19	#include <linux/init.h>
	20	#include <linux/of.h>
	21	#include <linux/smp.h>
	22
	23	#include <asm/cacheflush.h>
	24
	25	static phys_addr_t cpu_release_addr[NR_CPUS];
	26
	27	static int __init smp_spin_table_init_cpu(struct device_node *dn, int cpu)
	28	{
	29	/*
	30	* Determine the address from which the CPU is polling.
	31	*/
	32	if (of_property_read_u64(dn, "cpu-release-addr",
	33	&cpu_release_addr[cpu])) {
	34	pr_err("CPU %d: missing or invalid cpu-release-addr property\n",
	35	cpu);
	36
	37	return -1;
	38	}
	39
	40	return 0;
	41	}
	42
	43	static int __init smp_spin_table_prepare_cpu(int cpu)
	44	{
	45	void **release_addr;
	46
	47	if (!cpu_release_addr[cpu])
	48	return -ENODEV;
	49
	50	release_addr = __va(cpu_release_addr[cpu]);
	51	release_addr[0] = (void *)__pa(secondary_holding_pen);
	52	__flush_dcache_area(release_addr, sizeof(release_addr[0]));
	53
	54	/*
	55	* Send an event to wake up the secondary CPU.
	56	*/
	57	sev();
	58
	59	return 0;
	60	}
	61
	62	const struct smp_enable_ops smp_spin_table_ops __initconst = {
	63	.name = "spin-table",
	64	.init_cpu = smp_spin_table_init_cpu,
	65	.prepare_cpu = smp_spin_table_prepare_cpu,
	66	};
	67

arch/arm64/mm/mmu.c

Diff comments View file @ 79a69d3

 /*
  * Based on arch/arm/mm/mmu.c
  *
  * Copyright (C) 1995-2005 Russell King
  * Copyright (C) 2012 ARM Ltd.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #include <linux/export.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/mman.h>
 #include <linux/nodemask.h>
 #include <linux/memblock.h>
 #include <linux/fs.h>
+#include <linux/io.h>
 #include <asm/cputype.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sizes.h>
 #include <asm/tlb.h>
 #include <asm/mmu_context.h>
 #include "mm.h"
 /*
  * Empty_zero_page is a special page that is used for zero-initialized data
  * and COW.
  */
 struct page *empty_zero_page;
 EXPORT_SYMBOL(empty_zero_page);
 pgprot_t pgprot_default;
 EXPORT_SYMBOL(pgprot_default);
 static pmdval_t prot_sect_kernel;
 struct cachepolicy {
 	const char	policy[16];
 	u64		mair;
 	u64		tcr;
 };
 static struct cachepolicy cache_policies[] __initdata = {
 	{
 		.policy		= "uncached",
 		.mair		= 0x44,			/* inner, outer non-cacheable */
 		.tcr		= TCR_IRGN_NC | TCR_ORGN_NC,
 	}, {
 		.policy		= "writethrough",
 		.mair		= 0xaa,			/* inner, outer write-through, read-allocate */
 		.tcr		= TCR_IRGN_WT | TCR_ORGN_WT,
 	}, {
 		.policy		= "writeback",
 		.mair		= 0xee,			/* inner, outer write-back, read-allocate */
 		.tcr		= TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
 	}
 };
 /*
  * These are useful for identifying cache coherency problems by allowing the
  * cache or the cache and writebuffer to be turned off. It changes the Normal
  * memory caching attributes in the MAIR_EL1 register.
  */
 static int __init early_cachepolicy(char *p)
 {
 	int i;
 	u64 tmp;
 	for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
 		int len = strlen(cache_policies[i].policy);
 		if (memcmp(p, cache_policies[i].policy, len) == 0)
 			break;
 	}
 	if (i == ARRAY_SIZE(cache_policies)) {
 		pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
 		return 0;
 	}
 	flush_cache_all();
 	/*
 	 * Modify MT_NORMAL attributes in MAIR_EL1.
 	 */
 	asm volatile(
 	"	mrs	%0, mair_el1\n"
 	"	bfi	%0, %1, #%2, #8\n"
 	"	msr	mair_el1, %0\n"
 	"	isb\n"
 	: "=&r" (tmp)
 	: "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
 	/*
 	 * Modify TCR PTW cacheability attributes.
 	 */
 	asm volatile(
 	"	mrs	%0, tcr_el1\n"
 	"	bic	%0, %0, %2\n"
 	"	orr	%0, %0, %1\n"
 	"	msr	tcr_el1, %0\n"
 	"	isb\n"
 	: "=&r" (tmp)
 	: "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
 	flush_cache_all();
 	return 0;
 }
 early_param("cachepolicy", early_cachepolicy);
 /*
  * Adjust the PMD section entries according to the CPU in use.
  */
 static void __init init_mem_pgprot(void)
 {
 	pteval_t default_pgprot;
 	int i;
 	default_pgprot = PTE_ATTRINDX(MT_NORMAL);
 	prot_sect_kernel = PMD_TYPE_SECT | PMD_SECT_AF | PMD_ATTRINDX(MT_NORMAL);
 #ifdef CONFIG_SMP
 	/*
 	 * Mark memory with the "shared" attribute for SMP systems
 	 */
 	default_pgprot |= PTE_SHARED;
 	prot_sect_kernel |= PMD_SECT_S;
 #endif
 	for (i = 0; i < 16; i++) {
 		unsigned long v = pgprot_val(protection_map[i]);
 		protection_map[i] = __pgprot(v | default_pgprot);
 	}
 	pgprot_default = __pgprot(PTE_TYPE_PAGE | PTE_AF | default_pgprot);
 }
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 			      unsigned long size, pgprot_t vma_prot)
 {
 	if (!pfn_valid(pfn))
 		return pgprot_noncached(vma_prot);
 	else if (file->f_flags & O_SYNC)
 		return pgprot_writecombine(vma_prot);
 	return vma_prot;
 }
 EXPORT_SYMBOL(phys_mem_access_prot);
 static void __init *early_alloc(unsigned long sz)
 {
 	void *ptr = __va(memblock_alloc(sz, sz));
 	memset(ptr, 0, sz);
 	return ptr;
 }
 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
 				  unsigned long end, unsigned long pfn)
 {
 	pte_t *pte;
 	if (pmd_none(*pmd)) {
 		pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
 		__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
 	}
 	BUG_ON(pmd_bad(*pmd));
 	pte = pte_offset_kernel(pmd, addr);
 	do {
 		set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
 		pfn++;
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 }
 static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
 				  unsigned long end, phys_addr_t phys)
 {
 	pmd_t *pmd;
 	unsigned long next;
 	/*
 	 * Check for initial section mappings in the pgd/pud and remove them.
 	 */
 	if (pud_none(*pud) || pud_bad(*pud)) {
 		pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
 		pud_populate(&init_mm, pud, pmd);
 	}
 	pmd = pmd_offset(pud, addr);
 	do {
 		next = pmd_addr_end(addr, end);
 		/* try section mapping first */
 		if (((addr | next | phys) & ~SECTION_MASK) == 0)
 			set_pmd(pmd, __pmd(phys | prot_sect_kernel));
 		else
 			alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys));
 		phys += next - addr;
 	} while (pmd++, addr = next, addr != end);
 }
 static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
 				  unsigned long end, unsigned long phys)
 {
 	pud_t *pud = pud_offset(pgd, addr);
 	unsigned long next;
 	do {
 		next = pud_addr_end(addr, end);
 		alloc_init_pmd(pud, addr, next, phys);
 		phys += next - addr;
 	} while (pud++, addr = next, addr != end);
 }
 /*
  * Create the page directory entries and any necessary page tables for the
  * mapping specified by 'md'.
  */
 static void __init create_mapping(phys_addr_t phys, unsigned long virt,
 				  phys_addr_t size)
 {
 	unsigned long addr, length, end, next;
 	pgd_t *pgd;
 	if (virt < VMALLOC_START) {
 		pr_warning("BUG: not creating mapping for 0x%016llx at 0x%016lx - outside kernel range\n",
 			   phys, virt);
 		return;
 	}
 	addr = virt & PAGE_MASK;
 	length = PAGE_ALIGN(size + (virt & ~PAGE_MASK));
 	pgd = pgd_offset_k(addr);
 	end = addr + length;
 	do {
 		next = pgd_addr_end(addr, end);
 		alloc_init_pud(pgd, addr, next, phys);
 		phys += next - addr;
 	} while (pgd++, addr = next, addr != end);
 }
+#ifdef CONFIG_EARLY_PRINTK
+/*
+ * Create an early I/O mapping using the pgd/pmd entries already populated
+ * in head.S as this function is called too early to allocated any memory. The
+ * mapping size is 2MB with 4KB pages or 64KB or 64KB pages.
+ */
+void __iomem * __init early_io_map(phys_addr_t phys, unsigned long virt)
+{
+	unsigned long size, mask;
+	bool page64k = IS_ENABLED(ARM64_64K_PAGES);
+	pgd_t *pgd;
+	pud_t *pud;
+	pmd_t *pmd;
+	pte_t *pte;
+	/*
+	 * No early pte entries with !ARM64_64K_PAGES configuration, so using
+	 * sections (pmd).
+	 */
+	size = page64k ? PAGE_SIZE : SECTION_SIZE;
+	mask = ~(size - 1);
+	pgd = pgd_offset_k(virt);
+	pud = pud_offset(pgd, virt);
+	if (pud_none(*pud))
+		return NULL;
+	pmd = pmd_offset(pud, virt);
+	if (page64k) {
+		if (pmd_none(*pmd))
+			return NULL;
+		pte = pte_offset_kernel(pmd, virt);
+		set_pte(pte, __pte((phys & mask) | PROT_DEVICE_nGnRE));
+	} else {
+		set_pmd(pmd, __pmd((phys & mask) | PROT_SECT_DEVICE_nGnRE));
+	}
+	return (void __iomem *)((virt & mask) + (phys & ~mask));
+}
+#endif
 static void __init map_mem(void)
 {
 	struct memblock_region *reg;
 	/* map all the memory banks */
 	for_each_memblock(memory, reg) {
 		phys_addr_t start = reg->base;
 		phys_addr_t end = start + reg->size;
 		if (start >= end)
 			break;
 		create_mapping(start, __phys_to_virt(start), end - start);
 	}
 }
 /*
  * paging_init() sets up the page tables, initialises the zone memory
  * maps and sets up the zero page.
  */
 void __init paging_init(void)
 {
 	void *zero_page;
 	/*
 	 * Maximum PGDIR_SIZE addressable via the initial direct kernel
 	 * mapping in swapper_pg_dir.
 	 */
 	memblock_set_current_limit((PHYS_OFFSET & PGDIR_MASK) + PGDIR_SIZE);
 	init_mem_pgprot();
 	map_mem();
 	/*
 	 * Finally flush the caches and tlb to ensure that we're in a
 	 * consistent state.
 	 */
 	flush_cache_all();
 	flush_tlb_all();
 	/* allocate the zero page. */
 	zero_page = early_alloc(PAGE_SIZE);
 	bootmem_init();
 	empty_zero_page = virt_to_page(zero_page);
 	__flush_dcache_page(empty_zero_page);
 	/*
 	 * TTBR0 is only used for the identity mapping at this stage. Make it
 	 * point to zero page to avoid speculatively fetching new entries.
 	 */
 	cpu_set_reserved_ttbr0();
 	flush_tlb_all();
 }
 /*
  * Enable the identity mapping to allow the MMU disabling.
  */
 void setup_mm_for_reboot(void)
 {
 	cpu_switch_mm(idmap_pg_dir, &init_mm);
 	flush_tlb_all();
 }
 /*
  * Check whether a kernel address is valid (derived from arch/x86/).
  */
 int kern_addr_valid(unsigned long addr)
 {
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	if ((((long)addr) >> VA_BITS) != -1UL)
 		return 0;
 	pgd = pgd_offset_k(addr);
 	if (pgd_none(*pgd))
 		return 0;
 	pud = pud_offset(pgd, addr);
 	if (pud_none(*pud))
 		return 0;
 	pmd = pmd_offset(pud, addr);
 	if (pmd_none(*pmd))
 		return 0;
 	pte = pte_offset_kernel(pmd, addr);
 	if (pte_none(*pte))
 		return 0;
 	return pfn_valid(pte_pfn(*pte));
 }
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 #ifdef CONFIG_ARM64_64K_PAGES
 int __meminit vmemmap_populate(struct page *start_page,
 			       unsigned long size, int node)
 {
 	return vmemmap_populate_basepages(start_page, size, node);
 }
 #else	/* !CONFIG_ARM64_64K_PAGES */
 int __meminit vmemmap_populate(struct page *start_page,
 			       unsigned long size, int node)
 {
 	unsigned long addr = (unsigned long)start_page;
 	unsigned long end = (unsigned long)(start_page + size);
 	unsigned long next;
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
 	do {
 		next = pmd_addr_end(addr, end);
 		pgd = vmemmap_pgd_populate(addr, node);
 		if (!pgd)
 			return -ENOMEM;
 		pud = vmemmap_pud_populate(pgd, addr, node);
 		if (!pud)
 			return -ENOMEM;
 		pmd = pmd_offset(pud, addr);
 		if (pmd_none(*pmd)) {
 			void *p = NULL;
 			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
 			if (!p)
 				return -ENOMEM;
 			set_pmd(pmd, __pmd(__pa(p) | prot_sect_kernel));
 		} else
 			vmemmap_verify((pte_t *)pmd, node, addr, next);
 	} while (addr = next, addr != end);
 	return 0;
 }
 #endif	/* CONFIG_ARM64_64K_PAGES */
 #endif	/* CONFIG_SPARSEMEM_VMEMMAP */