Commit 82b51734b4f228c76b6064b6e899d9d3d4c17c1a
Exists in
master
and in
16 other branches
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull ARM64 updates from Catalin Marinas: - CPU suspend support on top of PSCI (firmware Power State Coordination Interface) - jump label support - CMA can now be enabled on arm64 - HWCAP bits for crypto and CRC32 extensions - optimised percpu using tpidr_el1 register - code cleanup * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (42 commits) arm64: fix typo in entry.S arm64: kernel: restore HW breakpoint registers in cpu_suspend jump_label: use defined macros instead of hard-coding for better readability arm64, jump label: optimize jump label implementation arm64, jump label: detect %c support for ARM64 arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions arm64: move encode_insn_immediate() from module.c to insn.c arm64: introduce interfaces to hotpatch kernel and module code arm64: introduce basic aarch64 instruction decoding helpers arm64: dts: Reduce size of virtio block device for foundation model arm64: Remove unused __data_loc variable arm64: Enable CMA arm64: Warn on NULL device structure for dma APIs arm64: Add hwcaps for crypto and CRC32 extensions. arm64: drop redundant macros from read_cpuid() arm64: Remove outdated comment arm64: cmpxchg: update macros to prevent warnings arm64: support single-step and breakpoint handler hooks ARM64: fix framepointer check in unwind_frame ARM64: check stack pointer in get_wchan ...
Showing 50 changed files Side-by-side Diff
- arch/arm/kvm/arm.c
- arch/arm64/Kconfig
- arch/arm64/boot/dts/foundation-v8.dts
- arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
- arch/arm64/include/asm/Kbuild
- arch/arm64/include/asm/cmpxchg.h
- arch/arm64/include/asm/cpu_ops.h
- arch/arm64/include/asm/cputype.h
- arch/arm64/include/asm/debug-monitors.h
- arch/arm64/include/asm/dma-contiguous.h
- arch/arm64/include/asm/futex.h
- arch/arm64/include/asm/hardirq.h
- arch/arm64/include/asm/insn.h
- arch/arm64/include/asm/jump_label.h
- arch/arm64/include/asm/memory.h
- arch/arm64/include/asm/percpu.h
- arch/arm64/include/asm/proc-fns.h
- arch/arm64/include/asm/smp_plat.h
- arch/arm64/include/asm/suspend.h
- arch/arm64/include/asm/uaccess.h
- arch/arm64/include/asm/word-at-a-time.h
- arch/arm64/include/uapi/asm/hwcap.h
- arch/arm64/kernel/Makefile
- arch/arm64/kernel/arm64ksyms.c
- arch/arm64/kernel/asm-offsets.c
- arch/arm64/kernel/debug-monitors.c
- arch/arm64/kernel/entry.S
- arch/arm64/kernel/fpsimd.c
- arch/arm64/kernel/head.S
- arch/arm64/kernel/hw_breakpoint.c
- arch/arm64/kernel/insn.c
- arch/arm64/kernel/jump_label.c
- arch/arm64/kernel/module.c
- arch/arm64/kernel/perf_event.c
- arch/arm64/kernel/process.c
- arch/arm64/kernel/setup.c
- arch/arm64/kernel/sleep.S
- arch/arm64/kernel/smp.c
- arch/arm64/kernel/stacktrace.c
- arch/arm64/kernel/suspend.c
- arch/arm64/kernel/vmlinux.lds.S
- arch/arm64/lib/Makefile
- arch/arm64/lib/strncpy_from_user.S
- arch/arm64/lib/strnlen_user.S
- arch/arm64/mm/dma-mapping.c
- arch/arm64/mm/init.c
- arch/arm64/mm/proc.S
- include/linux/irqdesc.h
- include/linux/jump_label.h
- scripts/gcc-goto.sh
arch/arm/kvm/arm.c
... | ... | @@ -17,6 +17,7 @@ |
17 | 17 | */ |
18 | 18 | |
19 | 19 | #include <linux/cpu.h> |
20 | +#include <linux/cpu_pm.h> | |
20 | 21 | #include <linux/errno.h> |
21 | 22 | #include <linux/err.h> |
22 | 23 | #include <linux/kvm_host.h> |
... | ... | @@ -853,6 +854,33 @@ |
853 | 854 | .notifier_call = hyp_init_cpu_notify, |
854 | 855 | }; |
855 | 856 | |
857 | +#ifdef CONFIG_CPU_PM | |
858 | +static int hyp_init_cpu_pm_notifier(struct notifier_block *self, | |
859 | + unsigned long cmd, | |
860 | + void *v) | |
861 | +{ | |
862 | + if (cmd == CPU_PM_EXIT) { | |
863 | + cpu_init_hyp_mode(NULL); | |
864 | + return NOTIFY_OK; | |
865 | + } | |
866 | + | |
867 | + return NOTIFY_DONE; | |
868 | +} | |
869 | + | |
870 | +static struct notifier_block hyp_init_cpu_pm_nb = { | |
871 | + .notifier_call = hyp_init_cpu_pm_notifier, | |
872 | +}; | |
873 | + | |
874 | +static void __init hyp_cpu_pm_init(void) | |
875 | +{ | |
876 | + cpu_pm_register_notifier(&hyp_init_cpu_pm_nb); | |
877 | +} | |
878 | +#else | |
879 | +static inline void hyp_cpu_pm_init(void) | |
880 | +{ | |
881 | +} | |
882 | +#endif | |
883 | + | |
856 | 884 | /** |
857 | 885 | * Inits Hyp-mode on all online CPUs |
858 | 886 | */ |
... | ... | @@ -1012,6 +1040,8 @@ |
1012 | 1040 | kvm_err("Cannot register HYP init CPU notifier (%d)\n", err); |
1013 | 1041 | goto out_err; |
1014 | 1042 | } |
1043 | + | |
1044 | + hyp_cpu_pm_init(); | |
1015 | 1045 | |
1016 | 1046 | kvm_coproc_table_init(); |
1017 | 1047 | return 0; |
arch/arm64/Kconfig
... | ... | @@ -2,6 +2,7 @@ |
2 | 2 | def_bool y |
3 | 3 | select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE |
4 | 4 | select ARCH_USE_CMPXCHG_LOCKREF |
5 | + select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST | |
5 | 6 | select ARCH_WANT_OPTIONAL_GPIOLIB |
6 | 7 | select ARCH_WANT_COMPAT_IPC_PARSE_VERSION |
7 | 8 | select ARCH_WANT_FRAME_POINTERS |
8 | 9 | |
9 | 10 | |
10 | 11 | |
11 | 12 | |
... | ... | @@ -11,19 +12,27 @@ |
11 | 12 | select BUILDTIME_EXTABLE_SORT |
12 | 13 | select CLONE_BACKWARDS |
13 | 14 | select COMMON_CLK |
15 | + select CPU_PM if (SUSPEND || CPU_IDLE) | |
16 | + select DCACHE_WORD_ACCESS | |
14 | 17 | select GENERIC_CLOCKEVENTS |
18 | + select GENERIC_CLOCKEVENTS_BROADCAST if SMP | |
15 | 19 | select GENERIC_IOMAP |
16 | 20 | select GENERIC_IRQ_PROBE |
17 | 21 | select GENERIC_IRQ_SHOW |
18 | 22 | select GENERIC_SCHED_CLOCK |
19 | 23 | select GENERIC_SMP_IDLE_THREAD |
24 | + select GENERIC_STRNCPY_FROM_USER | |
25 | + select GENERIC_STRNLEN_USER | |
20 | 26 | select GENERIC_TIME_VSYSCALL |
21 | 27 | select HARDIRQS_SW_RESEND |
28 | + select HAVE_ARCH_JUMP_LABEL | |
22 | 29 | select HAVE_ARCH_TRACEHOOK |
23 | 30 | select HAVE_DEBUG_BUGVERBOSE |
24 | 31 | select HAVE_DEBUG_KMEMLEAK |
25 | 32 | select HAVE_DMA_API_DEBUG |
26 | 33 | select HAVE_DMA_ATTRS |
34 | + select HAVE_DMA_CONTIGUOUS | |
35 | + select HAVE_EFFICIENT_UNALIGNED_ACCESS | |
27 | 36 | select HAVE_GENERIC_DMA_COHERENT |
28 | 37 | select HAVE_HW_BREAKPOINT if PERF_EVENTS |
29 | 38 | select HAVE_MEMBLOCK |
... | ... | @@ -272,6 +281,24 @@ |
272 | 281 | config SYSVIPC_COMPAT |
273 | 282 | def_bool y |
274 | 283 | depends on COMPAT && SYSVIPC |
284 | + | |
285 | +endmenu | |
286 | + | |
287 | +menu "Power management options" | |
288 | + | |
289 | +source "kernel/power/Kconfig" | |
290 | + | |
291 | +config ARCH_SUSPEND_POSSIBLE | |
292 | + def_bool y | |
293 | + | |
294 | +config ARM64_CPU_SUSPEND | |
295 | + def_bool PM_SLEEP | |
296 | + | |
297 | +endmenu | |
298 | + | |
299 | +menu "CPU Power Management" | |
300 | + | |
301 | +source "drivers/cpuidle/Kconfig" | |
275 | 302 | |
276 | 303 | endmenu |
277 | 304 |
arch/arm64/boot/dts/foundation-v8.dts
arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
... | ... | @@ -183,6 +183,12 @@ |
183 | 183 | clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>; |
184 | 184 | clock-names = "clcdclk", "apb_pclk"; |
185 | 185 | }; |
186 | + | |
187 | + virtio_block@0130000 { | |
188 | + compatible = "virtio,mmio"; | |
189 | + reg = <0x130000 0x200>; | |
190 | + interrupts = <42>; | |
191 | + }; | |
186 | 192 | }; |
187 | 193 | |
188 | 194 | v2m_fixed_3v3: fixedregulator@0 { |
arch/arm64/include/asm/Kbuild
arch/arm64/include/asm/cmpxchg.h
... | ... | @@ -158,17 +158,23 @@ |
158 | 158 | return ret; |
159 | 159 | } |
160 | 160 | |
161 | -#define cmpxchg(ptr,o,n) \ | |
162 | - ((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ | |
163 | - (unsigned long)(o), \ | |
164 | - (unsigned long)(n), \ | |
165 | - sizeof(*(ptr)))) | |
161 | +#define cmpxchg(ptr, o, n) \ | |
162 | +({ \ | |
163 | + __typeof__(*(ptr)) __ret; \ | |
164 | + __ret = (__typeof__(*(ptr))) \ | |
165 | + __cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \ | |
166 | + sizeof(*(ptr))); \ | |
167 | + __ret; \ | |
168 | +}) | |
166 | 169 | |
167 | -#define cmpxchg_local(ptr,o,n) \ | |
168 | - ((__typeof__(*(ptr)))__cmpxchg((ptr), \ | |
169 | - (unsigned long)(o), \ | |
170 | - (unsigned long)(n), \ | |
171 | - sizeof(*(ptr)))) | |
170 | +#define cmpxchg_local(ptr, o, n) \ | |
171 | +({ \ | |
172 | + __typeof__(*(ptr)) __ret; \ | |
173 | + __ret = (__typeof__(*(ptr))) \ | |
174 | + __cmpxchg((ptr), (unsigned long)(o), \ | |
175 | + (unsigned long)(n), sizeof(*(ptr))); \ | |
176 | + __ret; \ | |
177 | +}) | |
172 | 178 | |
173 | 179 | #define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) |
174 | 180 | #define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) |
arch/arm64/include/asm/cpu_ops.h
... | ... | @@ -39,6 +39,9 @@ |
39 | 39 | * from the cpu to be killed. |
40 | 40 | * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the |
41 | 41 | * cpu being killed. |
42 | + * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing | |
43 | + * to wrong parameters or error conditions. Called from the | |
44 | + * CPU being suspended. Must be called with IRQs disabled. | |
42 | 45 | */ |
43 | 46 | struct cpu_operations { |
44 | 47 | const char *name; |
... | ... | @@ -49,6 +52,9 @@ |
49 | 52 | #ifdef CONFIG_HOTPLUG_CPU |
50 | 53 | int (*cpu_disable)(unsigned int cpu); |
51 | 54 | void (*cpu_die)(unsigned int cpu); |
55 | +#endif | |
56 | +#ifdef CONFIG_ARM64_CPU_SUSPEND | |
57 | + int (*cpu_suspend)(unsigned long); | |
52 | 58 | #endif |
53 | 59 | }; |
54 | 60 |
arch/arm64/include/asm/cputype.h
... | ... | @@ -16,23 +16,23 @@ |
16 | 16 | #ifndef __ASM_CPUTYPE_H |
17 | 17 | #define __ASM_CPUTYPE_H |
18 | 18 | |
19 | -#define ID_MIDR_EL1 "midr_el1" | |
20 | -#define ID_MPIDR_EL1 "mpidr_el1" | |
21 | -#define ID_CTR_EL0 "ctr_el0" | |
22 | - | |
23 | -#define ID_AA64PFR0_EL1 "id_aa64pfr0_el1" | |
24 | -#define ID_AA64DFR0_EL1 "id_aa64dfr0_el1" | |
25 | -#define ID_AA64AFR0_EL1 "id_aa64afr0_el1" | |
26 | -#define ID_AA64ISAR0_EL1 "id_aa64isar0_el1" | |
27 | -#define ID_AA64MMFR0_EL1 "id_aa64mmfr0_el1" | |
28 | - | |
29 | 19 | #define INVALID_HWID ULONG_MAX |
30 | 20 | |
31 | 21 | #define MPIDR_HWID_BITMASK 0xff00ffffff |
32 | 22 | |
23 | +#define MPIDR_LEVEL_BITS_SHIFT 3 | |
24 | +#define MPIDR_LEVEL_BITS (1 << MPIDR_LEVEL_BITS_SHIFT) | |
25 | +#define MPIDR_LEVEL_MASK ((1 << MPIDR_LEVEL_BITS) - 1) | |
26 | + | |
27 | +#define MPIDR_LEVEL_SHIFT(level) \ | |
28 | + (((1 << level) >> 1) << MPIDR_LEVEL_BITS_SHIFT) | |
29 | + | |
30 | +#define MPIDR_AFFINITY_LEVEL(mpidr, level) \ | |
31 | + ((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK) | |
32 | + | |
33 | 33 | #define read_cpuid(reg) ({ \ |
34 | 34 | u64 __val; \ |
35 | - asm("mrs %0, " reg : "=r" (__val)); \ | |
35 | + asm("mrs %0, " #reg : "=r" (__val)); \ | |
36 | 36 | __val; \ |
37 | 37 | }) |
38 | 38 | |
39 | 39 | |
... | ... | @@ -54,12 +54,12 @@ |
54 | 54 | */ |
55 | 55 | static inline u32 __attribute_const__ read_cpuid_id(void) |
56 | 56 | { |
57 | - return read_cpuid(ID_MIDR_EL1); | |
57 | + return read_cpuid(MIDR_EL1); | |
58 | 58 | } |
59 | 59 | |
60 | 60 | static inline u64 __attribute_const__ read_cpuid_mpidr(void) |
61 | 61 | { |
62 | - return read_cpuid(ID_MPIDR_EL1); | |
62 | + return read_cpuid(MPIDR_EL1); | |
63 | 63 | } |
64 | 64 | |
65 | 65 | static inline unsigned int __attribute_const__ read_cpuid_implementor(void) |
... | ... | @@ -74,7 +74,7 @@ |
74 | 74 | |
75 | 75 | static inline u32 __attribute_const__ read_cpuid_cachetype(void) |
76 | 76 | { |
77 | - return read_cpuid(ID_CTR_EL0); | |
77 | + return read_cpuid(CTR_EL0); | |
78 | 78 | } |
79 | 79 | |
80 | 80 | #endif /* __ASSEMBLY__ */ |
arch/arm64/include/asm/debug-monitors.h
... | ... | @@ -62,6 +62,27 @@ |
62 | 62 | |
63 | 63 | #define DBG_ARCH_ID_RESERVED 0 /* In case of ptrace ABI updates. */ |
64 | 64 | |
65 | +#define DBG_HOOK_HANDLED 0 | |
66 | +#define DBG_HOOK_ERROR 1 | |
67 | + | |
68 | +struct step_hook { | |
69 | + struct list_head node; | |
70 | + int (*fn)(struct pt_regs *regs, unsigned int esr); | |
71 | +}; | |
72 | + | |
73 | +void register_step_hook(struct step_hook *hook); | |
74 | +void unregister_step_hook(struct step_hook *hook); | |
75 | + | |
76 | +struct break_hook { | |
77 | + struct list_head node; | |
78 | + u32 esr_val; | |
79 | + u32 esr_mask; | |
80 | + int (*fn)(struct pt_regs *regs, unsigned int esr); | |
81 | +}; | |
82 | + | |
83 | +void register_break_hook(struct break_hook *hook); | |
84 | +void unregister_break_hook(struct break_hook *hook); | |
85 | + | |
65 | 86 | u8 debug_monitors_arch(void); |
66 | 87 | |
67 | 88 | void enable_debug_monitors(enum debug_el el); |
arch/arm64/include/asm/dma-contiguous.h
1 | +/* | |
2 | + * Copyright (c) 2013, The Linux Foundation. All rights reserved. | |
3 | + * | |
4 | + * This program is free software; you can redistribute it and/or modify | |
5 | + * it under the terms of the GNU General Public License version 2 and | |
6 | + * only version 2 as published by the Free Software Foundation. | |
7 | + * | |
8 | + * This program is distributed in the hope that it will be useful, | |
9 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | + * GNU General Public License for more details. | |
12 | + */ | |
13 | + | |
14 | +#ifndef _ASM_DMA_CONTIGUOUS_H | |
15 | +#define _ASM_DMA_CONTIGUOUS_H | |
16 | + | |
17 | +#ifdef __KERNEL__ | |
18 | +#ifdef CONFIG_DMA_CMA | |
19 | + | |
20 | +#include <linux/types.h> | |
21 | +#include <asm-generic/dma-contiguous.h> | |
22 | + | |
23 | +static inline void | |
24 | +dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { } | |
25 | + | |
26 | +#endif | |
27 | +#endif | |
28 | + | |
29 | +#endif |
arch/arm64/include/asm/futex.h
arch/arm64/include/asm/hardirq.h
arch/arm64/include/asm/insn.h
1 | +/* | |
2 | + * Copyright (C) 2013 Huawei Ltd. | |
3 | + * Author: Jiang Liu <liuj97@gmail.com> | |
4 | + * | |
5 | + * This program is free software; you can redistribute it and/or modify | |
6 | + * it under the terms of the GNU General Public License version 2 as | |
7 | + * published by the Free Software Foundation. | |
8 | + * | |
9 | + * This program is distributed in the hope that it will be useful, | |
10 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | + * GNU General Public License for more details. | |
13 | + * | |
14 | + * You should have received a copy of the GNU General Public License | |
15 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | + */ | |
17 | +#ifndef __ASM_INSN_H | |
18 | +#define __ASM_INSN_H | |
19 | +#include <linux/types.h> | |
20 | + | |
21 | +/* A64 instructions are always 32 bits. */ | |
22 | +#define AARCH64_INSN_SIZE 4 | |
23 | + | |
24 | +/* | |
25 | + * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a | |
26 | + * Section C3.1 "A64 instruction index by encoding": | |
27 | + * AArch64 main encoding table | |
28 | + * Bit position | |
29 | + * 28 27 26 25 Encoding Group | |
30 | + * 0 0 - - Unallocated | |
31 | + * 1 0 0 - Data processing, immediate | |
32 | + * 1 0 1 - Branch, exception generation and system instructions | |
33 | + * - 1 - 0 Loads and stores | |
34 | + * - 1 0 1 Data processing - register | |
35 | + * 0 1 1 1 Data processing - SIMD and floating point | |
36 | + * 1 1 1 1 Data processing - SIMD and floating point | |
37 | + * "-" means "don't care" | |
38 | + */ | |
39 | +enum aarch64_insn_encoding_class { | |
40 | + AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */ | |
41 | + AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */ | |
42 | + AARCH64_INSN_CLS_DP_REG, /* Data processing - register */ | |
43 | + AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */ | |
44 | + AARCH64_INSN_CLS_LDST, /* Loads and stores */ | |
45 | + AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and | |
46 | + * system instructions */ | |
47 | +}; | |
48 | + | |
49 | +enum aarch64_insn_hint_op { | |
50 | + AARCH64_INSN_HINT_NOP = 0x0 << 5, | |
51 | + AARCH64_INSN_HINT_YIELD = 0x1 << 5, | |
52 | + AARCH64_INSN_HINT_WFE = 0x2 << 5, | |
53 | + AARCH64_INSN_HINT_WFI = 0x3 << 5, | |
54 | + AARCH64_INSN_HINT_SEV = 0x4 << 5, | |
55 | + AARCH64_INSN_HINT_SEVL = 0x5 << 5, | |
56 | +}; | |
57 | + | |
58 | +enum aarch64_insn_imm_type { | |
59 | + AARCH64_INSN_IMM_ADR, | |
60 | + AARCH64_INSN_IMM_26, | |
61 | + AARCH64_INSN_IMM_19, | |
62 | + AARCH64_INSN_IMM_16, | |
63 | + AARCH64_INSN_IMM_14, | |
64 | + AARCH64_INSN_IMM_12, | |
65 | + AARCH64_INSN_IMM_9, | |
66 | + AARCH64_INSN_IMM_MAX | |
67 | +}; | |
68 | + | |
69 | +enum aarch64_insn_branch_type { | |
70 | + AARCH64_INSN_BRANCH_NOLINK, | |
71 | + AARCH64_INSN_BRANCH_LINK, | |
72 | +}; | |
73 | + | |
74 | +#define __AARCH64_INSN_FUNCS(abbr, mask, val) \ | |
75 | +static __always_inline bool aarch64_insn_is_##abbr(u32 code) \ | |
76 | +{ return (code & (mask)) == (val); } \ | |
77 | +static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \ | |
78 | +{ return (val); } | |
79 | + | |
80 | +__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) | |
81 | +__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) | |
82 | +__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) | |
83 | +__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) | |
84 | +__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003) | |
85 | +__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000) | |
86 | +__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F) | |
87 | + | |
88 | +#undef __AARCH64_INSN_FUNCS | |
89 | + | |
90 | +bool aarch64_insn_is_nop(u32 insn); | |
91 | + | |
92 | +int aarch64_insn_read(void *addr, u32 *insnp); | |
93 | +int aarch64_insn_write(void *addr, u32 insn); | |
94 | +enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); | |
95 | +u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, | |
96 | + u32 insn, u64 imm); | |
97 | +u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, | |
98 | + enum aarch64_insn_branch_type type); | |
99 | +u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op); | |
100 | +u32 aarch64_insn_gen_nop(void); | |
101 | + | |
102 | +bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); | |
103 | + | |
104 | +int aarch64_insn_patch_text_nosync(void *addr, u32 insn); | |
105 | +int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); | |
106 | +int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); | |
107 | + | |
108 | +#endif /* __ASM_INSN_H */ |
arch/arm64/include/asm/jump_label.h
1 | +/* | |
2 | + * Copyright (C) 2013 Huawei Ltd. | |
3 | + * Author: Jiang Liu <liuj97@gmail.com> | |
4 | + * | |
5 | + * Based on arch/arm/include/asm/jump_label.h | |
6 | + * | |
7 | + * This program is free software; you can redistribute it and/or modify | |
8 | + * it under the terms of the GNU General Public License version 2 as | |
9 | + * published by the Free Software Foundation. | |
10 | + * | |
11 | + * This program is distributed in the hope that it will be useful, | |
12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | + * GNU General Public License for more details. | |
15 | + * | |
16 | + * You should have received a copy of the GNU General Public License | |
17 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | + */ | |
19 | +#ifndef __ASM_JUMP_LABEL_H | |
20 | +#define __ASM_JUMP_LABEL_H | |
21 | +#include <linux/types.h> | |
22 | +#include <asm/insn.h> | |
23 | + | |
24 | +#ifdef __KERNEL__ | |
25 | + | |
26 | +#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE | |
27 | + | |
28 | +static __always_inline bool arch_static_branch(struct static_key *key) | |
29 | +{ | |
30 | + asm goto("1: nop\n\t" | |
31 | + ".pushsection __jump_table, \"aw\"\n\t" | |
32 | + ".align 3\n\t" | |
33 | + ".quad 1b, %l[l_yes], %c0\n\t" | |
34 | + ".popsection\n\t" | |
35 | + : : "i"(key) : : l_yes); | |
36 | + | |
37 | + return false; | |
38 | +l_yes: | |
39 | + return true; | |
40 | +} | |
41 | + | |
42 | +#endif /* __KERNEL__ */ | |
43 | + | |
44 | +typedef u64 jump_label_t; | |
45 | + | |
46 | +struct jump_entry { | |
47 | + jump_label_t code; | |
48 | + jump_label_t target; | |
49 | + jump_label_t key; | |
50 | +}; | |
51 | + | |
52 | +#endif /* __ASM_JUMP_LABEL_H */ |
arch/arm64/include/asm/memory.h
... | ... | @@ -146,8 +146,7 @@ |
146 | 146 | #define ARCH_PFN_OFFSET PHYS_PFN_OFFSET |
147 | 147 | |
148 | 148 | #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) |
149 | -#define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \ | |
150 | - ((void *)(kaddr) < (void *)high_memory)) | |
149 | +#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) | |
151 | 150 | |
152 | 151 | #endif |
153 | 152 |
arch/arm64/include/asm/percpu.h
1 | +/* | |
2 | + * Copyright (C) 2013 ARM Ltd. | |
3 | + * | |
4 | + * This program is free software; you can redistribute it and/or modify | |
5 | + * it under the terms of the GNU General Public License version 2 as | |
6 | + * published by the Free Software Foundation. | |
7 | + * | |
8 | + * This program is distributed in the hope that it will be useful, | |
9 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | + * GNU General Public License for more details. | |
12 | + * | |
13 | + * You should have received a copy of the GNU General Public License | |
14 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | + */ | |
16 | +#ifndef __ASM_PERCPU_H | |
17 | +#define __ASM_PERCPU_H | |
18 | + | |
19 | +static inline void set_my_cpu_offset(unsigned long off) | |
20 | +{ | |
21 | + asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory"); | |
22 | +} | |
23 | + | |
24 | +static inline unsigned long __my_cpu_offset(void) | |
25 | +{ | |
26 | + unsigned long off; | |
27 | + register unsigned long *sp asm ("sp"); | |
28 | + | |
29 | + /* | |
30 | + * We want to allow caching the value, so avoid using volatile and | |
31 | + * instead use a fake stack read to hazard against barrier(). | |
32 | + */ | |
33 | + asm("mrs %0, tpidr_el1" : "=r" (off) : "Q" (*sp)); | |
34 | + | |
35 | + return off; | |
36 | +} | |
37 | +#define __my_cpu_offset __my_cpu_offset() | |
38 | + | |
39 | +#include <asm-generic/percpu.h> | |
40 | + | |
41 | +#endif /* __ASM_PERCPU_H */ |
arch/arm64/include/asm/proc-fns.h
... | ... | @@ -26,11 +26,14 @@ |
26 | 26 | #include <asm/page.h> |
27 | 27 | |
28 | 28 | struct mm_struct; |
29 | +struct cpu_suspend_ctx; | |
29 | 30 | |
30 | 31 | extern void cpu_cache_off(void); |
31 | 32 | extern void cpu_do_idle(void); |
32 | 33 | extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); |
33 | 34 | extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); |
35 | +extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr); | |
36 | +extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr); | |
34 | 37 | |
35 | 38 | #include <asm/memory.h> |
36 | 39 |
arch/arm64/include/asm/smp_plat.h
... | ... | @@ -21,6 +21,19 @@ |
21 | 21 | |
22 | 22 | #include <asm/types.h> |
23 | 23 | |
24 | +struct mpidr_hash { | |
25 | + u64 mask; | |
26 | + u32 shift_aff[4]; | |
27 | + u32 bits; | |
28 | +}; | |
29 | + | |
30 | +extern struct mpidr_hash mpidr_hash; | |
31 | + | |
32 | +static inline u32 mpidr_hash_size(void) | |
33 | +{ | |
34 | + return 1 << mpidr_hash.bits; | |
35 | +} | |
36 | + | |
24 | 37 | /* |
25 | 38 | * Logical CPU mapping. |
26 | 39 | */ |
arch/arm64/include/asm/suspend.h
1 | +#ifndef __ASM_SUSPEND_H | |
2 | +#define __ASM_SUSPEND_H | |
3 | + | |
4 | +#define NR_CTX_REGS 11 | |
5 | + | |
6 | +/* | |
7 | + * struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on | |
8 | + * the stack, which must be 16-byte aligned on v8 | |
9 | + */ | |
10 | +struct cpu_suspend_ctx { | |
11 | + /* | |
12 | + * This struct must be kept in sync with | |
13 | + * cpu_do_{suspend/resume} in mm/proc.S | |
14 | + */ | |
15 | + u64 ctx_regs[NR_CTX_REGS]; | |
16 | + u64 sp; | |
17 | +} __aligned(16); | |
18 | + | |
19 | +struct sleep_save_sp { | |
20 | + phys_addr_t *save_ptr_stash; | |
21 | + phys_addr_t save_ptr_stash_phys; | |
22 | +}; | |
23 | + | |
24 | +extern void cpu_resume(void); | |
25 | +extern int cpu_suspend(unsigned long); | |
26 | + | |
27 | +#endif |
arch/arm64/include/asm/uaccess.h
... | ... | @@ -100,6 +100,7 @@ |
100 | 100 | }) |
101 | 101 | |
102 | 102 | #define access_ok(type, addr, size) __range_ok(addr, size) |
103 | +#define user_addr_max get_fs | |
103 | 104 | |
104 | 105 | /* |
105 | 106 | * The "__xxx" versions of the user access functions do not verify the address |
... | ... | @@ -240,9 +241,6 @@ |
240 | 241 | extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n); |
241 | 242 | extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); |
242 | 243 | |
243 | -extern unsigned long __must_check __strncpy_from_user(char *to, const char __user *from, unsigned long count); | |
244 | -extern unsigned long __must_check __strnlen_user(const char __user *s, long n); | |
245 | - | |
246 | 244 | static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) |
247 | 245 | { |
248 | 246 | if (access_ok(VERIFY_READ, from, n)) |
249 | 247 | |
... | ... | @@ -276,25 +274,10 @@ |
276 | 274 | return n; |
277 | 275 | } |
278 | 276 | |
279 | -static inline long __must_check strncpy_from_user(char *dst, const char __user *src, long count) | |
280 | -{ | |
281 | - long res = -EFAULT; | |
282 | - if (access_ok(VERIFY_READ, src, 1)) | |
283 | - res = __strncpy_from_user(dst, src, count); | |
284 | - return res; | |
285 | -} | |
277 | +extern long strncpy_from_user(char *dest, const char __user *src, long count); | |
286 | 278 | |
287 | -#define strlen_user(s) strnlen_user(s, ~0UL >> 1) | |
288 | - | |
289 | -static inline long __must_check strnlen_user(const char __user *s, long n) | |
290 | -{ | |
291 | - unsigned long res = 0; | |
292 | - | |
293 | - if (__addr_ok(s)) | |
294 | - res = __strnlen_user(s, n); | |
295 | - | |
296 | - return res; | |
297 | -} | |
279 | +extern __must_check long strlen_user(const char __user *str); | |
280 | +extern __must_check long strnlen_user(const char __user *str, long n); | |
298 | 281 | |
299 | 282 | #endif /* __ASM_UACCESS_H */ |
arch/arm64/include/asm/word-at-a-time.h
1 | +/* | |
2 | + * Copyright (C) 2013 ARM Ltd. | |
3 | + * | |
4 | + * This program is free software; you can redistribute it and/or modify | |
5 | + * it under the terms of the GNU General Public License version 2 as | |
6 | + * published by the Free Software Foundation. | |
7 | + * | |
8 | + * This program is distributed in the hope that it will be useful, | |
9 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | + * GNU General Public License for more details. | |
12 | + * | |
13 | + * You should have received a copy of the GNU General Public License | |
14 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
15 | + */ | |
16 | +#ifndef __ASM_WORD_AT_A_TIME_H | |
17 | +#define __ASM_WORD_AT_A_TIME_H | |
18 | + | |
19 | +#ifndef __AARCH64EB__ | |
20 | + | |
21 | +#include <linux/kernel.h> | |
22 | + | |
23 | +struct word_at_a_time { | |
24 | + const unsigned long one_bits, high_bits; | |
25 | +}; | |
26 | + | |
27 | +#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } | |
28 | + | |
29 | +static inline unsigned long has_zero(unsigned long a, unsigned long *bits, | |
30 | + const struct word_at_a_time *c) | |
31 | +{ | |
32 | + unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits; | |
33 | + *bits = mask; | |
34 | + return mask; | |
35 | +} | |
36 | + | |
37 | +#define prep_zero_mask(a, bits, c) (bits) | |
38 | + | |
39 | +static inline unsigned long create_zero_mask(unsigned long bits) | |
40 | +{ | |
41 | + bits = (bits - 1) & ~bits; | |
42 | + return bits >> 7; | |
43 | +} | |
44 | + | |
45 | +static inline unsigned long find_zero(unsigned long mask) | |
46 | +{ | |
47 | + return fls64(mask) >> 3; | |
48 | +} | |
49 | + | |
50 | +#define zero_bytemask(mask) (mask) | |
51 | + | |
52 | +#else /* __AARCH64EB__ */ | |
53 | +#include <asm-generic/word-at-a-time.h> | |
54 | +#endif | |
55 | + | |
56 | +/* | |
57 | + * Load an unaligned word from kernel space. | |
58 | + * | |
59 | + * In the (very unlikely) case of the word being a page-crosser | |
60 | + * and the next page not being mapped, take the exception and | |
61 | + * return zeroes in the non-existing part. | |
62 | + */ | |
63 | +static inline unsigned long load_unaligned_zeropad(const void *addr) | |
64 | +{ | |
65 | + unsigned long ret, offset; | |
66 | + | |
67 | + /* Load word from unaligned pointer addr */ | |
68 | + asm( | |
69 | + "1: ldr %0, %3\n" | |
70 | + "2:\n" | |
71 | + " .pushsection .fixup,\"ax\"\n" | |
72 | + " .align 2\n" | |
73 | + "3: and %1, %2, #0x7\n" | |
74 | + " bic %2, %2, #0x7\n" | |
75 | + " ldr %0, [%2]\n" | |
76 | + " lsl %1, %1, #0x3\n" | |
77 | +#ifndef __AARCH64EB__ | |
78 | + " lsr %0, %0, %1\n" | |
79 | +#else | |
80 | + " lsl %0, %0, %1\n" | |
81 | +#endif | |
82 | + " b 2b\n" | |
83 | + " .popsection\n" | |
84 | + " .pushsection __ex_table,\"a\"\n" | |
85 | + " .align 3\n" | |
86 | + " .quad 1b, 3b\n" | |
87 | + " .popsection" | |
88 | + : "=&r" (ret), "=&r" (offset) | |
89 | + : "r" (addr), "Q" (*(unsigned long *)addr)); | |
90 | + | |
91 | + return ret; | |
92 | +} | |
93 | + | |
94 | +#endif /* __ASM_WORD_AT_A_TIME_H */ |
arch/arm64/include/uapi/asm/hwcap.h
... | ... | @@ -22,7 +22,11 @@ |
22 | 22 | #define HWCAP_FP (1 << 0) |
23 | 23 | #define HWCAP_ASIMD (1 << 1) |
24 | 24 | #define HWCAP_EVTSTRM (1 << 2) |
25 | - | |
25 | +#define HWCAP_AES (1 << 3) | |
26 | +#define HWCAP_PMULL (1 << 4) | |
27 | +#define HWCAP_SHA1 (1 << 5) | |
28 | +#define HWCAP_SHA2 (1 << 6) | |
29 | +#define HWCAP_CRC32 (1 << 7) | |
26 | 30 | |
27 | 31 | #endif /* _UAPI__ASM_HWCAP_H */ |
arch/arm64/kernel/Makefile
... | ... | @@ -9,7 +9,7 @@ |
9 | 9 | arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ |
10 | 10 | entry-fpsimd.o process.o ptrace.o setup.o signal.o \ |
11 | 11 | sys.o stacktrace.o time.o traps.o io.o vdso.o \ |
12 | - hyp-stub.o psci.o cpu_ops.o | |
12 | + hyp-stub.o psci.o cpu_ops.o insn.o | |
13 | 13 | |
14 | 14 | arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ |
15 | 15 | sys_compat.o |
... | ... | @@ -18,6 +18,8 @@ |
18 | 18 | arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o |
19 | 19 | arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o |
20 | 20 | arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o |
21 | +arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o | |
22 | +arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o | |
21 | 23 | |
22 | 24 | obj-y += $(arm64-obj-y) vdso/ |
23 | 25 | obj-m += $(arm64-obj-m) |
arch/arm64/kernel/arm64ksyms.c
... | ... | @@ -29,13 +29,10 @@ |
29 | 29 | |
30 | 30 | #include <asm/checksum.h> |
31 | 31 | |
32 | - /* user mem (segment) */ | |
33 | -EXPORT_SYMBOL(__strnlen_user); | |
34 | -EXPORT_SYMBOL(__strncpy_from_user); | |
35 | - | |
36 | 32 | EXPORT_SYMBOL(copy_page); |
37 | 33 | EXPORT_SYMBOL(clear_page); |
38 | 34 | |
35 | + /* user mem (segment) */ | |
39 | 36 | EXPORT_SYMBOL(__copy_from_user); |
40 | 37 | EXPORT_SYMBOL(__copy_to_user); |
41 | 38 | EXPORT_SYMBOL(__clear_user); |
arch/arm64/kernel/asm-offsets.c
... | ... | @@ -25,6 +25,8 @@ |
25 | 25 | #include <asm/thread_info.h> |
26 | 26 | #include <asm/memory.h> |
27 | 27 | #include <asm/cputable.h> |
28 | +#include <asm/smp_plat.h> | |
29 | +#include <asm/suspend.h> | |
28 | 30 | #include <asm/vdso_datapage.h> |
29 | 31 | #include <linux/kbuild.h> |
30 | 32 | |
... | ... | @@ -137,6 +139,15 @@ |
137 | 139 | DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr)); |
138 | 140 | DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); |
139 | 141 | DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); |
142 | +#endif | |
143 | +#ifdef CONFIG_ARM64_CPU_SUSPEND | |
144 | + DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); | |
145 | + DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); | |
146 | + DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); | |
147 | + DEFINE(MPIDR_HASH_SHIFTS, offsetof(struct mpidr_hash, shift_aff)); | |
148 | + DEFINE(SLEEP_SAVE_SP_SZ, sizeof(struct sleep_save_sp)); | |
149 | + DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys)); | |
150 | + DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash)); | |
140 | 151 | #endif |
141 | 152 | return 0; |
142 | 153 | } |
arch/arm64/kernel/debug-monitors.c
... | ... | @@ -187,6 +187,48 @@ |
187 | 187 | regs->pstate = spsr; |
188 | 188 | } |
189 | 189 | |
190 | +/* EL1 Single Step Handler hooks */ | |
191 | +static LIST_HEAD(step_hook); | |
192 | +DEFINE_RWLOCK(step_hook_lock); | |
193 | + | |
194 | +void register_step_hook(struct step_hook *hook) | |
195 | +{ | |
196 | + write_lock(&step_hook_lock); | |
197 | + list_add(&hook->node, &step_hook); | |
198 | + write_unlock(&step_hook_lock); | |
199 | +} | |
200 | + | |
201 | +void unregister_step_hook(struct step_hook *hook) | |
202 | +{ | |
203 | + write_lock(&step_hook_lock); | |
204 | + list_del(&hook->node); | |
205 | + write_unlock(&step_hook_lock); | |
206 | +} | |
207 | + | |
208 | +/* | |
209 | + * Call registered single step handers | |
210 | + * There is no Syndrome info to check for determining the handler. | |
211 | + * So we call all the registered handlers, until the right handler is | |
212 | + * found which returns zero. | |
213 | + */ | |
214 | +static int call_step_hook(struct pt_regs *regs, unsigned int esr) | |
215 | +{ | |
216 | + struct step_hook *hook; | |
217 | + int retval = DBG_HOOK_ERROR; | |
218 | + | |
219 | + read_lock(&step_hook_lock); | |
220 | + | |
221 | + list_for_each_entry(hook, &step_hook, node) { | |
222 | + retval = hook->fn(regs, esr); | |
223 | + if (retval == DBG_HOOK_HANDLED) | |
224 | + break; | |
225 | + } | |
226 | + | |
227 | + read_unlock(&step_hook_lock); | |
228 | + | |
229 | + return retval; | |
230 | +} | |
231 | + | |
190 | 232 | static int single_step_handler(unsigned long addr, unsigned int esr, |
191 | 233 | struct pt_regs *regs) |
192 | 234 | { |
... | ... | @@ -214,7 +256,9 @@ |
214 | 256 | */ |
215 | 257 | user_rewind_single_step(current); |
216 | 258 | } else { |
217 | - /* TODO: route to KGDB */ | |
259 | + if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED) | |
260 | + return 0; | |
261 | + | |
218 | 262 | pr_warning("Unexpected kernel single-step exception at EL1\n"); |
219 | 263 | /* |
220 | 264 | * Re-enable stepping since we know that we will be |
221 | 265 | |
... | ... | @@ -226,10 +270,52 @@ |
226 | 270 | return 0; |
227 | 271 | } |
228 | 272 | |
273 | +/* | |
274 | + * Breakpoint handler is re-entrant as another breakpoint can | |
275 | + * hit within breakpoint handler, especically in kprobes. | |
276 | + * Use reader/writer locks instead of plain spinlock. | |
277 | + */ | |
278 | +static LIST_HEAD(break_hook); | |
279 | +DEFINE_RWLOCK(break_hook_lock); | |
280 | + | |
281 | +void register_break_hook(struct break_hook *hook) | |
282 | +{ | |
283 | + write_lock(&break_hook_lock); | |
284 | + list_add(&hook->node, &break_hook); | |
285 | + write_unlock(&break_hook_lock); | |
286 | +} | |
287 | + | |
288 | +void unregister_break_hook(struct break_hook *hook) | |
289 | +{ | |
290 | + write_lock(&break_hook_lock); | |
291 | + list_del(&hook->node); | |
292 | + write_unlock(&break_hook_lock); | |
293 | +} | |
294 | + | |
295 | +static int call_break_hook(struct pt_regs *regs, unsigned int esr) | |
296 | +{ | |
297 | + struct break_hook *hook; | |
298 | + int (*fn)(struct pt_regs *regs, unsigned int esr) = NULL; | |
299 | + | |
300 | + read_lock(&break_hook_lock); | |
301 | + list_for_each_entry(hook, &break_hook, node) | |
302 | + if ((esr & hook->esr_mask) == hook->esr_val) | |
303 | + fn = hook->fn; | |
304 | + read_unlock(&break_hook_lock); | |
305 | + | |
306 | + return fn ? fn(regs, esr) : DBG_HOOK_ERROR; | |
307 | +} | |
308 | + | |
229 | 309 | static int brk_handler(unsigned long addr, unsigned int esr, |
230 | 310 | struct pt_regs *regs) |
231 | 311 | { |
232 | 312 | siginfo_t info; |
313 | + | |
314 | + if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED) | |
315 | + return 0; | |
316 | + | |
317 | + pr_warn("unexpected brk exception at %lx, esr=0x%x\n", | |
318 | + (long)instruction_pointer(regs), esr); | |
233 | 319 | |
234 | 320 | if (!user_mode(regs)) |
235 | 321 | return -EFAULT; |
arch/arm64/kernel/entry.S
... | ... | @@ -288,6 +288,8 @@ |
288 | 288 | /* |
289 | 289 | * Debug exception handling |
290 | 290 | */ |
291 | + cmp x24, #ESR_EL1_EC_BRK64 // if BRK64 | |
292 | + cinc x24, x24, eq // set bit '0' | |
291 | 293 | tbz x24, #0, el1_inv // EL1 only |
292 | 294 | mrs x0, far_el1 |
293 | 295 | mov x2, sp // struct pt_regs |
... | ... | @@ -314,7 +316,7 @@ |
314 | 316 | |
315 | 317 | #ifdef CONFIG_PREEMPT |
316 | 318 | get_thread_info tsk |
317 | - ldr w24, [tsk, #TI_PREEMPT] // restore preempt count | |
319 | + ldr w24, [tsk, #TI_PREEMPT] // get preempt count | |
318 | 320 | cbnz w24, 1f // preempt count != 0 |
319 | 321 | ldr x0, [tsk, #TI_FLAGS] // get flags |
320 | 322 | tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling? |
arch/arm64/kernel/fpsimd.c
... | ... | @@ -17,6 +17,7 @@ |
17 | 17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
18 | 18 | */ |
19 | 19 | |
20 | +#include <linux/cpu_pm.h> | |
20 | 21 | #include <linux/kernel.h> |
21 | 22 | #include <linux/init.h> |
22 | 23 | #include <linux/sched.h> |
... | ... | @@ -113,6 +114,39 @@ |
113 | 114 | |
114 | 115 | #endif /* CONFIG_KERNEL_MODE_NEON */ |
115 | 116 | |
117 | +#ifdef CONFIG_CPU_PM | |
118 | +static int fpsimd_cpu_pm_notifier(struct notifier_block *self, | |
119 | + unsigned long cmd, void *v) | |
120 | +{ | |
121 | + switch (cmd) { | |
122 | + case CPU_PM_ENTER: | |
123 | + if (current->mm) | |
124 | + fpsimd_save_state(¤t->thread.fpsimd_state); | |
125 | + break; | |
126 | + case CPU_PM_EXIT: | |
127 | + if (current->mm) | |
128 | + fpsimd_load_state(¤t->thread.fpsimd_state); | |
129 | + break; | |
130 | + case CPU_PM_ENTER_FAILED: | |
131 | + default: | |
132 | + return NOTIFY_DONE; | |
133 | + } | |
134 | + return NOTIFY_OK; | |
135 | +} | |
136 | + | |
137 | +static struct notifier_block fpsimd_cpu_pm_notifier_block = { | |
138 | + .notifier_call = fpsimd_cpu_pm_notifier, | |
139 | +}; | |
140 | + | |
141 | +static void fpsimd_pm_init(void) | |
142 | +{ | |
143 | + cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); | |
144 | +} | |
145 | + | |
146 | +#else | |
147 | +static inline void fpsimd_pm_init(void) { } | |
148 | +#endif /* CONFIG_CPU_PM */ | |
149 | + | |
116 | 150 | /* |
117 | 151 | * FP/SIMD support code initialisation. |
118 | 152 | */ |
... | ... | @@ -130,6 +164,8 @@ |
130 | 164 | pr_notice("Advanced SIMD is not implemented\n"); |
131 | 165 | else |
132 | 166 | elf_hwcap |= HWCAP_ASIMD; |
167 | + | |
168 | + fpsimd_pm_init(); | |
133 | 169 | |
134 | 170 | return 0; |
135 | 171 | } |
arch/arm64/kernel/head.S
... | ... | @@ -482,8 +482,6 @@ |
482 | 482 | .type __switch_data, %object |
483 | 483 | __switch_data: |
484 | 484 | .quad __mmap_switched |
485 | - .quad __data_loc // x4 | |
486 | - .quad _data // x5 | |
487 | 485 | .quad __bss_start // x6 |
488 | 486 | .quad _end // x7 |
489 | 487 | .quad processor_id // x4 |
490 | 488 | |
... | ... | @@ -498,15 +496,7 @@ |
498 | 496 | __mmap_switched: |
499 | 497 | adr x3, __switch_data + 8 |
500 | 498 | |
501 | - ldp x4, x5, [x3], #16 | |
502 | 499 | ldp x6, x7, [x3], #16 |
503 | - cmp x4, x5 // Copy data segment if needed | |
504 | -1: ccmp x5, x6, #4, ne | |
505 | - b.eq 2f | |
506 | - ldr x16, [x4], #8 | |
507 | - str x16, [x5], #8 | |
508 | - b 1b | |
509 | -2: | |
510 | 500 | 1: cmp x6, x7 |
511 | 501 | b.hs 2f |
512 | 502 | str xzr, [x6], #8 // Clear BSS |
arch/arm64/kernel/hw_breakpoint.c
... | ... | @@ -20,6 +20,7 @@ |
20 | 20 | |
21 | 21 | #define pr_fmt(fmt) "hw-breakpoint: " fmt |
22 | 22 | |
23 | +#include <linux/cpu_pm.h> | |
23 | 24 | #include <linux/errno.h> |
24 | 25 | #include <linux/hw_breakpoint.h> |
25 | 26 | #include <linux/perf_event.h> |
26 | 27 | |
27 | 28 | |
28 | 29 | |
29 | 30 | |
... | ... | @@ -169,15 +170,68 @@ |
169 | 170 | } |
170 | 171 | } |
171 | 172 | |
172 | -/* | |
173 | - * Install a perf counter breakpoint. | |
173 | +enum hw_breakpoint_ops { | |
174 | + HW_BREAKPOINT_INSTALL, | |
175 | + HW_BREAKPOINT_UNINSTALL, | |
176 | + HW_BREAKPOINT_RESTORE | |
177 | +}; | |
178 | + | |
179 | +/** | |
180 | + * hw_breakpoint_slot_setup - Find and setup a perf slot according to | |
181 | + * operations | |
182 | + * | |
183 | + * @slots: pointer to array of slots | |
184 | + * @max_slots: max number of slots | |
185 | + * @bp: perf_event to setup | |
186 | + * @ops: operation to be carried out on the slot | |
187 | + * | |
188 | + * Return: | |
189 | + * slot index on success | |
190 | + * -ENOSPC if no slot is available/matches | |
191 | + * -EINVAL on wrong operations parameter | |
174 | 192 | */ |
175 | -int arch_install_hw_breakpoint(struct perf_event *bp) | |
193 | +static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, | |
194 | + struct perf_event *bp, | |
195 | + enum hw_breakpoint_ops ops) | |
176 | 196 | { |
197 | + int i; | |
198 | + struct perf_event **slot; | |
199 | + | |
200 | + for (i = 0; i < max_slots; ++i) { | |
201 | + slot = &slots[i]; | |
202 | + switch (ops) { | |
203 | + case HW_BREAKPOINT_INSTALL: | |
204 | + if (!*slot) { | |
205 | + *slot = bp; | |
206 | + return i; | |
207 | + } | |
208 | + break; | |
209 | + case HW_BREAKPOINT_UNINSTALL: | |
210 | + if (*slot == bp) { | |
211 | + *slot = NULL; | |
212 | + return i; | |
213 | + } | |
214 | + break; | |
215 | + case HW_BREAKPOINT_RESTORE: | |
216 | + if (*slot == bp) | |
217 | + return i; | |
218 | + break; | |
219 | + default: | |
220 | + pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); | |
221 | + return -EINVAL; | |
222 | + } | |
223 | + } | |
224 | + return -ENOSPC; | |
225 | +} | |
226 | + | |
227 | +static int hw_breakpoint_control(struct perf_event *bp, | |
228 | + enum hw_breakpoint_ops ops) | |
229 | +{ | |
177 | 230 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); |
178 | - struct perf_event **slot, **slots; | |
231 | + struct perf_event **slots; | |
179 | 232 | struct debug_info *debug_info = ¤t->thread.debug; |
180 | 233 | int i, max_slots, ctrl_reg, val_reg, reg_enable; |
234 | + enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege); | |
181 | 235 | u32 ctrl; |
182 | 236 | |
183 | 237 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { |
184 | 238 | |
185 | 239 | |
186 | 240 | |
187 | 241 | |
188 | 242 | |
189 | 243 | |
190 | 244 | |
191 | 245 | |
... | ... | @@ -196,67 +250,54 @@ |
196 | 250 | reg_enable = !debug_info->wps_disabled; |
197 | 251 | } |
198 | 252 | |
199 | - for (i = 0; i < max_slots; ++i) { | |
200 | - slot = &slots[i]; | |
253 | + i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); | |
201 | 254 | |
202 | - if (!*slot) { | |
203 | - *slot = bp; | |
204 | - break; | |
205 | - } | |
206 | - } | |
255 | + if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) | |
256 | + return i; | |
207 | 257 | |
208 | - if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot")) | |
209 | - return -ENOSPC; | |
258 | + switch (ops) { | |
259 | + case HW_BREAKPOINT_INSTALL: | |
260 | + /* | |
261 | + * Ensure debug monitors are enabled at the correct exception | |
262 | + * level. | |
263 | + */ | |
264 | + enable_debug_monitors(dbg_el); | |
265 | + /* Fall through */ | |
266 | + case HW_BREAKPOINT_RESTORE: | |
267 | + /* Setup the address register. */ | |
268 | + write_wb_reg(val_reg, i, info->address); | |
210 | 269 | |
211 | - /* Ensure debug monitors are enabled at the correct exception level. */ | |
212 | - enable_debug_monitors(debug_exception_level(info->ctrl.privilege)); | |
270 | + /* Setup the control register. */ | |
271 | + ctrl = encode_ctrl_reg(info->ctrl); | |
272 | + write_wb_reg(ctrl_reg, i, | |
273 | + reg_enable ? ctrl | 0x1 : ctrl & ~0x1); | |
274 | + break; | |
275 | + case HW_BREAKPOINT_UNINSTALL: | |
276 | + /* Reset the control register. */ | |
277 | + write_wb_reg(ctrl_reg, i, 0); | |
213 | 278 | |
214 | - /* Setup the address register. */ | |
215 | - write_wb_reg(val_reg, i, info->address); | |
279 | + /* | |
280 | + * Release the debug monitors for the correct exception | |
281 | + * level. | |
282 | + */ | |
283 | + disable_debug_monitors(dbg_el); | |
284 | + break; | |
285 | + } | |
216 | 286 | |
217 | - /* Setup the control register. */ | |
218 | - ctrl = encode_ctrl_reg(info->ctrl); | |
219 | - write_wb_reg(ctrl_reg, i, reg_enable ? ctrl | 0x1 : ctrl & ~0x1); | |
220 | - | |
221 | 287 | return 0; |
222 | 288 | } |
223 | 289 | |
224 | -void arch_uninstall_hw_breakpoint(struct perf_event *bp) | |
290 | +/* | |
291 | + * Install a perf counter breakpoint. | |
292 | + */ | |
293 | +int arch_install_hw_breakpoint(struct perf_event *bp) | |
225 | 294 | { |
226 | - struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
227 | - struct perf_event **slot, **slots; | |
228 | - int i, max_slots, base; | |
295 | + return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); | |
296 | +} | |
229 | 297 | |
230 | - if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { | |
231 | - /* Breakpoint */ | |
232 | - base = AARCH64_DBG_REG_BCR; | |
233 | - slots = this_cpu_ptr(bp_on_reg); | |
234 | - max_slots = core_num_brps; | |
235 | - } else { | |
236 | - /* Watchpoint */ | |
237 | - base = AARCH64_DBG_REG_WCR; | |
238 | - slots = this_cpu_ptr(wp_on_reg); | |
239 | - max_slots = core_num_wrps; | |
240 | - } | |
241 | - | |
242 | - /* Remove the breakpoint. */ | |
243 | - for (i = 0; i < max_slots; ++i) { | |
244 | - slot = &slots[i]; | |
245 | - | |
246 | - if (*slot == bp) { | |
247 | - *slot = NULL; | |
248 | - break; | |
249 | - } | |
250 | - } | |
251 | - | |
252 | - if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot")) | |
253 | - return; | |
254 | - | |
255 | - /* Reset the control register. */ | |
256 | - write_wb_reg(base, i, 0); | |
257 | - | |
258 | - /* Release the debug monitors for the correct exception level. */ | |
259 | - disable_debug_monitors(debug_exception_level(info->ctrl.privilege)); | |
298 | +void arch_uninstall_hw_breakpoint(struct perf_event *bp) | |
299 | +{ | |
300 | + hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); | |
260 | 301 | } |
261 | 302 | |
262 | 303 | static int get_hbp_len(u8 hbp_len) |
263 | 304 | |
264 | 305 | |
... | ... | @@ -806,18 +847,36 @@ |
806 | 847 | /* |
807 | 848 | * CPU initialisation. |
808 | 849 | */ |
809 | -static void reset_ctrl_regs(void *unused) | |
850 | +static void hw_breakpoint_reset(void *unused) | |
810 | 851 | { |
811 | 852 | int i; |
812 | - | |
813 | - for (i = 0; i < core_num_brps; ++i) { | |
814 | - write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); | |
815 | - write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); | |
853 | + struct perf_event **slots; | |
854 | + /* | |
855 | + * When a CPU goes through cold-boot, it does not have any installed | |
856 | + * slot, so it is safe to share the same function for restoring and | |
857 | + * resetting breakpoints; when a CPU is hotplugged in, it goes | |
858 | + * through the slots, which are all empty, hence it just resets control | |
859 | + * and value for debug registers. | |
860 | + * When this function is triggered on warm-boot through a CPU PM | |
861 | + * notifier some slots might be initialized; if so they are | |
862 | + * reprogrammed according to the debug slots content. | |
863 | + */ | |
864 | + for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { | |
865 | + if (slots[i]) { | |
866 | + hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); | |
867 | + } else { | |
868 | + write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); | |
869 | + write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); | |
870 | + } | |
816 | 871 | } |
817 | 872 | |
818 | - for (i = 0; i < core_num_wrps; ++i) { | |
819 | - write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); | |
820 | - write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); | |
873 | + for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) { | |
874 | + if (slots[i]) { | |
875 | + hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); | |
876 | + } else { | |
877 | + write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); | |
878 | + write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); | |
879 | + } | |
821 | 880 | } |
822 | 881 | } |
823 | 882 | |
... | ... | @@ -827,7 +886,7 @@ |
827 | 886 | { |
828 | 887 | int cpu = (long)hcpu; |
829 | 888 | if (action == CPU_ONLINE) |
830 | - smp_call_function_single(cpu, reset_ctrl_regs, NULL, 1); | |
889 | + smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1); | |
831 | 890 | return NOTIFY_OK; |
832 | 891 | } |
833 | 892 | |
... | ... | @@ -835,6 +894,14 @@ |
835 | 894 | .notifier_call = hw_breakpoint_reset_notify, |
836 | 895 | }; |
837 | 896 | |
897 | +#ifdef CONFIG_ARM64_CPU_SUSPEND | |
898 | +extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)); | |
899 | +#else | |
900 | +static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) | |
901 | +{ | |
902 | +} | |
903 | +#endif | |
904 | + | |
838 | 905 | /* |
839 | 906 | * One-time initialisation. |
840 | 907 | */ |
... | ... | @@ -850,8 +917,8 @@ |
850 | 917 | * Reset the breakpoint resources. We assume that a halting |
851 | 918 | * debugger will leave the world in a nice state for us. |
852 | 919 | */ |
853 | - smp_call_function(reset_ctrl_regs, NULL, 1); | |
854 | - reset_ctrl_regs(NULL); | |
920 | + smp_call_function(hw_breakpoint_reset, NULL, 1); | |
921 | + hw_breakpoint_reset(NULL); | |
855 | 922 | |
856 | 923 | /* Register debug fault handlers. */ |
857 | 924 | hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, |
... | ... | @@ -861,6 +928,8 @@ |
861 | 928 | |
862 | 929 | /* Register hotplug notifier. */ |
863 | 930 | register_cpu_notifier(&hw_breakpoint_reset_nb); |
931 | + /* Register cpu_suspend hw breakpoint restore hook */ | |
932 | + cpu_suspend_set_dbg_restorer(hw_breakpoint_reset); | |
864 | 933 | |
865 | 934 | return 0; |
866 | 935 | } |
arch/arm64/kernel/insn.c
1 | +/* | |
2 | + * Copyright (C) 2013 Huawei Ltd. | |
3 | + * Author: Jiang Liu <liuj97@gmail.com> | |
4 | + * | |
5 | + * This program is free software; you can redistribute it and/or modify | |
6 | + * it under the terms of the GNU General Public License version 2 as | |
7 | + * published by the Free Software Foundation. | |
8 | + * | |
9 | + * This program is distributed in the hope that it will be useful, | |
10 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | + * GNU General Public License for more details. | |
13 | + * | |
14 | + * You should have received a copy of the GNU General Public License | |
15 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
16 | + */ | |
17 | +#include <linux/bitops.h> | |
18 | +#include <linux/compiler.h> | |
19 | +#include <linux/kernel.h> | |
20 | +#include <linux/smp.h> | |
21 | +#include <linux/stop_machine.h> | |
22 | +#include <linux/uaccess.h> | |
23 | +#include <asm/cacheflush.h> | |
24 | +#include <asm/insn.h> | |
25 | + | |
26 | +static int aarch64_insn_encoding_class[] = { | |
27 | + AARCH64_INSN_CLS_UNKNOWN, | |
28 | + AARCH64_INSN_CLS_UNKNOWN, | |
29 | + AARCH64_INSN_CLS_UNKNOWN, | |
30 | + AARCH64_INSN_CLS_UNKNOWN, | |
31 | + AARCH64_INSN_CLS_LDST, | |
32 | + AARCH64_INSN_CLS_DP_REG, | |
33 | + AARCH64_INSN_CLS_LDST, | |
34 | + AARCH64_INSN_CLS_DP_FPSIMD, | |
35 | + AARCH64_INSN_CLS_DP_IMM, | |
36 | + AARCH64_INSN_CLS_DP_IMM, | |
37 | + AARCH64_INSN_CLS_BR_SYS, | |
38 | + AARCH64_INSN_CLS_BR_SYS, | |
39 | + AARCH64_INSN_CLS_LDST, | |
40 | + AARCH64_INSN_CLS_DP_REG, | |
41 | + AARCH64_INSN_CLS_LDST, | |
42 | + AARCH64_INSN_CLS_DP_FPSIMD, | |
43 | +}; | |
44 | + | |
45 | +enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn) | |
46 | +{ | |
47 | + return aarch64_insn_encoding_class[(insn >> 25) & 0xf]; | |
48 | +} | |
49 | + | |
50 | +/* NOP is an alias of HINT */ | |
51 | +bool __kprobes aarch64_insn_is_nop(u32 insn) | |
52 | +{ | |
53 | + if (!aarch64_insn_is_hint(insn)) | |
54 | + return false; | |
55 | + | |
56 | + switch (insn & 0xFE0) { | |
57 | + case AARCH64_INSN_HINT_YIELD: | |
58 | + case AARCH64_INSN_HINT_WFE: | |
59 | + case AARCH64_INSN_HINT_WFI: | |
60 | + case AARCH64_INSN_HINT_SEV: | |
61 | + case AARCH64_INSN_HINT_SEVL: | |
62 | + return false; | |
63 | + default: | |
64 | + return true; | |
65 | + } | |
66 | +} | |
67 | + | |
68 | +/* | |
69 | + * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always | |
70 | + * little-endian. | |
71 | + */ | |
72 | +int __kprobes aarch64_insn_read(void *addr, u32 *insnp) | |
73 | +{ | |
74 | + int ret; | |
75 | + u32 val; | |
76 | + | |
77 | + ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE); | |
78 | + if (!ret) | |
79 | + *insnp = le32_to_cpu(val); | |
80 | + | |
81 | + return ret; | |
82 | +} | |
83 | + | |
84 | +int __kprobes aarch64_insn_write(void *addr, u32 insn) | |
85 | +{ | |
86 | + insn = cpu_to_le32(insn); | |
87 | + return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE); | |
88 | +} | |
89 | + | |
90 | +static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) | |
91 | +{ | |
92 | + if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS) | |
93 | + return false; | |
94 | + | |
95 | + return aarch64_insn_is_b(insn) || | |
96 | + aarch64_insn_is_bl(insn) || | |
97 | + aarch64_insn_is_svc(insn) || | |
98 | + aarch64_insn_is_hvc(insn) || | |
99 | + aarch64_insn_is_smc(insn) || | |
100 | + aarch64_insn_is_brk(insn) || | |
101 | + aarch64_insn_is_nop(insn); | |
102 | +} | |
103 | + | |
104 | +/* | |
105 | + * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a | |
106 | + * Section B2.6.5 "Concurrent modification and execution of instructions": | |
107 | + * Concurrent modification and execution of instructions can lead to the | |
108 | + * resulting instruction performing any behavior that can be achieved by | |
109 | + * executing any sequence of instructions that can be executed from the | |
110 | + * same Exception level, except where the instruction before modification | |
111 | + * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC, | |
112 | + * or SMC instruction. | |
113 | + */ | |
114 | +bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn) | |
115 | +{ | |
116 | + return __aarch64_insn_hotpatch_safe(old_insn) && | |
117 | + __aarch64_insn_hotpatch_safe(new_insn); | |
118 | +} | |
119 | + | |
120 | +int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) | |
121 | +{ | |
122 | + u32 *tp = addr; | |
123 | + int ret; | |
124 | + | |
125 | + /* A64 instructions must be word aligned */ | |
126 | + if ((uintptr_t)tp & 0x3) | |
127 | + return -EINVAL; | |
128 | + | |
129 | + ret = aarch64_insn_write(tp, insn); | |
130 | + if (ret == 0) | |
131 | + flush_icache_range((uintptr_t)tp, | |
132 | + (uintptr_t)tp + AARCH64_INSN_SIZE); | |
133 | + | |
134 | + return ret; | |
135 | +} | |
136 | + | |
137 | +struct aarch64_insn_patch { | |
138 | + void **text_addrs; | |
139 | + u32 *new_insns; | |
140 | + int insn_cnt; | |
141 | + atomic_t cpu_count; | |
142 | +}; | |
143 | + | |
144 | +static int __kprobes aarch64_insn_patch_text_cb(void *arg) | |
145 | +{ | |
146 | + int i, ret = 0; | |
147 | + struct aarch64_insn_patch *pp = arg; | |
148 | + | |
149 | + /* The first CPU becomes master */ | |
150 | + if (atomic_inc_return(&pp->cpu_count) == 1) { | |
151 | + for (i = 0; ret == 0 && i < pp->insn_cnt; i++) | |
152 | + ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], | |
153 | + pp->new_insns[i]); | |
154 | + /* | |
155 | + * aarch64_insn_patch_text_nosync() calls flush_icache_range(), | |
156 | + * which ends with "dsb; isb" pair guaranteeing global | |
157 | + * visibility. | |
158 | + */ | |
159 | + atomic_set(&pp->cpu_count, -1); | |
160 | + } else { | |
161 | + while (atomic_read(&pp->cpu_count) != -1) | |
162 | + cpu_relax(); | |
163 | + isb(); | |
164 | + } | |
165 | + | |
166 | + return ret; | |
167 | +} | |
168 | + | |
169 | +int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt) | |
170 | +{ | |
171 | + struct aarch64_insn_patch patch = { | |
172 | + .text_addrs = addrs, | |
173 | + .new_insns = insns, | |
174 | + .insn_cnt = cnt, | |
175 | + .cpu_count = ATOMIC_INIT(0), | |
176 | + }; | |
177 | + | |
178 | + if (cnt <= 0) | |
179 | + return -EINVAL; | |
180 | + | |
181 | + return stop_machine(aarch64_insn_patch_text_cb, &patch, | |
182 | + cpu_online_mask); | |
183 | +} | |
184 | + | |
185 | +int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) | |
186 | +{ | |
187 | + int ret; | |
188 | + u32 insn; | |
189 | + | |
190 | + /* Unsafe to patch multiple instructions without synchronizaiton */ | |
191 | + if (cnt == 1) { | |
192 | + ret = aarch64_insn_read(addrs[0], &insn); | |
193 | + if (ret) | |
194 | + return ret; | |
195 | + | |
196 | + if (aarch64_insn_hotpatch_safe(insn, insns[0])) { | |
197 | + /* | |
198 | + * ARMv8 architecture doesn't guarantee all CPUs see | |
199 | + * the new instruction after returning from function | |
200 | + * aarch64_insn_patch_text_nosync(). So send IPIs to | |
201 | + * all other CPUs to achieve instruction | |
202 | + * synchronization. | |
203 | + */ | |
204 | + ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]); | |
205 | + kick_all_cpus_sync(); | |
206 | + return ret; | |
207 | + } | |
208 | + } | |
209 | + | |
210 | + return aarch64_insn_patch_text_sync(addrs, insns, cnt); | |
211 | +} | |
212 | + | |
213 | +u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, | |
214 | + u32 insn, u64 imm) | |
215 | +{ | |
216 | + u32 immlo, immhi, lomask, himask, mask; | |
217 | + int shift; | |
218 | + | |
219 | + switch (type) { | |
220 | + case AARCH64_INSN_IMM_ADR: | |
221 | + lomask = 0x3; | |
222 | + himask = 0x7ffff; | |
223 | + immlo = imm & lomask; | |
224 | + imm >>= 2; | |
225 | + immhi = imm & himask; | |
226 | + imm = (immlo << 24) | (immhi); | |
227 | + mask = (lomask << 24) | (himask); | |
228 | + shift = 5; | |
229 | + break; | |
230 | + case AARCH64_INSN_IMM_26: | |
231 | + mask = BIT(26) - 1; | |
232 | + shift = 0; | |
233 | + break; | |
234 | + case AARCH64_INSN_IMM_19: | |
235 | + mask = BIT(19) - 1; | |
236 | + shift = 5; | |
237 | + break; | |
238 | + case AARCH64_INSN_IMM_16: | |
239 | + mask = BIT(16) - 1; | |
240 | + shift = 5; | |
241 | + break; | |
242 | + case AARCH64_INSN_IMM_14: | |
243 | + mask = BIT(14) - 1; | |
244 | + shift = 5; | |
245 | + break; | |
246 | + case AARCH64_INSN_IMM_12: | |
247 | + mask = BIT(12) - 1; | |
248 | + shift = 10; | |
249 | + break; | |
250 | + case AARCH64_INSN_IMM_9: | |
251 | + mask = BIT(9) - 1; | |
252 | + shift = 12; | |
253 | + break; | |
254 | + default: | |
255 | + pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", | |
256 | + type); | |
257 | + return 0; | |
258 | + } | |
259 | + | |
260 | + /* Update the immediate field. */ | |
261 | + insn &= ~(mask << shift); | |
262 | + insn |= (imm & mask) << shift; | |
263 | + | |
264 | + return insn; | |
265 | +} | |
266 | + | |
267 | +u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, | |
268 | + enum aarch64_insn_branch_type type) | |
269 | +{ | |
270 | + u32 insn; | |
271 | + long offset; | |
272 | + | |
273 | + /* | |
274 | + * PC: A 64-bit Program Counter holding the address of the current | |
275 | + * instruction. A64 instructions must be word-aligned. | |
276 | + */ | |
277 | + BUG_ON((pc & 0x3) || (addr & 0x3)); | |
278 | + | |
279 | + /* | |
280 | + * B/BL support [-128M, 128M) offset | |
281 | + * ARM64 virtual address arrangement guarantees all kernel and module | |
282 | + * texts are within +/-128M. | |
283 | + */ | |
284 | + offset = ((long)addr - (long)pc); | |
285 | + BUG_ON(offset < -SZ_128M || offset >= SZ_128M); | |
286 | + | |
287 | + if (type == AARCH64_INSN_BRANCH_LINK) | |
288 | + insn = aarch64_insn_get_bl_value(); | |
289 | + else | |
290 | + insn = aarch64_insn_get_b_value(); | |
291 | + | |
292 | + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, | |
293 | + offset >> 2); | |
294 | +} | |
295 | + | |
296 | +u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op) | |
297 | +{ | |
298 | + return aarch64_insn_get_hint_value() | op; | |
299 | +} | |
300 | + | |
301 | +u32 __kprobes aarch64_insn_gen_nop(void) | |
302 | +{ | |
303 | + return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP); | |
304 | +} |
arch/arm64/kernel/jump_label.c
1 | +/* | |
2 | + * Copyright (C) 2013 Huawei Ltd. | |
3 | + * Author: Jiang Liu <liuj97@gmail.com> | |
4 | + * | |
5 | + * Based on arch/arm/kernel/jump_label.c | |
6 | + * | |
7 | + * This program is free software; you can redistribute it and/or modify | |
8 | + * it under the terms of the GNU General Public License version 2 as | |
9 | + * published by the Free Software Foundation. | |
10 | + * | |
11 | + * This program is distributed in the hope that it will be useful, | |
12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | + * GNU General Public License for more details. | |
15 | + * | |
16 | + * You should have received a copy of the GNU General Public License | |
17 | + * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | + */ | |
19 | +#include <linux/kernel.h> | |
20 | +#include <linux/jump_label.h> | |
21 | +#include <asm/insn.h> | |
22 | + | |
23 | +#ifdef HAVE_JUMP_LABEL | |
24 | + | |
25 | +static void __arch_jump_label_transform(struct jump_entry *entry, | |
26 | + enum jump_label_type type, | |
27 | + bool is_static) | |
28 | +{ | |
29 | + void *addr = (void *)entry->code; | |
30 | + u32 insn; | |
31 | + | |
32 | + if (type == JUMP_LABEL_ENABLE) { | |
33 | + insn = aarch64_insn_gen_branch_imm(entry->code, | |
34 | + entry->target, | |
35 | + AARCH64_INSN_BRANCH_NOLINK); | |
36 | + } else { | |
37 | + insn = aarch64_insn_gen_nop(); | |
38 | + } | |
39 | + | |
40 | + if (is_static) | |
41 | + aarch64_insn_patch_text_nosync(addr, insn); | |
42 | + else | |
43 | + aarch64_insn_patch_text(&addr, &insn, 1); | |
44 | +} | |
45 | + | |
46 | +void arch_jump_label_transform(struct jump_entry *entry, | |
47 | + enum jump_label_type type) | |
48 | +{ | |
49 | + __arch_jump_label_transform(entry, type, false); | |
50 | +} | |
51 | + | |
52 | +void arch_jump_label_transform_static(struct jump_entry *entry, | |
53 | + enum jump_label_type type) | |
54 | +{ | |
55 | + __arch_jump_label_transform(entry, type, true); | |
56 | +} | |
57 | + | |
58 | +#endif /* HAVE_JUMP_LABEL */ |
arch/arm64/kernel/module.c
... | ... | @@ -25,7 +25,11 @@ |
25 | 25 | #include <linux/mm.h> |
26 | 26 | #include <linux/moduleloader.h> |
27 | 27 | #include <linux/vmalloc.h> |
28 | +#include <asm/insn.h> | |
28 | 29 | |
30 | +#define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX | |
31 | +#define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16 | |
32 | + | |
29 | 33 | void *module_alloc(unsigned long size) |
30 | 34 | { |
31 | 35 | return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, |
32 | 36 | |
33 | 37 | |
34 | 38 | |
... | ... | @@ -94,28 +98,18 @@ |
94 | 98 | return 0; |
95 | 99 | } |
96 | 100 | |
97 | -enum aarch64_imm_type { | |
98 | - INSN_IMM_MOVNZ, | |
99 | - INSN_IMM_MOVK, | |
100 | - INSN_IMM_ADR, | |
101 | - INSN_IMM_26, | |
102 | - INSN_IMM_19, | |
103 | - INSN_IMM_16, | |
104 | - INSN_IMM_14, | |
105 | - INSN_IMM_12, | |
106 | - INSN_IMM_9, | |
107 | -}; | |
108 | - | |
109 | -static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm) | |
101 | +static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val, | |
102 | + int lsb, enum aarch64_insn_imm_type imm_type) | |
110 | 103 | { |
111 | - u32 immlo, immhi, lomask, himask, mask; | |
112 | - int shift; | |
104 | + u64 imm, limit = 0; | |
105 | + s64 sval; | |
106 | + u32 insn = le32_to_cpu(*(u32 *)place); | |
113 | 107 | |
114 | - /* The instruction stream is always little endian. */ | |
115 | - insn = le32_to_cpu(insn); | |
108 | + sval = do_reloc(op, place, val); | |
109 | + sval >>= lsb; | |
110 | + imm = sval & 0xffff; | |
116 | 111 | |
117 | - switch (type) { | |
118 | - case INSN_IMM_MOVNZ: | |
112 | + if (imm_type == AARCH64_INSN_IMM_MOVNZ) { | |
119 | 113 | /* |
120 | 114 | * For signed MOVW relocations, we have to manipulate the |
121 | 115 | * instruction encoding depending on whether or not the |
122 | 116 | |
123 | 117 | |
... | ... | @@ -134,70 +128,12 @@ |
134 | 128 | */ |
135 | 129 | imm = ~imm; |
136 | 130 | } |
137 | - case INSN_IMM_MOVK: | |
138 | - mask = BIT(16) - 1; | |
139 | - shift = 5; | |
140 | - break; | |
141 | - case INSN_IMM_ADR: | |
142 | - lomask = 0x3; | |
143 | - himask = 0x7ffff; | |
144 | - immlo = imm & lomask; | |
145 | - imm >>= 2; | |
146 | - immhi = imm & himask; | |
147 | - imm = (immlo << 24) | (immhi); | |
148 | - mask = (lomask << 24) | (himask); | |
149 | - shift = 5; | |
150 | - break; | |
151 | - case INSN_IMM_26: | |
152 | - mask = BIT(26) - 1; | |
153 | - shift = 0; | |
154 | - break; | |
155 | - case INSN_IMM_19: | |
156 | - mask = BIT(19) - 1; | |
157 | - shift = 5; | |
158 | - break; | |
159 | - case INSN_IMM_16: | |
160 | - mask = BIT(16) - 1; | |
161 | - shift = 5; | |
162 | - break; | |
163 | - case INSN_IMM_14: | |
164 | - mask = BIT(14) - 1; | |
165 | - shift = 5; | |
166 | - break; | |
167 | - case INSN_IMM_12: | |
168 | - mask = BIT(12) - 1; | |
169 | - shift = 10; | |
170 | - break; | |
171 | - case INSN_IMM_9: | |
172 | - mask = BIT(9) - 1; | |
173 | - shift = 12; | |
174 | - break; | |
175 | - default: | |
176 | - pr_err("encode_insn_immediate: unknown immediate encoding %d\n", | |
177 | - type); | |
178 | - return 0; | |
131 | + imm_type = AARCH64_INSN_IMM_MOVK; | |
179 | 132 | } |
180 | 133 | |
181 | - /* Update the immediate field. */ | |
182 | - insn &= ~(mask << shift); | |
183 | - insn |= (imm & mask) << shift; | |
184 | - | |
185 | - return cpu_to_le32(insn); | |
186 | -} | |
187 | - | |
188 | -static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val, | |
189 | - int lsb, enum aarch64_imm_type imm_type) | |
190 | -{ | |
191 | - u64 imm, limit = 0; | |
192 | - s64 sval; | |
193 | - u32 insn = *(u32 *)place; | |
194 | - | |
195 | - sval = do_reloc(op, place, val); | |
196 | - sval >>= lsb; | |
197 | - imm = sval & 0xffff; | |
198 | - | |
199 | 134 | /* Update the instruction with the new encoding. */ |
200 | - *(u32 *)place = encode_insn_immediate(imm_type, insn, imm); | |
135 | + insn = aarch64_insn_encode_immediate(imm_type, insn, imm); | |
136 | + *(u32 *)place = cpu_to_le32(insn); | |
201 | 137 | |
202 | 138 | /* Shift out the immediate field. */ |
203 | 139 | sval >>= 16; |
204 | 140 | |
... | ... | @@ -206,9 +142,9 @@ |
206 | 142 | * For unsigned immediates, the overflow check is straightforward. |
207 | 143 | * For signed immediates, the sign bit is actually the bit past the |
208 | 144 | * most significant bit of the field. |
209 | - * The INSN_IMM_16 immediate type is unsigned. | |
145 | + * The AARCH64_INSN_IMM_16 immediate type is unsigned. | |
210 | 146 | */ |
211 | - if (imm_type != INSN_IMM_16) { | |
147 | + if (imm_type != AARCH64_INSN_IMM_16) { | |
212 | 148 | sval++; |
213 | 149 | limit++; |
214 | 150 | } |
215 | 151 | |
... | ... | @@ -221,11 +157,11 @@ |
221 | 157 | } |
222 | 158 | |
223 | 159 | static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val, |
224 | - int lsb, int len, enum aarch64_imm_type imm_type) | |
160 | + int lsb, int len, enum aarch64_insn_imm_type imm_type) | |
225 | 161 | { |
226 | 162 | u64 imm, imm_mask; |
227 | 163 | s64 sval; |
228 | - u32 insn = *(u32 *)place; | |
164 | + u32 insn = le32_to_cpu(*(u32 *)place); | |
229 | 165 | |
230 | 166 | /* Calculate the relocation value. */ |
231 | 167 | sval = do_reloc(op, place, val); |
... | ... | @@ -236,7 +172,8 @@ |
236 | 172 | imm = sval & imm_mask; |
237 | 173 | |
238 | 174 | /* Update the instruction's immediate field. */ |
239 | - *(u32 *)place = encode_insn_immediate(imm_type, insn, imm); | |
175 | + insn = aarch64_insn_encode_immediate(imm_type, insn, imm); | |
176 | + *(u32 *)place = cpu_to_le32(insn); | |
240 | 177 | |
241 | 178 | /* |
242 | 179 | * Extract the upper value bits (including the sign bit) and |
243 | 180 | |
244 | 181 | |
245 | 182 | |
246 | 183 | |
247 | 184 | |
248 | 185 | |
249 | 186 | |
250 | 187 | |
251 | 188 | |
252 | 189 | |
253 | 190 | |
254 | 191 | |
255 | 192 | |
256 | 193 | |
257 | 194 | |
258 | 195 | |
259 | 196 | |
260 | 197 | |
261 | 198 | |
262 | 199 | |
263 | 200 | |
264 | 201 | |
265 | 202 | |
266 | 203 | |
... | ... | @@ -318,125 +255,125 @@ |
318 | 255 | overflow_check = false; |
319 | 256 | case R_AARCH64_MOVW_UABS_G0: |
320 | 257 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, |
321 | - INSN_IMM_16); | |
258 | + AARCH64_INSN_IMM_16); | |
322 | 259 | break; |
323 | 260 | case R_AARCH64_MOVW_UABS_G1_NC: |
324 | 261 | overflow_check = false; |
325 | 262 | case R_AARCH64_MOVW_UABS_G1: |
326 | 263 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, |
327 | - INSN_IMM_16); | |
264 | + AARCH64_INSN_IMM_16); | |
328 | 265 | break; |
329 | 266 | case R_AARCH64_MOVW_UABS_G2_NC: |
330 | 267 | overflow_check = false; |
331 | 268 | case R_AARCH64_MOVW_UABS_G2: |
332 | 269 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, |
333 | - INSN_IMM_16); | |
270 | + AARCH64_INSN_IMM_16); | |
334 | 271 | break; |
335 | 272 | case R_AARCH64_MOVW_UABS_G3: |
336 | 273 | /* We're using the top bits so we can't overflow. */ |
337 | 274 | overflow_check = false; |
338 | 275 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48, |
339 | - INSN_IMM_16); | |
276 | + AARCH64_INSN_IMM_16); | |
340 | 277 | break; |
341 | 278 | case R_AARCH64_MOVW_SABS_G0: |
342 | 279 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0, |
343 | - INSN_IMM_MOVNZ); | |
280 | + AARCH64_INSN_IMM_MOVNZ); | |
344 | 281 | break; |
345 | 282 | case R_AARCH64_MOVW_SABS_G1: |
346 | 283 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16, |
347 | - INSN_IMM_MOVNZ); | |
284 | + AARCH64_INSN_IMM_MOVNZ); | |
348 | 285 | break; |
349 | 286 | case R_AARCH64_MOVW_SABS_G2: |
350 | 287 | ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32, |
351 | - INSN_IMM_MOVNZ); | |
288 | + AARCH64_INSN_IMM_MOVNZ); | |
352 | 289 | break; |
353 | 290 | case R_AARCH64_MOVW_PREL_G0_NC: |
354 | 291 | overflow_check = false; |
355 | 292 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, |
356 | - INSN_IMM_MOVK); | |
293 | + AARCH64_INSN_IMM_MOVK); | |
357 | 294 | break; |
358 | 295 | case R_AARCH64_MOVW_PREL_G0: |
359 | 296 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0, |
360 | - INSN_IMM_MOVNZ); | |
297 | + AARCH64_INSN_IMM_MOVNZ); | |
361 | 298 | break; |
362 | 299 | case R_AARCH64_MOVW_PREL_G1_NC: |
363 | 300 | overflow_check = false; |
364 | 301 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, |
365 | - INSN_IMM_MOVK); | |
302 | + AARCH64_INSN_IMM_MOVK); | |
366 | 303 | break; |
367 | 304 | case R_AARCH64_MOVW_PREL_G1: |
368 | 305 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16, |
369 | - INSN_IMM_MOVNZ); | |
306 | + AARCH64_INSN_IMM_MOVNZ); | |
370 | 307 | break; |
371 | 308 | case R_AARCH64_MOVW_PREL_G2_NC: |
372 | 309 | overflow_check = false; |
373 | 310 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, |
374 | - INSN_IMM_MOVK); | |
311 | + AARCH64_INSN_IMM_MOVK); | |
375 | 312 | break; |
376 | 313 | case R_AARCH64_MOVW_PREL_G2: |
377 | 314 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32, |
378 | - INSN_IMM_MOVNZ); | |
315 | + AARCH64_INSN_IMM_MOVNZ); | |
379 | 316 | break; |
380 | 317 | case R_AARCH64_MOVW_PREL_G3: |
381 | 318 | /* We're using the top bits so we can't overflow. */ |
382 | 319 | overflow_check = false; |
383 | 320 | ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48, |
384 | - INSN_IMM_MOVNZ); | |
321 | + AARCH64_INSN_IMM_MOVNZ); | |
385 | 322 | break; |
386 | 323 | |
387 | 324 | /* Immediate instruction relocations. */ |
388 | 325 | case R_AARCH64_LD_PREL_LO19: |
389 | 326 | ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, |
390 | - INSN_IMM_19); | |
327 | + AARCH64_INSN_IMM_19); | |
391 | 328 | break; |
392 | 329 | case R_AARCH64_ADR_PREL_LO21: |
393 | 330 | ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21, |
394 | - INSN_IMM_ADR); | |
331 | + AARCH64_INSN_IMM_ADR); | |
395 | 332 | break; |
396 | 333 | case R_AARCH64_ADR_PREL_PG_HI21_NC: |
397 | 334 | overflow_check = false; |
398 | 335 | case R_AARCH64_ADR_PREL_PG_HI21: |
399 | 336 | ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21, |
400 | - INSN_IMM_ADR); | |
337 | + AARCH64_INSN_IMM_ADR); | |
401 | 338 | break; |
402 | 339 | case R_AARCH64_ADD_ABS_LO12_NC: |
403 | 340 | case R_AARCH64_LDST8_ABS_LO12_NC: |
404 | 341 | overflow_check = false; |
405 | 342 | ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12, |
406 | - INSN_IMM_12); | |
343 | + AARCH64_INSN_IMM_12); | |
407 | 344 | break; |
408 | 345 | case R_AARCH64_LDST16_ABS_LO12_NC: |
409 | 346 | overflow_check = false; |
410 | 347 | ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11, |
411 | - INSN_IMM_12); | |
348 | + AARCH64_INSN_IMM_12); | |
412 | 349 | break; |
413 | 350 | case R_AARCH64_LDST32_ABS_LO12_NC: |
414 | 351 | overflow_check = false; |
415 | 352 | ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10, |
416 | - INSN_IMM_12); | |
353 | + AARCH64_INSN_IMM_12); | |
417 | 354 | break; |
418 | 355 | case R_AARCH64_LDST64_ABS_LO12_NC: |
419 | 356 | overflow_check = false; |
420 | 357 | ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9, |
421 | - INSN_IMM_12); | |
358 | + AARCH64_INSN_IMM_12); | |
422 | 359 | break; |
423 | 360 | case R_AARCH64_LDST128_ABS_LO12_NC: |
424 | 361 | overflow_check = false; |
425 | 362 | ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8, |
426 | - INSN_IMM_12); | |
363 | + AARCH64_INSN_IMM_12); | |
427 | 364 | break; |
428 | 365 | case R_AARCH64_TSTBR14: |
429 | 366 | ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14, |
430 | - INSN_IMM_14); | |
367 | + AARCH64_INSN_IMM_14); | |
431 | 368 | break; |
432 | 369 | case R_AARCH64_CONDBR19: |
433 | 370 | ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19, |
434 | - INSN_IMM_19); | |
371 | + AARCH64_INSN_IMM_19); | |
435 | 372 | break; |
436 | 373 | case R_AARCH64_JUMP26: |
437 | 374 | case R_AARCH64_CALL26: |
438 | 375 | ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26, |
439 | - INSN_IMM_26); | |
376 | + AARCH64_INSN_IMM_26); | |
440 | 377 | break; |
441 | 378 | |
442 | 379 | default: |
arch/arm64/kernel/perf_event.c
... | ... | @@ -22,6 +22,7 @@ |
22 | 22 | |
23 | 23 | #include <linux/bitmap.h> |
24 | 24 | #include <linux/interrupt.h> |
25 | +#include <linux/irq.h> | |
25 | 26 | #include <linux/kernel.h> |
26 | 27 | #include <linux/export.h> |
27 | 28 | #include <linux/perf_event.h> |
28 | 29 | |
29 | 30 | |
30 | 31 | |
31 | 32 | |
32 | 33 | |
... | ... | @@ -363,26 +364,53 @@ |
363 | 364 | } |
364 | 365 | |
365 | 366 | static void |
367 | +armpmu_disable_percpu_irq(void *data) | |
368 | +{ | |
369 | + unsigned int irq = *(unsigned int *)data; | |
370 | + disable_percpu_irq(irq); | |
371 | +} | |
372 | + | |
373 | +static void | |
366 | 374 | armpmu_release_hardware(struct arm_pmu *armpmu) |
367 | 375 | { |
368 | - int i, irq, irqs; | |
376 | + int irq; | |
377 | + unsigned int i, irqs; | |
369 | 378 | struct platform_device *pmu_device = armpmu->plat_device; |
370 | 379 | |
371 | 380 | irqs = min(pmu_device->num_resources, num_possible_cpus()); |
381 | + if (!irqs) | |
382 | + return; | |
372 | 383 | |
373 | - for (i = 0; i < irqs; ++i) { | |
374 | - if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) | |
375 | - continue; | |
376 | - irq = platform_get_irq(pmu_device, i); | |
377 | - if (irq >= 0) | |
378 | - free_irq(irq, armpmu); | |
384 | + irq = platform_get_irq(pmu_device, 0); | |
385 | + if (irq <= 0) | |
386 | + return; | |
387 | + | |
388 | + if (irq_is_percpu(irq)) { | |
389 | + on_each_cpu(armpmu_disable_percpu_irq, &irq, 1); | |
390 | + free_percpu_irq(irq, &cpu_hw_events); | |
391 | + } else { | |
392 | + for (i = 0; i < irqs; ++i) { | |
393 | + if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) | |
394 | + continue; | |
395 | + irq = platform_get_irq(pmu_device, i); | |
396 | + if (irq > 0) | |
397 | + free_irq(irq, armpmu); | |
398 | + } | |
379 | 399 | } |
380 | 400 | } |
381 | 401 | |
402 | +static void | |
403 | +armpmu_enable_percpu_irq(void *data) | |
404 | +{ | |
405 | + unsigned int irq = *(unsigned int *)data; | |
406 | + enable_percpu_irq(irq, IRQ_TYPE_NONE); | |
407 | +} | |
408 | + | |
382 | 409 | static int |
383 | 410 | armpmu_reserve_hardware(struct arm_pmu *armpmu) |
384 | 411 | { |
385 | - int i, err, irq, irqs; | |
412 | + int err, irq; | |
413 | + unsigned int i, irqs; | |
386 | 414 | struct platform_device *pmu_device = armpmu->plat_device; |
387 | 415 | |
388 | 416 | if (!pmu_device) { |
389 | 417 | |
390 | 418 | |
391 | 419 | |
392 | 420 | |
393 | 421 | |
... | ... | @@ -391,39 +419,59 @@ |
391 | 419 | } |
392 | 420 | |
393 | 421 | irqs = min(pmu_device->num_resources, num_possible_cpus()); |
394 | - if (irqs < 1) { | |
422 | + if (!irqs) { | |
395 | 423 | pr_err("no irqs for PMUs defined\n"); |
396 | 424 | return -ENODEV; |
397 | 425 | } |
398 | 426 | |
399 | - for (i = 0; i < irqs; ++i) { | |
400 | - err = 0; | |
401 | - irq = platform_get_irq(pmu_device, i); | |
402 | - if (irq < 0) | |
403 | - continue; | |
427 | + irq = platform_get_irq(pmu_device, 0); | |
428 | + if (irq <= 0) { | |
429 | + pr_err("failed to get valid irq for PMU device\n"); | |
430 | + return -ENODEV; | |
431 | + } | |
404 | 432 | |
405 | - /* | |
406 | - * If we have a single PMU interrupt that we can't shift, | |
407 | - * assume that we're running on a uniprocessor machine and | |
408 | - * continue. Otherwise, continue without this interrupt. | |
409 | - */ | |
410 | - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { | |
411 | - pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", | |
412 | - irq, i); | |
413 | - continue; | |
414 | - } | |
433 | + if (irq_is_percpu(irq)) { | |
434 | + err = request_percpu_irq(irq, armpmu->handle_irq, | |
435 | + "arm-pmu", &cpu_hw_events); | |
415 | 436 | |
416 | - err = request_irq(irq, armpmu->handle_irq, | |
417 | - IRQF_NOBALANCING, | |
418 | - "arm-pmu", armpmu); | |
419 | 437 | if (err) { |
420 | - pr_err("unable to request IRQ%d for ARM PMU counters\n", | |
421 | - irq); | |
438 | + pr_err("unable to request percpu IRQ%d for ARM PMU counters\n", | |
439 | + irq); | |
422 | 440 | armpmu_release_hardware(armpmu); |
423 | 441 | return err; |
424 | 442 | } |
425 | 443 | |
426 | - cpumask_set_cpu(i, &armpmu->active_irqs); | |
444 | + on_each_cpu(armpmu_enable_percpu_irq, &irq, 1); | |
445 | + } else { | |
446 | + for (i = 0; i < irqs; ++i) { | |
447 | + err = 0; | |
448 | + irq = platform_get_irq(pmu_device, i); | |
449 | + if (irq <= 0) | |
450 | + continue; | |
451 | + | |
452 | + /* | |
453 | + * If we have a single PMU interrupt that we can't shift, | |
454 | + * assume that we're running on a uniprocessor machine and | |
455 | + * continue. Otherwise, continue without this interrupt. | |
456 | + */ | |
457 | + if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { | |
458 | + pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", | |
459 | + irq, i); | |
460 | + continue; | |
461 | + } | |
462 | + | |
463 | + err = request_irq(irq, armpmu->handle_irq, | |
464 | + IRQF_NOBALANCING, | |
465 | + "arm-pmu", armpmu); | |
466 | + if (err) { | |
467 | + pr_err("unable to request IRQ%d for ARM PMU counters\n", | |
468 | + irq); | |
469 | + armpmu_release_hardware(armpmu); | |
470 | + return err; | |
471 | + } | |
472 | + | |
473 | + cpumask_set_cpu(i, &armpmu->active_irqs); | |
474 | + } | |
427 | 475 | } |
428 | 476 | |
429 | 477 | return 0; |
arch/arm64/kernel/process.c
... | ... | @@ -33,6 +33,7 @@ |
33 | 33 | #include <linux/kallsyms.h> |
34 | 34 | #include <linux/init.h> |
35 | 35 | #include <linux/cpu.h> |
36 | +#include <linux/cpuidle.h> | |
36 | 37 | #include <linux/elfcore.h> |
37 | 38 | #include <linux/pm.h> |
38 | 39 | #include <linux/tick.h> |
... | ... | @@ -98,8 +99,10 @@ |
98 | 99 | * This should do all the clock switching and wait for interrupt |
99 | 100 | * tricks |
100 | 101 | */ |
101 | - cpu_do_idle(); | |
102 | - local_irq_enable(); | |
102 | + if (cpuidle_idle_call()) { | |
103 | + cpu_do_idle(); | |
104 | + local_irq_enable(); | |
105 | + } | |
103 | 106 | } |
104 | 107 | |
105 | 108 | #ifdef CONFIG_HOTPLUG_CPU |
... | ... | @@ -308,6 +311,7 @@ |
308 | 311 | unsigned long get_wchan(struct task_struct *p) |
309 | 312 | { |
310 | 313 | struct stackframe frame; |
314 | + unsigned long stack_page; | |
311 | 315 | int count = 0; |
312 | 316 | if (!p || p == current || p->state == TASK_RUNNING) |
313 | 317 | return 0; |
314 | 318 | |
... | ... | @@ -315,9 +319,11 @@ |
315 | 319 | frame.fp = thread_saved_fp(p); |
316 | 320 | frame.sp = thread_saved_sp(p); |
317 | 321 | frame.pc = thread_saved_pc(p); |
322 | + stack_page = (unsigned long)task_stack_page(p); | |
318 | 323 | do { |
319 | - int ret = unwind_frame(&frame); | |
320 | - if (ret < 0) | |
324 | + if (frame.sp < stack_page || | |
325 | + frame.sp >= stack_page + THREAD_SIZE || | |
326 | + unwind_frame(&frame)) | |
321 | 327 | return 0; |
322 | 328 | if (!in_sched_functions(frame.pc)) |
323 | 329 | return frame.pc; |
arch/arm64/kernel/setup.c
... | ... | @@ -108,20 +108,95 @@ |
108 | 108 | printk("%s", buf); |
109 | 109 | } |
110 | 110 | |
111 | +void __init smp_setup_processor_id(void) | |
112 | +{ | |
113 | + /* | |
114 | + * clear __my_cpu_offset on boot CPU to avoid hang caused by | |
115 | + * using percpu variable early, for example, lockdep will | |
116 | + * access percpu variable inside lock_release | |
117 | + */ | |
118 | + set_my_cpu_offset(0); | |
119 | +} | |
120 | + | |
111 | 121 | bool arch_match_cpu_phys_id(int cpu, u64 phys_id) |
112 | 122 | { |
113 | 123 | return phys_id == cpu_logical_map(cpu); |
114 | 124 | } |
115 | 125 | |
126 | +struct mpidr_hash mpidr_hash; | |
127 | +#ifdef CONFIG_SMP | |
128 | +/** | |
129 | + * smp_build_mpidr_hash - Pre-compute shifts required at each affinity | |
130 | + * level in order to build a linear index from an | |
131 | + * MPIDR value. Resulting algorithm is a collision | |
132 | + * free hash carried out through shifting and ORing | |
133 | + */ | |
134 | +static void __init smp_build_mpidr_hash(void) | |
135 | +{ | |
136 | + u32 i, affinity, fs[4], bits[4], ls; | |
137 | + u64 mask = 0; | |
138 | + /* | |
139 | + * Pre-scan the list of MPIDRS and filter out bits that do | |
140 | + * not contribute to affinity levels, ie they never toggle. | |
141 | + */ | |
142 | + for_each_possible_cpu(i) | |
143 | + mask |= (cpu_logical_map(i) ^ cpu_logical_map(0)); | |
144 | + pr_debug("mask of set bits %#llx\n", mask); | |
145 | + /* | |
146 | + * Find and stash the last and first bit set at all affinity levels to | |
147 | + * check how many bits are required to represent them. | |
148 | + */ | |
149 | + for (i = 0; i < 4; i++) { | |
150 | + affinity = MPIDR_AFFINITY_LEVEL(mask, i); | |
151 | + /* | |
152 | + * Find the MSB bit and LSB bits position | |
153 | + * to determine how many bits are required | |
154 | + * to express the affinity level. | |
155 | + */ | |
156 | + ls = fls(affinity); | |
157 | + fs[i] = affinity ? ffs(affinity) - 1 : 0; | |
158 | + bits[i] = ls - fs[i]; | |
159 | + } | |
160 | + /* | |
161 | + * An index can be created from the MPIDR_EL1 by isolating the | |
162 | + * significant bits at each affinity level and by shifting | |
163 | + * them in order to compress the 32 bits values space to a | |
164 | + * compressed set of values. This is equivalent to hashing | |
165 | + * the MPIDR_EL1 through shifting and ORing. It is a collision free | |
166 | + * hash though not minimal since some levels might contain a number | |
167 | + * of CPUs that is not an exact power of 2 and their bit | |
168 | + * representation might contain holes, eg MPIDR_EL1[7:0] = {0x2, 0x80}. | |
169 | + */ | |
170 | + mpidr_hash.shift_aff[0] = MPIDR_LEVEL_SHIFT(0) + fs[0]; | |
171 | + mpidr_hash.shift_aff[1] = MPIDR_LEVEL_SHIFT(1) + fs[1] - bits[0]; | |
172 | + mpidr_hash.shift_aff[2] = MPIDR_LEVEL_SHIFT(2) + fs[2] - | |
173 | + (bits[1] + bits[0]); | |
174 | + mpidr_hash.shift_aff[3] = MPIDR_LEVEL_SHIFT(3) + | |
175 | + fs[3] - (bits[2] + bits[1] + bits[0]); | |
176 | + mpidr_hash.mask = mask; | |
177 | + mpidr_hash.bits = bits[3] + bits[2] + bits[1] + bits[0]; | |
178 | + pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] aff3[%u] mask[%#llx] bits[%u]\n", | |
179 | + mpidr_hash.shift_aff[0], | |
180 | + mpidr_hash.shift_aff[1], | |
181 | + mpidr_hash.shift_aff[2], | |
182 | + mpidr_hash.shift_aff[3], | |
183 | + mpidr_hash.mask, | |
184 | + mpidr_hash.bits); | |
185 | + /* | |
186 | + * 4x is an arbitrary value used to warn on a hash table much bigger | |
187 | + * than expected on most systems. | |
188 | + */ | |
189 | + if (mpidr_hash_size() > 4 * num_possible_cpus()) | |
190 | + pr_warn("Large number of MPIDR hash buckets detected\n"); | |
191 | + __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash)); | |
192 | +} | |
193 | +#endif | |
194 | + | |
116 | 195 | static void __init setup_processor(void) |
117 | 196 | { |
118 | 197 | struct cpu_info *cpu_info; |
198 | + u64 features, block; | |
119 | 199 | |
120 | - /* | |
121 | - * locate processor in the list of supported processor | |
122 | - * types. The linker builds this table for us from the | |
123 | - * entries in arch/arm/mm/proc.S | |
124 | - */ | |
125 | 200 | cpu_info = lookup_processor_type(read_cpuid_id()); |
126 | 201 | if (!cpu_info) { |
127 | 202 | printk("CPU configuration botched (ID %08x), unable to continue.\n", |
... | ... | @@ -136,6 +211,37 @@ |
136 | 211 | |
137 | 212 | sprintf(init_utsname()->machine, ELF_PLATFORM); |
138 | 213 | elf_hwcap = 0; |
214 | + | |
215 | + /* | |
216 | + * ID_AA64ISAR0_EL1 contains 4-bit wide signed feature blocks. | |
217 | + * The blocks we test below represent incremental functionality | |
218 | + * for non-negative values. Negative values are reserved. | |
219 | + */ | |
220 | + features = read_cpuid(ID_AA64ISAR0_EL1); | |
221 | + block = (features >> 4) & 0xf; | |
222 | + if (!(block & 0x8)) { | |
223 | + switch (block) { | |
224 | + default: | |
225 | + case 2: | |
226 | + elf_hwcap |= HWCAP_PMULL; | |
227 | + case 1: | |
228 | + elf_hwcap |= HWCAP_AES; | |
229 | + case 0: | |
230 | + break; | |
231 | + } | |
232 | + } | |
233 | + | |
234 | + block = (features >> 8) & 0xf; | |
235 | + if (block && !(block & 0x8)) | |
236 | + elf_hwcap |= HWCAP_SHA1; | |
237 | + | |
238 | + block = (features >> 12) & 0xf; | |
239 | + if (block && !(block & 0x8)) | |
240 | + elf_hwcap |= HWCAP_SHA2; | |
241 | + | |
242 | + block = (features >> 16) & 0xf; | |
243 | + if (block && !(block & 0x8)) | |
244 | + elf_hwcap |= HWCAP_CRC32; | |
139 | 245 | } |
140 | 246 | |
141 | 247 | static void __init setup_machine_fdt(phys_addr_t dt_phys) |
... | ... | @@ -236,6 +342,7 @@ |
236 | 342 | cpu_read_bootcpu_ops(); |
237 | 343 | #ifdef CONFIG_SMP |
238 | 344 | smp_init_cpus(); |
345 | + smp_build_mpidr_hash(); | |
239 | 346 | #endif |
240 | 347 | |
241 | 348 | #ifdef CONFIG_VT |
... | ... | @@ -275,6 +382,11 @@ |
275 | 382 | "fp", |
276 | 383 | "asimd", |
277 | 384 | "evtstrm", |
385 | + "aes", | |
386 | + "pmull", | |
387 | + "sha1", | |
388 | + "sha2", | |
389 | + "crc32", | |
278 | 390 | NULL |
279 | 391 | }; |
280 | 392 |
arch/arm64/kernel/sleep.S
1 | +#include <linux/errno.h> | |
2 | +#include <linux/linkage.h> | |
3 | +#include <asm/asm-offsets.h> | |
4 | +#include <asm/assembler.h> | |
5 | + | |
6 | + .text | |
7 | +/* | |
8 | + * Implementation of MPIDR_EL1 hash algorithm through shifting | |
9 | + * and OR'ing. | |
10 | + * | |
11 | + * @dst: register containing hash result | |
12 | + * @rs0: register containing affinity level 0 bit shift | |
13 | + * @rs1: register containing affinity level 1 bit shift | |
14 | + * @rs2: register containing affinity level 2 bit shift | |
15 | + * @rs3: register containing affinity level 3 bit shift | |
16 | + * @mpidr: register containing MPIDR_EL1 value | |
17 | + * @mask: register containing MPIDR mask | |
18 | + * | |
19 | + * Pseudo C-code: | |
20 | + * | |
21 | + *u32 dst; | |
22 | + * | |
23 | + *compute_mpidr_hash(u32 rs0, u32 rs1, u32 rs2, u32 rs3, u64 mpidr, u64 mask) { | |
24 | + * u32 aff0, aff1, aff2, aff3; | |
25 | + * u64 mpidr_masked = mpidr & mask; | |
26 | + * aff0 = mpidr_masked & 0xff; | |
27 | + * aff1 = mpidr_masked & 0xff00; | |
28 | + * aff2 = mpidr_masked & 0xff0000; | |
29 | + * aff2 = mpidr_masked & 0xff00000000; | |
30 | + * dst = (aff0 >> rs0 | aff1 >> rs1 | aff2 >> rs2 | aff3 >> rs3); | |
31 | + *} | |
32 | + * Input registers: rs0, rs1, rs2, rs3, mpidr, mask | |
33 | + * Output register: dst | |
34 | + * Note: input and output registers must be disjoint register sets | |
35 | + (eg: a macro instance with mpidr = x1 and dst = x1 is invalid) | |
36 | + */ | |
37 | + .macro compute_mpidr_hash dst, rs0, rs1, rs2, rs3, mpidr, mask | |
38 | + and \mpidr, \mpidr, \mask // mask out MPIDR bits | |
39 | + and \dst, \mpidr, #0xff // mask=aff0 | |
40 | + lsr \dst ,\dst, \rs0 // dst=aff0>>rs0 | |
41 | + and \mask, \mpidr, #0xff00 // mask = aff1 | |
42 | + lsr \mask ,\mask, \rs1 | |
43 | + orr \dst, \dst, \mask // dst|=(aff1>>rs1) | |
44 | + and \mask, \mpidr, #0xff0000 // mask = aff2 | |
45 | + lsr \mask ,\mask, \rs2 | |
46 | + orr \dst, \dst, \mask // dst|=(aff2>>rs2) | |
47 | + and \mask, \mpidr, #0xff00000000 // mask = aff3 | |
48 | + lsr \mask ,\mask, \rs3 | |
49 | + orr \dst, \dst, \mask // dst|=(aff3>>rs3) | |
50 | + .endm | |
51 | +/* | |
52 | + * Save CPU state for a suspend. This saves callee registers, and allocates | |
53 | + * space on the kernel stack to save the CPU specific registers + some | |
54 | + * other data for resume. | |
55 | + * | |
56 | + * x0 = suspend finisher argument | |
57 | + */ | |
58 | +ENTRY(__cpu_suspend) | |
59 | + stp x29, lr, [sp, #-96]! | |
60 | + stp x19, x20, [sp,#16] | |
61 | + stp x21, x22, [sp,#32] | |
62 | + stp x23, x24, [sp,#48] | |
63 | + stp x25, x26, [sp,#64] | |
64 | + stp x27, x28, [sp,#80] | |
65 | + mov x2, sp | |
66 | + sub sp, sp, #CPU_SUSPEND_SZ // allocate cpu_suspend_ctx | |
67 | + mov x1, sp | |
68 | + /* | |
69 | + * x1 now points to struct cpu_suspend_ctx allocated on the stack | |
70 | + */ | |
71 | + str x2, [x1, #CPU_CTX_SP] | |
72 | + ldr x2, =sleep_save_sp | |
73 | + ldr x2, [x2, #SLEEP_SAVE_SP_VIRT] | |
74 | +#ifdef CONFIG_SMP | |
75 | + mrs x7, mpidr_el1 | |
76 | + ldr x9, =mpidr_hash | |
77 | + ldr x10, [x9, #MPIDR_HASH_MASK] | |
78 | + /* | |
79 | + * Following code relies on the struct mpidr_hash | |
80 | + * members size. | |
81 | + */ | |
82 | + ldp w3, w4, [x9, #MPIDR_HASH_SHIFTS] | |
83 | + ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)] | |
84 | + compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10 | |
85 | + add x2, x2, x8, lsl #3 | |
86 | +#endif | |
87 | + bl __cpu_suspend_finisher | |
88 | + /* | |
89 | + * Never gets here, unless suspend fails. | |
90 | + * Successful cpu_suspend should return from cpu_resume, returning | |
91 | + * through this code path is considered an error | |
92 | + * If the return value is set to 0 force x0 = -EOPNOTSUPP | |
93 | + * to make sure a proper error condition is propagated | |
94 | + */ | |
95 | + cmp x0, #0 | |
96 | + mov x3, #-EOPNOTSUPP | |
97 | + csel x0, x3, x0, eq | |
98 | + add sp, sp, #CPU_SUSPEND_SZ // rewind stack pointer | |
99 | + ldp x19, x20, [sp, #16] | |
100 | + ldp x21, x22, [sp, #32] | |
101 | + ldp x23, x24, [sp, #48] | |
102 | + ldp x25, x26, [sp, #64] | |
103 | + ldp x27, x28, [sp, #80] | |
104 | + ldp x29, lr, [sp], #96 | |
105 | + ret | |
106 | +ENDPROC(__cpu_suspend) | |
107 | + .ltorg | |
108 | + | |
109 | +/* | |
110 | + * x0 must contain the sctlr value retrieved from restored context | |
111 | + */ | |
112 | +ENTRY(cpu_resume_mmu) | |
113 | + ldr x3, =cpu_resume_after_mmu | |
114 | + msr sctlr_el1, x0 // restore sctlr_el1 | |
115 | + isb | |
116 | + br x3 // global jump to virtual address | |
117 | +ENDPROC(cpu_resume_mmu) | |
118 | +cpu_resume_after_mmu: | |
119 | + mov x0, #0 // return zero on success | |
120 | + ldp x19, x20, [sp, #16] | |
121 | + ldp x21, x22, [sp, #32] | |
122 | + ldp x23, x24, [sp, #48] | |
123 | + ldp x25, x26, [sp, #64] | |
124 | + ldp x27, x28, [sp, #80] | |
125 | + ldp x29, lr, [sp], #96 | |
126 | + ret | |
127 | +ENDPROC(cpu_resume_after_mmu) | |
128 | + | |
129 | + .data | |
130 | +ENTRY(cpu_resume) | |
131 | + bl el2_setup // if in EL2 drop to EL1 cleanly | |
132 | +#ifdef CONFIG_SMP | |
133 | + mrs x1, mpidr_el1 | |
134 | + adr x4, mpidr_hash_ptr | |
135 | + ldr x5, [x4] | |
136 | + add x8, x4, x5 // x8 = struct mpidr_hash phys address | |
137 | + /* retrieve mpidr_hash members to compute the hash */ | |
138 | + ldr x2, [x8, #MPIDR_HASH_MASK] | |
139 | + ldp w3, w4, [x8, #MPIDR_HASH_SHIFTS] | |
140 | + ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)] | |
141 | + compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2 | |
142 | + /* x7 contains hash index, let's use it to grab context pointer */ | |
143 | +#else | |
144 | + mov x7, xzr | |
145 | +#endif | |
146 | + adr x0, sleep_save_sp | |
147 | + ldr x0, [x0, #SLEEP_SAVE_SP_PHYS] | |
148 | + ldr x0, [x0, x7, lsl #3] | |
149 | + /* load sp from context */ | |
150 | + ldr x2, [x0, #CPU_CTX_SP] | |
151 | + adr x1, sleep_idmap_phys | |
152 | + /* load physical address of identity map page table in x1 */ | |
153 | + ldr x1, [x1] | |
154 | + mov sp, x2 | |
155 | + /* | |
156 | + * cpu_do_resume expects x0 to contain context physical address | |
157 | + * pointer and x1 to contain physical address of 1:1 page tables | |
158 | + */ | |
159 | + bl cpu_do_resume // PC relative jump, MMU off | |
160 | + b cpu_resume_mmu // Resume MMU, never returns | |
161 | +ENDPROC(cpu_resume) | |
162 | + | |
163 | + .align 3 | |
164 | +mpidr_hash_ptr: | |
165 | + /* | |
166 | + * offset of mpidr_hash symbol from current location | |
167 | + * used to obtain run-time mpidr_hash address with MMU off | |
168 | + */ | |
169 | + .quad mpidr_hash - . | |
170 | +/* | |
171 | + * physical address of identity mapped page tables | |
172 | + */ | |
173 | + .type sleep_idmap_phys, #object | |
174 | +ENTRY(sleep_idmap_phys) | |
175 | + .quad 0 | |
176 | +/* | |
177 | + * struct sleep_save_sp { | |
178 | + * phys_addr_t *save_ptr_stash; | |
179 | + * phys_addr_t save_ptr_stash_phys; | |
180 | + * }; | |
181 | + */ | |
182 | + .type sleep_save_sp, #object | |
183 | +ENTRY(sleep_save_sp) | |
184 | + .space SLEEP_SAVE_SP_SZ // struct sleep_save_sp |
arch/arm64/kernel/smp.c
... | ... | @@ -61,6 +61,7 @@ |
61 | 61 | IPI_CALL_FUNC, |
62 | 62 | IPI_CALL_FUNC_SINGLE, |
63 | 63 | IPI_CPU_STOP, |
64 | + IPI_TIMER, | |
64 | 65 | }; |
65 | 66 | |
66 | 67 | /* |
... | ... | @@ -122,8 +123,6 @@ |
122 | 123 | struct mm_struct *mm = &init_mm; |
123 | 124 | unsigned int cpu = smp_processor_id(); |
124 | 125 | |
125 | - printk("CPU%u: Booted secondary processor\n", cpu); | |
126 | - | |
127 | 126 | /* |
128 | 127 | * All kernel threads share the same mm context; grab a |
129 | 128 | * reference and switch to it. |
... | ... | @@ -132,6 +131,9 @@ |
132 | 131 | current->active_mm = mm; |
133 | 132 | cpumask_set_cpu(cpu, mm_cpumask(mm)); |
134 | 133 | |
134 | + set_my_cpu_offset(per_cpu_offset(smp_processor_id())); | |
135 | + printk("CPU%u: Booted secondary processor\n", cpu); | |
136 | + | |
135 | 137 | /* |
136 | 138 | * TTBR0 is only used for the identity mapping at this stage. Make it |
137 | 139 | * point to zero page to avoid speculatively fetching new entries. |
... | ... | @@ -271,6 +273,7 @@ |
271 | 273 | |
272 | 274 | void __init smp_prepare_boot_cpu(void) |
273 | 275 | { |
276 | + set_my_cpu_offset(per_cpu_offset(smp_processor_id())); | |
274 | 277 | } |
275 | 278 | |
276 | 279 | static void (*smp_cross_call)(const struct cpumask *, unsigned int); |
... | ... | @@ -447,6 +450,7 @@ |
447 | 450 | S(IPI_CALL_FUNC, "Function call interrupts"), |
448 | 451 | S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), |
449 | 452 | S(IPI_CPU_STOP, "CPU stop interrupts"), |
453 | + S(IPI_TIMER, "Timer broadcast interrupts"), | |
450 | 454 | }; |
451 | 455 | |
452 | 456 | void show_ipi_list(struct seq_file *p, int prec) |
... | ... | @@ -532,6 +536,14 @@ |
532 | 536 | irq_exit(); |
533 | 537 | break; |
534 | 538 | |
539 | +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST | |
540 | + case IPI_TIMER: | |
541 | + irq_enter(); | |
542 | + tick_receive_broadcast(); | |
543 | + irq_exit(); | |
544 | + break; | |
545 | +#endif | |
546 | + | |
535 | 547 | default: |
536 | 548 | pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); |
537 | 549 | break; |
... | ... | @@ -543,6 +555,13 @@ |
543 | 555 | { |
544 | 556 | smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); |
545 | 557 | } |
558 | + | |
559 | +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST | |
560 | +void tick_broadcast(const struct cpumask *mask) | |
561 | +{ | |
562 | + smp_cross_call(mask, IPI_TIMER); | |
563 | +} | |
564 | +#endif | |
546 | 565 | |
547 | 566 | void smp_send_stop(void) |
548 | 567 | { |
arch/arm64/kernel/stacktrace.c
arch/arm64/kernel/suspend.c
1 | +#include <linux/slab.h> | |
2 | +#include <asm/cacheflush.h> | |
3 | +#include <asm/cpu_ops.h> | |
4 | +#include <asm/debug-monitors.h> | |
5 | +#include <asm/pgtable.h> | |
6 | +#include <asm/memory.h> | |
7 | +#include <asm/smp_plat.h> | |
8 | +#include <asm/suspend.h> | |
9 | +#include <asm/tlbflush.h> | |
10 | + | |
11 | +extern int __cpu_suspend(unsigned long); | |
12 | +/* | |
13 | + * This is called by __cpu_suspend() to save the state, and do whatever | |
14 | + * flushing is required to ensure that when the CPU goes to sleep we have | |
15 | + * the necessary data available when the caches are not searched. | |
16 | + * | |
17 | + * @arg: Argument to pass to suspend operations | |
18 | + * @ptr: CPU context virtual address | |
19 | + * @save_ptr: address of the location where the context physical address | |
20 | + * must be saved | |
21 | + */ | |
22 | +int __cpu_suspend_finisher(unsigned long arg, struct cpu_suspend_ctx *ptr, | |
23 | + phys_addr_t *save_ptr) | |
24 | +{ | |
25 | + int cpu = smp_processor_id(); | |
26 | + | |
27 | + *save_ptr = virt_to_phys(ptr); | |
28 | + | |
29 | + cpu_do_suspend(ptr); | |
30 | + /* | |
31 | + * Only flush the context that must be retrieved with the MMU | |
32 | + * off. VA primitives ensure the flush is applied to all | |
33 | + * cache levels so context is pushed to DRAM. | |
34 | + */ | |
35 | + __flush_dcache_area(ptr, sizeof(*ptr)); | |
36 | + __flush_dcache_area(save_ptr, sizeof(*save_ptr)); | |
37 | + | |
38 | + return cpu_ops[cpu]->cpu_suspend(arg); | |
39 | +} | |
40 | + | |
41 | +/* | |
42 | + * This hook is provided so that cpu_suspend code can restore HW | |
43 | + * breakpoints as early as possible in the resume path, before reenabling | |
44 | + * debug exceptions. Code cannot be run from a CPU PM notifier since by the | |
45 | + * time the notifier runs debug exceptions might have been enabled already, | |
46 | + * with HW breakpoints registers content still in an unknown state. | |
47 | + */ | |
48 | +void (*hw_breakpoint_restore)(void *); | |
49 | +void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) | |
50 | +{ | |
51 | + /* Prevent multiple restore hook initializations */ | |
52 | + if (WARN_ON(hw_breakpoint_restore)) | |
53 | + return; | |
54 | + hw_breakpoint_restore = hw_bp_restore; | |
55 | +} | |
56 | + | |
57 | +/** | |
58 | + * cpu_suspend | |
59 | + * | |
60 | + * @arg: argument to pass to the finisher function | |
61 | + */ | |
62 | +int cpu_suspend(unsigned long arg) | |
63 | +{ | |
64 | + struct mm_struct *mm = current->active_mm; | |
65 | + int ret, cpu = smp_processor_id(); | |
66 | + unsigned long flags; | |
67 | + | |
68 | + /* | |
69 | + * If cpu_ops have not been registered or suspend | |
70 | + * has not been initialized, cpu_suspend call fails early. | |
71 | + */ | |
72 | + if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) | |
73 | + return -EOPNOTSUPP; | |
74 | + | |
75 | + /* | |
76 | + * From this point debug exceptions are disabled to prevent | |
77 | + * updates to mdscr register (saved and restored along with | |
78 | + * general purpose registers) from kernel debuggers. | |
79 | + */ | |
80 | + local_dbg_save(flags); | |
81 | + | |
82 | + /* | |
83 | + * mm context saved on the stack, it will be restored when | |
84 | + * the cpu comes out of reset through the identity mapped | |
85 | + * page tables, so that the thread address space is properly | |
86 | + * set-up on function return. | |
87 | + */ | |
88 | + ret = __cpu_suspend(arg); | |
89 | + if (ret == 0) { | |
90 | + cpu_switch_mm(mm->pgd, mm); | |
91 | + flush_tlb_all(); | |
92 | + /* | |
93 | + * Restore HW breakpoint registers to sane values | |
94 | + * before debug exceptions are possibly reenabled | |
95 | + * through local_dbg_restore. | |
96 | + */ | |
97 | + if (hw_breakpoint_restore) | |
98 | + hw_breakpoint_restore(NULL); | |
99 | + } | |
100 | + | |
101 | + /* | |
102 | + * Restore pstate flags. OS lock and mdscr have been already | |
103 | + * restored, so from this point onwards, debugging is fully | |
104 | + * renabled if it was enabled when core started shutdown. | |
105 | + */ | |
106 | + local_dbg_restore(flags); | |
107 | + | |
108 | + return ret; | |
109 | +} | |
110 | + | |
111 | +extern struct sleep_save_sp sleep_save_sp; | |
112 | +extern phys_addr_t sleep_idmap_phys; | |
113 | + | |
114 | +static int cpu_suspend_init(void) | |
115 | +{ | |
116 | + void *ctx_ptr; | |
117 | + | |
118 | + /* ctx_ptr is an array of physical addresses */ | |
119 | + ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(phys_addr_t), GFP_KERNEL); | |
120 | + | |
121 | + if (WARN_ON(!ctx_ptr)) | |
122 | + return -ENOMEM; | |
123 | + | |
124 | + sleep_save_sp.save_ptr_stash = ctx_ptr; | |
125 | + sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr); | |
126 | + sleep_idmap_phys = virt_to_phys(idmap_pg_dir); | |
127 | + __flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp)); | |
128 | + __flush_dcache_area(&sleep_idmap_phys, sizeof(sleep_idmap_phys)); | |
129 | + | |
130 | + return 0; | |
131 | +} | |
132 | +early_initcall(cpu_suspend_init); |
arch/arm64/kernel/vmlinux.lds.S
... | ... | @@ -99,17 +99,14 @@ |
99 | 99 | |
100 | 100 | . = ALIGN(PAGE_SIZE); |
101 | 101 | _data = .; |
102 | - __data_loc = _data - LOAD_OFFSET; | |
103 | 102 | _sdata = .; |
104 | 103 | RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE) |
105 | 104 | _edata = .; |
106 | - _edata_loc = __data_loc + SIZEOF(.data); | |
107 | 105 | |
108 | 106 | BSS_SECTION(0, 0, 0) |
109 | 107 | _end = .; |
110 | 108 | |
111 | 109 | STABS_DEBUG |
112 | - .comment 0 : { *(.comment) } | |
113 | 110 | } |
114 | 111 | |
115 | 112 | /* |
arch/arm64/lib/Makefile
1 | -lib-y := bitops.o delay.o \ | |
2 | - strncpy_from_user.o strnlen_user.o clear_user.o \ | |
3 | - copy_from_user.o copy_to_user.o copy_in_user.o \ | |
4 | - copy_page.o clear_page.o \ | |
5 | - memchr.o memcpy.o memmove.o memset.o \ | |
1 | +lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ | |
2 | + copy_to_user.o copy_in_user.o copy_page.o \ | |
3 | + clear_page.o memchr.o memcpy.o memmove.o memset.o \ | |
6 | 4 | strchr.o strrchr.o |
arch/arm64/lib/strncpy_from_user.S
1 | -/* | |
2 | - * Based on arch/arm/lib/strncpy_from_user.S | |
3 | - * | |
4 | - * Copyright (C) 1995-2000 Russell King | |
5 | - * Copyright (C) 2012 ARM Ltd. | |
6 | - * | |
7 | - * This program is free software; you can redistribute it and/or modify | |
8 | - * it under the terms of the GNU General Public License version 2 as | |
9 | - * published by the Free Software Foundation. | |
10 | - * | |
11 | - * This program is distributed in the hope that it will be useful, | |
12 | - * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | - * GNU General Public License for more details. | |
15 | - * | |
16 | - * You should have received a copy of the GNU General Public License | |
17 | - * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | - */ | |
19 | - | |
20 | -#include <linux/linkage.h> | |
21 | -#include <asm/assembler.h> | |
22 | -#include <asm/errno.h> | |
23 | - | |
24 | - .text | |
25 | - .align 5 | |
26 | - | |
27 | -/* | |
28 | - * Copy a string from user space to kernel space. | |
29 | - * x0 = dst, x1 = src, x2 = byte length | |
30 | - * returns the number of characters copied (strlen of copied string), | |
31 | - * -EFAULT on exception, or "len" if we fill the whole buffer | |
32 | - */ | |
33 | -ENTRY(__strncpy_from_user) | |
34 | - mov x4, x1 | |
35 | -1: subs x2, x2, #1 | |
36 | - bmi 2f | |
37 | -USER(9f, ldrb w3, [x1], #1 ) | |
38 | - strb w3, [x0], #1 | |
39 | - cbnz w3, 1b | |
40 | - sub x1, x1, #1 // take NUL character out of count | |
41 | -2: sub x0, x1, x4 | |
42 | - ret | |
43 | -ENDPROC(__strncpy_from_user) | |
44 | - | |
45 | - .section .fixup,"ax" | |
46 | - .align 0 | |
47 | -9: strb wzr, [x0] // null terminate | |
48 | - mov x0, #-EFAULT | |
49 | - ret | |
50 | - .previous |
arch/arm64/lib/strnlen_user.S
1 | -/* | |
2 | - * Based on arch/arm/lib/strnlen_user.S | |
3 | - * | |
4 | - * Copyright (C) 1995-2000 Russell King | |
5 | - * Copyright (C) 2012 ARM Ltd. | |
6 | - * | |
7 | - * This program is free software; you can redistribute it and/or modify | |
8 | - * it under the terms of the GNU General Public License version 2 as | |
9 | - * published by the Free Software Foundation. | |
10 | - * | |
11 | - * This program is distributed in the hope that it will be useful, | |
12 | - * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | - * GNU General Public License for more details. | |
15 | - * | |
16 | - * You should have received a copy of the GNU General Public License | |
17 | - * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | - */ | |
19 | - | |
20 | -#include <linux/linkage.h> | |
21 | -#include <asm/assembler.h> | |
22 | -#include <asm/errno.h> | |
23 | - | |
24 | - .text | |
25 | - .align 5 | |
26 | - | |
27 | -/* Prototype: unsigned long __strnlen_user(const char *str, long n) | |
28 | - * Purpose : get length of a string in user memory | |
29 | - * Params : str - address of string in user memory | |
30 | - * Returns : length of string *including terminator* | |
31 | - * or zero on exception, or n if too long | |
32 | - */ | |
33 | -ENTRY(__strnlen_user) | |
34 | - mov x2, x0 | |
35 | -1: subs x1, x1, #1 | |
36 | - b.mi 2f | |
37 | -USER(9f, ldrb w3, [x0], #1 ) | |
38 | - cbnz w3, 1b | |
39 | -2: sub x0, x0, x2 | |
40 | - ret | |
41 | -ENDPROC(__strnlen_user) | |
42 | - | |
43 | - .section .fixup,"ax" | |
44 | - .align 0 | |
45 | -9: mov x0, #0 | |
46 | - ret | |
47 | - .previous |
arch/arm64/mm/dma-mapping.c
... | ... | @@ -21,6 +21,7 @@ |
21 | 21 | #include <linux/export.h> |
22 | 22 | #include <linux/slab.h> |
23 | 23 | #include <linux/dma-mapping.h> |
24 | +#include <linux/dma-contiguous.h> | |
24 | 25 | #include <linux/vmalloc.h> |
25 | 26 | #include <linux/swiotlb.h> |
26 | 27 | |
27 | 28 | |
28 | 29 | |
... | ... | @@ -33,17 +34,47 @@ |
33 | 34 | dma_addr_t *dma_handle, gfp_t flags, |
34 | 35 | struct dma_attrs *attrs) |
35 | 36 | { |
37 | + if (dev == NULL) { | |
38 | + WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); | |
39 | + return NULL; | |
40 | + } | |
41 | + | |
36 | 42 | if (IS_ENABLED(CONFIG_ZONE_DMA32) && |
37 | 43 | dev->coherent_dma_mask <= DMA_BIT_MASK(32)) |
38 | 44 | flags |= GFP_DMA32; |
39 | - return swiotlb_alloc_coherent(dev, size, dma_handle, flags); | |
45 | + if (IS_ENABLED(CONFIG_DMA_CMA)) { | |
46 | + struct page *page; | |
47 | + | |
48 | + page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, | |
49 | + get_order(size)); | |
50 | + if (!page) | |
51 | + return NULL; | |
52 | + | |
53 | + *dma_handle = phys_to_dma(dev, page_to_phys(page)); | |
54 | + return page_address(page); | |
55 | + } else { | |
56 | + return swiotlb_alloc_coherent(dev, size, dma_handle, flags); | |
57 | + } | |
40 | 58 | } |
41 | 59 | |
42 | 60 | static void arm64_swiotlb_free_coherent(struct device *dev, size_t size, |
43 | 61 | void *vaddr, dma_addr_t dma_handle, |
44 | 62 | struct dma_attrs *attrs) |
45 | 63 | { |
46 | - swiotlb_free_coherent(dev, size, vaddr, dma_handle); | |
64 | + if (dev == NULL) { | |
65 | + WARN_ONCE(1, "Use an actual device structure for DMA allocation\n"); | |
66 | + return; | |
67 | + } | |
68 | + | |
69 | + if (IS_ENABLED(CONFIG_DMA_CMA)) { | |
70 | + phys_addr_t paddr = dma_to_phys(dev, dma_handle); | |
71 | + | |
72 | + dma_release_from_contiguous(dev, | |
73 | + phys_to_page(paddr), | |
74 | + size >> PAGE_SHIFT); | |
75 | + } else { | |
76 | + swiotlb_free_coherent(dev, size, vaddr, dma_handle); | |
77 | + } | |
47 | 78 | } |
48 | 79 | |
49 | 80 | static struct dma_map_ops arm64_swiotlb_dma_ops = { |
arch/arm64/mm/init.c
... | ... | @@ -30,6 +30,7 @@ |
30 | 30 | #include <linux/memblock.h> |
31 | 31 | #include <linux/sort.h> |
32 | 32 | #include <linux/of_fdt.h> |
33 | +#include <linux/dma-contiguous.h> | |
33 | 34 | |
34 | 35 | #include <asm/sections.h> |
35 | 36 | #include <asm/setup.h> |
... | ... | @@ -158,6 +159,8 @@ |
158 | 159 | break; |
159 | 160 | memblock_reserve(base, size); |
160 | 161 | } |
162 | + | |
163 | + dma_contiguous_reserve(0); | |
161 | 164 | |
162 | 165 | memblock_allow_resize(); |
163 | 166 | memblock_dump_all(); |
arch/arm64/mm/proc.S
... | ... | @@ -80,6 +80,75 @@ |
80 | 80 | ret |
81 | 81 | ENDPROC(cpu_do_idle) |
82 | 82 | |
83 | +#ifdef CONFIG_ARM64_CPU_SUSPEND | |
84 | +/** | |
85 | + * cpu_do_suspend - save CPU registers context | |
86 | + * | |
87 | + * x0: virtual address of context pointer | |
88 | + */ | |
89 | +ENTRY(cpu_do_suspend) | |
90 | + mrs x2, tpidr_el0 | |
91 | + mrs x3, tpidrro_el0 | |
92 | + mrs x4, contextidr_el1 | |
93 | + mrs x5, mair_el1 | |
94 | + mrs x6, cpacr_el1 | |
95 | + mrs x7, ttbr1_el1 | |
96 | + mrs x8, tcr_el1 | |
97 | + mrs x9, vbar_el1 | |
98 | + mrs x10, mdscr_el1 | |
99 | + mrs x11, oslsr_el1 | |
100 | + mrs x12, sctlr_el1 | |
101 | + stp x2, x3, [x0] | |
102 | + stp x4, x5, [x0, #16] | |
103 | + stp x6, x7, [x0, #32] | |
104 | + stp x8, x9, [x0, #48] | |
105 | + stp x10, x11, [x0, #64] | |
106 | + str x12, [x0, #80] | |
107 | + ret | |
108 | +ENDPROC(cpu_do_suspend) | |
109 | + | |
110 | +/** | |
111 | + * cpu_do_resume - restore CPU register context | |
112 | + * | |
113 | + * x0: Physical address of context pointer | |
114 | + * x1: ttbr0_el1 to be restored | |
115 | + * | |
116 | + * Returns: | |
117 | + * sctlr_el1 value in x0 | |
118 | + */ | |
119 | +ENTRY(cpu_do_resume) | |
120 | + /* | |
121 | + * Invalidate local tlb entries before turning on MMU | |
122 | + */ | |
123 | + tlbi vmalle1 | |
124 | + ldp x2, x3, [x0] | |
125 | + ldp x4, x5, [x0, #16] | |
126 | + ldp x6, x7, [x0, #32] | |
127 | + ldp x8, x9, [x0, #48] | |
128 | + ldp x10, x11, [x0, #64] | |
129 | + ldr x12, [x0, #80] | |
130 | + msr tpidr_el0, x2 | |
131 | + msr tpidrro_el0, x3 | |
132 | + msr contextidr_el1, x4 | |
133 | + msr mair_el1, x5 | |
134 | + msr cpacr_el1, x6 | |
135 | + msr ttbr0_el1, x1 | |
136 | + msr ttbr1_el1, x7 | |
137 | + msr tcr_el1, x8 | |
138 | + msr vbar_el1, x9 | |
139 | + msr mdscr_el1, x10 | |
140 | + /* | |
141 | + * Restore oslsr_el1 by writing oslar_el1 | |
142 | + */ | |
143 | + ubfx x11, x11, #1, #1 | |
144 | + msr oslar_el1, x11 | |
145 | + mov x0, x12 | |
146 | + dsb nsh // Make sure local tlb invalidation completed | |
147 | + isb | |
148 | + ret | |
149 | +ENDPROC(cpu_do_resume) | |
150 | +#endif | |
151 | + | |
83 | 152 | /* |
84 | 153 | * cpu_switch_mm(pgd_phys, tsk) |
85 | 154 | * |
include/linux/irqdesc.h
... | ... | @@ -152,6 +152,14 @@ |
152 | 152 | return desc->status_use_accessors & IRQ_NO_BALANCING_MASK; |
153 | 153 | } |
154 | 154 | |
155 | +static inline int irq_is_percpu(unsigned int irq) | |
156 | +{ | |
157 | + struct irq_desc *desc; | |
158 | + | |
159 | + desc = irq_to_desc(irq); | |
160 | + return desc->status_use_accessors & IRQ_PER_CPU; | |
161 | +} | |
162 | + | |
155 | 163 | static inline void |
156 | 164 | irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class) |
157 | 165 | { |
include/linux/jump_label.h
... | ... | @@ -81,18 +81,21 @@ |
81 | 81 | #include <linux/atomic.h> |
82 | 82 | #ifdef HAVE_JUMP_LABEL |
83 | 83 | |
84 | -#define JUMP_LABEL_TRUE_BRANCH 1UL | |
84 | +#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL | |
85 | +#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL | |
86 | +#define JUMP_LABEL_TYPE_MASK 1UL | |
85 | 87 | |
86 | 88 | static |
87 | 89 | inline struct jump_entry *jump_label_get_entries(struct static_key *key) |
88 | 90 | { |
89 | 91 | return (struct jump_entry *)((unsigned long)key->entries |
90 | - & ~JUMP_LABEL_TRUE_BRANCH); | |
92 | + & ~JUMP_LABEL_TYPE_MASK); | |
91 | 93 | } |
92 | 94 | |
93 | 95 | static inline bool jump_label_get_branch_default(struct static_key *key) |
94 | 96 | { |
95 | - if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH) | |
97 | + if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) == | |
98 | + JUMP_LABEL_TYPE_TRUE_BRANCH) | |
96 | 99 | return true; |
97 | 100 | return false; |
98 | 101 | } |
... | ... | @@ -122,10 +125,12 @@ |
122 | 125 | extern void static_key_slow_dec(struct static_key *key); |
123 | 126 | extern void jump_label_apply_nops(struct module *mod); |
124 | 127 | |
125 | -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ | |
126 | - { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) | |
127 | -#define STATIC_KEY_INIT_FALSE ((struct static_key) \ | |
128 | - { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) | |
128 | +#define STATIC_KEY_INIT_TRUE ((struct static_key) \ | |
129 | + { .enabled = ATOMIC_INIT(1), \ | |
130 | + .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) | |
131 | +#define STATIC_KEY_INIT_FALSE ((struct static_key) \ | |
132 | + { .enabled = ATOMIC_INIT(0), \ | |
133 | + .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) | |
129 | 134 | |
130 | 135 | #else /* !HAVE_JUMP_LABEL */ |
131 | 136 |
scripts/gcc-goto.sh
... | ... | @@ -5,7 +5,7 @@ |
5 | 5 | cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" |
6 | 6 | int main(void) |
7 | 7 | { |
8 | -#ifdef __arm__ | |
8 | +#if defined(__arm__) || defined(__aarch64__) | |
9 | 9 | /* |
10 | 10 | * Not related to asm goto, but used by jump label |
11 | 11 | * and broken on some ARM GCC versions (see GCC Bug 48637). |