ARM: EXYNOS: Add MCPM call-back functions

Add machine-dependent MCPM call-backs for Exynos5420. These are used to power up/down the secondary CPUs during boot, shutdown, s2r and switching. Signed-off-by: Thomas Abraham <thomas.ab@samsung.com> Signed-off-by: Inderpal Singh <inderpal.s@samsung.com> Signed-off-by: Andrew Bresticker <abrestic@chromium.org> Signed-off-by: Abhilash Kesavan <a.kesavan@samsung.com> Reviewed-by: Nicolas Pitre <nicolas.pitre@linaro.org> Signed-off-by: Kukjin Kim <kgene.kim@samsung.com>

ARM: EXYNOS: Add MCPM call-back functions
Add machine-dependent MCPM call-backs for Exynos5420. These are used to power up/down the secondary CPUs during boot, shutdown, s2r and switching. Signed-off-by: Thomas Abraham <thomas.ab@samsung.com> Signed-off-by: Inderpal Singh <inderpal.s@samsung.com> Signed-off-by: Andrew Bresticker <abrestic@chromium.org> Signed-off-by: Abhilash Kesavan <a.kesavan@samsung.com> Reviewed-by: Nicolas Pitre <nicolas.pitre@linaro.org> Signed-off-by: Kukjin Kim <kgene.kim@samsung.com>
Abhilash Kesavan · Kukjin Kim
1 parent 5b56642bd8
Showing 4 changed files with 364 additions and 0 deletions Side-by-side Diff
arch/arm/mach-exynos/Kconfig
arch/arm/mach-exynos/Makefile
arch/arm/mach-exynos/mcpm-exynos.c
arch/arm/mach-exynos/regs-pmu.h
@@ -110,5 +110,13 @@
  
 endmenu
  
+config EXYNOS5420_MCPM
+	bool "Exynos5420 Multi-Cluster PM support"
+	depends on MCPM && SOC_EXYNOS5420
+	select ARM_CCI
+	help
+	  This is needed to provide CPU and cluster power management
+	  on Exynos5420 implementing big.LITTLE.
+
 endif
@@ -29,4 +29,6 @@
  
 plus_sec := $(call as-instr,.arch_extension sec,+sec)
 AFLAGS_exynos-smc.o		:=-Wa,-march=armv7-a$(plus_sec)
+
+obj-$(CONFIG_EXYNOS5420_MCPM)	+= mcpm-exynos.o
+/*
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd.
+ *		http://www.samsung.com
+ *
+ * arch/arm/mach-exynos/mcpm-exynos.c
+ *
+ * Based on arch/arm/mach-vexpress/dcscb.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/arm-cci.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+
+#include <asm/cputype.h>
+#include <asm/cp15.h>
+#include <asm/mcpm.h>
+
+#include "regs-pmu.h"
+#include "common.h"
+
+#define EXYNOS5420_CPUS_PER_CLUSTER	4
+#define EXYNOS5420_NR_CLUSTERS		2
+#define MCPM_BOOT_ADDR_OFFSET		0x1c
+
+/*
+ * The common v7_exit_coherency_flush API could not be used because of the
+ * Erratum 799270 workaround. This macro is the same as the common one (in
+ * arch/arm/include/asm/cacheflush.h) except for the erratum handling.
+ */
+#define exynos_v7_exit_coherency_flush(level) \
+	asm volatile( \
+	"stmfd	sp!, {fp, ip}\n\t"\
+	"mrc	p15, 0, r0, c1, c0, 0	@ get SCTLR\n\t" \
+	"bic	r0, r0, #"__stringify(CR_C)"\n\t" \
+	"mcr	p15, 0, r0, c1, c0, 0	@ set SCTLR\n\t" \
+	"isb\n\t"\
+	"bl	v7_flush_dcache_"__stringify(level)"\n\t" \
+	"clrex\n\t"\
+	"mrc	p15, 0, r0, c1, c0, 1	@ get ACTLR\n\t" \
+	"bic	r0, r0, #(1 << 6)	@ disable local coherency\n\t" \
+	/* Dummy Load of a device register to avoid Erratum 799270 */ \
+	"ldr	r4, [%0]\n\t" \
+	"and	r4, r4, #0\n\t" \
+	"orr	r0, r0, r4\n\t" \
+	"mcr	p15, 0, r0, c1, c0, 1	@ set ACTLR\n\t" \
+	"isb\n\t" \
+	"dsb\n\t" \
+	"ldmfd	sp!, {fp, ip}" \
+	: \
+	: "Ir" (S5P_INFORM0) \
+	: "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \
+	  "r9", "r10", "lr", "memory")
+
+/*
+ * We can't use regular spinlocks. In the switcher case, it is possible
+ * for an outbound CPU to call power_down() after its inbound counterpart
+ * is already live using the same logical CPU number which trips lockdep
+ * debugging.
+ */
+static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
+static int
+cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS];
+
+#define exynos_cluster_usecnt(cluster) \
+	(cpu_use_count[0][cluster] +   \
+	 cpu_use_count[1][cluster] +   \
+	 cpu_use_count[2][cluster] +   \
+	 cpu_use_count[3][cluster])
+
+#define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster)
+
+static int exynos_cluster_power_control(unsigned int cluster, int enable)
+{
+	unsigned int tries = 100;
+	unsigned int val;
+
+	if (enable) {
+		exynos_cluster_power_up(cluster);
+		val = S5P_CORE_LOCAL_PWR_EN;
+	} else {
+		exynos_cluster_power_down(cluster);
+		val = 0;
+	}
+
+	/* Wait until cluster power control is applied */
+	while (tries--) {
+		if (exynos_cluster_power_state(cluster) == val)
+			return 0;
+
+		cpu_relax();
+	}
+	pr_debug("timed out waiting for cluster %u to power %s\n", cluster,
+		enable ? "on" : "off");
+
+	return -ETIMEDOUT;
+}
+
+static int exynos_power_up(unsigned int cpu, unsigned int cluster)
+{
+	unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
+	int err = 0;
+
+	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
+	if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
+		cluster >= EXYNOS5420_NR_CLUSTERS)
+		return -EINVAL;
+
+	/*
+	 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
+	 * variant exists, we need to disable IRQs manually here.
+	 */
+	local_irq_disable();
+	arch_spin_lock(&exynos_mcpm_lock);
+
+	cpu_use_count[cpu][cluster]++;
+	if (cpu_use_count[cpu][cluster] == 1) {
+		bool was_cluster_down =
+			(exynos_cluster_usecnt(cluster) == 1);
+
+		/*
+		 * Turn on the cluster (L2/COMMON) and then power on the
+		 * cores.
+		 */
+		if (was_cluster_down)
+			err = exynos_cluster_power_control(cluster, 1);
+
+		if (!err)
+			exynos_cpu_power_up(cpunr);
+		else
+			exynos_cluster_power_control(cluster, 0);
+	} else if (cpu_use_count[cpu][cluster] != 2) {
+		/*
+		 * The only possible values are:
+		 * 0 = CPU down
+		 * 1 = CPU (still) up
+		 * 2 = CPU requested to be up before it had a chance
+		 *     to actually make itself down.
+		 * Any other value is a bug.
+		 */
+		BUG();
+	}
+
+	arch_spin_unlock(&exynos_mcpm_lock);
+	local_irq_enable();
+
+	return err;
+}
+
+/*
+ * NOTE: This function requires the stack data to be visible through power down
+ * and can only be executed on processors like A15 and A7 that hit the cache
+ * with the C bit clear in the SCTLR register.
+ */
+static void exynos_power_down(void)
+{
+	unsigned int mpidr, cpu, cluster;
+	bool last_man = false, skip_wfi = false;
+	unsigned int cpunr;
+
+	mpidr = read_cpuid_mpidr();
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+	cpunr =  cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
+
+	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
+	BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
+			cluster >= EXYNOS5420_NR_CLUSTERS);
+
+	__mcpm_cpu_going_down(cpu, cluster);
+
+	arch_spin_lock(&exynos_mcpm_lock);
+	BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
+	cpu_use_count[cpu][cluster]--;
+	if (cpu_use_count[cpu][cluster] == 0) {
+		exynos_cpu_power_down(cpunr);
+
+		if (exynos_cluster_unused(cluster))
+			/* TODO: Turn off the cluster here to save power. */
+			last_man = true;
+	} else if (cpu_use_count[cpu][cluster] == 1) {
+		/*
+		 * A power_up request went ahead of us.
+		 * Even if we do not want to shut this CPU down,
+		 * the caller expects a certain state as if the WFI
+		 * was aborted.  So let's continue with cache cleaning.
+		 */
+		skip_wfi = true;
+	} else {
+		BUG();
+	}
+
+	if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
+		arch_spin_unlock(&exynos_mcpm_lock);
+
+		if (read_cpuid_part_number() == ARM_CPU_PART_CORTEX_A15) {
+			/*
+			 * On the Cortex-A15 we need to disable
+			 * L2 prefetching before flushing the cache.
+			 */
+			asm volatile(
+			"mcr	p15, 1, %0, c15, c0, 3\n\t"
+			"isb\n\t"
+			"dsb"
+			: : "r" (0x400));
+		}
+
+		/* Flush all cache levels for this cluster. */
+		exynos_v7_exit_coherency_flush(all);
+
+		/*
+		 * Disable cluster-level coherency by masking
+		 * incoming snoops and DVM messages:
+		 */
+		cci_disable_port_by_cpu(mpidr);
+
+		__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
+	} else {
+		arch_spin_unlock(&exynos_mcpm_lock);
+
+		/* Disable and flush the local CPU cache. */
+		exynos_v7_exit_coherency_flush(louis);
+	}
+
+	__mcpm_cpu_down(cpu, cluster);
+
+	/* Now we are prepared for power-down, do it: */
+	if (!skip_wfi)
+		wfi();
+
+	/* Not dead at this point?  Let our caller cope. */
+}
+
+static int exynos_power_down_finish(unsigned int cpu, unsigned int cluster)
+{
+	unsigned int tries = 100;
+	unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
+
+	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
+	BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER ||
+			cluster >= EXYNOS5420_NR_CLUSTERS);
+
+	/* Wait for the core state to be OFF */
+	while (tries--) {
+		if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) {
+			if ((exynos_cpu_power_state(cpunr) == 0))
+				return 0; /* success: the CPU is halted */
+		}
+
+		/* Otherwise, wait and retry: */
+		msleep(1);
+	}
+
+	return -ETIMEDOUT; /* timeout */
+}
+
+static const struct mcpm_platform_ops exynos_power_ops = {
+	.power_up		= exynos_power_up,
+	.power_down		= exynos_power_down,
+	.power_down_finish	= exynos_power_down_finish,
+};
+
+static void __init exynos_mcpm_usage_count_init(void)
+{
+	unsigned int mpidr, cpu, cluster;
+
+	mpidr = read_cpuid_mpidr();
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+
+	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
+	BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER  ||
+			cluster >= EXYNOS5420_NR_CLUSTERS);
+
+	cpu_use_count[cpu][cluster] = 1;
+}
+
+/*
+ * Enable cluster-level coherency, in preparation for turning on the MMU.
+ */
+static void __naked exynos_pm_power_up_setup(unsigned int affinity_level)
+{
+	asm volatile ("\n"
+	"cmp	r0, #1\n"
+	"bxne	lr\n"
+	"b	cci_enable_port_for_self");
+}
+
+static int __init exynos_mcpm_init(void)
+{
+	struct device_node *node;
+	void __iomem *ns_sram_base_addr;
+	int ret;
+
+	node = of_find_compatible_node(NULL, NULL, "samsung,exynos5420");
+	if (!node)
+		return -ENODEV;
+	of_node_put(node);
+
+	if (!cci_probed())
+		return -ENODEV;
+
+	node = of_find_compatible_node(NULL, NULL,
+			"samsung,exynos4210-sysram-ns");
+	if (!node)
+		return -ENODEV;
+
+	ns_sram_base_addr = of_iomap(node, 0);
+	of_node_put(node);
+	if (!ns_sram_base_addr) {
+		pr_err("failed to map non-secure iRAM base address\n");
+		return -ENOMEM;
+	}
+
+	/*
+	 * To increase the stability of KFC reset we need to program
+	 * the PMU SPARE3 register
+	 */
+	__raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3);
+
+	exynos_mcpm_usage_count_init();
+
+	ret = mcpm_platform_register(&exynos_power_ops);
+	if (!ret)
+		ret = mcpm_sync_init(exynos_pm_power_up_setup);
+	if (ret) {
+		iounmap(ns_sram_base_addr);
+		return ret;
+	}
+
+	mcpm_smp_set_ops();
+
+	pr_info("Exynos MCPM support installed\n");
+
+	/*
+	 * Future entries into the kernel can now go
+	 * through the cluster entry vectors.
+	 */
+	__raw_writel(virt_to_phys(mcpm_entry_point),
+			ns_sram_base_addr + MCPM_BOOT_ADDR_OFFSET);
+
+	iounmap(ns_sram_base_addr);
+
+	return ret;
+}
+
+early_initcall(exynos_mcpm_init);
@@ -38,6 +38,7 @@
 #define S5P_INFORM5				S5P_PMUREG(0x0814)
 #define S5P_INFORM6				S5P_PMUREG(0x0818)
 #define S5P_INFORM7				S5P_PMUREG(0x081C)
+#define S5P_PMU_SPARE3				S5P_PMUREG(0x090C)
  
 #define S5P_ARM_CORE0_LOWPWR			S5P_PMUREG(0x1000)
 #define S5P_DIS_IRQ_CORE0			S5P_PMUREG(0x1004)
@@ -321,6 +322,8 @@
 #define EXYNOS5_OPTION_USE_STANDBYWFI				(1 << 16)
  
 #define EXYNOS5_OPTION_USE_RETENTION				(1 << 4)
+
+#define EXYNOS5420_SWRESET_KFC_SEL				0x3
  
 #endif /* __ASM_ARCH_REGS_PMU_H */
...	...	@@ -110,5 +110,13 @@
110	110
111	111	endmenu
112	112
	113	+config EXYNOS5420_MCPM
	114	+ bool "Exynos5420 Multi-Cluster PM support"
	115	+ depends on MCPM && SOC_EXYNOS5420
	116	+ select ARM_CCI
	117	+ help
	118	+ This is needed to provide CPU and cluster power management
	119	+ on Exynos5420 implementing big.LITTLE.
	120	+
113	121	endif
...	...	@@ -29,4 +29,6 @@
29	29
30	30	plus_sec := $(call as-instr,.arch_extension sec,+sec)
31	31	AFLAGS_exynos-smc.o :=-Wa,-march=armv7-a$(plus_sec)
	32	+
	33	+obj-$(CONFIG_EXYNOS5420_MCPM) += mcpm-exynos.o
	1	+/*
	2	+ * Copyright (c) 2014 Samsung Electronics Co., Ltd.
	3	+ * http://www.samsung.com
	4	+ *
	5	+ * arch/arm/mach-exynos/mcpm-exynos.c
	6	+ *
	7	+ * Based on arch/arm/mach-vexpress/dcscb.c
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or modify
	10	+ * it under the terms of the GNU General Public License version 2 as
	11	+ * published by the Free Software Foundation.
	12	+ */
	13	+
	14	+#include <linux/arm-cci.h>
	15	+#include <linux/delay.h>
	16	+#include <linux/io.h>
	17	+#include <linux/of_address.h>
	18	+
	19	+#include <asm/cputype.h>
	20	+#include <asm/cp15.h>
	21	+#include <asm/mcpm.h>
	22	+
	23	+#include "regs-pmu.h"
	24	+#include "common.h"
	25	+
	26	+#define EXYNOS5420_CPUS_PER_CLUSTER 4
	27	+#define EXYNOS5420_NR_CLUSTERS 2
	28	+#define MCPM_BOOT_ADDR_OFFSET 0x1c
	29	+
	30	+/*
	31	+ * The common v7_exit_coherency_flush API could not be used because of the
	32	+ * Erratum 799270 workaround. This macro is the same as the common one (in
	33	+ * arch/arm/include/asm/cacheflush.h) except for the erratum handling.
	34	+ */
	35	+#define exynos_v7_exit_coherency_flush(level) \
	36	+ asm volatile( \
	37	+ "stmfd sp!, {fp, ip}\n\t"\
	38	+ "mrc p15, 0, r0, c1, c0, 0 @ get SCTLR\n\t" \
	39	+ "bic r0, r0, #"__stringify(CR_C)"\n\t" \
	40	+ "mcr p15, 0, r0, c1, c0, 0 @ set SCTLR\n\t" \
	41	+ "isb\n\t"\
	42	+ "bl v7_flush_dcache_"__stringify(level)"\n\t" \
	43	+ "clrex\n\t"\
	44	+ "mrc p15, 0, r0, c1, c0, 1 @ get ACTLR\n\t" \
	45	+ "bic r0, r0, #(1 << 6) @ disable local coherency\n\t" \
	46	+ /* Dummy Load of a device register to avoid Erratum 799270 */ \
	47	+ "ldr r4, [%0]\n\t" \
	48	+ "and r4, r4, #0\n\t" \
	49	+ "orr r0, r0, r4\n\t" \
	50	+ "mcr p15, 0, r0, c1, c0, 1 @ set ACTLR\n\t" \
	51	+ "isb\n\t" \
	52	+ "dsb\n\t" \
	53	+ "ldmfd sp!, {fp, ip}" \
	54	+ : \
	55	+ : "Ir" (S5P_INFORM0) \
	56	+ : "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", \
	57	+ "r9", "r10", "lr", "memory")
	58	+
	59	+/*
	60	+ * We can't use regular spinlocks. In the switcher case, it is possible
	61	+ * for an outbound CPU to call power_down() after its inbound counterpart
	62	+ * is already live using the same logical CPU number which trips lockdep
	63	+ * debugging.
	64	+ */
	65	+static arch_spinlock_t exynos_mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
	66	+static int
	67	+cpu_use_count[EXYNOS5420_CPUS_PER_CLUSTER][EXYNOS5420_NR_CLUSTERS];
	68	+
	69	+#define exynos_cluster_usecnt(cluster) \
	70	+ (cpu_use_count[0][cluster] + \
	71	+ cpu_use_count[1][cluster] + \
	72	+ cpu_use_count[2][cluster] + \
	73	+ cpu_use_count[3][cluster])
	74	+
	75	+#define exynos_cluster_unused(cluster) !exynos_cluster_usecnt(cluster)
	76	+
	77	+static int exynos_cluster_power_control(unsigned int cluster, int enable)
	78	+{
	79	+ unsigned int tries = 100;
	80	+ unsigned int val;
	81	+
	82	+ if (enable) {
	83	+ exynos_cluster_power_up(cluster);
	84	+ val = S5P_CORE_LOCAL_PWR_EN;
	85	+ } else {
	86	+ exynos_cluster_power_down(cluster);
	87	+ val = 0;
	88	+ }
	89	+
	90	+ /* Wait until cluster power control is applied */
	91	+ while (tries--) {
	92	+ if (exynos_cluster_power_state(cluster) == val)
	93	+ return 0;
	94	+
	95	+ cpu_relax();
	96	+ }
	97	+ pr_debug("timed out waiting for cluster %u to power %s\n", cluster,
	98	+ enable ? "on" : "off");
	99	+
	100	+ return -ETIMEDOUT;
	101	+}
	102	+
	103	+static int exynos_power_up(unsigned int cpu, unsigned int cluster)
	104	+{
	105	+ unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
	106	+ int err = 0;
	107	+
	108	+ pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	109	+ if (cpu >= EXYNOS5420_CPUS_PER_CLUSTER \|\|
	110	+ cluster >= EXYNOS5420_NR_CLUSTERS)
	111	+ return -EINVAL;
	112	+
	113	+ /*
	114	+ * Since this is called with IRQs enabled, and no arch_spin_lock_irq
	115	+ * variant exists, we need to disable IRQs manually here.
	116	+ */
	117	+ local_irq_disable();
	118	+ arch_spin_lock(&exynos_mcpm_lock);
	119	+
	120	+ cpu_use_count[cpu][cluster]++;
	121	+ if (cpu_use_count[cpu][cluster] == 1) {
	122	+ bool was_cluster_down =
	123	+ (exynos_cluster_usecnt(cluster) == 1);
	124	+
	125	+ /*
	126	+ * Turn on the cluster (L2/COMMON) and then power on the
	127	+ * cores.
	128	+ */
	129	+ if (was_cluster_down)
	130	+ err = exynos_cluster_power_control(cluster, 1);
	131	+
	132	+ if (!err)
	133	+ exynos_cpu_power_up(cpunr);
	134	+ else
	135	+ exynos_cluster_power_control(cluster, 0);
	136	+ } else if (cpu_use_count[cpu][cluster] != 2) {
	137	+ /*
	138	+ * The only possible values are:
	139	+ * 0 = CPU down
	140	+ * 1 = CPU (still) up
	141	+ * 2 = CPU requested to be up before it had a chance
	142	+ * to actually make itself down.
	143	+ * Any other value is a bug.
	144	+ */
	145	+ BUG();
	146	+ }
	147	+
	148	+ arch_spin_unlock(&exynos_mcpm_lock);
	149	+ local_irq_enable();
	150	+
	151	+ return err;
	152	+}
	153	+
	154	+/*
	155	+ * NOTE: This function requires the stack data to be visible through power down
	156	+ * and can only be executed on processors like A15 and A7 that hit the cache
	157	+ * with the C bit clear in the SCTLR register.
	158	+ */
	159	+static void exynos_power_down(void)
	160	+{
	161	+ unsigned int mpidr, cpu, cluster;
	162	+ bool last_man = false, skip_wfi = false;
	163	+ unsigned int cpunr;
	164	+
	165	+ mpidr = read_cpuid_mpidr();
	166	+ cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	167	+ cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	168	+ cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
	169	+
	170	+ pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	171	+ BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER \|\|
	172	+ cluster >= EXYNOS5420_NR_CLUSTERS);
	173	+
	174	+ __mcpm_cpu_going_down(cpu, cluster);
	175	+
	176	+ arch_spin_lock(&exynos_mcpm_lock);
	177	+ BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
	178	+ cpu_use_count[cpu][cluster]--;
	179	+ if (cpu_use_count[cpu][cluster] == 0) {
	180	+ exynos_cpu_power_down(cpunr);
	181	+
	182	+ if (exynos_cluster_unused(cluster))
	183	+ /* TODO: Turn off the cluster here to save power. */
	184	+ last_man = true;
	185	+ } else if (cpu_use_count[cpu][cluster] == 1) {
	186	+ /*
	187	+ * A power_up request went ahead of us.
	188	+ * Even if we do not want to shut this CPU down,
	189	+ * the caller expects a certain state as if the WFI
	190	+ * was aborted. So let's continue with cache cleaning.
	191	+ */
	192	+ skip_wfi = true;
	193	+ } else {
	194	+ BUG();
	195	+ }
	196	+
	197	+ if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
	198	+ arch_spin_unlock(&exynos_mcpm_lock);
	199	+
	200	+ if (read_cpuid_part_number() == ARM_CPU_PART_CORTEX_A15) {
	201	+ /*
	202	+ * On the Cortex-A15 we need to disable
	203	+ * L2 prefetching before flushing the cache.
	204	+ */
	205	+ asm volatile(
	206	+ "mcr p15, 1, %0, c15, c0, 3\n\t"
	207	+ "isb\n\t"
	208	+ "dsb"
	209	+ : : "r" (0x400));
	210	+ }
	211	+
	212	+ /* Flush all cache levels for this cluster. */
	213	+ exynos_v7_exit_coherency_flush(all);
	214	+
	215	+ /*
	216	+ * Disable cluster-level coherency by masking
	217	+ * incoming snoops and DVM messages:
	218	+ */
	219	+ cci_disable_port_by_cpu(mpidr);
	220	+
	221	+ __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
	222	+ } else {
	223	+ arch_spin_unlock(&exynos_mcpm_lock);
	224	+
	225	+ /* Disable and flush the local CPU cache. */
	226	+ exynos_v7_exit_coherency_flush(louis);
	227	+ }
	228	+
	229	+ __mcpm_cpu_down(cpu, cluster);
	230	+
	231	+ /* Now we are prepared for power-down, do it: */
	232	+ if (!skip_wfi)
	233	+ wfi();
	234	+
	235	+ /* Not dead at this point? Let our caller cope. */
	236	+}
	237	+
	238	+static int exynos_power_down_finish(unsigned int cpu, unsigned int cluster)
	239	+{
	240	+ unsigned int tries = 100;
	241	+ unsigned int cpunr = cpu + (cluster * EXYNOS5420_CPUS_PER_CLUSTER);
	242	+
	243	+ pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	244	+ BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER \|\|
	245	+ cluster >= EXYNOS5420_NR_CLUSTERS);
	246	+
	247	+ /* Wait for the core state to be OFF */
	248	+ while (tries--) {
	249	+ if (ACCESS_ONCE(cpu_use_count[cpu][cluster]) == 0) {
	250	+ if ((exynos_cpu_power_state(cpunr) == 0))
	251	+ return 0; /* success: the CPU is halted */
	252	+ }
	253	+
	254	+ /* Otherwise, wait and retry: */
	255	+ msleep(1);
	256	+ }
	257	+
	258	+ return -ETIMEDOUT; /* timeout */
	259	+}
	260	+
	261	+static const struct mcpm_platform_ops exynos_power_ops = {
	262	+ .power_up = exynos_power_up,
	263	+ .power_down = exynos_power_down,
	264	+ .power_down_finish = exynos_power_down_finish,
	265	+};
	266	+
	267	+static void __init exynos_mcpm_usage_count_init(void)
	268	+{
	269	+ unsigned int mpidr, cpu, cluster;
	270	+
	271	+ mpidr = read_cpuid_mpidr();
	272	+ cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
	273	+ cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
	274	+
	275	+ pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
	276	+ BUG_ON(cpu >= EXYNOS5420_CPUS_PER_CLUSTER \|\|
	277	+ cluster >= EXYNOS5420_NR_CLUSTERS);
	278	+
	279	+ cpu_use_count[cpu][cluster] = 1;
	280	+}
	281	+
	282	+/*
	283	+ * Enable cluster-level coherency, in preparation for turning on the MMU.
	284	+ */
	285	+static void __naked exynos_pm_power_up_setup(unsigned int affinity_level)
	286	+{
	287	+ asm volatile ("\n"
	288	+ "cmp r0, #1\n"
	289	+ "bxne lr\n"
	290	+ "b cci_enable_port_for_self");
	291	+}
	292	+
	293	+static int __init exynos_mcpm_init(void)
	294	+{
	295	+ struct device_node *node;
	296	+ void __iomem *ns_sram_base_addr;
	297	+ int ret;
	298	+
	299	+ node = of_find_compatible_node(NULL, NULL, "samsung,exynos5420");
	300	+ if (!node)
	301	+ return -ENODEV;
	302	+ of_node_put(node);
	303	+
	304	+ if (!cci_probed())
	305	+ return -ENODEV;
	306	+
	307	+ node = of_find_compatible_node(NULL, NULL,
	308	+ "samsung,exynos4210-sysram-ns");
	309	+ if (!node)
	310	+ return -ENODEV;
	311	+
	312	+ ns_sram_base_addr = of_iomap(node, 0);
	313	+ of_node_put(node);
	314	+ if (!ns_sram_base_addr) {
	315	+ pr_err("failed to map non-secure iRAM base address\n");
	316	+ return -ENOMEM;
	317	+ }
	318	+
	319	+ /*
	320	+ * To increase the stability of KFC reset we need to program
	321	+ * the PMU SPARE3 register
	322	+ */
	323	+ __raw_writel(EXYNOS5420_SWRESET_KFC_SEL, S5P_PMU_SPARE3);
	324	+
	325	+ exynos_mcpm_usage_count_init();
	326	+
	327	+ ret = mcpm_platform_register(&exynos_power_ops);
	328	+ if (!ret)
	329	+ ret = mcpm_sync_init(exynos_pm_power_up_setup);
	330	+ if (ret) {
	331	+ iounmap(ns_sram_base_addr);
	332	+ return ret;
	333	+ }
	334	+
	335	+ mcpm_smp_set_ops();
	336	+
	337	+ pr_info("Exynos MCPM support installed\n");
	338	+
	339	+ /*
	340	+ * Future entries into the kernel can now go
	341	+ * through the cluster entry vectors.
	342	+ */
	343	+ __raw_writel(virt_to_phys(mcpm_entry_point),
	344	+ ns_sram_base_addr + MCPM_BOOT_ADDR_OFFSET);
	345	+
	346	+ iounmap(ns_sram_base_addr);
	347	+
	348	+ return ret;
	349	+}
	350	+
	351	+early_initcall(exynos_mcpm_init);
...	...	@@ -38,6 +38,7 @@
38	38	#define S5P_INFORM5 S5P_PMUREG(0x0814)
39	39	#define S5P_INFORM6 S5P_PMUREG(0x0818)
40	40	#define S5P_INFORM7 S5P_PMUREG(0x081C)
	41	+#define S5P_PMU_SPARE3 S5P_PMUREG(0x090C)
41	42
42	43	#define S5P_ARM_CORE0_LOWPWR S5P_PMUREG(0x1000)
43	44	#define S5P_DIS_IRQ_CORE0 S5P_PMUREG(0x1004)
...	...	@@ -321,6 +322,8 @@
321	322	#define EXYNOS5_OPTION_USE_STANDBYWFI (1 << 16)
322	323
323	324	#define EXYNOS5_OPTION_USE_RETENTION (1 << 4)
	325	+
	326	+#define EXYNOS5420_SWRESET_KFC_SEL 0x3
324	327
325	328	#endif /* __ASM_ARCH_REGS_PMU_H */