armv7: cache maintenance operations for armv7

- Add a framework for layered cache maintenance - separate out SOC specific outer cache maintenance from maintenance of caches known to CPU - Add generic ARMv7 cache maintenance operations that affect all caches known to ARMv7 CPUs. For instance in Cortex-A8 these opertions will affect both L1 and L2 caches. In Cortex-A9 these will affect only L1 cache - D-cache operations supported: - Invalidate entire D-cache - Invalidate D-cache range - Flush(clean & invalidate) entire D-cache - Flush D-cache range - I-cache operations supported: - Invalidate entire I-cache - Add maintenance functions for TLB, branch predictor array etc. - Enable -march=armv7-a so that armv7 assembly instructions can be used Signed-off-by: Aneesh V <aneesh@ti.com>

armv7: cache maintenance operations for armv7
- Add a framework for layered cache maintenance - separate out SOC specific outer cache maintenance from maintenance of caches known to CPU - Add generic ARMv7 cache maintenance operations that affect all caches known to ARMv7 CPUs. For instance in Cortex-A8 these opertions will affect both L1 and L2 caches. In Cortex-A9 these will affect only L1 cache - D-cache operations supported: - Invalidate entire D-cache - Invalidate D-cache range - Flush(clean & invalidate) entire D-cache - Flush D-cache range - I-cache operations supported: - Invalidate entire I-cache - Add maintenance functions for TLB, branch predictor array etc. - Enable -march=armv7-a so that armv7 assembly instructions can be used Signed-off-by: Aneesh V <aneesh@ti.com>
Aneesh V · Albert ARIBAUD
1 parent 4c93da7c39
Showing 6 changed files with 527 additions and 2 deletions Side-by-side Diff
README
arch/arm/cpu/armv7/Makefile
arch/arm/cpu/armv7/cache_v7.c
arch/arm/include/asm/armv7.h
arch/arm/include/asm/utils.h
include/common.h
@@ -460,6 +460,11 @@
 		Note: If a "bootargs" environment is defined, it will overwride
 		the defaults discussed just above.
  
+- Cache Configuration:
+		CONFIG_SYS_ICACHE_OFF - Do not enable instruction cache in U-Boot
+		CONFIG_SYS_DCACHE_OFF - Do not enable data cache in U-Boot
+		CONFIG_SYS_L2CACHE_OFF- Do not enable L2 cache in U-Boot
+
 - Serial Ports:
 		CONFIG_PL010_SERIAL
  
@@ -26,7 +26,7 @@
 LIB	= $(obj)lib$(CPU).o
  
 START	:= start.o
-COBJS	:= cpu.o
+COBJS	:= cpu.o cache_v7.o
 COBJS  += syslib.o
  
 SRCS	:= $(START:.o=.S) $(COBJS:.o=.c)
+/*
+ * (C) Copyright 2010
+ * Texas Instruments, <www.ti.com>
+ * Aneesh V <aneesh@ti.com>
+ *
+ * See file CREDITS for list of people who contributed to this
+ * project.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+#include <linux/types.h>
+#include <common.h>
+#include <asm/armv7.h>
+#include <asm/utils.h>
+
+#define ARMV7_DCACHE_INVAL_ALL		1
+#define ARMV7_DCACHE_CLEAN_INVAL_ALL	2
+#define ARMV7_DCACHE_INVAL_RANGE	3
+#define ARMV7_DCACHE_CLEAN_INVAL_RANGE	4
+
+#ifndef CONFIG_SYS_DCACHE_OFF
+/*
+ * Write the level and type you want to Cache Size Selection Register(CSSELR)
+ * to get size details from Current Cache Size ID Register(CCSIDR)
+ */
+static void set_csselr(u32 level, u32 type)
+{	u32 csselr = level << 1 | type;
+
+	/* Write to Cache Size Selection Register(CSSELR) */
+	asm volatile ("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr));
+}
+
+static u32 get_ccsidr(void)
+{
+	u32 ccsidr;
+
+	/* Read current CP15 Cache Size ID Register */
+	asm volatile ("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr));
+	return ccsidr;
+}
+
+static u32 get_clidr(void)
+{
+	u32 clidr;
+
+	/* Read current CP15 Cache Level ID Register */
+	asm volatile ("mrc p15,1,%0,c0,c0,1" : "=r" (clidr));
+	return clidr;
+}
+
+static void v7_inval_dcache_level_setway(u32 level, u32 num_sets,
+					 u32 num_ways, u32 way_shift,
+					 u32 log2_line_len)
+{
+	int way, set, setway;
+
+	/*
+	 * For optimal assembly code:
+	 *	a. count down
+	 *	b. have bigger loop inside
+	 */
+	for (way = num_ways - 1; way >= 0 ; way--) {
+		for (set = num_sets - 1; set >= 0; set--) {
+			setway = (level << 1) | (set << log2_line_len) |
+				 (way << way_shift);
+			/* Invalidate data/unified cache line by set/way */
+			asm volatile ("	mcr p15, 0, %0, c7, c6, 2"
+					: : "r" (setway));
+		}
+	}
+	/* DMB to make sure the operation is complete */
+	CP15DMB;
+}
+
+static void v7_clean_inval_dcache_level_setway(u32 level, u32 num_sets,
+					       u32 num_ways, u32 way_shift,
+					       u32 log2_line_len)
+{
+	int way, set, setway;
+
+	/*
+	 * For optimal assembly code:
+	 *	a. count down
+	 *	b. have bigger loop inside
+	 */
+	for (way = num_ways - 1; way >= 0 ; way--) {
+		for (set = num_sets - 1; set >= 0; set--) {
+			setway = (level << 1) | (set << log2_line_len) |
+				 (way << way_shift);
+			/*
+			 * Clean & Invalidate data/unified
+			 * cache line by set/way
+			 */
+			asm volatile ("	mcr p15, 0, %0, c7, c14, 2"
+					: : "r" (setway));
+		}
+	}
+	/* DMB to make sure the operation is complete */
+	CP15DMB;
+}
+
+static void v7_maint_dcache_level_setway(u32 level, u32 operation)
+{
+	u32 ccsidr;
+	u32 num_sets, num_ways, log2_line_len, log2_num_ways;
+	u32 way_shift;
+
+	set_csselr(level, ARMV7_CSSELR_IND_DATA_UNIFIED);
+
+	ccsidr = get_ccsidr();
+
+	log2_line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >>
+				CCSIDR_LINE_SIZE_OFFSET) + 2;
+	/* Converting from words to bytes */
+	log2_line_len += 2;
+
+	num_ways  = ((ccsidr & CCSIDR_ASSOCIATIVITY_MASK) >>
+			CCSIDR_ASSOCIATIVITY_OFFSET) + 1;
+	num_sets  = ((ccsidr & CCSIDR_NUM_SETS_MASK) >>
+			CCSIDR_NUM_SETS_OFFSET) + 1;
+	/*
+	 * According to ARMv7 ARM number of sets and number of ways need
+	 * not be a power of 2
+	 */
+	log2_num_ways = log_2_n_round_up(num_ways);
+
+	way_shift = (32 - log2_num_ways);
+	if (operation == ARMV7_DCACHE_INVAL_ALL) {
+		v7_inval_dcache_level_setway(level, num_sets, num_ways,
+				      way_shift, log2_line_len);
+	} else if (operation == ARMV7_DCACHE_CLEAN_INVAL_ALL) {
+		v7_clean_inval_dcache_level_setway(level, num_sets, num_ways,
+						   way_shift, log2_line_len);
+	}
+}
+
+static void v7_maint_dcache_all(u32 operation)
+{
+	u32 level, cache_type, level_start_bit = 0;
+
+	u32 clidr = get_clidr();
+
+	for (level = 0; level < 7; level++) {
+		cache_type = (clidr >> level_start_bit) & 0x7;
+		if ((cache_type == ARMV7_CLIDR_CTYPE_DATA_ONLY) ||
+		    (cache_type == ARMV7_CLIDR_CTYPE_INSTRUCTION_DATA) ||
+		    (cache_type == ARMV7_CLIDR_CTYPE_UNIFIED))
+			v7_maint_dcache_level_setway(level, operation);
+		level_start_bit += 3;
+	}
+}
+
+static void v7_dcache_clean_inval_range(u32 start,
+					u32 stop, u32 line_len)
+{
+	u32 mva;
+
+	/* Align start to cache line boundary */
+	start &= ~(line_len - 1);
+	for (mva = start; mva < stop; mva = mva + line_len) {
+		/* DCCIMVAC - Clean & Invalidate data cache by MVA to PoC */
+		asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva));
+	}
+}
+
+static void v7_dcache_inval_range(u32 start, u32 stop, u32 line_len)
+{
+	u32 mva;
+
+	/*
+	 * If start address is not aligned to cache-line flush the first
+	 * line to prevent affecting somebody else's buffer
+	 */
+	if (start & (line_len - 1)) {
+		v7_dcache_clean_inval_range(start, start + 1, line_len);
+		/* move to next cache line */
+		start = (start + line_len - 1) & ~(line_len - 1);
+	}
+
+	/*
+	 * If stop address is not aligned to cache-line flush the last
+	 * line to prevent affecting somebody else's buffer
+	 */
+	if (stop & (line_len - 1)) {
+		v7_dcache_clean_inval_range(stop, stop + 1, line_len);
+		/* align to the beginning of this cache line */
+		stop &= ~(line_len - 1);
+	}
+
+	for (mva = start; mva < stop; mva = mva + line_len) {
+		/* DCIMVAC - Invalidate data cache by MVA to PoC */
+		asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva));
+	}
+}
+
+static void v7_dcache_maint_range(u32 start, u32 stop, u32 range_op)
+{
+	u32 line_len, ccsidr;
+
+	ccsidr = get_ccsidr();
+	line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >>
+			CCSIDR_LINE_SIZE_OFFSET) + 2;
+	/* Converting from words to bytes */
+	line_len += 2;
+	/* converting from log2(linelen) to linelen */
+	line_len = 1 << line_len;
+
+	switch (range_op) {
+	case ARMV7_DCACHE_CLEAN_INVAL_RANGE:
+		v7_dcache_clean_inval_range(start, stop, line_len);
+		break;
+	case ARMV7_DCACHE_INVAL_RANGE:
+		v7_dcache_inval_range(start, stop, line_len);
+		break;
+	}
+
+	/* DMB to make sure the operation is complete */
+	CP15DMB;
+}
+
+/* Invalidate TLB */
+static void v7_inval_tlb(void)
+{
+	/* Invalidate entire unified TLB */
+	asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0));
+	/* Invalidate entire data TLB */
+	asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0));
+	/* Invalidate entire instruction TLB */
+	asm volatile ("mcr p15, 0, %0, c8, c5, 0" : : "r" (0));
+	/* Full system DSB - make sure that the invalidation is complete */
+	CP15DSB;
+	/* Full system ISB - make sure the instruction stream sees it */
+	CP15ISB;
+}
+
+void invalidate_dcache_all(void)
+{
+	v7_maint_dcache_all(ARMV7_DCACHE_INVAL_ALL);
+
+	v7_outer_cache_inval_all();
+}
+
+/*
+ * Performs a clean & invalidation of the entire data cache
+ * at all levels
+ */
+void flush_dcache_all(void)
+{
+	v7_maint_dcache_all(ARMV7_DCACHE_CLEAN_INVAL_ALL);
+
+	v7_outer_cache_flush_all();
+}
+
+/*
+ * Invalidates range in all levels of D-cache/unified cache used:
+ * Affects the range [start, stop - 1]
+ */
+void invalidate_dcache_range(unsigned long start, unsigned long stop)
+{
+
+	v7_dcache_maint_range(start, stop, ARMV7_DCACHE_INVAL_RANGE);
+
+	v7_outer_cache_inval_range(start, stop);
+}
+
+/*
+ * Flush range(clean & invalidate) from all levels of D-cache/unified
+ * cache used:
+ * Affects the range [start, stop - 1]
+ */
+void flush_dcache_range(unsigned long start, unsigned long stop)
+{
+	v7_dcache_maint_range(start, stop, ARMV7_DCACHE_CLEAN_INVAL_RANGE);
+
+	v7_outer_cache_flush_range(start, stop);
+}
+
+void arm_init_before_mmu(void)
+{
+	v7_outer_cache_enable();
+	invalidate_dcache_all();
+	v7_inval_tlb();
+}
+
+/*
+ * Flush range from all levels of d-cache/unified-cache used:
+ * Affects the range [start, start + size - 1]
+ */
+void  flush_cache(unsigned long start, unsigned long size)
+{
+	flush_dcache_range(start, start + size);
+}
+#else /* #ifndef CONFIG_SYS_DCACHE_OFF */
+void invalidate_dcache_all(void)
+{
+}
+
+void flush_dcache_all(void)
+{
+}
+
+void invalidate_dcache_range(unsigned long start, unsigned long stop)
+{
+}
+
+void flush_dcache_range(unsigned long start, unsigned long stop)
+{
+}
+
+void arm_init_before_mmu(void)
+{
+}
+
+void  flush_cache(unsigned long start, unsigned long size)
+{
+}
+#endif /* #ifndef CONFIG_SYS_DCACHE_OFF */
+
+#ifndef CONFIG_SYS_ICACHE_OFF
+/* Invalidate entire I-cache and branch predictor array */
+void invalidate_icache_all(void)
+{
+	/*
+	 * Invalidate all instruction caches to PoU.
+	 * Also flushes branch target cache.
+	 */
+	asm volatile ("mcr p15, 0, %0, c7, c5, 0" : : "r" (0));
+
+	/* Invalidate entire branch predictor array */
+	asm volatile ("mcr p15, 0, %0, c7, c5, 6" : : "r" (0));
+
+	/* Full system DSB - make sure that the invalidation is complete */
+	CP15DSB;
+
+	/* ISB - make sure the instruction stream sees it */
+	CP15ISB;
+}
+#else
+void invalidate_icache_all(void)
+{
+}
+#endif
+
+/*
+ * Stub implementations for outer cache operations
+ */
+void __v7_outer_cache_enable(void)
+{
+}
+void v7_outer_cache_enable(void)
+	__attribute__((weak, alias("__v7_outer_cache_enable")));
+
+void __v7_outer_cache_disable(void)
+{
+}
+void v7_outer_cache_disable(void)
+	__attribute__((weak, alias("__v7_outer_cache_disable")));
+
+void __v7_outer_cache_flush_all(void)
+{
+}
+void v7_outer_cache_flush_all(void)
+	__attribute__((weak, alias("__v7_outer_cache_flush_all")));
+
+void __v7_outer_cache_inval_all(void)
+{
+}
+void v7_outer_cache_inval_all(void)
+	__attribute__((weak, alias("__v7_outer_cache_inval_all")));
+
+void __v7_outer_cache_flush_range(u32 start, u32 end)
+{
+}
+void v7_outer_cache_flush_range(u32 start, u32 end)
+	__attribute__((weak, alias("__v7_outer_cache_flush_range")));
+
+void __v7_outer_cache_inval_range(u32 start, u32 end)
+{
+}
+void v7_outer_cache_inval_range(u32 start, u32 end)
+	__attribute__((weak, alias("__v7_outer_cache_inval_range")));
+/*
+ * (C) Copyright 2010
+ * Texas Instruments, <www.ti.com>
+ * Aneesh V <aneesh@ti.com>
+ *
+ * See file CREDITS for list of people who contributed to this
+ * project.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+#ifndef ARMV7_H
+#define ARMV7_H
+#include <linux/types.h>
+
+/* CCSIDR */
+#define CCSIDR_LINE_SIZE_OFFSET		0
+#define CCSIDR_LINE_SIZE_MASK		0x7
+#define CCSIDR_ASSOCIATIVITY_OFFSET	3
+#define CCSIDR_ASSOCIATIVITY_MASK	(0x3FF << 3)
+#define CCSIDR_NUM_SETS_OFFSET		13
+#define CCSIDR_NUM_SETS_MASK		(0x7FFF << 13)
+
+/*
+ * Values for InD field in CSSELR
+ * Selects the type of cache
+ */
+#define ARMV7_CSSELR_IND_DATA_UNIFIED	0
+#define ARMV7_CSSELR_IND_INSTRUCTION	1
+
+/* Values for Ctype fields in CLIDR */
+#define ARMV7_CLIDR_CTYPE_NO_CACHE		0
+#define ARMV7_CLIDR_CTYPE_INSTRUCTION_ONLY	1
+#define ARMV7_CLIDR_CTYPE_DATA_ONLY		2
+#define ARMV7_CLIDR_CTYPE_INSTRUCTION_DATA	3
+#define ARMV7_CLIDR_CTYPE_UNIFIED		4
+
+/*
+ * CP15 Barrier instructions
+ * Please note that we have separate barrier instructions in ARMv7
+ * However, we use the CP15 based instructtions because we use
+ * -march=armv5 in U-Boot
+ */
+#define CP15ISB	asm volatile ("mcr     p15, 0, %0, c7, c5, 4" : : "r" (0))
+#define CP15DSB	asm volatile ("mcr     p15, 0, %0, c7, c10, 4" : : "r" (0))
+#define CP15DMB	asm volatile ("mcr     p15, 0, %0, c7, c10, 5" : : "r" (0))
+
+void v7_outer_cache_enable(void);
+void v7_outer_cache_disable(void);
+void v7_outer_cache_flush_all(void);
+void v7_outer_cache_inval_all(void);
+void v7_outer_cache_flush_range(u32 start, u32 end);
+void v7_outer_cache_inval_range(u32 start, u32 end);
+
+#endif
+/*
+ * (C) Copyright 2010
+ * Texas Instruments, <www.ti.com>
+ * Aneesh V <aneesh@ti.com>
+ *
+ * See file CREDITS for list of people who contributed to this
+ * project.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+ * MA 02111-1307 USA
+ */
+#ifndef _UTILS_H_
+#define _UTILS_H_
+
+static inline s32 log_2_n_round_up(u32 n)
+{
+	s32 log2n = -1;
+	u32 temp = n;
+
+	while (temp) {
+		log2n++;
+		temp >>= 1;
+	}
+
+	if (n & (n - 1))
+		return log2n + 1; /* not power of 2 - round up */
+	else
+		return log2n; /* power of 2 */
+}
+
+static inline s32 log_2_n_round_down(u32 n)
+{
+	s32 log2n = -1;
+	u32 temp = n;
+
+	while (temp) {
+		log2n++;
+		temp >>= 1;
+	}
+
+	return log2n;
+}
+
+#endif
@@ -413,6 +413,7 @@
 int	dcache_status (void);
 void	dcache_enable (void);
 void	dcache_disable(void);
+void	mmu_disable(void);
 void	relocate_code (ulong, gd_t *, ulong) __attribute__ ((noreturn));
 ulong	get_endaddr   (void);
 void	trap_init     (ulong);
  
@@ -611,9 +612,11 @@
  
 /* arch/$(ARCH)/lib/cache.c */
 void	flush_cache   (unsigned long, unsigned long);
+void	flush_dcache_all(void);
 void	flush_dcache_range(unsigned long start, unsigned long stop);
 void	invalidate_dcache_range(unsigned long start, unsigned long stop);
-
+void	invalidate_dcache_all(void);
+void	invalidate_icache_all(void);
  
 /* arch/$(ARCH)/lib/ticks.S */
 unsigned long long get_ticks(void);
...	...	@@ -460,6 +460,11 @@
460	460	Note: If a "bootargs" environment is defined, it will overwride
461	461	the defaults discussed just above.
462	462
	463	+- Cache Configuration:
	464	+ CONFIG_SYS_ICACHE_OFF - Do not enable instruction cache in U-Boot
	465	+ CONFIG_SYS_DCACHE_OFF - Do not enable data cache in U-Boot
	466	+ CONFIG_SYS_L2CACHE_OFF- Do not enable L2 cache in U-Boot
	467	+
463	468	- Serial Ports:
464	469	CONFIG_PL010_SERIAL
465	470
...	...	@@ -26,7 +26,7 @@
26	26	LIB = $(obj)lib$(CPU).o
27	27
28	28	START := start.o
29		-COBJS := cpu.o
	29	+COBJS := cpu.o cache_v7.o
30	30	COBJS += syslib.o
31	31
32	32	SRCS := $(START:.o=.S) $(COBJS:.o=.c)
	1	+/*
	2	+ * (C) Copyright 2010
	3	+ * Texas Instruments, <www.ti.com>
	4	+ * Aneesh V <aneesh@ti.com>
	5	+ *
	6	+ * See file CREDITS for list of people who contributed to this
	7	+ * project.
	8	+ *
	9	+ * This program is free software; you can redistribute it and/or
	10	+ * modify it under the terms of the GNU General Public License as
	11	+ * published by the Free Software Foundation; either version 2 of
	12	+ * the License, or (at your option) any later version.
	13	+ *
	14	+ * This program is distributed in the hope that it will be useful,
	15	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	16	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	17	+ * GNU General Public License for more details.
	18	+ *
	19	+ * You should have received a copy of the GNU General Public License
	20	+ * along with this program; if not, write to the Free Software
	21	+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
	22	+ * MA 02111-1307 USA
	23	+ */
	24	+#include <linux/types.h>
	25	+#include <common.h>
	26	+#include <asm/armv7.h>
	27	+#include <asm/utils.h>
	28	+
	29	+#define ARMV7_DCACHE_INVAL_ALL 1
	30	+#define ARMV7_DCACHE_CLEAN_INVAL_ALL 2
	31	+#define ARMV7_DCACHE_INVAL_RANGE 3
	32	+#define ARMV7_DCACHE_CLEAN_INVAL_RANGE 4
	33	+
	34	+#ifndef CONFIG_SYS_DCACHE_OFF
	35	+/*
	36	+ * Write the level and type you want to Cache Size Selection Register(CSSELR)
	37	+ * to get size details from Current Cache Size ID Register(CCSIDR)
	38	+ */
	39	+static void set_csselr(u32 level, u32 type)
	40	+{ u32 csselr = level << 1 \| type;
	41	+
	42	+ /* Write to Cache Size Selection Register(CSSELR) */
	43	+ asm volatile ("mcr p15, 2, %0, c0, c0, 0" : : "r" (csselr));
	44	+}
	45	+
	46	+static u32 get_ccsidr(void)
	47	+{
	48	+ u32 ccsidr;
	49	+
	50	+ /* Read current CP15 Cache Size ID Register */
	51	+ asm volatile ("mrc p15, 1, %0, c0, c0, 0" : "=r" (ccsidr));
	52	+ return ccsidr;
	53	+}
	54	+
	55	+static u32 get_clidr(void)
	56	+{
	57	+ u32 clidr;
	58	+
	59	+ /* Read current CP15 Cache Level ID Register */
	60	+ asm volatile ("mrc p15,1,%0,c0,c0,1" : "=r" (clidr));
	61	+ return clidr;
	62	+}
	63	+
	64	+static void v7_inval_dcache_level_setway(u32 level, u32 num_sets,
	65	+ u32 num_ways, u32 way_shift,
	66	+ u32 log2_line_len)
	67	+{
	68	+ int way, set, setway;
	69	+
	70	+ /*
	71	+ * For optimal assembly code:
	72	+ * a. count down
	73	+ * b. have bigger loop inside
	74	+ */
	75	+ for (way = num_ways - 1; way >= 0 ; way--) {
	76	+ for (set = num_sets - 1; set >= 0; set--) {
	77	+ setway = (level << 1) \| (set << log2_line_len) \|
	78	+ (way << way_shift);
	79	+ /* Invalidate data/unified cache line by set/way */
	80	+ asm volatile (" mcr p15, 0, %0, c7, c6, 2"
	81	+ : : "r" (setway));
	82	+ }
	83	+ }
	84	+ /* DMB to make sure the operation is complete */
	85	+ CP15DMB;
	86	+}
	87	+
	88	+static void v7_clean_inval_dcache_level_setway(u32 level, u32 num_sets,
	89	+ u32 num_ways, u32 way_shift,
	90	+ u32 log2_line_len)
	91	+{
	92	+ int way, set, setway;
	93	+
	94	+ /*
	95	+ * For optimal assembly code:
	96	+ * a. count down
	97	+ * b. have bigger loop inside
	98	+ */
	99	+ for (way = num_ways - 1; way >= 0 ; way--) {
	100	+ for (set = num_sets - 1; set >= 0; set--) {
	101	+ setway = (level << 1) \| (set << log2_line_len) \|
	102	+ (way << way_shift);
	103	+ /*
	104	+ * Clean & Invalidate data/unified
	105	+ * cache line by set/way
	106	+ */
	107	+ asm volatile (" mcr p15, 0, %0, c7, c14, 2"
	108	+ : : "r" (setway));
	109	+ }
	110	+ }
	111	+ /* DMB to make sure the operation is complete */
	112	+ CP15DMB;
	113	+}
	114	+
	115	+static void v7_maint_dcache_level_setway(u32 level, u32 operation)
	116	+{
	117	+ u32 ccsidr;
	118	+ u32 num_sets, num_ways, log2_line_len, log2_num_ways;
	119	+ u32 way_shift;
	120	+
	121	+ set_csselr(level, ARMV7_CSSELR_IND_DATA_UNIFIED);
	122	+
	123	+ ccsidr = get_ccsidr();
	124	+
	125	+ log2_line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >>
	126	+ CCSIDR_LINE_SIZE_OFFSET) + 2;
	127	+ /* Converting from words to bytes */
	128	+ log2_line_len += 2;
	129	+
	130	+ num_ways = ((ccsidr & CCSIDR_ASSOCIATIVITY_MASK) >>
	131	+ CCSIDR_ASSOCIATIVITY_OFFSET) + 1;
	132	+ num_sets = ((ccsidr & CCSIDR_NUM_SETS_MASK) >>
	133	+ CCSIDR_NUM_SETS_OFFSET) + 1;
	134	+ /*
	135	+ * According to ARMv7 ARM number of sets and number of ways need
	136	+ * not be a power of 2
	137	+ */
	138	+ log2_num_ways = log_2_n_round_up(num_ways);
	139	+
	140	+ way_shift = (32 - log2_num_ways);
	141	+ if (operation == ARMV7_DCACHE_INVAL_ALL) {
	142	+ v7_inval_dcache_level_setway(level, num_sets, num_ways,
	143	+ way_shift, log2_line_len);
	144	+ } else if (operation == ARMV7_DCACHE_CLEAN_INVAL_ALL) {
	145	+ v7_clean_inval_dcache_level_setway(level, num_sets, num_ways,
	146	+ way_shift, log2_line_len);
	147	+ }
	148	+}
	149	+
	150	+static void v7_maint_dcache_all(u32 operation)
	151	+{
	152	+ u32 level, cache_type, level_start_bit = 0;
	153	+
	154	+ u32 clidr = get_clidr();
	155	+
	156	+ for (level = 0; level < 7; level++) {
	157	+ cache_type = (clidr >> level_start_bit) & 0x7;
	158	+ if ((cache_type == ARMV7_CLIDR_CTYPE_DATA_ONLY) \|\|
	159	+ (cache_type == ARMV7_CLIDR_CTYPE_INSTRUCTION_DATA) \|\|
	160	+ (cache_type == ARMV7_CLIDR_CTYPE_UNIFIED))
	161	+ v7_maint_dcache_level_setway(level, operation);
	162	+ level_start_bit += 3;
	163	+ }
	164	+}
	165	+
	166	+static void v7_dcache_clean_inval_range(u32 start,
	167	+ u32 stop, u32 line_len)
	168	+{
	169	+ u32 mva;
	170	+
	171	+ /* Align start to cache line boundary */
	172	+ start &= ~(line_len - 1);
	173	+ for (mva = start; mva < stop; mva = mva + line_len) {
	174	+ /* DCCIMVAC - Clean & Invalidate data cache by MVA to PoC */
	175	+ asm volatile ("mcr p15, 0, %0, c7, c14, 1" : : "r" (mva));
	176	+ }
	177	+}
	178	+
	179	+static void v7_dcache_inval_range(u32 start, u32 stop, u32 line_len)
	180	+{
	181	+ u32 mva;
	182	+
	183	+ /*
	184	+ * If start address is not aligned to cache-line flush the first
	185	+ * line to prevent affecting somebody else's buffer
	186	+ */
	187	+ if (start & (line_len - 1)) {
	188	+ v7_dcache_clean_inval_range(start, start + 1, line_len);
	189	+ /* move to next cache line */
	190	+ start = (start + line_len - 1) & ~(line_len - 1);
	191	+ }
	192	+
	193	+ /*
	194	+ * If stop address is not aligned to cache-line flush the last
	195	+ * line to prevent affecting somebody else's buffer
	196	+ */
	197	+ if (stop & (line_len - 1)) {
	198	+ v7_dcache_clean_inval_range(stop, stop + 1, line_len);
	199	+ /* align to the beginning of this cache line */
	200	+ stop &= ~(line_len - 1);
	201	+ }
	202	+
	203	+ for (mva = start; mva < stop; mva = mva + line_len) {
	204	+ /* DCIMVAC - Invalidate data cache by MVA to PoC */
	205	+ asm volatile ("mcr p15, 0, %0, c7, c6, 1" : : "r" (mva));
	206	+ }
	207	+}
	208	+
	209	+static void v7_dcache_maint_range(u32 start, u32 stop, u32 range_op)
	210	+{
	211	+ u32 line_len, ccsidr;
	212	+
	213	+ ccsidr = get_ccsidr();
	214	+ line_len = ((ccsidr & CCSIDR_LINE_SIZE_MASK) >>
	215	+ CCSIDR_LINE_SIZE_OFFSET) + 2;
	216	+ /* Converting from words to bytes */
	217	+ line_len += 2;
	218	+ /* converting from log2(linelen) to linelen */
	219	+ line_len = 1 << line_len;
	220	+
	221	+ switch (range_op) {
	222	+ case ARMV7_DCACHE_CLEAN_INVAL_RANGE:
	223	+ v7_dcache_clean_inval_range(start, stop, line_len);
	224	+ break;
	225	+ case ARMV7_DCACHE_INVAL_RANGE:
	226	+ v7_dcache_inval_range(start, stop, line_len);
	227	+ break;
	228	+ }
	229	+
	230	+ /* DMB to make sure the operation is complete */
	231	+ CP15DMB;
	232	+}
	233	+
	234	+/* Invalidate TLB */
	235	+static void v7_inval_tlb(void)
	236	+{
	237	+ /* Invalidate entire unified TLB */
	238	+ asm volatile ("mcr p15, 0, %0, c8, c7, 0" : : "r" (0));
	239	+ /* Invalidate entire data TLB */
	240	+ asm volatile ("mcr p15, 0, %0, c8, c6, 0" : : "r" (0));
	241	+ /* Invalidate entire instruction TLB */
	242	+ asm volatile ("mcr p15, 0, %0, c8, c5, 0" : : "r" (0));
	243	+ /* Full system DSB - make sure that the invalidation is complete */
	244	+ CP15DSB;
	245	+ /* Full system ISB - make sure the instruction stream sees it */
	246	+ CP15ISB;
	247	+}
	248	+
	249	+void invalidate_dcache_all(void)
	250	+{
	251	+ v7_maint_dcache_all(ARMV7_DCACHE_INVAL_ALL);
	252	+
	253	+ v7_outer_cache_inval_all();
	254	+}
	255	+
	256	+/*
	257	+ * Performs a clean & invalidation of the entire data cache
	258	+ * at all levels
	259	+ */
	260	+void flush_dcache_all(void)
	261	+{
	262	+ v7_maint_dcache_all(ARMV7_DCACHE_CLEAN_INVAL_ALL);
	263	+
	264	+ v7_outer_cache_flush_all();
	265	+}
	266	+
	267	+/*
	268	+ * Invalidates range in all levels of D-cache/unified cache used:
	269	+ * Affects the range [start, stop - 1]
	270	+ */
	271	+void invalidate_dcache_range(unsigned long start, unsigned long stop)
	272	+{
	273	+
	274	+ v7_dcache_maint_range(start, stop, ARMV7_DCACHE_INVAL_RANGE);
	275	+
	276	+ v7_outer_cache_inval_range(start, stop);
	277	+}
	278	+
	279	+/*
	280	+ * Flush range(clean & invalidate) from all levels of D-cache/unified
	281	+ * cache used:
	282	+ * Affects the range [start, stop - 1]
	283	+ */
	284	+void flush_dcache_range(unsigned long start, unsigned long stop)
	285	+{
	286	+ v7_dcache_maint_range(start, stop, ARMV7_DCACHE_CLEAN_INVAL_RANGE);
	287	+
	288	+ v7_outer_cache_flush_range(start, stop);
	289	+}
	290	+
	291	+void arm_init_before_mmu(void)
	292	+{
	293	+ v7_outer_cache_enable();
	294	+ invalidate_dcache_all();
	295	+ v7_inval_tlb();
	296	+}
	297	+
	298	+/*
	299	+ * Flush range from all levels of d-cache/unified-cache used:
	300	+ * Affects the range [start, start + size - 1]
	301	+ */
	302	+void flush_cache(unsigned long start, unsigned long size)
	303	+{
	304	+ flush_dcache_range(start, start + size);
	305	+}
	306	+#else /* #ifndef CONFIG_SYS_DCACHE_OFF */
	307	+void invalidate_dcache_all(void)
	308	+{
	309	+}
	310	+
	311	+void flush_dcache_all(void)
	312	+{
	313	+}
	314	+
	315	+void invalidate_dcache_range(unsigned long start, unsigned long stop)
	316	+{
	317	+}
	318	+
	319	+void flush_dcache_range(unsigned long start, unsigned long stop)
	320	+{
	321	+}
	322	+
	323	+void arm_init_before_mmu(void)
	324	+{
	325	+}
	326	+
	327	+void flush_cache(unsigned long start, unsigned long size)
	328	+{
	329	+}
	330	+#endif /* #ifndef CONFIG_SYS_DCACHE_OFF */
	331	+
	332	+#ifndef CONFIG_SYS_ICACHE_OFF
	333	+/* Invalidate entire I-cache and branch predictor array */
	334	+void invalidate_icache_all(void)
	335	+{
	336	+ /*
	337	+ * Invalidate all instruction caches to PoU.
	338	+ * Also flushes branch target cache.
	339	+ */
	340	+ asm volatile ("mcr p15, 0, %0, c7, c5, 0" : : "r" (0));
	341	+
	342	+ /* Invalidate entire branch predictor array */
	343	+ asm volatile ("mcr p15, 0, %0, c7, c5, 6" : : "r" (0));
	344	+
	345	+ /* Full system DSB - make sure that the invalidation is complete */
	346	+ CP15DSB;
	347	+
	348	+ /* ISB - make sure the instruction stream sees it */
	349	+ CP15ISB;
	350	+}
	351	+#else
	352	+void invalidate_icache_all(void)
	353	+{
	354	+}
	355	+#endif
	356	+
	357	+/*
	358	+ * Stub implementations for outer cache operations
	359	+ */
	360	+void __v7_outer_cache_enable(void)
	361	+{
	362	+}
	363	+void v7_outer_cache_enable(void)
	364	+ __attribute__((weak, alias("__v7_outer_cache_enable")));
	365	+
	366	+void __v7_outer_cache_disable(void)
	367	+{
	368	+}
	369	+void v7_outer_cache_disable(void)
	370	+ __attribute__((weak, alias("__v7_outer_cache_disable")));
	371	+
	372	+void __v7_outer_cache_flush_all(void)
	373	+{
	374	+}
	375	+void v7_outer_cache_flush_all(void)
	376	+ __attribute__((weak, alias("__v7_outer_cache_flush_all")));
	377	+
	378	+void __v7_outer_cache_inval_all(void)
	379	+{
	380	+}
	381	+void v7_outer_cache_inval_all(void)
	382	+ __attribute__((weak, alias("__v7_outer_cache_inval_all")));
	383	+
	384	+void __v7_outer_cache_flush_range(u32 start, u32 end)
	385	+{
	386	+}
	387	+void v7_outer_cache_flush_range(u32 start, u32 end)
	388	+ __attribute__((weak, alias("__v7_outer_cache_flush_range")));
	389	+
	390	+void __v7_outer_cache_inval_range(u32 start, u32 end)
	391	+{
	392	+}
	393	+void v7_outer_cache_inval_range(u32 start, u32 end)
	394	+ __attribute__((weak, alias("__v7_outer_cache_inval_range")));
...	...	@@ -413,6 +413,7 @@
413	413	int dcache_status (void);
414	414	void dcache_enable (void);
415	415	void dcache_disable(void);
	416	+void mmu_disable(void);
416	417	void relocate_code (ulong, gd_t *, ulong) __attribute__ ((noreturn));
417	418	ulong get_endaddr (void);
418	419	void trap_init (ulong);
419	420
...	...	@@ -611,9 +612,11 @@
611	612
612	613	/* arch/$(ARCH)/lib/cache.c */
613	614	void flush_cache (unsigned long, unsigned long);
	615	+void flush_dcache_all(void);
614	616	void flush_dcache_range(unsigned long start, unsigned long stop);
615	617	void invalidate_dcache_range(unsigned long start, unsigned long stop);
616		-
	618	+void invalidate_dcache_all(void);
	619	+void invalidate_icache_all(void);
617	620
618	621	/* arch/$(ARCH)/lib/ticks.S */
619	622	unsigned long long get_ticks(void);