ARC: [SMP] ASID allocation

-Track a Per CPU ASID counter -mm-per-cpu ASID (multiple threads, or mm migrated around) Signed-off-by: Vineet Gupta <vgupta@synopsys.com>

ARC: [SMP] ASID allocation
-Track a Per CPU ASID counter -mm-per-cpu ASID (multiple threads, or mm migrated around) Signed-off-by: Vineet Gupta <vgupta@synopsys.com>
Vineet Gupta
1 parent b6fe8e7c01
Showing 3 changed files with 37 additions and 23 deletions Side-by-side Diff
arch/arc/include/asm/mmu.h
arch/arc/include/asm/mmu_context.h
arch/arc/mm/tlb.c
@@ -48,7 +48,7 @@
 #ifndef __ASSEMBLY__
  
 typedef struct {
-	unsigned long asid;	/* 8 bit MMU PID + Generation cycle */
+	unsigned long asid[NR_CPUS];	/* 8 bit MMU PID + Generation cycle */
 } mm_context_t;
  
 #ifdef CONFIG_ARC_DBG_TLB_PARANOIA
@@ -30,13 +30,13 @@
  * "Fast Context Switch" i.e. no TLB flush on ctxt-switch
  *
  * Linux assigns each task a unique ASID. A simple round-robin allocation
- * of H/w ASID is done using software tracker @asid_cache.
+ * of H/w ASID is done using software tracker @asid_cpu.
  * When it reaches max 255, the allocation cycle starts afresh by flushing
  * the entire TLB and wrapping ASID back to zero.
  *
  * A new allocation cycle, post rollover, could potentially reassign an ASID
  * to a different task. Thus the rule is to refresh the ASID in a new cycle.
- * The 32 bit @asid_cache (and mm->asid) have 8 bits MMU PID and rest 24 bits
+ * The 32 bit @asid_cpu (and mm->asid) have 8 bits MMU PID and rest 24 bits
  * serve as cycle/generation indicator and natural 32 bit unsigned math
  * automagically increments the generation when lower 8 bits rollover.
  */
  
@@ -47,9 +47,11 @@
 #define MM_CTXT_FIRST_CYCLE	(MM_CTXT_ASID_MASK + 1)
 #define MM_CTXT_NO_ASID		0UL
  
-#define hw_pid(mm)		(mm->context.asid & MM_CTXT_ASID_MASK)
+#define asid_mm(mm, cpu)	mm->context.asid[cpu]
+#define hw_pid(mm, cpu)		(asid_mm(mm, cpu) & MM_CTXT_ASID_MASK)
  
-extern unsigned int asid_cache;
+DECLARE_PER_CPU(unsigned int, asid_cache);
+#define asid_cpu(cpu)		per_cpu(asid_cache, cpu)
  
 /*
  * Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
@@ -57,6 +59,7 @@
  */
 static inline void get_new_mmu_context(struct mm_struct *mm)
 {
+	const unsigned int cpu = smp_processor_id();
 	unsigned long flags;
  
 	local_irq_save(flags);
  
@@ -71,11 +74,11 @@
 	 * 	 first need to destroy the context, setting it to invalid
 	 * 	 value.
 	 */
-	if (!((mm->context.asid ^ asid_cache) & MM_CTXT_CYCLE_MASK))
+	if (!((asid_mm(mm, cpu) ^ asid_cpu(cpu)) & MM_CTXT_CYCLE_MASK))
 		goto set_hw;
  
 	/* move to new ASID and handle rollover */
-	if (unlikely(!(++asid_cache & MM_CTXT_ASID_MASK))) {
+	if (unlikely(!(++asid_cpu(cpu) & MM_CTXT_ASID_MASK))) {
  
 		flush_tlb_all();
  
  
  
@@ -84,15 +87,15 @@
 		 * If the container itself wrapped around, set it to a non zero
 		 * "generation" to distinguish from no context
 		 */
-		if (!asid_cache)
-			asid_cache = MM_CTXT_FIRST_CYCLE;
+		if (!asid_cpu(cpu))
+			asid_cpu(cpu) = MM_CTXT_FIRST_CYCLE;
 	}
  
 	/* Assign new ASID to tsk */
-	mm->context.asid = asid_cache;
+	asid_mm(mm, cpu) = asid_cpu(cpu);
  
 set_hw:
-	write_aux_reg(ARC_REG_PID, hw_pid(mm) | MMU_ENABLE);
+	write_aux_reg(ARC_REG_PID, hw_pid(mm, cpu) | MMU_ENABLE);
  
 	local_irq_restore(flags);
 }
  
@@ -104,10 +107,24 @@
 static inline int
 init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
-	mm->context.asid = MM_CTXT_NO_ASID;
+	int i;
+
+	for_each_possible_cpu(i)
+		asid_mm(mm, i) = MM_CTXT_NO_ASID;
+
 	return 0;
 }
  
+static inline void destroy_context(struct mm_struct *mm)
+{
+	unsigned long flags;
+
+	/* Needed to elide CONFIG_DEBUG_PREEMPT warning */
+	local_irq_save(flags);
+	asid_mm(mm, smp_processor_id()) = MM_CTXT_NO_ASID;
+	local_irq_restore(flags);
+}
+
 /* Prepare the MMU for task: setup PID reg with allocated ASID
     If task doesn't have an ASID (never alloc or stolen, get a new ASID)
 */
@@ -130,11 +147,6 @@
  * only if it was unallocated
  */
 #define activate_mm(prev, next)		switch_mm(prev, next, NULL)
-
-static inline void destroy_context(struct mm_struct *mm)
-{
-	mm->context.asid = MM_CTXT_NO_ASID;
-}
  
 /* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
  * for retiring-mm. However destroy_context( ) still needs to do that because
@@ -100,7 +100,7 @@
  
  
 /* A copy of the ASID from the PID reg is kept in asid_cache */
-unsigned int asid_cache = MM_CTXT_FIRST_CYCLE;
+DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE;
  
 /*
  * Utility Routine to erase a J-TLB entry
@@ -274,6 +274,7 @@
 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 			   unsigned long end)
 {
+	const unsigned int cpu = smp_processor_id();
 	unsigned long flags;
  
 	/* If range @start to @end is more than 32 TLB entries deep,
  
@@ -297,9 +298,9 @@
  
 	local_irq_save(flags);
  
-	if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
+	if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
 		while (start < end) {
-			tlb_entry_erase(start | hw_pid(vma->vm_mm));
+			tlb_entry_erase(start | hw_pid(vma->vm_mm, cpu));
 			start += PAGE_SIZE;
 		}
 	}
@@ -346,6 +347,7 @@
  
 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 {
+	const unsigned int cpu = smp_processor_id();
 	unsigned long flags;
  
 	/* Note that it is critical that interrupts are DISABLED between
@@ -353,8 +355,8 @@
 	 */
 	local_irq_save(flags);
  
-	if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
-		tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm));
+	if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
+		tlb_entry_erase((page & PAGE_MASK) | hw_pid(vma->vm_mm, cpu));
 		utlb_invalidate();
 	}
  
@@ -400,7 +402,7 @@
  
 	local_irq_save(flags);
  
-	tlb_paranoid_check(vma->vm_mm->context.asid, address);
+	tlb_paranoid_check(asid_mm(vma->vm_mm, smp_processor_id()), address);
  
 	address &= PAGE_MASK;
...	...	@@ -48,7 +48,7 @@
48	48	#ifndef __ASSEMBLY__
49	49
50	50	typedef struct {
51		- unsigned long asid; /* 8 bit MMU PID + Generation cycle */
	51	+ unsigned long asid[NR_CPUS]; /* 8 bit MMU PID + Generation cycle */
52	52	} mm_context_t;
53	53
54	54	#ifdef CONFIG_ARC_DBG_TLB_PARANOIA
...	...	@@ -30,13 +30,13 @@
30	30	* "Fast Context Switch" i.e. no TLB flush on ctxt-switch
31	31	*
32	32	* Linux assigns each task a unique ASID. A simple round-robin allocation
33		- * of H/w ASID is done using software tracker @asid_cache.
	33	+ * of H/w ASID is done using software tracker @asid_cpu.
34	34	* When it reaches max 255, the allocation cycle starts afresh by flushing
35	35	* the entire TLB and wrapping ASID back to zero.
36	36	*
37	37	* A new allocation cycle, post rollover, could potentially reassign an ASID
38	38	* to a different task. Thus the rule is to refresh the ASID in a new cycle.
39		- * The 32 bit @asid_cache (and mm->asid) have 8 bits MMU PID and rest 24 bits
	39	+ * The 32 bit @asid_cpu (and mm->asid) have 8 bits MMU PID and rest 24 bits
40	40	* serve as cycle/generation indicator and natural 32 bit unsigned math
41	41	* automagically increments the generation when lower 8 bits rollover.
42	42	*/
43	43
...	...	@@ -47,9 +47,11 @@
47	47	#define MM_CTXT_FIRST_CYCLE (MM_CTXT_ASID_MASK + 1)
48	48	#define MM_CTXT_NO_ASID 0UL
49	49
50		-#define hw_pid(mm) (mm->context.asid & MM_CTXT_ASID_MASK)
	50	+#define asid_mm(mm, cpu) mm->context.asid[cpu]
	51	+#define hw_pid(mm, cpu) (asid_mm(mm, cpu) & MM_CTXT_ASID_MASK)
51	52
52		-extern unsigned int asid_cache;
	53	+DECLARE_PER_CPU(unsigned int, asid_cache);
	54	+#define asid_cpu(cpu) per_cpu(asid_cache, cpu)
53	55
54	56	/*
55	57	* Get a new ASID if task doesn't have a valid one (unalloc or from prev cycle)
...	...	@@ -57,6 +59,7 @@
57	59	*/
58	60	static inline void get_new_mmu_context(struct mm_struct *mm)
59	61	{
	62	+ const unsigned int cpu = smp_processor_id();
60	63	unsigned long flags;
61	64
62	65	local_irq_save(flags);
63	66
...	...	@@ -71,11 +74,11 @@
71	74	* first need to destroy the context, setting it to invalid
72	75	* value.
73	76	*/
74		- if (!((mm->context.asid ^ asid_cache) & MM_CTXT_CYCLE_MASK))
	77	+ if (!((asid_mm(mm, cpu) ^ asid_cpu(cpu)) & MM_CTXT_CYCLE_MASK))
75	78	goto set_hw;
76	79
77	80	/* move to new ASID and handle rollover */
78		- if (unlikely(!(++asid_cache & MM_CTXT_ASID_MASK))) {
	81	+ if (unlikely(!(++asid_cpu(cpu) & MM_CTXT_ASID_MASK))) {
79	82
80	83	flush_tlb_all();
81	84
82	85
83	86
...	...	@@ -84,15 +87,15 @@
84	87	* If the container itself wrapped around, set it to a non zero
85	88	* "generation" to distinguish from no context
86	89	*/
87		- if (!asid_cache)
88		- asid_cache = MM_CTXT_FIRST_CYCLE;
	90	+ if (!asid_cpu(cpu))
	91	+ asid_cpu(cpu) = MM_CTXT_FIRST_CYCLE;
89	92	}
90	93
91	94	/* Assign new ASID to tsk */
92		- mm->context.asid = asid_cache;
	95	+ asid_mm(mm, cpu) = asid_cpu(cpu);
93	96
94	97	set_hw:
95		- write_aux_reg(ARC_REG_PID, hw_pid(mm) \| MMU_ENABLE);
	98	+ write_aux_reg(ARC_REG_PID, hw_pid(mm, cpu) \| MMU_ENABLE);
96	99
97	100	local_irq_restore(flags);
98	101	}
99	102
...	...	@@ -104,10 +107,24 @@
104	107	static inline int
105	108	init_new_context(struct task_struct tsk, struct mm_struct mm)
106	109	{
107		- mm->context.asid = MM_CTXT_NO_ASID;
	110	+ int i;
	111	+
	112	+ for_each_possible_cpu(i)
	113	+ asid_mm(mm, i) = MM_CTXT_NO_ASID;
	114	+
108	115	return 0;
109	116	}
110	117
	118	+static inline void destroy_context(struct mm_struct *mm)
	119	+{
	120	+ unsigned long flags;
	121	+
	122	+ /* Needed to elide CONFIG_DEBUG_PREEMPT warning */
	123	+ local_irq_save(flags);
	124	+ asid_mm(mm, smp_processor_id()) = MM_CTXT_NO_ASID;
	125	+ local_irq_restore(flags);
	126	+}
	127	+
111	128	/* Prepare the MMU for task: setup PID reg with allocated ASID
112	129	If task doesn't have an ASID (never alloc or stolen, get a new ASID)
113	130	*/
...	...	@@ -130,11 +147,6 @@
130	147	* only if it was unallocated
131	148	*/
132	149	#define activate_mm(prev, next) switch_mm(prev, next, NULL)
133		-
134		-static inline void destroy_context(struct mm_struct *mm)
135		-{
136		- mm->context.asid = MM_CTXT_NO_ASID;
137		-}
138	150
139	151	/* it seemed that deactivate_mm( ) is a reasonable place to do book-keeping
140	152	* for retiring-mm. However destroy_context( ) still needs to do that because
...	...	@@ -100,7 +100,7 @@
100	100
101	101
102	102	/* A copy of the ASID from the PID reg is kept in asid_cache */
103		-unsigned int asid_cache = MM_CTXT_FIRST_CYCLE;
	103	+DEFINE_PER_CPU(unsigned int, asid_cache) = MM_CTXT_FIRST_CYCLE;
104	104
105	105	/*
106	106	* Utility Routine to erase a J-TLB entry
...	...	@@ -274,6 +274,7 @@
274	274	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
275	275	unsigned long end)
276	276	{
	277	+ const unsigned int cpu = smp_processor_id();
277	278	unsigned long flags;
278	279
279	280	/* If range @start to @end is more than 32 TLB entries deep,
280	281
...	...	@@ -297,9 +298,9 @@
297	298
298	299	local_irq_save(flags);
299	300
300		- if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
	301	+ if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
301	302	while (start < end) {
302		- tlb_entry_erase(start \| hw_pid(vma->vm_mm));
	303	+ tlb_entry_erase(start \| hw_pid(vma->vm_mm, cpu));
303	304	start += PAGE_SIZE;
304	305	}
305	306	}
...	...	@@ -346,6 +347,7 @@
346	347
347	348	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
348	349	{
	350	+ const unsigned int cpu = smp_processor_id();
349	351	unsigned long flags;
350	352
351	353	/* Note that it is critical that interrupts are DISABLED between
...	...	@@ -353,8 +355,8 @@
353	355	*/
354	356	local_irq_save(flags);
355	357
356		- if (vma->vm_mm->context.asid != MM_CTXT_NO_ASID) {
357		- tlb_entry_erase((page & PAGE_MASK) \| hw_pid(vma->vm_mm));
	358	+ if (asid_mm(vma->vm_mm, cpu) != MM_CTXT_NO_ASID) {
	359	+ tlb_entry_erase((page & PAGE_MASK) \| hw_pid(vma->vm_mm, cpu));
358	360	utlb_invalidate();
359	361	}
360	362
...	...	@@ -400,7 +402,7 @@
400	402
401	403	local_irq_save(flags);
402	404
403		- tlb_paranoid_check(vma->vm_mm->context.asid, address);
	405	+ tlb_paranoid_check(asid_mm(vma->vm_mm, smp_processor_id()), address);
404	406
405	407	address &= PAGE_MASK;
406	408