core: Replace __get_cpu_var with __this_cpu_read if not used for an address.

__get_cpu_var() can be replaced with this_cpu_read and will then use a single read instruction with implied address calculation to access the correct per cpu instance. However, the address of a per cpu variable passed to __this_cpu_read() cannot be determined (since it's an implied address conversion through segment prefixes). Therefore apply this only to uses of __get_cpu_var where the address of the variable is not used. Cc: Pekka Enberg <penberg@cs.helsinki.fi> Cc: Hugh Dickins <hughd@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Acked-by: H. Peter Anvin <hpa@zytor.com> Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Tejun Heo <tj@kernel.org>

core: Replace __get_cpu_var with __this_cpu_read if not used for an address.
__get_cpu_var() can be replaced with this_cpu_read and will then use a single read instruction with implied address calculation to access the correct per cpu instance. However, the address of a per cpu variable passed to __this_cpu_read() cannot be determined (since it's an implied address conversion through segment prefixes). Therefore apply this only to uses of __get_cpu_var where the address of the variable is not used. Cc: Pekka Enberg <penberg@cs.helsinki.fi> Cc: Hugh Dickins <hughd@google.com> Cc: Thomas Gleixner <tglx@linutronix.de> Acked-by: H. Peter Anvin <hpa@zytor.com> Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Tejun Heo <tj@kernel.org>
Christoph Lameter · Tejun Heo
1 parent 780f36d8b3
Showing 13 changed files with 60 additions and 66 deletions Side-by-side Diff
include/asm-generic/irq_regs.h
include/linux/elevator.h
include/linux/kernel_stat.h
kernel/exit.c
kernel/fork.c
kernel/hrtimer.c
kernel/printk.c
kernel/rcutree.c
kernel/softirq.c
kernel/time/tick-common.c
kernel/time/tick-oneshot.c
kernel/watchdog.c
mm/slab.c
@@ -22,15 +22,15 @@
  
 static inline struct pt_regs *get_irq_regs(void)
 {
-	return __get_cpu_var(__irq_regs);
+	return __this_cpu_read(__irq_regs);
 }
  
 static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
 {
-	struct pt_regs *old_regs, **pp_regs = &__get_cpu_var(__irq_regs);
+	struct pt_regs *old_regs;
  
-	old_regs = *pp_regs;
-	*pp_regs = new_regs;
+	old_regs = __this_cpu_read(__irq_regs);
+	__this_cpu_write(__irq_regs, new_regs);
 	return old_regs;
 }
  
@@ -195,15 +195,9 @@
 /*
  * io context count accounting
  */
-#define elv_ioc_count_mod(name, __val)				\
-	do {							\
-		preempt_disable();				\
-		__get_cpu_var(name) += (__val);			\
-		preempt_enable();				\
-	} while (0)
-
-#define elv_ioc_count_inc(name)	elv_ioc_count_mod(name, 1)
-#define elv_ioc_count_dec(name)	elv_ioc_count_mod(name, -1)
+#define elv_ioc_count_mod(name, __val) this_cpu_add(name, __val)
+#define elv_ioc_count_inc(name)	this_cpu_inc(name)
+#define elv_ioc_count_dec(name)	this_cpu_dec(name)
  
 #define elv_ioc_count_read(name)				\
 ({								\
@@ -47,7 +47,7 @@
  
 #ifndef CONFIG_GENERIC_HARDIRQS
 #define kstat_irqs_this_cpu(irq) \
-	(kstat_this_cpu.irqs[irq])
+	(this_cpu_read(kstat.irqs[irq])
  
 struct irq_desc;
  
@@ -69,7 +69,7 @@
  
 		list_del_rcu(&p->tasks);
 		list_del_init(&p->sibling);
-		__get_cpu_var(process_counts)--;
+		__this_cpu_dec(process_counts);
 	}
 	list_del_rcu(&p->thread_group);
 }
@@ -1282,7 +1282,7 @@
 			attach_pid(p, PIDTYPE_SID, task_session(current));
 			list_add_tail(&p->sibling, &p->real_parent->children);
 			list_add_tail_rcu(&p->tasks, &init_task.tasks);
-			__get_cpu_var(process_counts)++;
+			__this_cpu_inc(process_counts);
 		}
 		attach_pid(p, PIDTYPE_PID, pid);
 		nr_threads++;
@@ -497,7 +497,7 @@
  */
 static inline int hrtimer_hres_active(void)
 {
-	return __get_cpu_var(hrtimer_bases).hres_active;
+	return __this_cpu_read(hrtimer_bases.hres_active);
 }
  
 /*
@@ -1074,8 +1074,8 @@
  
 void printk_tick(void)
 {
-	if (__get_cpu_var(printk_pending)) {
-		__get_cpu_var(printk_pending) = 0;
+	if (__this_cpu_read(printk_pending)) {
+		__this_cpu_write(printk_pending, 0);
 		wake_up_interruptible(&log_wait);
 	}
 }
@@ -367,8 +367,8 @@
 	WARN_ON_ONCE(rdtp->dynticks & 0x1);
  
 	/* If the interrupt queued a callback, get out of dyntick mode. */
-	if (__get_cpu_var(rcu_sched_data).nxtlist ||
-	    __get_cpu_var(rcu_bh_data).nxtlist)
+	if (__this_cpu_read(rcu_sched_data.nxtlist) ||
+	    __this_cpu_read(rcu_bh_data.nxtlist))
 		set_need_resched();
 }
  
@@ -70,7 +70,7 @@
 static void wakeup_softirqd(void)
 {
 	/* Interrupts are disabled: no need to stop preemption */
-	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
+	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  
 	if (tsk && tsk->state != TASK_RUNNING)
 		wake_up_process(tsk);
@@ -388,8 +388,8 @@
  
 	local_irq_save(flags);
 	t->next = NULL;
-	*__get_cpu_var(tasklet_vec).tail = t;
-	__get_cpu_var(tasklet_vec).tail = &(t->next);
+	*__this_cpu_read(tasklet_vec.tail) = t;
+	__this_cpu_write(tasklet_vec.tail, &(t->next));
 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
 	local_irq_restore(flags);
 }
@@ -402,8 +402,8 @@
  
 	local_irq_save(flags);
 	t->next = NULL;
-	*__get_cpu_var(tasklet_hi_vec).tail = t;
-	__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
+	*__this_cpu_read(tasklet_hi_vec.tail) = t;
+	__this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 	raise_softirq_irqoff(HI_SOFTIRQ);
 	local_irq_restore(flags);
 }
@@ -414,8 +414,8 @@
 {
 	BUG_ON(!irqs_disabled());
  
-	t->next = __get_cpu_var(tasklet_hi_vec).head;
-	__get_cpu_var(tasklet_hi_vec).head = t;
+	t->next = __this_cpu_read(tasklet_hi_vec.head);
+	__this_cpu_write(tasklet_hi_vec.head, t);
 	__raise_softirq_irqoff(HI_SOFTIRQ);
 }
  
@@ -426,9 +426,9 @@
 	struct tasklet_struct *list;
  
 	local_irq_disable();
-	list = __get_cpu_var(tasklet_vec).head;
-	__get_cpu_var(tasklet_vec).head = NULL;
-	__get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
+	list = __this_cpu_read(tasklet_vec.head);
+	__this_cpu_write(tasklet_vec.head, NULL);
+	__this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 	local_irq_enable();
  
 	while (list) {
@@ -449,8 +449,8 @@
  
 		local_irq_disable();
 		t->next = NULL;
-		*__get_cpu_var(tasklet_vec).tail = t;
-		__get_cpu_var(tasklet_vec).tail = &(t->next);
+		*__this_cpu_read(tasklet_vec.tail) = t;
+		__this_cpu_write(tasklet_vec.tail, &(t->next));
 		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
 		local_irq_enable();
 	}
@@ -461,9 +461,9 @@
 	struct tasklet_struct *list;
  
 	local_irq_disable();
-	list = __get_cpu_var(tasklet_hi_vec).head;
-	__get_cpu_var(tasklet_hi_vec).head = NULL;
-	__get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
+	list = __this_cpu_read(tasklet_hi_vec.head);
+	__this_cpu_write(tasklet_hi_vec.head, NULL);
+	__this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 	local_irq_enable();
  
 	while (list) {
@@ -484,8 +484,8 @@
  
 		local_irq_disable();
 		t->next = NULL;
-		*__get_cpu_var(tasklet_hi_vec).tail = t;
-		__get_cpu_var(tasklet_hi_vec).tail = &(t->next);
+		*__this_cpu_read(tasklet_hi_vec.tail) = t;
+		__this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 		__raise_softirq_irqoff(HI_SOFTIRQ);
 		local_irq_enable();
 	}
  
@@ -802,16 +802,16 @@
  
 	/* Find end, append list for that CPU. */
 	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
-		*(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
-		__get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
+		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
+		this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 		per_cpu(tasklet_vec, cpu).head = NULL;
 		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 	}
 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
  
 	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
-		*__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
-		__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
+		*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
+		__this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 		per_cpu(tasklet_hi_vec, cpu).head = NULL;
 		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 	}
@@ -49,7 +49,7 @@
  */
 int tick_is_oneshot_available(void)
 {
-	struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
+	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  
 	return dev && (dev->features & CLOCK_EVT_FEAT_ONESHOT);
 }
@@ -95,7 +95,7 @@
  */
 int tick_program_event(ktime_t expires, int force)
 {
-	struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
+	struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  
 	return tick_dev_program_event(dev, expires, force);
 }
@@ -167,7 +167,7 @@
 	int ret;
  
 	local_irq_save(flags);
-	ret = __get_cpu_var(tick_cpu_device).mode == TICKDEV_MODE_ONESHOT;
+	ret = __this_cpu_read(tick_cpu_device.mode) == TICKDEV_MODE_ONESHOT;
 	local_irq_restore(flags);
  
 	return ret;
@@ -116,12 +116,12 @@
 {
 	int this_cpu = smp_processor_id();
  
-	__get_cpu_var(watchdog_touch_ts) = get_timestamp(this_cpu);
+	__this_cpu_write(watchdog_touch_ts, get_timestamp(this_cpu));
 }
  
 void touch_softlockup_watchdog(void)
 {
-	__raw_get_cpu_var(watchdog_touch_ts) = 0;
+	__this_cpu_write(watchdog_touch_ts, 0);
 }
 EXPORT_SYMBOL(touch_softlockup_watchdog);
  
  
  
@@ -165,12 +165,12 @@
 /* watchdog detector functions */
 static int is_hardlockup(void)
 {
-	unsigned long hrint = __get_cpu_var(hrtimer_interrupts);
+	unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
  
-	if (__get_cpu_var(hrtimer_interrupts_saved) == hrint)
+	if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
 		return 1;
  
-	__get_cpu_var(hrtimer_interrupts_saved) = hrint;
+	__this_cpu_write(hrtimer_interrupts_saved, hrint);
 	return 0;
 }
 #endif
@@ -203,8 +203,8 @@
 	/* Ensure the watchdog never gets throttled */
 	event->hw.interrupts = 0;
  
-	if (__get_cpu_var(watchdog_nmi_touch) == true) {
-		__get_cpu_var(watchdog_nmi_touch) = false;
+	if (__this_cpu_read(watchdog_nmi_touch) == true) {
+		__this_cpu_write(watchdog_nmi_touch, false);
 		return;
 	}
  
@@ -218,7 +218,7 @@
 		int this_cpu = smp_processor_id();
  
 		/* only print hardlockups once */
-		if (__get_cpu_var(hard_watchdog_warn) == true)
+		if (__this_cpu_read(hard_watchdog_warn) == true)
 			return;
  
 		if (hardlockup_panic)
  
  
@@ -226,16 +226,16 @@
 		else
 			WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu);
  
-		__get_cpu_var(hard_watchdog_warn) = true;
+		__this_cpu_write(hard_watchdog_warn, true);
 		return;
 	}
  
-	__get_cpu_var(hard_watchdog_warn) = false;
+	__this_cpu_write(hard_watchdog_warn, false);
 	return;
 }
 static void watchdog_interrupt_count(void)
 {
-	__get_cpu_var(hrtimer_interrupts)++;
+	__this_cpu_inc(hrtimer_interrupts);
 }
 #else
 static inline void watchdog_interrupt_count(void) { return; }
@@ -244,7 +244,7 @@
 /* watchdog kicker functions */
 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 {
-	unsigned long touch_ts = __get_cpu_var(watchdog_touch_ts);
+	unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
 	struct pt_regs *regs = get_irq_regs();
 	int duration;
  
  
  
@@ -252,18 +252,18 @@
 	watchdog_interrupt_count();
  
 	/* kick the softlockup detector */
-	wake_up_process(__get_cpu_var(softlockup_watchdog));
+	wake_up_process(__this_cpu_read(softlockup_watchdog));
  
 	/* .. and repeat */
 	hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
  
 	if (touch_ts == 0) {
-		if (unlikely(__get_cpu_var(softlockup_touch_sync))) {
+		if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
 			/*
 			 * If the time stamp was touched atomically
 			 * make sure the scheduler tick is up to date.
 			 */
-			__get_cpu_var(softlockup_touch_sync) = false;
+			__this_cpu_write(softlockup_touch_sync, false);
 			sched_clock_tick();
 		}
 		__touch_watchdog();
@@ -279,7 +279,7 @@
 	duration = is_softlockup(touch_ts);
 	if (unlikely(duration)) {
 		/* only warn once */
-		if (__get_cpu_var(soft_watchdog_warn) == true)
+		if (__this_cpu_read(soft_watchdog_warn) == true)
 			return HRTIMER_RESTART;
  
 		printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
  
@@ -294,9 +294,9 @@
  
 		if (softlockup_panic)
 			panic("softlockup: hung tasks");
-		__get_cpu_var(soft_watchdog_warn) = true;
+		__this_cpu_write(soft_watchdog_warn, true);
 	} else
-		__get_cpu_var(soft_watchdog_warn) = false;
+		__this_cpu_write(soft_watchdog_warn, false);
  
 	return HRTIMER_RESTART;
 }
@@ -829,12 +829,12 @@
  
 static void next_reap_node(void)
 {
-	int node = __get_cpu_var(slab_reap_node);
+	int node = __this_cpu_read(slab_reap_node);
  
 	node = next_node(node, node_online_map);
 	if (unlikely(node >= MAX_NUMNODES))
 		node = first_node(node_online_map);
-	__get_cpu_var(slab_reap_node) = node;
+	__this_cpu_write(slab_reap_node, node);
 }
  
 #else
@@ -1012,7 +1012,7 @@
  */
 static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
 {
-	int node = __get_cpu_var(slab_reap_node);
+	int node = __this_cpu_read(slab_reap_node);
  
 	if (l3->alien) {
 		struct array_cache *ac = l3->alien[node];
...	...	@@ -22,15 +22,15 @@
22	22
23	23	static inline struct pt_regs *get_irq_regs(void)
24	24	{
25		- return __get_cpu_var(__irq_regs);
	25	+ return __this_cpu_read(__irq_regs);
26	26	}
27	27
28	28	static inline struct pt_regs set_irq_regs(struct pt_regs new_regs)
29	29	{
30		- struct pt_regs old_regs, *pp_regs = &__get_cpu_var(__irq_regs);
	30	+ struct pt_regs *old_regs;
31	31
32		- old_regs = *pp_regs;
33		- *pp_regs = new_regs;
	32	+ old_regs = __this_cpu_read(__irq_regs);
	33	+ __this_cpu_write(__irq_regs, new_regs);
34	34	return old_regs;
35	35	}
36	36
...	...	@@ -195,15 +195,9 @@
195	195	/*
196	196	* io context count accounting
197	197	*/
198		-#define elv_ioc_count_mod(name, __val) \
199		- do { \
200		- preempt_disable(); \
201		- __get_cpu_var(name) += (__val); \
202		- preempt_enable(); \
203		- } while (0)
204		-
205		-#define elv_ioc_count_inc(name) elv_ioc_count_mod(name, 1)
206		-#define elv_ioc_count_dec(name) elv_ioc_count_mod(name, -1)
	198	+#define elv_ioc_count_mod(name, __val) this_cpu_add(name, __val)
	199	+#define elv_ioc_count_inc(name) this_cpu_inc(name)
	200	+#define elv_ioc_count_dec(name) this_cpu_dec(name)
207	201
208	202	#define elv_ioc_count_read(name) \
209	203	({ \
...	...	@@ -47,7 +47,7 @@
47	47
48	48	#ifndef CONFIG_GENERIC_HARDIRQS
49	49	#define kstat_irqs_this_cpu(irq) \
50		- (kstat_this_cpu.irqs[irq])
	50	+ (this_cpu_read(kstat.irqs[irq])
51	51
52	52	struct irq_desc;
53	53
...	...	@@ -69,7 +69,7 @@
69	69
70	70	list_del_rcu(&p->tasks);
71	71	list_del_init(&p->sibling);
72		- __get_cpu_var(process_counts)--;
	72	+ __this_cpu_dec(process_counts);
73	73	}
74	74	list_del_rcu(&p->thread_group);
75	75	}
...	...	@@ -1282,7 +1282,7 @@
1282	1282	attach_pid(p, PIDTYPE_SID, task_session(current));
1283	1283	list_add_tail(&p->sibling, &p->real_parent->children);
1284	1284	list_add_tail_rcu(&p->tasks, &init_task.tasks);
1285		- __get_cpu_var(process_counts)++;
	1285	+ __this_cpu_inc(process_counts);
1286	1286	}
1287	1287	attach_pid(p, PIDTYPE_PID, pid);
1288	1288	nr_threads++;
...	...	@@ -497,7 +497,7 @@
497	497	*/
498	498	static inline int hrtimer_hres_active(void)
499	499	{
500		- return __get_cpu_var(hrtimer_bases).hres_active;
	500	+ return __this_cpu_read(hrtimer_bases.hres_active);
501	501	}
502	502
503	503	/*
...	...	@@ -1074,8 +1074,8 @@
1074	1074
1075	1075	void printk_tick(void)
1076	1076	{
1077		- if (__get_cpu_var(printk_pending)) {
1078		- __get_cpu_var(printk_pending) = 0;
	1077	+ if (__this_cpu_read(printk_pending)) {
	1078	+ __this_cpu_write(printk_pending, 0);
1079	1079	wake_up_interruptible(&log_wait);
1080	1080	}
1081	1081	}
...	...	@@ -367,8 +367,8 @@
367	367	WARN_ON_ONCE(rdtp->dynticks & 0x1);
368	368
369	369	/* If the interrupt queued a callback, get out of dyntick mode. */
370		- if (__get_cpu_var(rcu_sched_data).nxtlist \|\|
371		- __get_cpu_var(rcu_bh_data).nxtlist)
	370	+ if (__this_cpu_read(rcu_sched_data.nxtlist) \|\|
	371	+ __this_cpu_read(rcu_bh_data.nxtlist))
372	372	set_need_resched();
373	373	}
374	374
...	...	@@ -70,7 +70,7 @@
70	70	static void wakeup_softirqd(void)
71	71	{
72	72	/* Interrupts are disabled: no need to stop preemption */
73		- struct task_struct *tsk = __get_cpu_var(ksoftirqd);
	73	+ struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74	74
75	75	if (tsk && tsk->state != TASK_RUNNING)
76	76	wake_up_process(tsk);
...	...	@@ -388,8 +388,8 @@
388	388
389	389	local_irq_save(flags);
390	390	t->next = NULL;
391		- *__get_cpu_var(tasklet_vec).tail = t;
392		- __get_cpu_var(tasklet_vec).tail = &(t->next);
	391	+ *__this_cpu_read(tasklet_vec.tail) = t;
	392	+ __this_cpu_write(tasklet_vec.tail, &(t->next));
393	393	raise_softirq_irqoff(TASKLET_SOFTIRQ);
394	394	local_irq_restore(flags);
395	395	}
...	...	@@ -402,8 +402,8 @@
402	402
403	403	local_irq_save(flags);
404	404	t->next = NULL;
405		- *__get_cpu_var(tasklet_hi_vec).tail = t;
406		- __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	405	+ *__this_cpu_read(tasklet_hi_vec.tail) = t;
	406	+ __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
407	407	raise_softirq_irqoff(HI_SOFTIRQ);
408	408	local_irq_restore(flags);
409	409	}
...	...	@@ -414,8 +414,8 @@
414	414	{
415	415	BUG_ON(!irqs_disabled());
416	416
417		- t->next = __get_cpu_var(tasklet_hi_vec).head;
418		- __get_cpu_var(tasklet_hi_vec).head = t;
	417	+ t->next = __this_cpu_read(tasklet_hi_vec.head);
	418	+ __this_cpu_write(tasklet_hi_vec.head, t);
419	419	__raise_softirq_irqoff(HI_SOFTIRQ);
420	420	}
421	421
...	...	@@ -426,9 +426,9 @@
426	426	struct tasklet_struct *list;
427	427
428	428	local_irq_disable();
429		- list = __get_cpu_var(tasklet_vec).head;
430		- __get_cpu_var(tasklet_vec).head = NULL;
431		- __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
	429	+ list = __this_cpu_read(tasklet_vec.head);
	430	+ __this_cpu_write(tasklet_vec.head, NULL);
	431	+ __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
432	432	local_irq_enable();
433	433
434	434	while (list) {
...	...	@@ -449,8 +449,8 @@
449	449
450	450	local_irq_disable();
451	451	t->next = NULL;
452		- *__get_cpu_var(tasklet_vec).tail = t;
453		- __get_cpu_var(tasklet_vec).tail = &(t->next);
	452	+ *__this_cpu_read(tasklet_vec.tail) = t;
	453	+ __this_cpu_write(tasklet_vec.tail, &(t->next));
454	454	__raise_softirq_irqoff(TASKLET_SOFTIRQ);
455	455	local_irq_enable();
456	456	}
...	...	@@ -461,9 +461,9 @@
461	461	struct tasklet_struct *list;
462	462
463	463	local_irq_disable();
464		- list = __get_cpu_var(tasklet_hi_vec).head;
465		- __get_cpu_var(tasklet_hi_vec).head = NULL;
466		- __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
	464	+ list = __this_cpu_read(tasklet_hi_vec.head);
	465	+ __this_cpu_write(tasklet_hi_vec.head, NULL);
	466	+ __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
467	467	local_irq_enable();
468	468
469	469	while (list) {
...	...	@@ -484,8 +484,8 @@
484	484
485	485	local_irq_disable();
486	486	t->next = NULL;
487		- *__get_cpu_var(tasklet_hi_vec).tail = t;
488		- __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
	487	+ *__this_cpu_read(tasklet_hi_vec.tail) = t;
	488	+ __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
489	489	__raise_softirq_irqoff(HI_SOFTIRQ);
490	490	local_irq_enable();
491	491	}
492	492
...	...	@@ -802,16 +802,16 @@
802	802
803	803	/* Find end, append list for that CPU. */
804	804	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
805		- *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
806		- __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
	805	+ *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
	806	+ this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
807	807	per_cpu(tasklet_vec, cpu).head = NULL;
808	808	per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
809	809	}
810	810	raise_softirq_irqoff(TASKLET_SOFTIRQ);
811	811
812	812	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
813		- *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
814		- __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
	813	+ *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
	814	+ __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
815	815	per_cpu(tasklet_hi_vec, cpu).head = NULL;
816	816	per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
817	817	}
...	...	@@ -49,7 +49,7 @@
49	49	*/
50	50	int tick_is_oneshot_available(void)
51	51	{
52		- struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
	52	+ struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
53	53
54	54	return dev && (dev->features & CLOCK_EVT_FEAT_ONESHOT);
55	55	}