Eric Lee / linux-smarc-t335x-v3.2

Download as
Browse Code ยป

Commit 909ea96468096b07fbb41aaf69be060d92bd9271

Authored by Christoph Lameter
Committed by Tejun Heo
1 parent 780f36d8b3
Exists in master and in 4 other branches v3.2_AM335xPSP_04.06.00.11, v3.2_NEWT335xPSP_04.06.00.11, v3.2_SBC_SMARTMEN, v3.2_SMARCT335xPSP_04.06.00.11

core: Replace __get_cpu_var with __this_cpu_read if not used for an address. 

__get_cpu_var() can be replaced with this_cpu_read and will then use a
single read instruction with implied address calculation to access the
correct per cpu instance.

However, the address of a per cpu variable passed to __this_cpu_read()
cannot be determined (since it's an implied address conversion through
segment prefixes).  Therefore apply this only to uses of __get_cpu_var
where the address of the variable is not used.

Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Hugh Dickins <hughd@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Acked-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

Showing 13 changed files with 60 additions and 66 deletions Side-by-side Diff

include/asm-generic/irq_regs.h
Diff comments   View file @ 909ea96
... ... @@ -22,15 +22,15 @@
22 22  
23 23 static inline struct pt_regs *get_irq_regs(void)
24 24 {
25   - return __get_cpu_var(__irq_regs);
  25 + return __this_cpu_read(__irq_regs);
26 26 }
27 27  
28 28 static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
29 29 {
30   - struct pt_regs *old_regs, **pp_regs = &__get_cpu_var(__irq_regs);
  30 + struct pt_regs *old_regs;
31 31  
32   - old_regs = *pp_regs;
33   - *pp_regs = new_regs;
  32 + old_regs = __this_cpu_read(__irq_regs);
  33 + __this_cpu_write(__irq_regs, new_regs);
34 34 return old_regs;
35 35 }
36 36  
include/linux/elevator.h
Diff comments   View file @ 909ea96
... ... @@ -195,15 +195,9 @@
195 195 /*
196 196 * io context count accounting
197 197 */
198   -#define elv_ioc_count_mod(name, __val) \
199   - do { \
200   - preempt_disable(); \
201   - __get_cpu_var(name) += (__val); \
202   - preempt_enable(); \
203   - } while (0)
204   -
205   -#define elv_ioc_count_inc(name) elv_ioc_count_mod(name, 1)
206   -#define elv_ioc_count_dec(name) elv_ioc_count_mod(name, -1)
  198 +#define elv_ioc_count_mod(name, __val) this_cpu_add(name, __val)
  199 +#define elv_ioc_count_inc(name) this_cpu_inc(name)
  200 +#define elv_ioc_count_dec(name) this_cpu_dec(name)
207 201  
208 202 #define elv_ioc_count_read(name) \
209 203 ({ \
include/linux/kernel_stat.h
Diff comments   View file @ 909ea96
... ... @@ -47,7 +47,7 @@
47 47  
48 48 #ifndef CONFIG_GENERIC_HARDIRQS
49 49 #define kstat_irqs_this_cpu(irq) \
50   - (kstat_this_cpu.irqs[irq])
  50 + (this_cpu_read(kstat.irqs[irq])
51 51  
52 52 struct irq_desc;
53 53  
... ... @@ -69,7 +69,7 @@
69 69  
70 70 list_del_rcu(&p->tasks);
71 71 list_del_init(&p->sibling);
72   - __get_cpu_var(process_counts)--;
  72 + __this_cpu_dec(process_counts);
73 73 }
74 74 list_del_rcu(&p->thread_group);
75 75 }
... ... @@ -1282,7 +1282,7 @@
1282 1282 attach_pid(p, PIDTYPE_SID, task_session(current));
1283 1283 list_add_tail(&p->sibling, &p->real_parent->children);
1284 1284 list_add_tail_rcu(&p->tasks, &init_task.tasks);
1285   - __get_cpu_var(process_counts)++;
  1285 + __this_cpu_inc(process_counts);
1286 1286 }
1287 1287 attach_pid(p, PIDTYPE_PID, pid);
1288 1288 nr_threads++;
... ... @@ -497,7 +497,7 @@
497 497 */
498 498 static inline int hrtimer_hres_active(void)
499 499 {
500   - return __get_cpu_var(hrtimer_bases).hres_active;
  500 + return __this_cpu_read(hrtimer_bases.hres_active);
501 501 }
502 502  
503 503 /*
... ... @@ -1074,8 +1074,8 @@
1074 1074  
1075 1075 void printk_tick(void)
1076 1076 {
1077   - if (__get_cpu_var(printk_pending)) {
1078   - __get_cpu_var(printk_pending) = 0;
  1077 + if (__this_cpu_read(printk_pending)) {
  1078 + __this_cpu_write(printk_pending, 0);
1079 1079 wake_up_interruptible(&log_wait);
1080 1080 }
1081 1081 }
... ... @@ -367,8 +367,8 @@
367 367 WARN_ON_ONCE(rdtp->dynticks & 0x1);
368 368  
369 369 /* If the interrupt queued a callback, get out of dyntick mode. */
370   - if (__get_cpu_var(rcu_sched_data).nxtlist ||
371   - __get_cpu_var(rcu_bh_data).nxtlist)
  370 + if (__this_cpu_read(rcu_sched_data.nxtlist) ||
  371 + __this_cpu_read(rcu_bh_data.nxtlist))
372 372 set_need_resched();
373 373 }
374 374  
... ... @@ -70,7 +70,7 @@
70 70 static void wakeup_softirqd(void)
71 71 {
72 72 /* Interrupts are disabled: no need to stop preemption */
73   - struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  73 + struct task_struct *tsk = __this_cpu_read(ksoftirqd);
74 74  
75 75 if (tsk && tsk->state != TASK_RUNNING)
76 76 wake_up_process(tsk);
... ... @@ -388,8 +388,8 @@
388 388  
389 389 local_irq_save(flags);
390 390 t->next = NULL;
391   - *__get_cpu_var(tasklet_vec).tail = t;
392   - __get_cpu_var(tasklet_vec).tail = &(t->next);
  391 + *__this_cpu_read(tasklet_vec.tail) = t;
  392 + __this_cpu_write(tasklet_vec.tail, &(t->next));
393 393 raise_softirq_irqoff(TASKLET_SOFTIRQ);
394 394 local_irq_restore(flags);
395 395 }
... ... @@ -402,8 +402,8 @@
402 402  
403 403 local_irq_save(flags);
404 404 t->next = NULL;
405   - *__get_cpu_var(tasklet_hi_vec).tail = t;
406   - __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
  405 + *__this_cpu_read(tasklet_hi_vec.tail) = t;
  406 + __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
407 407 raise_softirq_irqoff(HI_SOFTIRQ);
408 408 local_irq_restore(flags);
409 409 }
... ... @@ -414,8 +414,8 @@
414 414 {
415 415 BUG_ON(!irqs_disabled());
416 416  
417   - t->next = __get_cpu_var(tasklet_hi_vec).head;
418   - __get_cpu_var(tasklet_hi_vec).head = t;
  417 + t->next = __this_cpu_read(tasklet_hi_vec.head);
  418 + __this_cpu_write(tasklet_hi_vec.head, t);
419 419 __raise_softirq_irqoff(HI_SOFTIRQ);
420 420 }
421 421  
... ... @@ -426,9 +426,9 @@
426 426 struct tasklet_struct *list;
427 427  
428 428 local_irq_disable();
429   - list = __get_cpu_var(tasklet_vec).head;
430   - __get_cpu_var(tasklet_vec).head = NULL;
431   - __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
  429 + list = __this_cpu_read(tasklet_vec.head);
  430 + __this_cpu_write(tasklet_vec.head, NULL);
  431 + __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
432 432 local_irq_enable();
433 433  
434 434 while (list) {
... ... @@ -449,8 +449,8 @@
449 449  
450 450 local_irq_disable();
451 451 t->next = NULL;
452   - *__get_cpu_var(tasklet_vec).tail = t;
453   - __get_cpu_var(tasklet_vec).tail = &(t->next);
  452 + *__this_cpu_read(tasklet_vec.tail) = t;
  453 + __this_cpu_write(tasklet_vec.tail, &(t->next));
454 454 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
455 455 local_irq_enable();
456 456 }
... ... @@ -461,9 +461,9 @@
461 461 struct tasklet_struct *list;
462 462  
463 463 local_irq_disable();
464   - list = __get_cpu_var(tasklet_hi_vec).head;
465   - __get_cpu_var(tasklet_hi_vec).head = NULL;
466   - __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
  464 + list = __this_cpu_read(tasklet_hi_vec.head);
  465 + __this_cpu_write(tasklet_hi_vec.head, NULL);
  466 + __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
467 467 local_irq_enable();
468 468  
469 469 while (list) {
... ... @@ -484,8 +484,8 @@
484 484  
485 485 local_irq_disable();
486 486 t->next = NULL;
487   - *__get_cpu_var(tasklet_hi_vec).tail = t;
488   - __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
  487 + *__this_cpu_read(tasklet_hi_vec.tail) = t;
  488 + __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
489 489 __raise_softirq_irqoff(HI_SOFTIRQ);
490 490 local_irq_enable();
491 491 }
492 492  
... ... @@ -802,16 +802,16 @@
802 802  
803 803 /* Find end, append list for that CPU. */
804 804 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
805   - *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
806   - __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
  805 + *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
  806 + this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
807 807 per_cpu(tasklet_vec, cpu).head = NULL;
808 808 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
809 809 }
810 810 raise_softirq_irqoff(TASKLET_SOFTIRQ);
811 811  
812 812 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
813   - *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
814   - __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
  813 + *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
  814 + __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
815 815 per_cpu(tasklet_hi_vec, cpu).head = NULL;
816 816 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
817 817 }
kernel/time/tick-common.c
Diff comments   View file @ 909ea96
... ... @@ -49,7 +49,7 @@
49 49 */
50 50 int tick_is_oneshot_available(void)
51 51 {
52   - struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
  52 + struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
53 53  
54 54 return dev && (dev->features & CLOCK_EVT_FEAT_ONESHOT);
55 55 }
kernel/time/tick-oneshot.c
Diff comments   View file @ 909ea96
... ... @@ -95,7 +95,7 @@
95 95 */
96 96 int tick_program_event(ktime_t expires, int force)
97 97 {
98   - struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
  98 + struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
99 99  
100 100 return tick_dev_program_event(dev, expires, force);
101 101 }
... ... @@ -167,7 +167,7 @@
167 167 int ret;
168 168  
169 169 local_irq_save(flags);
170   - ret = __get_cpu_var(tick_cpu_device).mode == TICKDEV_MODE_ONESHOT;
  170 + ret = __this_cpu_read(tick_cpu_device.mode) == TICKDEV_MODE_ONESHOT;
171 171 local_irq_restore(flags);
172 172  
173 173 return ret;
... ... @@ -116,12 +116,12 @@
116 116 {
117 117 int this_cpu = smp_processor_id();
118 118  
119   - __get_cpu_var(watchdog_touch_ts) = get_timestamp(this_cpu);
  119 + __this_cpu_write(watchdog_touch_ts, get_timestamp(this_cpu));
120 120 }
121 121  
122 122 void touch_softlockup_watchdog(void)
123 123 {
124   - __raw_get_cpu_var(watchdog_touch_ts) = 0;
  124 + __this_cpu_write(watchdog_touch_ts, 0);
125 125 }
126 126 EXPORT_SYMBOL(touch_softlockup_watchdog);
127 127  
128 128  
129 129  
... ... @@ -165,12 +165,12 @@
165 165 /* watchdog detector functions */
166 166 static int is_hardlockup(void)
167 167 {
168   - unsigned long hrint = __get_cpu_var(hrtimer_interrupts);
  168 + unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
169 169  
170   - if (__get_cpu_var(hrtimer_interrupts_saved) == hrint)
  170 + if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
171 171 return 1;
172 172  
173   - __get_cpu_var(hrtimer_interrupts_saved) = hrint;
  173 + __this_cpu_write(hrtimer_interrupts_saved, hrint);
174 174 return 0;
175 175 }
176 176 #endif
... ... @@ -203,8 +203,8 @@
203 203 /* Ensure the watchdog never gets throttled */
204 204 event->hw.interrupts = 0;
205 205  
206   - if (__get_cpu_var(watchdog_nmi_touch) == true) {
207   - __get_cpu_var(watchdog_nmi_touch) = false;
  206 + if (__this_cpu_read(watchdog_nmi_touch) == true) {
  207 + __this_cpu_write(watchdog_nmi_touch, false);
208 208 return;
209 209 }
210 210  
... ... @@ -218,7 +218,7 @@
218 218 int this_cpu = smp_processor_id();
219 219  
220 220 /* only print hardlockups once */
221   - if (__get_cpu_var(hard_watchdog_warn) == true)
  221 + if (__this_cpu_read(hard_watchdog_warn) == true)
222 222 return;
223 223  
224 224 if (hardlockup_panic)
225 225  
226 226  
... ... @@ -226,16 +226,16 @@
226 226 else
227 227 WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu);
228 228  
229   - __get_cpu_var(hard_watchdog_warn) = true;
  229 + __this_cpu_write(hard_watchdog_warn, true);
230 230 return;
231 231 }
232 232  
233   - __get_cpu_var(hard_watchdog_warn) = false;
  233 + __this_cpu_write(hard_watchdog_warn, false);
234 234 return;
235 235 }
236 236 static void watchdog_interrupt_count(void)
237 237 {
238   - __get_cpu_var(hrtimer_interrupts)++;
  238 + __this_cpu_inc(hrtimer_interrupts);
239 239 }
240 240 #else
241 241 static inline void watchdog_interrupt_count(void) { return; }
... ... @@ -244,7 +244,7 @@
244 244 /* watchdog kicker functions */
245 245 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
246 246 {
247   - unsigned long touch_ts = __get_cpu_var(watchdog_touch_ts);
  247 + unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
248 248 struct pt_regs *regs = get_irq_regs();
249 249 int duration;
250 250  
251 251  
252 252  
... ... @@ -252,18 +252,18 @@
252 252 watchdog_interrupt_count();
253 253  
254 254 /* kick the softlockup detector */
255   - wake_up_process(__get_cpu_var(softlockup_watchdog));
  255 + wake_up_process(__this_cpu_read(softlockup_watchdog));
256 256  
257 257 /* .. and repeat */
258 258 hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
259 259  
260 260 if (touch_ts == 0) {
261   - if (unlikely(__get_cpu_var(softlockup_touch_sync))) {
  261 + if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
262 262 /*
263 263 * If the time stamp was touched atomically
264 264 * make sure the scheduler tick is up to date.
265 265 */
266   - __get_cpu_var(softlockup_touch_sync) = false;
  266 + __this_cpu_write(softlockup_touch_sync, false);
267 267 sched_clock_tick();
268 268 }
269 269 __touch_watchdog();
... ... @@ -279,7 +279,7 @@
279 279 duration = is_softlockup(touch_ts);
280 280 if (unlikely(duration)) {
281 281 /* only warn once */
282   - if (__get_cpu_var(soft_watchdog_warn) == true)
  282 + if (__this_cpu_read(soft_watchdog_warn) == true)
283 283 return HRTIMER_RESTART;
284 284  
285 285 printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
286 286  
... ... @@ -294,9 +294,9 @@
294 294  
295 295 if (softlockup_panic)
296 296 panic("softlockup: hung tasks");
297   - __get_cpu_var(soft_watchdog_warn) = true;
  297 + __this_cpu_write(soft_watchdog_warn, true);
298 298 } else
299   - __get_cpu_var(soft_watchdog_warn) = false;
  299 + __this_cpu_write(soft_watchdog_warn, false);
300 300  
301 301 return HRTIMER_RESTART;
302 302 }
... ... @@ -829,12 +829,12 @@
829 829  
830 830 static void next_reap_node(void)
831 831 {
832   - int node = __get_cpu_var(slab_reap_node);
  832 + int node = __this_cpu_read(slab_reap_node);
833 833  
834 834 node = next_node(node, node_online_map);
835 835 if (unlikely(node >= MAX_NUMNODES))
836 836 node = first_node(node_online_map);
837   - __get_cpu_var(slab_reap_node) = node;
  837 + __this_cpu_write(slab_reap_node, node);
838 838 }
839 839  
840 840 #else
... ... @@ -1012,7 +1012,7 @@
1012 1012 */
1013 1013 static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
1014 1014 {
1015   - int node = __get_cpu_var(slab_reap_node);
  1015 + int node = __this_cpu_read(slab_reap_node);
1016 1016  
1017 1017 if (l3->alien) {
1018 1018 struct array_cache *ac = l3->alien[node];