Commit c0f489d2c6fec8994c642c2ec925eb858727dc7b

Authored by Paul E. McKenney
1 parent abaa93d9e1

rcu: Bind grace-period kthreads to non-NO_HZ_FULL CPUs

Binding the grace-period kthreads to the timekeeping CPU resulted in
significant performance decreases for some workloads.  For more detail,
see:

https://lkml.org/lkml/2014/6/3/395 for benchmark numbers

https://lkml.org/lkml/2014/6/4/218 for CPU statistics

It turns out that it is necessary to bind the grace-period kthreads
to the timekeeping CPU only when all but CPU 0 is a nohz_full CPU
on the one hand or if CONFIG_NO_HZ_FULL_SYSIDLE=y on the other.
In other cases, it suffices to bind the grace-period kthreads to the
set of non-nohz_full CPUs.

This commit therefore creates a tick_nohz_not_full_mask that is the
complement of tick_nohz_full_mask, and then binds the grace-period
kthread to the set of CPUs indicated by this new mask, which covers
the CONFIG_NO_HZ_FULL_SYSIDLE=n case.  The CONFIG_NO_HZ_FULL_SYSIDLE=y
case still binds the grace-period kthreads to the timekeeping CPU.
This commit also includes the tick_nohz_full_enabled() check suggested
by Frederic Weisbecker.

Reported-by: Jet Chen <jet.chen@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Created housekeeping_affine() and housekeeping_mask per
  fweisbec feedback. ]

Showing 3 changed files with 39 additions and 5 deletions Side-by-side Diff

include/linux/tick.h
... ... @@ -12,6 +12,7 @@
12 12 #include <linux/hrtimer.h>
13 13 #include <linux/context_tracking_state.h>
14 14 #include <linux/cpumask.h>
  15 +#include <linux/sched.h>
15 16  
16 17 #ifdef CONFIG_GENERIC_CLOCKEVENTS
17 18  
... ... @@ -162,6 +163,7 @@
162 163 #ifdef CONFIG_NO_HZ_FULL
163 164 extern bool tick_nohz_full_running;
164 165 extern cpumask_var_t tick_nohz_full_mask;
  166 +extern cpumask_var_t housekeeping_mask;
165 167  
166 168 static inline bool tick_nohz_full_enabled(void)
167 169 {
... ... @@ -193,6 +195,24 @@
193 195 static inline void tick_nohz_full_kick_all(void) { }
194 196 static inline void __tick_nohz_task_switch(struct task_struct *tsk) { }
195 197 #endif
  198 +
  199 +static inline bool is_housekeeping_cpu(int cpu)
  200 +{
  201 +#ifdef CONFIG_NO_HZ_FULL
  202 + if (tick_nohz_full_enabled())
  203 + return cpumask_test_cpu(cpu, housekeeping_mask);
  204 +#endif
  205 + return true;
  206 +}
  207 +
  208 +static inline void housekeeping_affine(struct task_struct *t)
  209 +{
  210 +#ifdef CONFIG_NO_HZ_FULL
  211 + if (tick_nohz_full_enabled())
  212 + set_cpus_allowed_ptr(t, housekeeping_mask);
  213 +
  214 +#endif
  215 +}
196 216  
197 217 static inline void tick_nohz_full_check(void)
198 218 {
kernel/rcu/tree_plugin.h
... ... @@ -2846,13 +2846,17 @@
2846 2846 */
2847 2847 static void rcu_bind_gp_kthread(void)
2848 2848 {
2849   -#ifdef CONFIG_NO_HZ_FULL
2850   - int cpu = tick_do_timer_cpu;
  2849 + int __maybe_unused cpu;
2851 2850  
2852   - if (cpu < 0 || cpu >= nr_cpu_ids)
  2851 + if (!tick_nohz_full_enabled())
2853 2852 return;
2854   - if (raw_smp_processor_id() != cpu)
  2853 +#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
  2854 + cpu = tick_do_timer_cpu;
  2855 + if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu)
2855 2856 set_cpus_allowed_ptr(current, cpumask_of(cpu));
2856   -#endif /* #ifdef CONFIG_NO_HZ_FULL */
  2857 +#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
  2858 + if (!is_housekeeping_cpu(raw_smp_processor_id()))
  2859 + housekeeping_affine(current);
  2860 +#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
2857 2861 }
kernel/time/tick-sched.c
... ... @@ -154,6 +154,7 @@
154 154  
155 155 #ifdef CONFIG_NO_HZ_FULL
156 156 cpumask_var_t tick_nohz_full_mask;
  157 +cpumask_var_t housekeeping_mask;
157 158 bool tick_nohz_full_running;
158 159  
159 160 static bool can_stop_full_tick(void)
... ... @@ -281,6 +282,7 @@
281 282 int cpu;
282 283  
283 284 alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
  285 + alloc_bootmem_cpumask_var(&housekeeping_mask);
284 286 if (cpulist_parse(str, tick_nohz_full_mask) < 0) {
285 287 pr_warning("NOHZ: Incorrect nohz_full cpumask\n");
286 288 return 1;
... ... @@ -291,6 +293,8 @@
291 293 pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu);
292 294 cpumask_clear_cpu(cpu, tick_nohz_full_mask);
293 295 }
  296 + cpumask_andnot(housekeeping_mask,
  297 + cpu_possible_mask, tick_nohz_full_mask);
294 298 tick_nohz_full_running = true;
295 299  
296 300 return 1;
297 301  
... ... @@ -332,9 +336,15 @@
332 336 pr_err("NO_HZ: Can't allocate full dynticks cpumask\n");
333 337 return err;
334 338 }
  339 + if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) {
  340 + pr_err("NO_HZ: Can't allocate not-full dynticks cpumask\n");
  341 + return err;
  342 + }
335 343 err = 0;
336 344 cpumask_setall(tick_nohz_full_mask);
337 345 cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask);
  346 + cpumask_clear(housekeeping_mask);
  347 + cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
338 348 tick_nohz_full_running = true;
339 349 #endif
340 350 return err;