Commit c0f489d2c6fec8994c642c2ec925eb858727dc7b
1 parent
abaa93d9e1
Exists in
ti-lsk-linux-4.1.y
and in
10 other branches
rcu: Bind grace-period kthreads to non-NO_HZ_FULL CPUs
Binding the grace-period kthreads to the timekeeping CPU resulted in significant performance decreases for some workloads. For more detail, see: https://lkml.org/lkml/2014/6/3/395 for benchmark numbers https://lkml.org/lkml/2014/6/4/218 for CPU statistics It turns out that it is necessary to bind the grace-period kthreads to the timekeeping CPU only when all but CPU 0 is a nohz_full CPU on the one hand or if CONFIG_NO_HZ_FULL_SYSIDLE=y on the other. In other cases, it suffices to bind the grace-period kthreads to the set of non-nohz_full CPUs. This commit therefore creates a tick_nohz_not_full_mask that is the complement of tick_nohz_full_mask, and then binds the grace-period kthread to the set of CPUs indicated by this new mask, which covers the CONFIG_NO_HZ_FULL_SYSIDLE=n case. The CONFIG_NO_HZ_FULL_SYSIDLE=y case still binds the grace-period kthreads to the timekeeping CPU. This commit also includes the tick_nohz_full_enabled() check suggested by Frederic Weisbecker. Reported-by: Jet Chen <jet.chen@intel.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> [ paulmck: Created housekeeping_affine() and housekeeping_mask per fweisbec feedback. ]
Showing 3 changed files with 39 additions and 5 deletions Side-by-side Diff
include/linux/tick.h
... | ... | @@ -12,6 +12,7 @@ |
12 | 12 | #include <linux/hrtimer.h> |
13 | 13 | #include <linux/context_tracking_state.h> |
14 | 14 | #include <linux/cpumask.h> |
15 | +#include <linux/sched.h> | |
15 | 16 | |
16 | 17 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
17 | 18 | |
... | ... | @@ -162,6 +163,7 @@ |
162 | 163 | #ifdef CONFIG_NO_HZ_FULL |
163 | 164 | extern bool tick_nohz_full_running; |
164 | 165 | extern cpumask_var_t tick_nohz_full_mask; |
166 | +extern cpumask_var_t housekeeping_mask; | |
165 | 167 | |
166 | 168 | static inline bool tick_nohz_full_enabled(void) |
167 | 169 | { |
... | ... | @@ -193,6 +195,24 @@ |
193 | 195 | static inline void tick_nohz_full_kick_all(void) { } |
194 | 196 | static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } |
195 | 197 | #endif |
198 | + | |
199 | +static inline bool is_housekeeping_cpu(int cpu) | |
200 | +{ | |
201 | +#ifdef CONFIG_NO_HZ_FULL | |
202 | + if (tick_nohz_full_enabled()) | |
203 | + return cpumask_test_cpu(cpu, housekeeping_mask); | |
204 | +#endif | |
205 | + return true; | |
206 | +} | |
207 | + | |
208 | +static inline void housekeeping_affine(struct task_struct *t) | |
209 | +{ | |
210 | +#ifdef CONFIG_NO_HZ_FULL | |
211 | + if (tick_nohz_full_enabled()) | |
212 | + set_cpus_allowed_ptr(t, housekeeping_mask); | |
213 | + | |
214 | +#endif | |
215 | +} | |
196 | 216 | |
197 | 217 | static inline void tick_nohz_full_check(void) |
198 | 218 | { |
kernel/rcu/tree_plugin.h
... | ... | @@ -2846,13 +2846,17 @@ |
2846 | 2846 | */ |
2847 | 2847 | static void rcu_bind_gp_kthread(void) |
2848 | 2848 | { |
2849 | -#ifdef CONFIG_NO_HZ_FULL | |
2850 | - int cpu = tick_do_timer_cpu; | |
2849 | + int __maybe_unused cpu; | |
2851 | 2850 | |
2852 | - if (cpu < 0 || cpu >= nr_cpu_ids) | |
2851 | + if (!tick_nohz_full_enabled()) | |
2853 | 2852 | return; |
2854 | - if (raw_smp_processor_id() != cpu) | |
2853 | +#ifdef CONFIG_NO_HZ_FULL_SYSIDLE | |
2854 | + cpu = tick_do_timer_cpu; | |
2855 | + if (cpu >= 0 && cpu < nr_cpu_ids && raw_smp_processor_id() != cpu) | |
2855 | 2856 | set_cpus_allowed_ptr(current, cpumask_of(cpu)); |
2856 | -#endif /* #ifdef CONFIG_NO_HZ_FULL */ | |
2857 | +#else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ | |
2858 | + if (!is_housekeeping_cpu(raw_smp_processor_id())) | |
2859 | + housekeeping_affine(current); | |
2860 | +#endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ | |
2857 | 2861 | } |
kernel/time/tick-sched.c
... | ... | @@ -154,6 +154,7 @@ |
154 | 154 | |
155 | 155 | #ifdef CONFIG_NO_HZ_FULL |
156 | 156 | cpumask_var_t tick_nohz_full_mask; |
157 | +cpumask_var_t housekeeping_mask; | |
157 | 158 | bool tick_nohz_full_running; |
158 | 159 | |
159 | 160 | static bool can_stop_full_tick(void) |
... | ... | @@ -281,6 +282,7 @@ |
281 | 282 | int cpu; |
282 | 283 | |
283 | 284 | alloc_bootmem_cpumask_var(&tick_nohz_full_mask); |
285 | + alloc_bootmem_cpumask_var(&housekeeping_mask); | |
284 | 286 | if (cpulist_parse(str, tick_nohz_full_mask) < 0) { |
285 | 287 | pr_warning("NOHZ: Incorrect nohz_full cpumask\n"); |
286 | 288 | return 1; |
... | ... | @@ -291,6 +293,8 @@ |
291 | 293 | pr_warning("NO_HZ: Clearing %d from nohz_full range for timekeeping\n", cpu); |
292 | 294 | cpumask_clear_cpu(cpu, tick_nohz_full_mask); |
293 | 295 | } |
296 | + cpumask_andnot(housekeeping_mask, | |
297 | + cpu_possible_mask, tick_nohz_full_mask); | |
294 | 298 | tick_nohz_full_running = true; |
295 | 299 | |
296 | 300 | return 1; |
297 | 301 | |
... | ... | @@ -332,9 +336,15 @@ |
332 | 336 | pr_err("NO_HZ: Can't allocate full dynticks cpumask\n"); |
333 | 337 | return err; |
334 | 338 | } |
339 | + if (!alloc_cpumask_var(&housekeeping_mask, GFP_KERNEL)) { | |
340 | + pr_err("NO_HZ: Can't allocate not-full dynticks cpumask\n"); | |
341 | + return err; | |
342 | + } | |
335 | 343 | err = 0; |
336 | 344 | cpumask_setall(tick_nohz_full_mask); |
337 | 345 | cpumask_clear_cpu(smp_processor_id(), tick_nohz_full_mask); |
346 | + cpumask_clear(housekeeping_mask); | |
347 | + cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | |
338 | 348 | tick_nohz_full_running = true; |
339 | 349 | #endif |
340 | 350 | return err; |