Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit ee1760b2b4920841f45b2ad07a1c9f99e08568e7

Authored by Mel Gorman
Committed by Jiri Slaby
1 parent f99bfd27bd
Exists in ti-linux-3.12.y and in 2 other branches newt-ti-linux-3.12.y, smarc-ti-linux-3.12.y

mm: page_alloc: use jump labels to avoid checking number_of_cpusets 

commit 664eeddeef6539247691197c1ac124d4aa872ab6 upstream.

If cpusets are not in use then we still check a global variable on every
page allocation.  Use jump labels to avoid the overhead.

Signed-off-by: Mel Gorman <mgorman@suse.de>
Reviewed-by: Rik van Riel <riel@redhat.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.cz>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Theodore Ts'o <tytso@mit.edu>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Mel Gorman <mgorman@suse.de>
Signed-off-by: Jiri Slaby <jslaby@suse.cz>

Showing 3 changed files with 32 additions and 14 deletions Side-by-side Diff

include/linux/cpuset.h
Diff comments   View file @ ee1760b
... ... @@ -12,11 +12,32 @@
12 12 #include <linux/cpumask.h>
13 13 #include <linux/nodemask.h>
14 14 #include <linux/mm.h>
  15 +#include <linux/jump_label.h>
15 16  
16 17 #ifdef CONFIG_CPUSETS
17 18  
18   -extern int number_of_cpusets; /* How many cpusets are defined in system? */
  19 +extern struct static_key cpusets_enabled_key;
  20 +static inline bool cpusets_enabled(void)
  21 +{
  22 + return static_key_false(&cpusets_enabled_key);
  23 +}
19 24  
  25 +static inline int nr_cpusets(void)
  26 +{
  27 + /* jump label reference count + the top-level cpuset */
  28 + return static_key_count(&cpusets_enabled_key) + 1;
  29 +}
  30 +
  31 +static inline void cpuset_inc(void)
  32 +{
  33 + static_key_slow_inc(&cpusets_enabled_key);
  34 +}
  35 +
  36 +static inline void cpuset_dec(void)
  37 +{
  38 + static_key_slow_dec(&cpusets_enabled_key);
  39 +}
  40 +
20 41 extern int cpuset_init(void);
21 42 extern void cpuset_init_smp(void);
22 43 extern void cpuset_update_active_cpus(bool cpu_online);
23 44  
... ... @@ -32,13 +53,13 @@
32 53  
33 54 static inline int cpuset_node_allowed_softwall(int node, gfp_t gfp_mask)
34 55 {
35   - return number_of_cpusets <= 1 ||
  56 + return nr_cpusets() <= 1 ||
36 57 __cpuset_node_allowed_softwall(node, gfp_mask);
37 58 }
38 59  
39 60 static inline int cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask)
40 61 {
41   - return number_of_cpusets <= 1 ||
  62 + return nr_cpusets() <= 1 ||
42 63 __cpuset_node_allowed_hardwall(node, gfp_mask);
43 64 }
44 65  
... ... @@ -119,6 +140,8 @@
119 140 }
120 141  
121 142 #else /* !CONFIG_CPUSETS */
  143 +
  144 +static inline bool cpusets_enabled(void) { return false; }
122 145  
123 146 static inline int cpuset_init(void) { return 0; }
124 147 static inline void cpuset_init_smp(void) {}
... ... @@ -61,12 +61,7 @@
61 61 #include <linux/cgroup.h>
62 62 #include <linux/wait.h>
63 63  
64   -/*
65   - * Tracks how many cpusets are currently defined in system.
66   - * When there is only one cpuset (the root cpuset) we can
67   - * short circuit some hooks.
68   - */
69   -int number_of_cpusets __read_mostly;
  64 +struct static_key cpusets_enabled_key __read_mostly = STATIC_KEY_INIT_FALSE;
70 65  
71 66 /* See "Frequency meter" comments, below. */
72 67  
... ... @@ -611,7 +606,7 @@
611 606 goto done;
612 607 }
613 608  
614   - csa = kmalloc(number_of_cpusets * sizeof(cp), GFP_KERNEL);
  609 + csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL);
615 610 if (!csa)
616 611 goto done;
617 612 csn = 0;
... ... @@ -1986,7 +1981,7 @@
1986 1981 if (is_spread_slab(parent))
1987 1982 set_bit(CS_SPREAD_SLAB, &cs->flags);
1988 1983  
1989   - number_of_cpusets++;
  1984 + cpuset_inc();
1990 1985  
1991 1986 if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
1992 1987 goto out_unlock;
... ... @@ -2037,7 +2032,7 @@
2037 2032 if (is_sched_load_balance(cs))
2038 2033 update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
2039 2034  
2040   - number_of_cpusets--;
  2035 + cpuset_dec();
2041 2036 clear_bit(CS_ONLINE, &cs->flags);
2042 2037  
2043 2038 mutex_unlock(&cpuset_mutex);
... ... @@ -2092,7 +2087,6 @@
2092 2087 if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL))
2093 2088 BUG();
2094 2089  
2095   - number_of_cpusets = 1;
2096 2090 return 0;
2097 2091 }
2098 2092  
... ... @@ -1920,7 +1920,8 @@
1920 1920 if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
1921 1921 !zlc_zone_worth_trying(zonelist, z, allowednodes))
1922 1922 continue;
1923   - if ((alloc_flags & ALLOC_CPUSET) &&
  1923 + if (cpusets_enabled() &&
  1924 + (alloc_flags & ALLOC_CPUSET) &&
1924 1925 !cpuset_zone_allowed_softwall(zone, gfp_mask))
1925 1926 continue;
1926 1927 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);