Commit a586df067afe0580bb02b7a6312ca2afe49bba03

Authored by Andi Kleen
Committed by Linus Torvalds
1 parent b520b85a96

x86: Support __attribute__((__cold__)) in gcc 4.3

gcc 4.3 supports a new __attribute__((__cold__)) to mark functions cold. Any
path directly leading to a call of this function will be unlikely. And gcc
will try to generate smaller code for the function itself.

Please use with care. The code generation advantage isn't large and in most
cases it is not worth uglifying code with this.

This patch marks some common error functions like panic(), printk()
as cold.  This will longer term make many unlikely()s unnecessary, although
we can keep them for now for older compilers.

BUG is not marked cold because there is currently no way to tell
gcc to mark a inline function told.

Also all __init and __exit functions are marked cold. With a non -Os
build this will tell the compiler to generate slightly smaller code
for them. I think it currently only uses less alignments for labels,
but that might change in the future.

One disadvantage over *likely() is that they cannot be easily instrumented
to verify them.

Another drawback is that only the latest gcc 4.3 snapshots support this.
Unfortunately we cannot detect this using the preprocessor. This means older
snapshots will fail now. I don't think that's a problem because they are
unreleased compilers that nobody should be using.

gcc also has a __hot__ attribute, but I don't see any sense in using
this in the kernel right now. But someday I hope gcc will be able
to use more aggressive optimizing for hot functions even in -Os,
if that happens it should be added.

Includes compile fix from Thomas Gleixner.

Cc: Jan Hubicka <jh@suse.cz>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 4 changed files with 35 additions and 8 deletions Side-by-side Diff

include/linux/compiler-gcc4.h
... ... @@ -23,4 +23,22 @@
23 23 * code
24 24 */
25 25 #define uninitialized_var(x) x = x
  26 +
  27 +#if !(__GNUC__ == 4 && __GNUC_MINOR__ < 3)
  28 +/* Mark functions as cold. gcc will assume any path leading to a call
  29 + to them will be unlikely. This means a lot of manual unlikely()s
  30 + are unnecessary now for any paths leading to the usual suspects
  31 + like BUG(), printk(), panic() etc. [but let's keep them for now for
  32 + older compilers]
  33 +
  34 + Early snapshots of gcc 4.3 don't support this and we can't detect this
  35 + in the preprocessor, but we can live with this because they're unreleased.
  36 + Maketime probing would be overkill here.
  37 +
  38 + gcc also has a __attribute__((__hot__)) to move hot functions into
  39 + a special section, but I don't see any sense in this right now in
  40 + the kernel context */
  41 +#define __cold __attribute__((__cold__))
  42 +
  43 +#endif
include/linux/compiler.h
... ... @@ -174,5 +174,14 @@
174 174 # define __attribute_const__ /* unimplemented */
175 175 #endif
176 176  
  177 +/*
  178 + * Tell gcc if a function is cold. The compiler will assume any path
  179 + * directly leading to the call is unlikely.
  180 + */
  181 +
  182 +#ifndef __cold
  183 +#define __cold
  184 +#endif
  185 +
177 186 #endif /* __LINUX_COMPILER_H */
include/linux/init.h
... ... @@ -40,10 +40,10 @@
40 40  
41 41 /* These are for everybody (although not all archs will actually
42 42 discard it in modules) */
43   -#define __init __attribute__ ((__section__ (".init.text")))
  43 +#define __init __attribute__ ((__section__ (".init.text"))) __cold
44 44 #define __initdata __attribute__ ((__section__ (".init.data")))
45 45 #define __exitdata __attribute__ ((__section__(".exit.data")))
46   -#define __exit_call __attribute_used__ __attribute__ ((__section__ (".exitcall.exit")))
  46 +#define __exit_call __attribute_used__ __attribute__ ((__section__ (".exitcall.exit"))) __cold
47 47  
48 48 /* modpost check for section mismatches during the kernel build.
49 49 * A section mismatch happens when there are references from a
50 50  
... ... @@ -59,9 +59,9 @@
59 59 #define __initdata_refok __attribute__ ((__section__ (".data.init.refok")))
60 60  
61 61 #ifdef MODULE
62   -#define __exit __attribute__ ((__section__(".exit.text")))
  62 +#define __exit __attribute__ ((__section__(".exit.text"))) __cold
63 63 #else
64   -#define __exit __attribute_used__ __attribute__ ((__section__(".exit.text")))
  64 +#define __exit __attribute_used__ __attribute__ ((__section__(".exit.text"))) __cold
65 65 #endif
66 66  
67 67 /* For assembly routines */
include/linux/kernel.h
... ... @@ -106,7 +106,7 @@
106 106 extern struct atomic_notifier_head panic_notifier_list;
107 107 extern long (*panic_blink)(long time);
108 108 NORET_TYPE void panic(const char * fmt, ...)
109   - __attribute__ ((NORET_AND format (printf, 1, 2)));
  109 + __attribute__ ((NORET_AND format (printf, 1, 2))) __cold;
110 110 extern void oops_enter(void);
111 111 extern void oops_exit(void);
112 112 extern int oops_may_print(void);
113 113  
... ... @@ -155,14 +155,14 @@
155 155 asmlinkage int vprintk(const char *fmt, va_list args)
156 156 __attribute__ ((format (printf, 1, 0)));
157 157 asmlinkage int printk(const char * fmt, ...)
158   - __attribute__ ((format (printf, 1, 2)));
  158 + __attribute__ ((format (printf, 1, 2))) __cold;
159 159 #else
160 160 static inline int vprintk(const char *s, va_list args)
161 161 __attribute__ ((format (printf, 1, 0)));
162 162 static inline int vprintk(const char *s, va_list args) { return 0; }
163 163 static inline int printk(const char *s, ...)
164 164 __attribute__ ((format (printf, 1, 2)));
165   -static inline int printk(const char *s, ...) { return 0; }
  165 +static inline int __cold printk(const char *s, ...) { return 0; }
166 166 #endif
167 167  
168 168 unsigned long int_sqrt(unsigned long);
... ... @@ -212,7 +212,7 @@
212 212 #define TAINT_USER (1<<6)
213 213 #define TAINT_DIE (1<<7)
214 214  
215   -extern void dump_stack(void);
  215 +extern void dump_stack(void) __cold;
216 216  
217 217 enum {
218 218 DUMP_PREFIX_NONE,