jump label: Base patch for jump label

base patch to implement 'jump labeling'. Based on a new 'asm goto' inline assembly gcc mechanism, we can now branch to labels from an 'asm goto' statment. This allows us to create a 'no-op' fastpath, which can subsequently be patched with a jump to the slowpath code. This is useful for code which might be rarely used, but which we'd like to be able to call, if needed. Tracepoints are the current usecase that these are being implemented for. Acked-by: David S. Miller <davem@davemloft.net> Signed-off-by: Jason Baron <jbaron@redhat.com> LKML-Reference: <ee8b3595967989fdaf84e698dc7447d315ce972a.1284733808.git.jbaron@redhat.com> [ cleaned up some formating ] Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

jump label: Base patch for jump label
base patch to implement 'jump labeling'. Based on a new 'asm goto' inline assembly gcc mechanism, we can now branch to labels from an 'asm goto' statment. This allows us to create a 'no-op' fastpath, which can subsequently be patched with a jump to the slowpath code. This is useful for code which might be rarely used, but which we'd like to be able to call, if needed. Tracepoints are the current usecase that these are being implemented for. Acked-by: David S. Miller <davem@davemloft.net> Signed-off-by: Jason Baron <jbaron@redhat.com> LKML-Reference: <ee8b3595967989fdaf84e698dc7447d315ce972a.1284733808.git.jbaron@redhat.com> [ cleaned up some formating ] Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Jason Baron · Steven Rostedt
1 parent fa6f2cc770
Showing 12 changed files with 442 additions and 4 deletions Side-by-side Diff
Makefile
arch/Kconfig
arch/x86/include/asm/alternative.h
arch/x86/kernel/alternative.c
include/asm-generic/vmlinux.lds.h
include/linux/jump_label.h
include/linux/module.h
kernel/Makefile
kernel/jump_label.c
kernel/kprobes.c
kernel/module.c
scripts/gcc-goto.sh
@@ -591,6 +591,11 @@
 # conserve stack if available
 KBUILD_CFLAGS   += $(call cc-option,-fconserve-stack)
  
+# check for 'asm goto'
+ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y)
+	KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
+endif
+
 # Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
 # But warn user when we do so
 warn-assign = \
@@ -158,5 +158,8 @@
 	  subsystem.  Also has support for calculating CPU cycle events
 	  to determine how many clock cycles in a given period.
  
+config HAVE_ARCH_JUMP_LABEL
+	bool
+
 source "kernel/gcov/Kconfig"
@@ -4,6 +4,7 @@
 #include <linux/types.h>
 #include <linux/stddef.h>
 #include <linux/stringify.h>
+#include <linux/jump_label.h>
 #include <asm/asm.h>
  
 /*
@@ -182,7 +183,7 @@
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
  
-#if defined(CONFIG_DYNAMIC_FTRACE)
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
 #define IDEAL_NOP_SIZE_5 5
 extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
 extern void arch_init_ideal_nop5(void);
@@ -641,7 +641,7 @@
 	return addr;
 }
  
-#if defined(CONFIG_DYNAMIC_FTRACE)
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
  
 unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
  
@@ -220,6 +220,8 @@
 									\
 	BUG_TABLE							\
 									\
+	JUMP_TABLE							\
+									\
 	/* PCI quirks */						\
 	.pci_fixup        : AT(ADDR(.pci_fixup) - LOAD_OFFSET) {	\
 		VMLINUX_SYMBOL(__start_pci_fixups_early) = .;		\
@@ -562,6 +564,14 @@
 #else
 #define BUG_TABLE
 #endif
+
+#define JUMP_TABLE							\
+	. = ALIGN(8);							\
+	__jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) {		\
+		VMLINUX_SYMBOL(__start___jump_table) = .;		\
+		*(__jump_table)						\
+		VMLINUX_SYMBOL(__stop___jump_table) = .;		\
+	}
  
 #ifdef CONFIG_PM_TRACE
 #define TRACEDATA							\
+#ifndef _LINUX_JUMP_LABEL_H
+#define _LINUX_JUMP_LABEL_H
+
+#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_HAVE_ARCH_JUMP_LABEL)
+# include <asm/jump_label.h>
+# define HAVE_JUMP_LABEL
+#endif
+
+enum jump_label_type {
+	JUMP_LABEL_ENABLE,
+	JUMP_LABEL_DISABLE
+};
+
+struct module;
+
+#ifdef HAVE_JUMP_LABEL
+
+extern struct jump_entry __start___jump_table[];
+extern struct jump_entry __stop___jump_table[];
+
+extern void arch_jump_label_transform(struct jump_entry *entry,
+				 enum jump_label_type type);
+extern void jump_label_update(unsigned long key, enum jump_label_type type);
+extern void jump_label_apply_nops(struct module *mod);
+extern void arch_jump_label_text_poke_early(jump_label_t addr);
+
+#define enable_jump_label(key) \
+	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
+
+#define disable_jump_label(key) \
+	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
+
+#else
+
+#define JUMP_LABEL(key, label)			\
+do {						\
+	if (unlikely(*key))			\
+		goto label;			\
+} while (0)
+
+#define enable_jump_label(cond_var)	\
+do {					\
+       *(cond_var) = 1;			\
+} while (0)
+
+#define disable_jump_label(cond_var)	\
+do {					\
+       *(cond_var) = 0;			\
+} while (0)
+
+static inline int jump_label_apply_nops(struct module *mod)
+{
+	return 0;
+}
+
+#endif
+
+#endif
@@ -350,7 +350,10 @@
 	struct tracepoint *tracepoints;
 	unsigned int num_tracepoints;
 #endif
-
+#ifdef HAVE_JUMP_LABEL
+	struct jump_entry *jump_entries;
+	unsigned int num_jump_entries;
+#endif
 #ifdef CONFIG_TRACING
 	const char **trace_bprintk_fmt_start;
 	unsigned int num_trace_bprintk_fmt;
@@ -10,7 +10,7 @@
 	    kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
 	    hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
 	    notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
-	    async.o range.o
+	    async.o range.o jump_label.o
 obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
 obj-y += groups.o
  
+/*
+ * jump label support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/slab.h>
+#include <linux/sort.h>
+#include <linux/err.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+#define JUMP_LABEL_HASH_BITS 6
+#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
+static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
+
+/* mutex to protect coming/going of the the jump_label table */
+static DEFINE_MUTEX(jump_label_mutex);
+
+struct jump_label_entry {
+	struct hlist_node hlist;
+	struct jump_entry *table;
+	int nr_entries;
+	/* hang modules off here */
+	struct hlist_head modules;
+	unsigned long key;
+};
+
+struct jump_label_module_entry {
+	struct hlist_node hlist;
+	struct jump_entry *table;
+	int nr_entries;
+	struct module *mod;
+};
+
+static int jump_label_cmp(const void *a, const void *b)
+{
+	const struct jump_entry *jea = a;
+	const struct jump_entry *jeb = b;
+
+	if (jea->key < jeb->key)
+		return -1;
+
+	if (jea->key > jeb->key)
+		return 1;
+
+	return 0;
+}
+
+static void
+sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
+{
+	unsigned long size;
+
+	size = (((unsigned long)stop - (unsigned long)start)
+					/ sizeof(struct jump_entry));
+	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
+}
+
+static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
+{
+	struct hlist_head *head;
+	struct hlist_node *node;
+	struct jump_label_entry *e;
+	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
+
+	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+	hlist_for_each_entry(e, node, head, hlist) {
+		if (key == e->key)
+			return e;
+	}
+	return NULL;
+}
+
+static struct jump_label_entry *
+add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
+{
+	struct hlist_head *head;
+	struct jump_label_entry *e;
+	u32 hash;
+
+	e = get_jump_label_entry(key);
+	if (e)
+		return ERR_PTR(-EEXIST);
+
+	e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+
+	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
+	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+	e->key = key;
+	e->table = table;
+	e->nr_entries = nr_entries;
+	INIT_HLIST_HEAD(&(e->modules));
+	hlist_add_head(&e->hlist, head);
+	return e;
+}
+
+static int
+build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
+{
+	struct jump_entry *iter, *iter_begin;
+	struct jump_label_entry *entry;
+	int count;
+
+	sort_jump_label_entries(start, stop);
+	iter = start;
+	while (iter < stop) {
+		entry = get_jump_label_entry(iter->key);
+		if (!entry) {
+			iter_begin = iter;
+			count = 0;
+			while ((iter < stop) &&
+				(iter->key == iter_begin->key)) {
+				iter++;
+				count++;
+			}
+			entry = add_jump_label_entry(iter_begin->key,
+							count, iter_begin);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+		 } else {
+			WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/***
+ * jump_label_update - update jump label text
+ * @key -  key value associated with a a jump label
+ * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
+ *
+ * Will enable/disable the jump for jump label @key, depending on the
+ * value of @type.
+ *
+ */
+
+void jump_label_update(unsigned long key, enum jump_label_type type)
+{
+	struct jump_entry *iter;
+	struct jump_label_entry *entry;
+	struct hlist_node *module_node;
+	struct jump_label_module_entry *e_module;
+	int count;
+
+	mutex_lock(&jump_label_mutex);
+	entry = get_jump_label_entry((jump_label_t)key);
+	if (entry) {
+		count = entry->nr_entries;
+		iter = entry->table;
+		while (count--) {
+			if (kernel_text_address(iter->code))
+				arch_jump_label_transform(iter, type);
+			iter++;
+		}
+		/* eanble/disable jump labels in modules */
+		hlist_for_each_entry(e_module, module_node, &(entry->modules),
+							hlist) {
+			count = e_module->nr_entries;
+			iter = e_module->table;
+			while (count--) {
+				if (kernel_text_address(iter->code))
+					arch_jump_label_transform(iter, type);
+				iter++;
+			}
+		}
+	}
+	mutex_unlock(&jump_label_mutex);
+}
+
+static __init int init_jump_label(void)
+{
+	int ret;
+	struct jump_entry *iter_start = __start___jump_table;
+	struct jump_entry *iter_stop = __stop___jump_table;
+	struct jump_entry *iter;
+
+	mutex_lock(&jump_label_mutex);
+	ret = build_jump_label_hashtable(__start___jump_table,
+					 __stop___jump_table);
+	iter = iter_start;
+	while (iter < iter_stop) {
+		arch_jump_label_text_poke_early(iter->code);
+		iter++;
+	}
+	mutex_unlock(&jump_label_mutex);
+	return ret;
+}
+early_initcall(init_jump_label);
+
+#ifdef CONFIG_MODULES
+
+static struct jump_label_module_entry *
+add_jump_label_module_entry(struct jump_label_entry *entry,
+			    struct jump_entry *iter_begin,
+			    int count, struct module *mod)
+{
+	struct jump_label_module_entry *e;
+
+	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+	e->mod = mod;
+	e->nr_entries = count;
+	e->table = iter_begin;
+	hlist_add_head(&e->hlist, &entry->modules);
+	return e;
+}
+
+static int add_jump_label_module(struct module *mod)
+{
+	struct jump_entry *iter, *iter_begin;
+	struct jump_label_entry *entry;
+	struct jump_label_module_entry *module_entry;
+	int count;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return 0;
+
+	sort_jump_label_entries(mod->jump_entries,
+				mod->jump_entries + mod->num_jump_entries);
+	iter = mod->jump_entries;
+	while (iter < mod->jump_entries + mod->num_jump_entries) {
+		entry = get_jump_label_entry(iter->key);
+		iter_begin = iter;
+		count = 0;
+		while ((iter < mod->jump_entries + mod->num_jump_entries) &&
+			(iter->key == iter_begin->key)) {
+				iter++;
+				count++;
+		}
+		if (!entry) {
+			entry = add_jump_label_entry(iter_begin->key, 0, NULL);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+		}
+		module_entry = add_jump_label_module_entry(entry, iter_begin,
+							   count, mod);
+		if (IS_ERR(module_entry))
+			return PTR_ERR(module_entry);
+	}
+	return 0;
+}
+
+static void remove_jump_label_module(struct module *mod)
+{
+	struct hlist_head *head;
+	struct hlist_node *node, *node_next, *module_node, *module_node_next;
+	struct jump_label_entry *e;
+	struct jump_label_module_entry *e_module;
+	int i;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return;
+
+	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+		head = &jump_label_table[i];
+		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+			hlist_for_each_entry_safe(e_module, module_node,
+						  module_node_next,
+						  &(e->modules), hlist) {
+				if (e_module->mod == mod) {
+					hlist_del(&e_module->hlist);
+					kfree(e_module);
+				}
+			}
+			if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
+				hlist_del(&e->hlist);
+				kfree(e);
+			}
+		}
+	}
+}
+
+static int
+jump_label_module_notify(struct notifier_block *self, unsigned long val,
+			 void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		mutex_lock(&jump_label_mutex);
+		ret = add_jump_label_module(mod);
+		if (ret)
+			remove_jump_label_module(mod);
+		mutex_unlock(&jump_label_mutex);
+		break;
+	case MODULE_STATE_GOING:
+		mutex_lock(&jump_label_mutex);
+		remove_jump_label_module(mod);
+		mutex_unlock(&jump_label_mutex);
+		break;
+	}
+	return ret;
+}
+
+/***
+ * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
+ * @mod: module to patch
+ *
+ * Allow for run-time selection of the optimal nops. Before the module
+ * loads patch these with arch_get_jump_label_nop(), which is specified by
+ * the arch specific jump label code.
+ */
+void jump_label_apply_nops(struct module *mod)
+{
+	struct jump_entry *iter;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return;
+
+	iter = mod->jump_entries;
+	while (iter < mod->jump_entries + mod->num_jump_entries) {
+		arch_jump_label_text_poke_early(iter->code);
+		iter++;
+	}
+}
+
+struct notifier_block jump_label_module_nb = {
+	.notifier_call = jump_label_module_notify,
+	.priority = 0,
+};
+
+static __init int init_jump_label_module(void)
+{
+	return register_module_notifier(&jump_label_module_nb);
+}
+early_initcall(init_jump_label_module);
+
+#endif /* CONFIG_MODULES */
+
+#endif
@@ -47,6 +47,7 @@
 #include <linux/memory.h>
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
+#include <linux/jump_label.h>
  
 #include <asm-generic/sections.h>
 #include <asm/cacheflush.h>
@@ -55,6 +55,7 @@
 #include <linux/async.h>
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
+#include <linux/jump_label.h>
  
 #define CREATE_TRACE_POINTS
 #include <trace/events/module.h>
@@ -2307,6 +2308,11 @@
 	mod->tracepoints = section_objs(info, "__tracepoints",
 					sizeof(*mod->tracepoints),
 					&mod->num_tracepoints);
+#endif
+#ifdef HAVE_JUMP_LABEL
+	mod->jump_entries = section_objs(info, "__jump_table",
+					sizeof(*mod->jump_entries),
+					&mod->num_jump_entries);
 #endif
 #ifdef CONFIG_EVENT_TRACING
 	mod->trace_events = section_objs(info, "_ftrace_events",
+#!/bin/sh
+# Test for gcc 'asm goto' suport
+# Copyright (C) 2010, Jason Baron <jbaron@redhat.com>
+
+echo "int main(void) { entry: asm goto (\"\"::::entry); return 0; }" | $1 -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
...	...	@@ -591,6 +591,11 @@
591	591	# conserve stack if available
592	592	KBUILD_CFLAGS += $(call cc-option,-fconserve-stack)
593	593
	594	+# check for 'asm goto'
	595	+ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y)
	596	+ KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
	597	+endif
	598	+
594	599	# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
595	600	# But warn user when we do so
596	601	warn-assign = \
...	...	@@ -158,5 +158,8 @@
158	158	subsystem. Also has support for calculating CPU cycle events
159	159	to determine how many clock cycles in a given period.
160	160
	161	+config HAVE_ARCH_JUMP_LABEL
	162	+ bool
	163	+
161	164	source "kernel/gcov/Kconfig"
...	...	@@ -4,6 +4,7 @@
4	4	#include <linux/types.h>
5	5	#include <linux/stddef.h>
6	6	#include <linux/stringify.h>
	7	+#include <linux/jump_label.h>
7	8	#include <asm/asm.h>
8	9
9	10	/*
...	...	@@ -182,7 +183,7 @@
182	183	extern void text_poke(void addr, const void *opcode, size_t len);
183	184	extern void text_poke_smp(void addr, const void *opcode, size_t len);
184	185
185		-#if defined(CONFIG_DYNAMIC_FTRACE)
	186	+#if defined(CONFIG_DYNAMIC_FTRACE) \|\| defined(HAVE_JUMP_LABEL)
186	187	#define IDEAL_NOP_SIZE_5 5
187	188	extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
188	189	extern void arch_init_ideal_nop5(void);
...	...	@@ -641,7 +641,7 @@
641	641	return addr;
642	642	}
643	643
644		-#if defined(CONFIG_DYNAMIC_FTRACE)
	644	+#if defined(CONFIG_DYNAMIC_FTRACE) \|\| defined(HAVE_JUMP_LABEL)
645	645
646	646	unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
647	647
...	...	@@ -220,6 +220,8 @@
220	220	\
221	221	BUG_TABLE \
222	222	\
	223	+ JUMP_TABLE \
	224	+ \
223	225	/* PCI quirks */ \
224	226	.pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \
225	227	VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
...	...	@@ -562,6 +564,14 @@
562	564	#else
563	565	#define BUG_TABLE
564	566	#endif
	567	+
	568	+#define JUMP_TABLE \
	569	+ . = ALIGN(8); \
	570	+ __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \
	571	+ VMLINUX_SYMBOL(__start___jump_table) = .; \
	572	+ *(__jump_table) \
	573	+ VMLINUX_SYMBOL(__stop___jump_table) = .; \
	574	+ }
565	575
566	576	#ifdef CONFIG_PM_TRACE
567	577	#define TRACEDATA \
	1	+#ifndef _LINUX_JUMP_LABEL_H
	2	+#define _LINUX_JUMP_LABEL_H
	3	+
	4	+#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_HAVE_ARCH_JUMP_LABEL)
	5	+# include <asm/jump_label.h>
	6	+# define HAVE_JUMP_LABEL
	7	+#endif
	8	+
	9	+enum jump_label_type {
	10	+ JUMP_LABEL_ENABLE,
	11	+ JUMP_LABEL_DISABLE
	12	+};
	13	+
	14	+struct module;
	15	+
	16	+#ifdef HAVE_JUMP_LABEL
	17	+
	18	+extern struct jump_entry __start___jump_table[];
	19	+extern struct jump_entry __stop___jump_table[];
	20	+
	21	+extern void arch_jump_label_transform(struct jump_entry *entry,
	22	+ enum jump_label_type type);
	23	+extern void jump_label_update(unsigned long key, enum jump_label_type type);
	24	+extern void jump_label_apply_nops(struct module *mod);
	25	+extern void arch_jump_label_text_poke_early(jump_label_t addr);
	26	+
	27	+#define enable_jump_label(key) \
	28	+ jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
	29	+
	30	+#define disable_jump_label(key) \
	31	+ jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
	32	+
	33	+#else
	34	+
	35	+#define JUMP_LABEL(key, label) \
	36	+do { \
	37	+ if (unlikely(*key)) \
	38	+ goto label; \
	39	+} while (0)
	40	+
	41	+#define enable_jump_label(cond_var) \
	42	+do { \
	43	+ *(cond_var) = 1; \
	44	+} while (0)
	45	+
	46	+#define disable_jump_label(cond_var) \
	47	+do { \
	48	+ *(cond_var) = 0; \
	49	+} while (0)
	50	+
	51	+static inline int jump_label_apply_nops(struct module *mod)
	52	+{
	53	+ return 0;
	54	+}
	55	+
	56	+#endif
	57	+
	58	+#endif
...	...	@@ -350,7 +350,10 @@
350	350	struct tracepoint *tracepoints;
351	351	unsigned int num_tracepoints;
352	352	#endif
353		-
	353	+#ifdef HAVE_JUMP_LABEL
	354	+ struct jump_entry *jump_entries;
	355	+ unsigned int num_jump_entries;
	356	+#endif
354	357	#ifdef CONFIG_TRACING
355	358	const char **trace_bprintk_fmt_start;
356	359	unsigned int num_trace_bprintk_fmt;
...	...	@@ -10,7 +10,7 @@
10	10	kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
11	11	hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12	12	notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
13		- async.o range.o
	13	+ async.o range.o jump_label.o
14	14	obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
15	15	obj-y += groups.o
16	16
	1	+/*
	2	+ * jump label support
	3	+ *
	4	+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
	5	+ *
	6	+ */
	7	+#include <linux/jump_label.h>
	8	+#include <linux/memory.h>
	9	+#include <linux/uaccess.h>
	10	+#include <linux/module.h>
	11	+#include <linux/list.h>
	12	+#include <linux/jhash.h>
	13	+#include <linux/slab.h>
	14	+#include <linux/sort.h>
	15	+#include <linux/err.h>
	16	+
	17	+#ifdef HAVE_JUMP_LABEL
	18	+
	19	+#define JUMP_LABEL_HASH_BITS 6
	20	+#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
	21	+static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
	22	+
	23	+/* mutex to protect coming/going of the the jump_label table */
	24	+static DEFINE_MUTEX(jump_label_mutex);
	25	+
	26	+struct jump_label_entry {
	27	+ struct hlist_node hlist;
	28	+ struct jump_entry *table;
	29	+ int nr_entries;
	30	+ /* hang modules off here */
	31	+ struct hlist_head modules;
	32	+ unsigned long key;
	33	+};
	34	+
	35	+struct jump_label_module_entry {
	36	+ struct hlist_node hlist;
	37	+ struct jump_entry *table;
	38	+ int nr_entries;
	39	+ struct module *mod;
	40	+};
	41	+
	42	+static int jump_label_cmp(const void a, const void b)
	43	+{
	44	+ const struct jump_entry *jea = a;
	45	+ const struct jump_entry *jeb = b;
	46	+
	47	+ if (jea->key < jeb->key)
	48	+ return -1;
	49	+
	50	+ if (jea->key > jeb->key)
	51	+ return 1;
	52	+
	53	+ return 0;
	54	+}
	55	+
	56	+static void
	57	+sort_jump_label_entries(struct jump_entry start, struct jump_entry stop)
	58	+{
	59	+ unsigned long size;
	60	+
	61	+ size = (((unsigned long)stop - (unsigned long)start)
	62	+ / sizeof(struct jump_entry));
	63	+ sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
	64	+}
	65	+
	66	+static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
	67	+{
	68	+ struct hlist_head *head;
	69	+ struct hlist_node *node;
	70	+ struct jump_label_entry *e;
	71	+ u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
	72	+
	73	+ head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	74	+ hlist_for_each_entry(e, node, head, hlist) {
	75	+ if (key == e->key)
	76	+ return e;
	77	+ }
	78	+ return NULL;
	79	+}
	80	+
	81	+static struct jump_label_entry *
	82	+add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
	83	+{
	84	+ struct hlist_head *head;
	85	+ struct jump_label_entry *e;
	86	+ u32 hash;
	87	+
	88	+ e = get_jump_label_entry(key);
	89	+ if (e)
	90	+ return ERR_PTR(-EEXIST);
	91	+
	92	+ e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
	93	+ if (!e)
	94	+ return ERR_PTR(-ENOMEM);
	95	+
	96	+ hash = jhash((void *)&key, sizeof(jump_label_t), 0);
	97	+ head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
	98	+ e->key = key;
	99	+ e->table = table;
	100	+ e->nr_entries = nr_entries;
	101	+ INIT_HLIST_HEAD(&(e->modules));
	102	+ hlist_add_head(&e->hlist, head);
	103	+ return e;
	104	+}
	105	+
	106	+static int
	107	+build_jump_label_hashtable(struct jump_entry start, struct jump_entry stop)
	108	+{
	109	+ struct jump_entry iter, iter_begin;
	110	+ struct jump_label_entry *entry;
	111	+ int count;
	112	+
	113	+ sort_jump_label_entries(start, stop);
	114	+ iter = start;
	115	+ while (iter < stop) {
	116	+ entry = get_jump_label_entry(iter->key);
	117	+ if (!entry) {
	118	+ iter_begin = iter;
	119	+ count = 0;
	120	+ while ((iter < stop) &&
	121	+ (iter->key == iter_begin->key)) {
	122	+ iter++;
	123	+ count++;
	124	+ }
	125	+ entry = add_jump_label_entry(iter_begin->key,
	126	+ count, iter_begin);
	127	+ if (IS_ERR(entry))
	128	+ return PTR_ERR(entry);
	129	+ } else {
	130	+ WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
	131	+ return -1;
	132	+ }
	133	+ }
	134	+ return 0;
	135	+}
	136	+
	137	+/***
	138	+ * jump_label_update - update jump label text
	139	+ * @key - key value associated with a a jump label
	140	+ * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
	141	+ *
	142	+ * Will enable/disable the jump for jump label @key, depending on the
	143	+ * value of @type.
	144	+ *
	145	+ */
	146	+
	147	+void jump_label_update(unsigned long key, enum jump_label_type type)
	148	+{
	149	+ struct jump_entry *iter;
	150	+ struct jump_label_entry *entry;
	151	+ struct hlist_node *module_node;
	152	+ struct jump_label_module_entry *e_module;
	153	+ int count;
	154	+
	155	+ mutex_lock(&jump_label_mutex);
	156	+ entry = get_jump_label_entry((jump_label_t)key);
	157	+ if (entry) {
	158	+ count = entry->nr_entries;
	159	+ iter = entry->table;
	160	+ while (count--) {
	161	+ if (kernel_text_address(iter->code))
	162	+ arch_jump_label_transform(iter, type);
	163	+ iter++;
	164	+ }
	165	+ /* eanble/disable jump labels in modules */
	166	+ hlist_for_each_entry(e_module, module_node, &(entry->modules),
	167	+ hlist) {
	168	+ count = e_module->nr_entries;
	169	+ iter = e_module->table;
	170	+ while (count--) {
	171	+ if (kernel_text_address(iter->code))
	172	+ arch_jump_label_transform(iter, type);
	173	+ iter++;
	174	+ }
	175	+ }
	176	+ }
	177	+ mutex_unlock(&jump_label_mutex);
	178	+}
	179	+
	180	+static __init int init_jump_label(void)
	181	+{
	182	+ int ret;
	183	+ struct jump_entry *iter_start = __start___jump_table;
	184	+ struct jump_entry *iter_stop = __stop___jump_table;
	185	+ struct jump_entry *iter;
	186	+
	187	+ mutex_lock(&jump_label_mutex);
	188	+ ret = build_jump_label_hashtable(__start___jump_table,
	189	+ __stop___jump_table);
	190	+ iter = iter_start;
	191	+ while (iter < iter_stop) {
	192	+ arch_jump_label_text_poke_early(iter->code);
	193	+ iter++;
	194	+ }
	195	+ mutex_unlock(&jump_label_mutex);
	196	+ return ret;
	197	+}
	198	+early_initcall(init_jump_label);
	199	+
	200	+#ifdef CONFIG_MODULES
	201	+
	202	+static struct jump_label_module_entry *
	203	+add_jump_label_module_entry(struct jump_label_entry *entry,
	204	+ struct jump_entry *iter_begin,
	205	+ int count, struct module *mod)
	206	+{
	207	+ struct jump_label_module_entry *e;
	208	+
	209	+ e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
	210	+ if (!e)
	211	+ return ERR_PTR(-ENOMEM);
	212	+ e->mod = mod;
	213	+ e->nr_entries = count;
	214	+ e->table = iter_begin;
	215	+ hlist_add_head(&e->hlist, &entry->modules);
	216	+ return e;
	217	+}
	218	+
	219	+static int add_jump_label_module(struct module *mod)
	220	+{
	221	+ struct jump_entry iter, iter_begin;
	222	+ struct jump_label_entry *entry;
	223	+ struct jump_label_module_entry *module_entry;
	224	+ int count;
	225	+
	226	+ /* if the module doesn't have jump label entries, just return */
	227	+ if (!mod->num_jump_entries)
	228	+ return 0;
	229	+
	230	+ sort_jump_label_entries(mod->jump_entries,
	231	+ mod->jump_entries + mod->num_jump_entries);
	232	+ iter = mod->jump_entries;
	233	+ while (iter < mod->jump_entries + mod->num_jump_entries) {
	234	+ entry = get_jump_label_entry(iter->key);
	235	+ iter_begin = iter;
	236	+ count = 0;
	237	+ while ((iter < mod->jump_entries + mod->num_jump_entries) &&
	238	+ (iter->key == iter_begin->key)) {
	239	+ iter++;
	240	+ count++;
	241	+ }
	242	+ if (!entry) {
	243	+ entry = add_jump_label_entry(iter_begin->key, 0, NULL);
	244	+ if (IS_ERR(entry))
	245	+ return PTR_ERR(entry);
	246	+ }
	247	+ module_entry = add_jump_label_module_entry(entry, iter_begin,
	248	+ count, mod);
	249	+ if (IS_ERR(module_entry))
	250	+ return PTR_ERR(module_entry);
	251	+ }
	252	+ return 0;
	253	+}
	254	+
	255	+static void remove_jump_label_module(struct module *mod)
	256	+{
	257	+ struct hlist_head *head;
	258	+ struct hlist_node node, node_next, module_node, module_node_next;
	259	+ struct jump_label_entry *e;
	260	+ struct jump_label_module_entry *e_module;
	261	+ int i;
	262	+
	263	+ /* if the module doesn't have jump label entries, just return */
	264	+ if (!mod->num_jump_entries)
	265	+ return;
	266	+
	267	+ for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
	268	+ head = &jump_label_table[i];
	269	+ hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
	270	+ hlist_for_each_entry_safe(e_module, module_node,
	271	+ module_node_next,
	272	+ &(e->modules), hlist) {
	273	+ if (e_module->mod == mod) {
	274	+ hlist_del(&e_module->hlist);
	275	+ kfree(e_module);
	276	+ }
	277	+ }
	278	+ if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
	279	+ hlist_del(&e->hlist);
	280	+ kfree(e);
	281	+ }
	282	+ }
	283	+ }
	284	+}
	285	+
	286	+static int
	287	+jump_label_module_notify(struct notifier_block *self, unsigned long val,
	288	+ void *data)
	289	+{
	290	+ struct module *mod = data;
	291	+ int ret = 0;
	292	+
	293	+ switch (val) {
	294	+ case MODULE_STATE_COMING:
	295	+ mutex_lock(&jump_label_mutex);
	296	+ ret = add_jump_label_module(mod);
	297	+ if (ret)
	298	+ remove_jump_label_module(mod);
	299	+ mutex_unlock(&jump_label_mutex);
	300	+ break;
	301	+ case MODULE_STATE_GOING:
	302	+ mutex_lock(&jump_label_mutex);
	303	+ remove_jump_label_module(mod);
	304	+ mutex_unlock(&jump_label_mutex);
	305	+ break;
	306	+ }
	307	+ return ret;
	308	+}
	309	+
	310	+/***
	311	+ * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
	312	+ * @mod: module to patch
	313	+ *
	314	+ * Allow for run-time selection of the optimal nops. Before the module
	315	+ * loads patch these with arch_get_jump_label_nop(), which is specified by
	316	+ * the arch specific jump label code.
	317	+ */
	318	+void jump_label_apply_nops(struct module *mod)
	319	+{
	320	+ struct jump_entry *iter;
	321	+
	322	+ /* if the module doesn't have jump label entries, just return */
	323	+ if (!mod->num_jump_entries)
	324	+ return;
	325	+
	326	+ iter = mod->jump_entries;
	327	+ while (iter < mod->jump_entries + mod->num_jump_entries) {
	328	+ arch_jump_label_text_poke_early(iter->code);
	329	+ iter++;
	330	+ }
	331	+}
	332	+
	333	+struct notifier_block jump_label_module_nb = {
	334	+ .notifier_call = jump_label_module_notify,
	335	+ .priority = 0,
	336	+};
	337	+
	338	+static __init int init_jump_label_module(void)
	339	+{
	340	+ return register_module_notifier(&jump_label_module_nb);
	341	+}
	342	+early_initcall(init_jump_label_module);
	343	+
	344	+#endif /* CONFIG_MODULES */
	345	+
	346	+#endif
...	...	@@ -47,6 +47,7 @@
47	47	#include <linux/memory.h>
48	48	#include <linux/ftrace.h>
49	49	#include <linux/cpu.h>
	50	+#include <linux/jump_label.h>
50	51
51	52	#include <asm-generic/sections.h>
52	53	#include <asm/cacheflush.h>