genirq: Provide forced interrupt threading

Add a commandline parameter "threadirqs" which forces all interrupts except those marked IRQF_NO_THREAD to run threaded. That's mostly a debug option to allow retrieving better debug data from crashing interrupt handlers. If "threadirqs" is not enabled on the kernel command line, then there is no impact in the interrupt hotpath. Architecture code needs to select CONFIG_IRQ_FORCED_THREADING after marking the interrupts which cant be threaded IRQF_NO_THREAD. All interrupts which have IRQF_TIMER set are implict marked IRQF_NO_THREAD. Also all PER_CPU interrupts are excluded. Forced threading hard interrupts also forces all soft interrupt handling into thread context. When enabled it might slow down things a bit, but for debugging problems in interrupt code it's a reasonable penalty as it does not immediately crash and burn the machine when an interrupt handler is buggy. Some test results on a Core2Duo machine: Cache cold run of: # time git grep irq_desc non-threaded threaded real 1m18.741s 1m19.061s user 0m1.874s 0m1.757s sys 0m5.843s 0m5.427s # iperf -c server non-threaded [ 3] 0.0-10.0 sec 1.09 GBytes 933 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 934 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 933 Mbits/sec threaded [ 3] 0.0-10.0 sec 1.09 GBytes 939 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 934 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 937 Mbits/sec Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Peter Zijlstra <peterz@infradead.org> LKML-Reference: <20110223234956.772668648@linutronix.de>

genirq: Provide forced interrupt threading
Add a commandline parameter "threadirqs" which forces all interrupts except those marked IRQF_NO_THREAD to run threaded. That's mostly a debug option to allow retrieving better debug data from crashing interrupt handlers. If "threadirqs" is not enabled on the kernel command line, then there is no impact in the interrupt hotpath. Architecture code needs to select CONFIG_IRQ_FORCED_THREADING after marking the interrupts which cant be threaded IRQF_NO_THREAD. All interrupts which have IRQF_TIMER set are implict marked IRQF_NO_THREAD. Also all PER_CPU interrupts are excluded. Forced threading hard interrupts also forces all soft interrupt handling into thread context. When enabled it might slow down things a bit, but for debugging problems in interrupt code it's a reasonable penalty as it does not immediately crash and burn the machine when an interrupt handler is buggy. Some test results on a Core2Duo machine: Cache cold run of: # time git grep irq_desc non-threaded threaded real 1m18.741s 1m19.061s user 0m1.874s 0m1.757s sys 0m5.843s 0m5.427s # iperf -c server non-threaded [ 3] 0.0-10.0 sec 1.09 GBytes 933 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 934 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 933 Mbits/sec threaded [ 3] 0.0-10.0 sec 1.09 GBytes 939 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 934 Mbits/sec [ 3] 0.0-10.0 sec 1.09 GBytes 937 Mbits/sec Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Peter Zijlstra <peterz@infradead.org> LKML-Reference: <20110223234956.772668648@linutronix.de>
Thomas Gleixner
1 parent 8eb90c30e0
Showing 6 changed files with 93 additions and 6 deletions Side-by-side Diff
Documentation/kernel-parameters.txt
include/linux/interrupt.h
kernel/irq/Kconfig
kernel/irq/internals.h
kernel/irq/manage.c
kernel/softirq.c
@@ -2436,6 +2436,10 @@
 			<deci-seconds>: poll all this frequency
 			0: no polling (default)
  
+	threadirqs	[KNL]
+			Force threading of all interrupt handlers except those
+			marked explicitely IRQF_NO_THREAD.
+
 	topology=	[S390]
 			Format: {off | on}
 			Specify if the kernel should make use of the cpu
@@ -383,6 +383,13 @@
 }
 #endif /* CONFIG_GENERIC_HARDIRQS */
  
+
+#ifdef CONFIG_IRQ_FORCED_THREADING
+extern bool force_irqthreads;
+#else
+#define force_irqthreads	(0)
+#endif
+
 #ifndef __ARCH_SET_SOFTIRQ_PENDING
 #define set_softirq_pending(x) (local_softirq_pending() = (x))
 #define or_softirq_pending(x)  (local_softirq_pending() |= (x))
@@ -38,6 +38,9 @@
 config IRQ_PREFLOW_FASTEOI
        bool
  
+config IRQ_FORCED_THREADING
+       bool
+
 config SPARSE_IRQ
 	bool "Support sparse irq numbering"
 	depends on HAVE_SPARSE_IRQ
@@ -27,12 +27,14 @@
  * IRQTF_DIED      - handler thread died
  * IRQTF_WARNED    - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
  * IRQTF_AFFINITY  - irq thread is requested to adjust affinity
+ * IRQTF_FORCED_THREAD  - irq action is force threaded
  */
 enum {
 	IRQTF_RUNTHREAD,
 	IRQTF_DIED,
 	IRQTF_WARNED,
 	IRQTF_AFFINITY,
+	IRQTF_FORCED_THREAD,
 };
  
 /*
@@ -17,6 +17,17 @@
  
 #include "internals.h"
  
+#ifdef CONFIG_IRQ_FORCED_THREADING
+__read_mostly bool force_irqthreads;
+
+static int __init setup_forced_irqthreads(char *arg)
+{
+	force_irqthreads = true;
+	return 0;
+}
+early_param("threadirqs", setup_forced_irqthreads);
+#endif
+
 /**
  *	synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  *	@irq: interrupt number to wait for
@@ -702,6 +713,32 @@
 #endif
  
 /*
+ * Interrupts which are not explicitely requested as threaded
+ * interrupts rely on the implicit bh/preempt disable of the hard irq
+ * context. So we need to disable bh here to avoid deadlocks and other
+ * side effects.
+ */
+static void
+irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
+{
+	local_bh_disable();
+	action->thread_fn(action->irq, action->dev_id);
+	irq_finalize_oneshot(desc, action, false);
+	local_bh_enable();
+}
+
+/*
+ * Interrupts explicitely requested as threaded interupts want to be
+ * preemtible - many of them need to sleep and wait for slow busses to
+ * complete.
+ */
+static void irq_thread_fn(struct irq_desc *desc, struct irqaction *action)
+{
+	action->thread_fn(action->irq, action->dev_id);
+	irq_finalize_oneshot(desc, action, false);
+}
+
+/*
  * Interrupt handler thread
  */
 static int irq_thread(void *data)
  
@@ -711,8 +748,15 @@
 	};
 	struct irqaction *action = data;
 	struct irq_desc *desc = irq_to_desc(action->irq);
+	void (*handler_fn)(struct irq_desc *desc, struct irqaction *action);
 	int wake;
  
+	if (force_irqthreads & test_bit(IRQTF_FORCED_THREAD,
+					&action->thread_flags))
+		handler_fn = irq_forced_thread_fn;
+	else
+		handler_fn = irq_thread_fn;
+
 	sched_setscheduler(current, SCHED_FIFO, &param);
 	current->irqaction = action;
  
@@ -736,10 +780,7 @@
 			raw_spin_unlock_irq(&desc->lock);
 		} else {
 			raw_spin_unlock_irq(&desc->lock);
-
-			action->thread_fn(action->irq, action->dev_id);
-
-			irq_finalize_oneshot(desc, action, false);
+			handler_fn(desc, action);
 		}
  
 		wake = atomic_dec_and_test(&desc->threads_active);
@@ -789,6 +830,22 @@
 	set_bit(IRQTF_DIED, &tsk->irqaction->flags);
 }
  
+static void irq_setup_forced_threading(struct irqaction *new)
+{
+	if (!force_irqthreads)
+		return;
+	if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
+		return;
+
+	new->flags |= IRQF_ONESHOT;
+
+	if (!new->thread_fn) {
+		set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
+		new->thread_fn = new->handler;
+		new->handler = irq_default_primary_handler;
+	}
+}
+
 /*
  * Internal function to register an irqaction - typically used to
  * allocate special interrupts that are part of the architecture.
@@ -838,6 +895,8 @@
 		 * dummy function which warns when called.
 		 */
 		new->handler = irq_nested_primary_handler;
+	} else {
+		irq_setup_forced_threading(new);
 	}
  
 	/*
@@ -311,9 +311,21 @@
 }
  
 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
-# define invoke_softirq()	__do_softirq()
+static inline void invoke_softirq(void)
+{
+	if (!force_irqthreads)
+		__do_softirq();
+	else
+		wakeup_softirqd();
+}
 #else
-# define invoke_softirq()	do_softirq()
+static inline void invoke_softirq(void)
+{
+	if (!force_irqthreads)
+		do_softirq();
+	else
+		wakeup_softirqd();
+}
 #endif
  
 /*
...	...	@@ -2436,6 +2436,10 @@
2436	2436	<deci-seconds>: poll all this frequency
2437	2437	0: no polling (default)
2438	2438
	2439	+ threadirqs [KNL]
	2440	+ Force threading of all interrupt handlers except those
	2441	+ marked explicitely IRQF_NO_THREAD.
	2442	+
2439	2443	topology= [S390]
2440	2444	Format: {off \| on}
2441	2445	Specify if the kernel should make use of the cpu
...	...	@@ -383,6 +383,13 @@
383	383	}
384	384	#endif /* CONFIG_GENERIC_HARDIRQS */
385	385
	386	+
	387	+#ifdef CONFIG_IRQ_FORCED_THREADING
	388	+extern bool force_irqthreads;
	389	+#else
	390	+#define force_irqthreads (0)
	391	+#endif
	392	+
386	393	#ifndef __ARCH_SET_SOFTIRQ_PENDING
387	394	#define set_softirq_pending(x) (local_softirq_pending() = (x))
388	395	#define or_softirq_pending(x) (local_softirq_pending() \|= (x))
...	...	@@ -38,6 +38,9 @@
38	38	config IRQ_PREFLOW_FASTEOI
39	39	bool
40	40
	41	+config IRQ_FORCED_THREADING
	42	+ bool
	43	+
41	44	config SPARSE_IRQ
42	45	bool "Support sparse irq numbering"
43	46	depends on HAVE_SPARSE_IRQ
...	...	@@ -27,12 +27,14 @@
27	27	* IRQTF_DIED - handler thread died
28	28	* IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed
29	29	* IRQTF_AFFINITY - irq thread is requested to adjust affinity
	30	+ * IRQTF_FORCED_THREAD - irq action is force threaded
30	31	*/
31	32	enum {
32	33	IRQTF_RUNTHREAD,
33	34	IRQTF_DIED,
34	35	IRQTF_WARNED,
35	36	IRQTF_AFFINITY,
	37	+ IRQTF_FORCED_THREAD,
36	38	};
37	39
38	40	/*
...	...	@@ -17,6 +17,17 @@
17	17
18	18	#include "internals.h"
19	19
	20	+#ifdef CONFIG_IRQ_FORCED_THREADING
	21	+__read_mostly bool force_irqthreads;
	22	+
	23	+static int __init setup_forced_irqthreads(char *arg)
	24	+{
	25	+ force_irqthreads = true;
	26	+ return 0;
	27	+}
	28	+early_param("threadirqs", setup_forced_irqthreads);
	29	+#endif
	30	+
20	31	/**
21	32	* synchronize_irq - wait for pending IRQ handlers (on other CPUs)
22	33	* @irq: interrupt number to wait for
...	...	@@ -702,6 +713,32 @@
702	713	#endif
703	714
704	715	/*
	716	+ * Interrupts which are not explicitely requested as threaded
	717	+ * interrupts rely on the implicit bh/preempt disable of the hard irq
	718	+ * context. So we need to disable bh here to avoid deadlocks and other
	719	+ * side effects.
	720	+ */
	721	+static void
	722	+irq_forced_thread_fn(struct irq_desc desc, struct irqaction action)
	723	+{
	724	+ local_bh_disable();
	725	+ action->thread_fn(action->irq, action->dev_id);
	726	+ irq_finalize_oneshot(desc, action, false);
	727	+ local_bh_enable();
	728	+}
	729	+
	730	+/*
	731	+ * Interrupts explicitely requested as threaded interupts want to be
	732	+ * preemtible - many of them need to sleep and wait for slow busses to
	733	+ * complete.
	734	+ */
	735	+static void irq_thread_fn(struct irq_desc desc, struct irqaction action)
	736	+{
	737	+ action->thread_fn(action->irq, action->dev_id);
	738	+ irq_finalize_oneshot(desc, action, false);
	739	+}
	740	+
	741	+/*
705	742	* Interrupt handler thread
706	743	*/
707	744	static int irq_thread(void *data)
708	745
...	...	@@ -711,8 +748,15 @@
711	748	};
712	749	struct irqaction *action = data;
713	750	struct irq_desc *desc = irq_to_desc(action->irq);
	751	+ void (handler_fn)(struct irq_desc desc, struct irqaction *action);
714	752	int wake;
715	753
	754	+ if (force_irqthreads & test_bit(IRQTF_FORCED_THREAD,
	755	+ &action->thread_flags))
	756	+ handler_fn = irq_forced_thread_fn;
	757	+ else
	758	+ handler_fn = irq_thread_fn;
	759	+
716	760	sched_setscheduler(current, SCHED_FIFO, &param);
717	761	current->irqaction = action;
718	762
...	...	@@ -736,10 +780,7 @@
736	780	raw_spin_unlock_irq(&desc->lock);
737	781	} else {
738	782	raw_spin_unlock_irq(&desc->lock);
739		-
740		- action->thread_fn(action->irq, action->dev_id);
741		-
742		- irq_finalize_oneshot(desc, action, false);
	783	+ handler_fn(desc, action);
743	784	}
744	785
745	786	wake = atomic_dec_and_test(&desc->threads_active);
...	...	@@ -789,6 +830,22 @@
789	830	set_bit(IRQTF_DIED, &tsk->irqaction->flags);
790	831	}
791	832
	833	+static void irq_setup_forced_threading(struct irqaction *new)
	834	+{
	835	+ if (!force_irqthreads)
	836	+ return;
	837	+ if (new->flags & (IRQF_NO_THREAD \| IRQF_PERCPU \| IRQF_ONESHOT))
	838	+ return;
	839	+
	840	+ new->flags \|= IRQF_ONESHOT;
	841	+
	842	+ if (!new->thread_fn) {
	843	+ set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
	844	+ new->thread_fn = new->handler;
	845	+ new->handler = irq_default_primary_handler;
	846	+ }
	847	+}
	848	+
792	849	/*
793	850	* Internal function to register an irqaction - typically used to
794	851	* allocate special interrupts that are part of the architecture.
...	...	@@ -838,6 +895,8 @@
838	895	* dummy function which warns when called.
839	896	*/
840	897	new->handler = irq_nested_primary_handler;
	898	+ } else {
	899	+ irq_setup_forced_threading(new);
841	900	}
842	901
843	902	/*
...	...	@@ -311,9 +311,21 @@
311	311	}
312	312
313	313	#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
314		-# define invoke_softirq() __do_softirq()
	314	+static inline void invoke_softirq(void)
	315	+{
	316	+ if (!force_irqthreads)
	317	+ __do_softirq();
	318	+ else
	319	+ wakeup_softirqd();
	320	+}
315	321	#else
316		-# define invoke_softirq() do_softirq()
	322	+static inline void invoke_softirq(void)
	323	+{
	324	+ if (!force_irqthreads)
	325	+ do_softirq();
	326	+ else
	327	+ wakeup_softirqd();
	328	+}
317	329	#endif
318	330
319	331	/*