genirq: Prepare the handling of shared oneshot interrupts

For level type interrupts we need to track how many threads are on flight to avoid useless interrupt storms when not all thread handlers have finished yet. Keep track of the woken threads and only unmask when there are no more threads in flight. Yes, I'm lazy and using a bitfield. But not only because I'm lazy, the main reason is that it's way simpler than using a refcount. A refcount based solution would need to keep track of various things like crashing the irq thread, spurious interrupts coming in, disables/enables, free_irq() and some more. The bitfield keeps the tracking simple and makes things just work. It's also nicely confined to the thread code pathes and does not require additional checks all over the place. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Peter Zijlstra <peterz@infradead.org> LKML-Reference: <20110223234956.388095876@linutronix.de>

genirq: Prepare the handling of shared oneshot interrupts
For level type interrupts we need to track how many threads are on flight to avoid useless interrupt storms when not all thread handlers have finished yet. Keep track of the woken threads and only unmask when there are no more threads in flight. Yes, I'm lazy and using a bitfield. But not only because I'm lazy, the main reason is that it's way simpler than using a refcount. A refcount based solution would need to keep track of various things like crashing the irq thread, spurious interrupts coming in, disables/enables, free_irq() and some more. The bitfield keeps the tracking simple and makes things just work. It's also nicely confined to the thread code pathes and does not require additional checks all over the place. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Peter Zijlstra <peterz@infradead.org> LKML-Reference: <20110223234956.388095876@linutronix.de>
Thomas Gleixner
1 parent 1204e95689
Showing 4 changed files with 115 additions and 19 deletions Side-by-side Diff
include/linux/interrupt.h
include/linux/irqdesc.h
kernel/irq/handle.c
kernel/irq/manage.c
@@ -99,6 +99,7 @@
  * @thread_fn:	interupt handler function for threaded interrupts
  * @thread:	thread pointer for threaded interrupts
  * @thread_flags:	flags related to @thread
+ * @thread_mask:	bitmask for keeping track of @thread activity
  */
 struct irqaction {
 	irq_handler_t handler;
@@ -109,6 +110,7 @@
 	irq_handler_t thread_fn;
 	struct task_struct *thread;
 	unsigned long thread_flags;
+	unsigned long thread_mask;
 	const char *name;
 	struct proc_dir_entry *dir;
 } ____cacheline_internodealigned_in_smp;
@@ -28,6 +28,7 @@
  * @lock:		locking for SMP
  * @affinity_notify:	context for notification of affinity changes
  * @pending_mask:	pending rebalanced interrupts
+ * @threads_oneshot:	bitfield to handle shared oneshot threads
  * @threads_active:	number of irqaction threads currently running
  * @wait_for_threads:	wait queue for sync_irq to wait for threaded handlers
  * @dir:		/proc/irq/ procfs entry
@@ -86,6 +87,7 @@
 	cpumask_var_t		pending_mask;
 #endif
 #endif
+	unsigned long		threads_oneshot;
 	atomic_t		threads_active;
 	wait_queue_head_t       wait_for_threads;
 #ifdef CONFIG_PROC_FS
@@ -51,6 +51,68 @@
 	       "but no thread function available.", irq, action->name);
 }
  
+static void irq_wake_thread(struct irq_desc *desc, struct irqaction *action)
+{
+	/*
+	 * Wake up the handler thread for this action. In case the
+	 * thread crashed and was killed we just pretend that we
+	 * handled the interrupt. The hardirq handler has disabled the
+	 * device interrupt, so no irq storm is lurking. If the
+	 * RUNTHREAD bit is already set, nothing to do.
+	 */
+	if (test_bit(IRQTF_DIED, &action->thread_flags) ||
+	    test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags))
+		return;
+
+	/*
+	 * It's safe to OR the mask lockless here. We have only two
+	 * places which write to threads_oneshot: This code and the
+	 * irq thread.
+	 *
+	 * This code is the hard irq context and can never run on two
+	 * cpus in parallel. If it ever does we have more serious
+	 * problems than this bitmask.
+	 *
+	 * The irq threads of this irq which clear their "running" bit
+	 * in threads_oneshot are serialized via desc->lock against
+	 * each other and they are serialized against this code by
+	 * IRQS_INPROGRESS.
+	 *
+	 * Hard irq handler:
+	 *
+	 *	spin_lock(desc->lock);
+	 *	desc->state |= IRQS_INPROGRESS;
+	 *	spin_unlock(desc->lock);
+	 *	set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
+	 *	desc->threads_oneshot |= mask;
+	 *	spin_lock(desc->lock);
+	 *	desc->state &= ~IRQS_INPROGRESS;
+	 *	spin_unlock(desc->lock);
+	 *
+	 * irq thread:
+	 *
+	 * again:
+	 *	spin_lock(desc->lock);
+	 *	if (desc->state & IRQS_INPROGRESS) {
+	 *		spin_unlock(desc->lock);
+	 *		while(desc->state & IRQS_INPROGRESS)
+	 *			cpu_relax();
+	 *		goto again;
+	 *	}
+	 *	if (!test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
+	 *		desc->threads_oneshot &= ~mask;
+	 *	spin_unlock(desc->lock);
+	 *
+	 * So either the thread waits for us to clear IRQS_INPROGRESS
+	 * or we are waiting in the flow handler for desc->lock to be
+	 * released before we reach this point. The thread also checks
+	 * IRQTF_RUNTHREAD under desc->lock. If set it leaves
+	 * threads_oneshot untouched and runs the thread another time.
+	 */
+	desc->threads_oneshot |= action->thread_mask;
+	wake_up_process(action->thread);
+}
+
 irqreturn_t
 handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action)
 {
@@ -85,19 +147,7 @@
 				break;
 			}
  
-			/*
-			 * Wake up the handler thread for this
-			 * action. In case the thread crashed and was
-			 * killed we just pretend that we handled the
-			 * interrupt. The hardirq handler above has
-			 * disabled the device interrupt, so no irq
-			 * storm is lurking.
-			 */
-			if (likely(!test_bit(IRQTF_DIED,
-					     &action->thread_flags))) {
-				set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
-				wake_up_process(action->thread);
-			}
+			irq_wake_thread(desc, action);
  
 			/* Fall through to add to randomness */
 		case IRQ_HANDLED:
@@ -617,8 +617,11 @@
  * handler finished. unmask if the interrupt has not been disabled and
  * is marked MASKED.
  */
-static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc)
+static void irq_finalize_oneshot(struct irq_desc *desc,
+				 struct irqaction *action, bool force)
 {
+	if (!(desc->istate & IRQS_ONESHOT))
+		return;
 again:
 	chip_bus_lock(desc);
 	raw_spin_lock_irq(&desc->lock);
@@ -631,6 +634,11 @@
 	 * on the other CPU. If we unmask the irq line then the
 	 * interrupt can come in again and masks the line, leaves due
 	 * to IRQS_INPROGRESS and the irq line is masked forever.
+	 *
+	 * This also serializes the state of shared oneshot handlers
+	 * versus "desc->threads_onehsot |= action->thread_mask;" in
+	 * irq_wake_thread(). See the comment there which explains the
+	 * serialization.
 	 */
 	if (unlikely(desc->istate & IRQS_INPROGRESS)) {
 		raw_spin_unlock_irq(&desc->lock);
  
@@ -639,11 +647,23 @@
 		goto again;
 	}
  
-	if (!(desc->istate & IRQS_DISABLED) && (desc->istate & IRQS_MASKED)) {
+	/*
+	 * Now check again, whether the thread should run. Otherwise
+	 * we would clear the threads_oneshot bit of this thread which
+	 * was just set.
+	 */
+	if (!force && test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
+		goto out_unlock;
+
+	desc->threads_oneshot &= ~action->thread_mask;
+
+	if (!desc->threads_oneshot && !(desc->istate & IRQS_DISABLED) &&
+	    (desc->istate & IRQS_MASKED)) {
 		irq_compat_clr_masked(desc);
 		desc->istate &= ~IRQS_MASKED;
 		desc->irq_data.chip->irq_unmask(&desc->irq_data);
 	}
+out_unlock:
 	raw_spin_unlock_irq(&desc->lock);
 	chip_bus_sync_unlock(desc);
 }
@@ -691,7 +711,7 @@
 	};
 	struct irqaction *action = data;
 	struct irq_desc *desc = irq_to_desc(action->irq);
-	int wake, oneshot = desc->istate & IRQS_ONESHOT;
+	int wake;
  
 	sched_setscheduler(current, SCHED_FIFO, &param);
 	current->irqaction = action;
@@ -719,8 +739,7 @@
  
 			action->thread_fn(action->irq, action->dev_id);
  
-			if (oneshot)
-				irq_finalize_oneshot(action->irq, desc);
+			irq_finalize_oneshot(desc, action, false);
 		}
  
 		wake = atomic_dec_and_test(&desc->threads_active);
@@ -729,6 +748,9 @@
 			wake_up(&desc->wait_for_threads);
 	}
  
+	/* Prevent a stale desc->threads_oneshot */
+	irq_finalize_oneshot(desc, action, true);
+
 	/*
 	 * Clear irqaction. Otherwise exit_irq_thread() would make
 	 * fuzz about an active irq thread going into nirvana.
@@ -743,6 +765,7 @@
 void exit_irq_thread(void)
 {
 	struct task_struct *tsk = current;
+	struct irq_desc *desc;
  
 	if (!tsk->irqaction)
 		return;
  
@@ -751,7 +774,15 @@
 	       "exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
 	       tsk->comm ? tsk->comm : "", tsk->pid, tsk->irqaction->irq);
  
+	desc = irq_to_desc(tsk->irqaction->irq);
+
 	/*
+	 * Prevent a stale desc->threads_oneshot. Must be called
+	 * before setting the IRQTF_DIED flag.
+	 */
+	irq_finalize_oneshot(desc, tsk->irqaction, true);
+
+	/*
 	 * Set the THREAD DIED flag to prevent further wakeups of the
 	 * soon to be gone threaded handler.
 	 */
@@ -767,7 +798,7 @@
 {
 	struct irqaction *old, **old_ptr;
 	const char *old_name = NULL;
-	unsigned long flags;
+	unsigned long flags, thread_mask = 0;
 	int ret, nested, shared = 0;
 	cpumask_var_t mask;
  
  
@@ -865,11 +896,22 @@
  
 		/* add new interrupt at end of irq queue */
 		do {
+			thread_mask |= old->thread_mask;
 			old_ptr = &old->next;
 			old = *old_ptr;
 		} while (old);
 		shared = 1;
 	}
+
+	/*
+	 * Setup the thread mask for this irqaction. Unlikely to have
+	 * 32 resp 64 irqs sharing one line, but who knows.
+	 */
+	if (new->flags & IRQF_ONESHOT && thread_mask == ~0UL) {
+		ret = -EBUSY;
+		goto out_mask;
+	}
+	new->thread_mask = 1 << ffz(thread_mask);
  
 	if (!shared) {
 		irq_chip_set_defaults(desc->irq_data.chip);
...	...	@@ -99,6 +99,7 @@
99	99	* @thread_fn: interupt handler function for threaded interrupts
100	100	* @thread: thread pointer for threaded interrupts
101	101	* @thread_flags: flags related to @thread
	102	+ * @thread_mask: bitmask for keeping track of @thread activity
102	103	*/
103	104	struct irqaction {
104	105	irq_handler_t handler;
...	...	@@ -109,6 +110,7 @@
109	110	irq_handler_t thread_fn;
110	111	struct task_struct *thread;
111	112	unsigned long thread_flags;
	113	+ unsigned long thread_mask;
112	114	const char *name;
113	115	struct proc_dir_entry *dir;
114	116	} ____cacheline_internodealigned_in_smp;
...	...	@@ -28,6 +28,7 @@
28	28	* @lock: locking for SMP
29	29	* @affinity_notify: context for notification of affinity changes
30	30	* @pending_mask: pending rebalanced interrupts
	31	+ * @threads_oneshot: bitfield to handle shared oneshot threads
31	32	* @threads_active: number of irqaction threads currently running
32	33	* @wait_for_threads: wait queue for sync_irq to wait for threaded handlers
33	34	* @dir: /proc/irq/ procfs entry
...	...	@@ -86,6 +87,7 @@
86	87	cpumask_var_t pending_mask;
87	88	#endif
88	89	#endif
	90	+ unsigned long threads_oneshot;
89	91	atomic_t threads_active;
90	92	wait_queue_head_t wait_for_threads;
91	93	#ifdef CONFIG_PROC_FS
...	...	@@ -51,6 +51,68 @@
51	51	"but no thread function available.", irq, action->name);
52	52	}
53	53
	54	+static void irq_wake_thread(struct irq_desc desc, struct irqaction action)
	55	+{
	56	+ /*
	57	+ * Wake up the handler thread for this action. In case the
	58	+ * thread crashed and was killed we just pretend that we
	59	+ * handled the interrupt. The hardirq handler has disabled the
	60	+ * device interrupt, so no irq storm is lurking. If the
	61	+ * RUNTHREAD bit is already set, nothing to do.
	62	+ */
	63	+ if (test_bit(IRQTF_DIED, &action->thread_flags) \|\|
	64	+ test_and_set_bit(IRQTF_RUNTHREAD, &action->thread_flags))
	65	+ return;
	66	+
	67	+ /*
	68	+ * It's safe to OR the mask lockless here. We have only two
	69	+ * places which write to threads_oneshot: This code and the
	70	+ * irq thread.
	71	+ *
	72	+ * This code is the hard irq context and can never run on two
	73	+ * cpus in parallel. If it ever does we have more serious
	74	+ * problems than this bitmask.
	75	+ *
	76	+ * The irq threads of this irq which clear their "running" bit
	77	+ * in threads_oneshot are serialized via desc->lock against
	78	+ * each other and they are serialized against this code by
	79	+ * IRQS_INPROGRESS.
	80	+ *
	81	+ * Hard irq handler:
	82	+ *
	83	+ * spin_lock(desc->lock);
	84	+ * desc->state \|= IRQS_INPROGRESS;
	85	+ * spin_unlock(desc->lock);
	86	+ * set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
	87	+ * desc->threads_oneshot \|= mask;
	88	+ * spin_lock(desc->lock);
	89	+ * desc->state &= ~IRQS_INPROGRESS;
	90	+ * spin_unlock(desc->lock);
	91	+ *
	92	+ * irq thread:
	93	+ *
	94	+ * again:
	95	+ * spin_lock(desc->lock);
	96	+ * if (desc->state & IRQS_INPROGRESS) {
	97	+ * spin_unlock(desc->lock);
	98	+ * while(desc->state & IRQS_INPROGRESS)
	99	+ * cpu_relax();
	100	+ * goto again;
	101	+ * }
	102	+ * if (!test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
	103	+ * desc->threads_oneshot &= ~mask;
	104	+ * spin_unlock(desc->lock);
	105	+ *
	106	+ * So either the thread waits for us to clear IRQS_INPROGRESS
	107	+ * or we are waiting in the flow handler for desc->lock to be
	108	+ * released before we reach this point. The thread also checks
	109	+ * IRQTF_RUNTHREAD under desc->lock. If set it leaves
	110	+ * threads_oneshot untouched and runs the thread another time.
	111	+ */
	112	+ desc->threads_oneshot \|= action->thread_mask;
	113	+ wake_up_process(action->thread);
	114	+}
	115	+
54	116	irqreturn_t
55	117	handle_irq_event_percpu(struct irq_desc desc, struct irqaction action)
56	118	{
...	...	@@ -85,19 +147,7 @@
85	147	break;
86	148	}
87	149
88		- /*
89		- * Wake up the handler thread for this
90		- * action. In case the thread crashed and was
91		- * killed we just pretend that we handled the
92		- * interrupt. The hardirq handler above has
93		- * disabled the device interrupt, so no irq
94		- * storm is lurking.
95		- */
96		- if (likely(!test_bit(IRQTF_DIED,
97		- &action->thread_flags))) {
98		- set_bit(IRQTF_RUNTHREAD, &action->thread_flags);
99		- wake_up_process(action->thread);
100		- }
	150	+ irq_wake_thread(desc, action);
101	151
102	152	/* Fall through to add to randomness */
103	153	case IRQ_HANDLED:
...	...	@@ -617,8 +617,11 @@
617	617	* handler finished. unmask if the interrupt has not been disabled and
618	618	* is marked MASKED.
619	619	*/
620		-static void irq_finalize_oneshot(unsigned int irq, struct irq_desc *desc)
	620	+static void irq_finalize_oneshot(struct irq_desc *desc,
	621	+ struct irqaction *action, bool force)
621	622	{
	623	+ if (!(desc->istate & IRQS_ONESHOT))
	624	+ return;
622	625	again:
623	626	chip_bus_lock(desc);
624	627	raw_spin_lock_irq(&desc->lock);
...	...	@@ -631,6 +634,11 @@
631	634	* on the other CPU. If we unmask the irq line then the
632	635	* interrupt can come in again and masks the line, leaves due
633	636	* to IRQS_INPROGRESS and the irq line is masked forever.
	637	+ *
	638	+ * This also serializes the state of shared oneshot handlers
	639	+ * versus "desc->threads_onehsot \|= action->thread_mask;" in
	640	+ * irq_wake_thread(). See the comment there which explains the
	641	+ * serialization.
634	642	*/
635	643	if (unlikely(desc->istate & IRQS_INPROGRESS)) {
636	644	raw_spin_unlock_irq(&desc->lock);
637	645
...	...	@@ -639,11 +647,23 @@
639	647	goto again;
640	648	}
641	649
642		- if (!(desc->istate & IRQS_DISABLED) && (desc->istate & IRQS_MASKED)) {
	650	+ /*
	651	+ * Now check again, whether the thread should run. Otherwise
	652	+ * we would clear the threads_oneshot bit of this thread which
	653	+ * was just set.
	654	+ */
	655	+ if (!force && test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
	656	+ goto out_unlock;
	657	+
	658	+ desc->threads_oneshot &= ~action->thread_mask;
	659	+
	660	+ if (!desc->threads_oneshot && !(desc->istate & IRQS_DISABLED) &&
	661	+ (desc->istate & IRQS_MASKED)) {
643	662	irq_compat_clr_masked(desc);
644	663	desc->istate &= ~IRQS_MASKED;
645	664	desc->irq_data.chip->irq_unmask(&desc->irq_data);
646	665	}
	666	+out_unlock:
647	667	raw_spin_unlock_irq(&desc->lock);
648	668	chip_bus_sync_unlock(desc);
649	669	}
...	...	@@ -691,7 +711,7 @@
691	711	};
692	712	struct irqaction *action = data;
693	713	struct irq_desc *desc = irq_to_desc(action->irq);
694		- int wake, oneshot = desc->istate & IRQS_ONESHOT;
	714	+ int wake;
695	715
696	716	sched_setscheduler(current, SCHED_FIFO, &param);
697	717	current->irqaction = action;
...	...	@@ -719,8 +739,7 @@
719	739
720	740	action->thread_fn(action->irq, action->dev_id);
721	741
722		- if (oneshot)
723		- irq_finalize_oneshot(action->irq, desc);
	742	+ irq_finalize_oneshot(desc, action, false);
724	743	}
725	744
726	745	wake = atomic_dec_and_test(&desc->threads_active);
...	...	@@ -729,6 +748,9 @@
729	748	wake_up(&desc->wait_for_threads);
730	749	}
731	750
	751	+ /* Prevent a stale desc->threads_oneshot */
	752	+ irq_finalize_oneshot(desc, action, true);
	753	+
732	754	/*
733	755	* Clear irqaction. Otherwise exit_irq_thread() would make
734	756	* fuzz about an active irq thread going into nirvana.
...	...	@@ -743,6 +765,7 @@
743	765	void exit_irq_thread(void)
744	766	{
745	767	struct task_struct *tsk = current;
	768	+ struct irq_desc *desc;
746	769
747	770	if (!tsk->irqaction)
748	771	return;
749	772
...	...	@@ -751,7 +774,15 @@
751	774	"exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
752	775	tsk->comm ? tsk->comm : "", tsk->pid, tsk->irqaction->irq);
753	776
	777	+ desc = irq_to_desc(tsk->irqaction->irq);
	778	+
754	779	/*
	780	+ * Prevent a stale desc->threads_oneshot. Must be called
	781	+ * before setting the IRQTF_DIED flag.
	782	+ */
	783	+ irq_finalize_oneshot(desc, tsk->irqaction, true);
	784	+
	785	+ /*
755	786	* Set the THREAD DIED flag to prevent further wakeups of the
756	787	* soon to be gone threaded handler.
757	788	*/
...	...	@@ -767,7 +798,7 @@
767	798	{
768	799	struct irqaction old, *old_ptr;
769	800	const char *old_name = NULL;
770		- unsigned long flags;
	801	+ unsigned long flags, thread_mask = 0;
771	802	int ret, nested, shared = 0;
772	803	cpumask_var_t mask;
773	804
774	805
...	...	@@ -865,11 +896,22 @@
865	896
866	897	/* add new interrupt at end of irq queue */
867	898	do {
	899	+ thread_mask \|= old->thread_mask;
868	900	old_ptr = &old->next;
869	901	old = *old_ptr;
870	902	} while (old);
871	903	shared = 1;
872	904	}
	905	+
	906	+ /*
	907	+ * Setup the thread mask for this irqaction. Unlikely to have
	908	+ * 32 resp 64 irqs sharing one line, but who knows.
	909	+ */
	910	+ if (new->flags & IRQF_ONESHOT && thread_mask == ~0UL) {
	911	+ ret = -EBUSY;
	912	+ goto out_mask;
	913	+ }
	914	+ new->thread_mask = 1 << ffz(thread_mask);
873	915
874	916	if (!shared) {
875	917	irq_chip_set_defaults(desc->irq_data.chip);