Blame view
kernel/irq_work.c
4.46 KB
e360adbe2 irq_work: Add gen... |
1 2 3 4 5 6 |
/* * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * * Provides a framework for enqueueing and running callbacks from hardirq * context. The enqueueing is NMI-safe. */ |
83e3fa6f0 irq_work: fix com... |
7 |
#include <linux/bug.h> |
e360adbe2 irq_work: Add gen... |
8 |
#include <linux/kernel.h> |
9984de1a5 kernel: Map most ... |
9 |
#include <linux/export.h> |
e360adbe2 irq_work: Add gen... |
10 |
#include <linux/irq_work.h> |
967d1f906 kernel: fix two i... |
11 |
#include <linux/percpu.h> |
e360adbe2 irq_work: Add gen... |
12 |
#include <linux/hardirq.h> |
ef1f09825 irq_work: fix com... |
13 |
#include <linux/irqflags.h> |
bc6679aef irq_work: Make se... |
14 15 |
#include <linux/sched.h> #include <linux/tick.h> |
c0e980a4b irq_work: Flush w... |
16 17 |
#include <linux/cpu.h> #include <linux/notifier.h> |
478850160 irq_work: Impleme... |
18 |
#include <linux/smp.h> |
967d1f906 kernel: fix two i... |
19 |
#include <asm/processor.h> |
e360adbe2 irq_work: Add gen... |
20 |
|
e360adbe2 irq_work: Add gen... |
21 |
|
b93e0b8fa irq_work: Split r... |
22 23 |
static DEFINE_PER_CPU(struct llist_head, raised_list); static DEFINE_PER_CPU(struct llist_head, lazy_list); |
e360adbe2 irq_work: Add gen... |
24 25 26 27 |
/* * Claim the entry so that no one else will poke at it. */ |
38aaf8090 irq_work: Use lli... |
28 |
static bool irq_work_claim(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
29 |
{ |
e0bbe2d80 irq_work: Fix rac... |
30 |
unsigned long flags, oflags, nflags; |
e360adbe2 irq_work: Add gen... |
31 |
|
e0bbe2d80 irq_work: Fix rac... |
32 33 34 35 36 |
/* * Start with our best wish as a premise but only trust any * flag value after cmpxchg() result. */ flags = work->flags & ~IRQ_WORK_PENDING; |
38aaf8090 irq_work: Use lli... |
37 |
for (;;) { |
38aaf8090 irq_work: Use lli... |
38 |
nflags = flags | IRQ_WORK_FLAGS; |
e0bbe2d80 irq_work: Fix rac... |
39 40 |
oflags = cmpxchg(&work->flags, flags, nflags); if (oflags == flags) |
38aaf8090 irq_work: Use lli... |
41 |
break; |
e0bbe2d80 irq_work: Fix rac... |
42 43 44 |
if (oflags & IRQ_WORK_PENDING) return false; flags = oflags; |
38aaf8090 irq_work: Use lli... |
45 46 |
cpu_relax(); } |
e360adbe2 irq_work: Add gen... |
47 48 49 |
return true; } |
e360adbe2 irq_work: Add gen... |
50 51 52 53 54 55 |
void __weak arch_irq_work_raise(void) { /* * Lame architectures will get the timer tick callback */ } |
478850160 irq_work: Impleme... |
56 |
#ifdef CONFIG_SMP |
e360adbe2 irq_work: Add gen... |
57 |
/* |
478850160 irq_work: Impleme... |
58 |
* Enqueue the irq_work @work on @cpu unless it's already pending |
c02cf5f8e irq_work: Remove ... |
59 60 61 |
* somewhere. * * Can be re-enqueued while the callback is still in progress. |
e360adbe2 irq_work: Add gen... |
62 |
*/ |
478850160 irq_work: Impleme... |
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
bool irq_work_queue_on(struct irq_work *work, int cpu) { /* All work should have been flushed before going offline */ WARN_ON_ONCE(cpu_is_offline(cpu)); /* Arch remote IPI send/receive backend aren't NMI safe */ WARN_ON_ONCE(in_nmi()); /* Only queue if not already pending */ if (!irq_work_claim(work)) return false; if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) arch_send_call_function_single_ipi(cpu); return true; } EXPORT_SYMBOL_GPL(irq_work_queue_on); #endif /* Enqueue the irq work @work on the current CPU */ |
cd578abb2 perf/x86: Warn to... |
84 |
bool irq_work_queue(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
85 |
{ |
c02cf5f8e irq_work: Remove ... |
86 87 |
/* Only queue if not already pending */ if (!irq_work_claim(work)) |
cd578abb2 perf/x86: Warn to... |
88 |
return false; |
c02cf5f8e irq_work: Remove ... |
89 90 |
/* Queue the entry and raise the IPI if needed. */ |
20b876918 irq_work: Use per... |
91 |
preempt_disable(); |
e360adbe2 irq_work: Add gen... |
92 |
|
b93e0b8fa irq_work: Split r... |
93 94 |
/* If the work is "lazy", handle it from next tick if any */ if (work->flags & IRQ_WORK_LAZY) { |
22127e93c time: Replace __g... |
95 |
if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && |
b93e0b8fa irq_work: Split r... |
96 97 98 |
tick_nohz_tick_stopped()) arch_irq_work_raise(); } else { |
22127e93c time: Replace __g... |
99 |
if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) |
bc6679aef irq_work: Make se... |
100 101 |
arch_irq_work_raise(); } |
e360adbe2 irq_work: Add gen... |
102 |
|
20b876918 irq_work: Use per... |
103 |
preempt_enable(); |
cd578abb2 perf/x86: Warn to... |
104 105 |
return true; |
e360adbe2 irq_work: Add gen... |
106 |
} |
e360adbe2 irq_work: Add gen... |
107 |
EXPORT_SYMBOL_GPL(irq_work_queue); |
00b429591 irq_work: Don't s... |
108 109 |
bool irq_work_needs_cpu(void) { |
b93e0b8fa irq_work: Split r... |
110 |
struct llist_head *raised, *lazy; |
00b429591 irq_work: Don't s... |
111 |
|
22127e93c time: Replace __g... |
112 113 |
raised = this_cpu_ptr(&raised_list); lazy = this_cpu_ptr(&lazy_list); |
76a33061b irq_work: Force r... |
114 115 116 117 |
if (llist_empty(raised) || arch_irq_work_has_interrupt()) if (llist_empty(lazy)) return false; |
00b429591 irq_work: Don't s... |
118 |
|
8aa2accee irq_work: Warn if... |
119 120 |
/* All work should have been flushed before going offline */ WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); |
00b429591 irq_work: Don't s... |
121 122 |
return true; } |
b93e0b8fa irq_work: Split r... |
123 |
static void irq_work_run_list(struct llist_head *list) |
e360adbe2 irq_work: Add gen... |
124 |
{ |
bc6679aef irq_work: Make se... |
125 |
unsigned long flags; |
38aaf8090 irq_work: Use lli... |
126 |
struct irq_work *work; |
38aaf8090 irq_work: Use lli... |
127 |
struct llist_node *llnode; |
e360adbe2 irq_work: Add gen... |
128 |
|
b93e0b8fa irq_work: Split r... |
129 |
BUG_ON(!irqs_disabled()); |
bc6679aef irq_work: Make se... |
130 |
|
b93e0b8fa irq_work: Split r... |
131 |
if (llist_empty(list)) |
e360adbe2 irq_work: Add gen... |
132 |
return; |
b93e0b8fa irq_work: Split r... |
133 |
llnode = llist_del_all(list); |
38aaf8090 irq_work: Use lli... |
134 135 |
while (llnode != NULL) { work = llist_entry(llnode, struct irq_work, llnode); |
e360adbe2 irq_work: Add gen... |
136 |
|
924f8f5af llist: Add llist_... |
137 |
llnode = llist_next(llnode); |
e360adbe2 irq_work: Add gen... |
138 139 |
/* |
38aaf8090 irq_work: Use lli... |
140 |
* Clear the PENDING bit, after this point the @work |
e360adbe2 irq_work: Add gen... |
141 |
* can be re-used. |
c8446b75b irq_work: Fix rac... |
142 143 144 |
* Make it immediately visible so that other CPUs trying * to claim that work don't rely on us to handle their data * while we are in the middle of the func. |
e360adbe2 irq_work: Add gen... |
145 |
*/ |
bc6679aef irq_work: Make se... |
146 147 |
flags = work->flags & ~IRQ_WORK_PENDING; xchg(&work->flags, flags); |
38aaf8090 irq_work: Use lli... |
148 |
work->func(work); |
e360adbe2 irq_work: Add gen... |
149 150 151 152 |
/* * Clear the BUSY bit and return to the free state if * no-one else claimed it meanwhile. */ |
bc6679aef irq_work: Make se... |
153 |
(void)cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); |
e360adbe2 irq_work: Add gen... |
154 155 |
} } |
c0e980a4b irq_work: Flush w... |
156 157 |
/* |
a77353e5e irq_work: Remove ... |
158 159 |
* hotplug calls this through: * hotplug_cfd() -> flush_smp_call_function_queue() |
c0e980a4b irq_work: Flush w... |
160 161 162 |
*/ void irq_work_run(void) { |
22127e93c time: Replace __g... |
163 164 |
irq_work_run_list(this_cpu_ptr(&raised_list)); irq_work_run_list(this_cpu_ptr(&lazy_list)); |
c0e980a4b irq_work: Flush w... |
165 |
} |
e360adbe2 irq_work: Add gen... |
166 |
EXPORT_SYMBOL_GPL(irq_work_run); |
76a33061b irq_work: Force r... |
167 168 |
void irq_work_tick(void) { |
56e4dea81 percpu: Convert r... |
169 |
struct llist_head *raised = this_cpu_ptr(&raised_list); |
76a33061b irq_work: Force r... |
170 171 172 |
if (!llist_empty(raised) && !arch_irq_work_has_interrupt()) irq_work_run_list(raised); |
56e4dea81 percpu: Convert r... |
173 |
irq_work_run_list(this_cpu_ptr(&lazy_list)); |
76a33061b irq_work: Force r... |
174 |
} |
e360adbe2 irq_work: Add gen... |
175 176 177 178 |
/* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use. */ |
38aaf8090 irq_work: Use lli... |
179 |
void irq_work_sync(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
180 181 |
{ WARN_ON_ONCE(irqs_disabled()); |
38aaf8090 irq_work: Use lli... |
182 |
while (work->flags & IRQ_WORK_BUSY) |
e360adbe2 irq_work: Add gen... |
183 184 185 |
cpu_relax(); } EXPORT_SYMBOL_GPL(irq_work_sync); |