Blame view
lib/irq_poll.c
5.6 KB
5e605b64a block: add blk-io... |
1 2 3 4 5 6 7 8 |
/* * Functions related to interrupt-poll handling in the block layer. This * is similar to NAPI for network devices. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> #include <linux/bio.h> |
5e605b64a block: add blk-io... |
9 10 |
#include <linux/interrupt.h> #include <linux/cpu.h> |
511cbce2f irq_poll: make bl... |
11 |
#include <linux/irq_poll.h> |
5e605b64a block: add blk-io... |
12 |
#include <linux/delay.h> |
511cbce2f irq_poll: make bl... |
13 |
static unsigned int irq_poll_budget __read_mostly = 256; |
37867ae7c block: adjust def... |
14 |
|
5e605b64a block: add blk-io... |
15 16 17 |
static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); /** |
511cbce2f irq_poll: make bl... |
18 |
* irq_poll_sched - Schedule a run of the iopoll handler |
5e605b64a block: add blk-io... |
19 20 21 |
* @iop: The parent iopoll structure * * Description: |
511cbce2f irq_poll: make bl... |
22 |
* Add this irq_poll structure to the pending poll list and trigger the |
ea51190c0 irq_poll: fold ir... |
23 |
* raise of the blk iopoll softirq. |
5e605b64a block: add blk-io... |
24 |
**/ |
511cbce2f irq_poll: make bl... |
25 |
void irq_poll_sched(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
26 27 |
{ unsigned long flags; |
ea51190c0 irq_poll: fold ir... |
28 29 |
if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) return; |
2ee177e94 irq_poll: Fix irq... |
30 |
if (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) |
ea51190c0 irq_poll: fold ir... |
31 |
return; |
5e605b64a block: add blk-io... |
32 |
local_irq_save(flags); |
170d800af block: Replace __... |
33 |
list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2f irq_poll: make bl... |
34 |
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64a block: add blk-io... |
35 36 |
local_irq_restore(flags); } |
511cbce2f irq_poll: make bl... |
37 |
EXPORT_SYMBOL(irq_poll_sched); |
5e605b64a block: add blk-io... |
38 39 |
/** |
511cbce2f irq_poll: make bl... |
40 |
* __irq_poll_complete - Mark this @iop as un-polled again |
5e605b64a block: add blk-io... |
41 42 43 |
* @iop: The parent iopoll structure * * Description: |
511cbce2f irq_poll: make bl... |
44 |
* See irq_poll_complete(). This function must be called with interrupts |
1badcfbd7 block: fix long l... |
45 |
* disabled. |
5e605b64a block: add blk-io... |
46 |
**/ |
83af187d1 irq_poll: mark __... |
47 |
static void __irq_poll_complete(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
48 49 |
{ list_del(&iop->list); |
4e857c58e arch: Mass conver... |
50 |
smp_mb__before_atomic(); |
511cbce2f irq_poll: make bl... |
51 |
clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64a block: add blk-io... |
52 |
} |
5e605b64a block: add blk-io... |
53 54 |
/** |
511cbce2f irq_poll: make bl... |
55 |
* irq_poll_complete - Mark this @iop as un-polled again |
5e605b64a block: add blk-io... |
56 57 58 |
* @iop: The parent iopoll structure * * Description: |
1badcfbd7 block: fix long l... |
59 60 |
* If a driver consumes less than the assigned budget in its run of the * iopoll handler, it'll end the polled mode by calling this function. The |
ea51190c0 irq_poll: fold ir... |
61 |
* iopoll handler will not be invoked again before irq_poll_sched() |
1badcfbd7 block: fix long l... |
62 |
* is called. |
5e605b64a block: add blk-io... |
63 |
**/ |
511cbce2f irq_poll: make bl... |
64 |
void irq_poll_complete(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
65 66 67 68 |
{ unsigned long flags; local_irq_save(flags); |
511cbce2f irq_poll: make bl... |
69 |
__irq_poll_complete(iop); |
5e605b64a block: add blk-io... |
70 71 |
local_irq_restore(flags); } |
511cbce2f irq_poll: make bl... |
72 |
EXPORT_SYMBOL(irq_poll_complete); |
5e605b64a block: add blk-io... |
73 |
|
511cbce2f irq_poll: make bl... |
74 |
static void irq_poll_softirq(struct softirq_action *h) |
5e605b64a block: add blk-io... |
75 |
{ |
170d800af block: Replace __... |
76 |
struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); |
511cbce2f irq_poll: make bl... |
77 |
int rearm = 0, budget = irq_poll_budget; |
5e605b64a block: add blk-io... |
78 |
unsigned long start_time = jiffies; |
5e605b64a block: add blk-io... |
79 80 81 82 |
local_irq_disable(); while (!list_empty(list)) { |
511cbce2f irq_poll: make bl... |
83 |
struct irq_poll *iop; |
5e605b64a block: add blk-io... |
84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
int work, weight; /* * If softirq window is exhausted then punt. */ if (budget <= 0 || time_after(jiffies, start_time)) { rearm = 1; break; } local_irq_enable(); /* Even though interrupts have been re-enabled, this * access is safe because interrupts can only add new * entries to the tail of this list, and only ->poll() * calls can remove this head entry from the list. */ |
511cbce2f irq_poll: make bl... |
101 |
iop = list_entry(list->next, struct irq_poll, list); |
5e605b64a block: add blk-io... |
102 103 104 |
weight = iop->weight; work = 0; |
511cbce2f irq_poll: make bl... |
105 |
if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64a block: add blk-io... |
106 107 108 109 110 |
work = iop->poll(iop, weight); budget -= work; local_irq_disable(); |
fca51d64c block: fix commen... |
111 112 113 114 115 116 |
/* * Drivers must not modify the iopoll state, if they * consume their assigned weight (or more, some drivers can't * easily just stop processing, they have to complete an * entire mask of commands).In such cases this code * still "owns" the iopoll instance and therefore can |
5e605b64a block: add blk-io... |
117 118 119 |
* move the instance around on the list at-will. */ if (work >= weight) { |
0bc92ace5 irq_poll: fold ir... |
120 |
if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) |
511cbce2f irq_poll: make bl... |
121 |
__irq_poll_complete(iop); |
5e605b64a block: add blk-io... |
122 123 124 125 126 127 |
else list_move_tail(&iop->list, list); } } if (rearm) |
511cbce2f irq_poll: make bl... |
128 |
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64a block: add blk-io... |
129 130 131 132 133 |
local_irq_enable(); } /** |
511cbce2f irq_poll: make bl... |
134 |
* irq_poll_disable - Disable iopoll on this @iop |
5e605b64a block: add blk-io... |
135 136 137 138 139 |
* @iop: The parent iopoll structure * * Description: * Disable io polling and wait for any pending callbacks to have completed. **/ |
511cbce2f irq_poll: make bl... |
140 |
void irq_poll_disable(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
141 |
{ |
511cbce2f irq_poll: make bl... |
142 143 |
set_bit(IRQ_POLL_F_DISABLE, &iop->state); while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64a block: add blk-io... |
144 |
msleep(1); |
511cbce2f irq_poll: make bl... |
145 |
clear_bit(IRQ_POLL_F_DISABLE, &iop->state); |
5e605b64a block: add blk-io... |
146 |
} |
511cbce2f irq_poll: make bl... |
147 |
EXPORT_SYMBOL(irq_poll_disable); |
5e605b64a block: add blk-io... |
148 149 |
/** |
511cbce2f irq_poll: make bl... |
150 |
* irq_poll_enable - Enable iopoll on this @iop |
5e605b64a block: add blk-io... |
151 152 153 |
* @iop: The parent iopoll structure * * Description: |
1badcfbd7 block: fix long l... |
154 155 |
* Enable iopoll on this @iop. Note that the handler run will not be * scheduled, it will only mark it as active. |
5e605b64a block: add blk-io... |
156 |
**/ |
511cbce2f irq_poll: make bl... |
157 |
void irq_poll_enable(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
158 |
{ |
511cbce2f irq_poll: make bl... |
159 |
BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); |
4e857c58e arch: Mass conver... |
160 |
smp_mb__before_atomic(); |
511cbce2f irq_poll: make bl... |
161 |
clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64a block: add blk-io... |
162 |
} |
511cbce2f irq_poll: make bl... |
163 |
EXPORT_SYMBOL(irq_poll_enable); |
5e605b64a block: add blk-io... |
164 165 |
/** |
511cbce2f irq_poll: make bl... |
166 |
* irq_poll_init - Initialize this @iop |
5e605b64a block: add blk-io... |
167 168 169 170 171 |
* @iop: The parent iopoll structure * @weight: The default weight (or command completion budget) * @poll_fn: The handler to invoke * * Description: |
78d0264eb irq_poll: don't d... |
172 |
* Initialize and enable this irq_poll structure. |
5e605b64a block: add blk-io... |
173 |
**/ |
511cbce2f irq_poll: make bl... |
174 |
void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) |
5e605b64a block: add blk-io... |
175 176 177 178 179 |
{ memset(iop, 0, sizeof(*iop)); INIT_LIST_HEAD(&iop->list); iop->weight = weight; iop->poll = poll_fn; |
5e605b64a block: add blk-io... |
180 |
} |
511cbce2f irq_poll: make bl... |
181 |
EXPORT_SYMBOL(irq_poll_init); |
5e605b64a block: add blk-io... |
182 |
|
511cbce2f irq_poll: make bl... |
183 |
static int irq_poll_cpu_notify(struct notifier_block *self, |
0b776b062 block: delete __c... |
184 |
unsigned long action, void *hcpu) |
5e605b64a block: add blk-io... |
185 186 187 188 189 190 191 192 193 194 |
{ /* * If a CPU goes away, splice its entries to the current CPU * and trigger a run of the softirq */ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { int cpu = (unsigned long) hcpu; local_irq_disable(); list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), |
170d800af block: Replace __... |
195 |
this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2f irq_poll: make bl... |
196 |
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64a block: add blk-io... |
197 198 199 200 201 |
local_irq_enable(); } return NOTIFY_OK; } |
511cbce2f irq_poll: make bl... |
202 203 |
static struct notifier_block irq_poll_cpu_notifier = { .notifier_call = irq_poll_cpu_notify, |
5e605b64a block: add blk-io... |
204 |
}; |
511cbce2f irq_poll: make bl... |
205 |
static __init int irq_poll_setup(void) |
5e605b64a block: add blk-io... |
206 207 208 209 210 |
{ int i; for_each_possible_cpu(i) INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); |
511cbce2f irq_poll: make bl... |
211 212 |
open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq); register_hotcpu_notifier(&irq_poll_cpu_notifier); |
5e605b64a block: add blk-io... |
213 214 |
return 0; } |
511cbce2f irq_poll: make bl... |
215 |
subsys_initcall(irq_poll_setup); |