Blame view
lib/irq_poll.c
5.44 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
5e605b64a block: add blk-io... |
2 3 4 5 6 7 8 9 |
/* * Functions related to interrupt-poll handling in the block layer. This * is similar to NAPI for network devices. */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> #include <linux/bio.h> |
5e605b64a block: add blk-io... |
10 11 |
#include <linux/interrupt.h> #include <linux/cpu.h> |
511cbce2f irq_poll: make bl... |
12 |
#include <linux/irq_poll.h> |
5e605b64a block: add blk-io... |
13 |
#include <linux/delay.h> |
511cbce2f irq_poll: make bl... |
14 |
static unsigned int irq_poll_budget __read_mostly = 256; |
37867ae7c block: adjust def... |
15 |
|
5e605b64a block: add blk-io... |
16 17 18 |
static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); /** |
511cbce2f irq_poll: make bl... |
19 |
* irq_poll_sched - Schedule a run of the iopoll handler |
5e605b64a block: add blk-io... |
20 21 22 |
* @iop: The parent iopoll structure * * Description: |
511cbce2f irq_poll: make bl... |
23 |
* Add this irq_poll structure to the pending poll list and trigger the |
ea51190c0 irq_poll: fold ir... |
24 |
* raise of the blk iopoll softirq. |
5e605b64a block: add blk-io... |
25 |
**/ |
511cbce2f irq_poll: make bl... |
26 |
void irq_poll_sched(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
27 28 |
{ unsigned long flags; |
ea51190c0 irq_poll: fold ir... |
29 30 |
if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) return; |
2ee177e94 irq_poll: Fix irq... |
31 |
if (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) |
ea51190c0 irq_poll: fold ir... |
32 |
return; |
5e605b64a block: add blk-io... |
33 |
local_irq_save(flags); |
170d800af block: Replace __... |
34 |
list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); |
511cbce2f irq_poll: make bl... |
35 |
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64a block: add blk-io... |
36 37 |
local_irq_restore(flags); } |
511cbce2f irq_poll: make bl... |
38 |
EXPORT_SYMBOL(irq_poll_sched); |
5e605b64a block: add blk-io... |
39 40 |
/** |
511cbce2f irq_poll: make bl... |
41 |
* __irq_poll_complete - Mark this @iop as un-polled again |
5e605b64a block: add blk-io... |
42 43 44 |
* @iop: The parent iopoll structure * * Description: |
511cbce2f irq_poll: make bl... |
45 |
* See irq_poll_complete(). This function must be called with interrupts |
1badcfbd7 block: fix long l... |
46 |
* disabled. |
5e605b64a block: add blk-io... |
47 |
**/ |
83af187d1 irq_poll: mark __... |
48 |
static void __irq_poll_complete(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
49 50 |
{ list_del(&iop->list); |
4e857c58e arch: Mass conver... |
51 |
smp_mb__before_atomic(); |
511cbce2f irq_poll: make bl... |
52 |
clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64a block: add blk-io... |
53 |
} |
5e605b64a block: add blk-io... |
54 55 |
/** |
511cbce2f irq_poll: make bl... |
56 |
* irq_poll_complete - Mark this @iop as un-polled again |
5e605b64a block: add blk-io... |
57 58 59 |
* @iop: The parent iopoll structure * * Description: |
1badcfbd7 block: fix long l... |
60 61 |
* If a driver consumes less than the assigned budget in its run of the * iopoll handler, it'll end the polled mode by calling this function. The |
ea51190c0 irq_poll: fold ir... |
62 |
* iopoll handler will not be invoked again before irq_poll_sched() |
1badcfbd7 block: fix long l... |
63 |
* is called. |
5e605b64a block: add blk-io... |
64 |
**/ |
511cbce2f irq_poll: make bl... |
65 |
void irq_poll_complete(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
66 67 68 69 |
{ unsigned long flags; local_irq_save(flags); |
511cbce2f irq_poll: make bl... |
70 |
__irq_poll_complete(iop); |
5e605b64a block: add blk-io... |
71 72 |
local_irq_restore(flags); } |
511cbce2f irq_poll: make bl... |
73 |
EXPORT_SYMBOL(irq_poll_complete); |
5e605b64a block: add blk-io... |
74 |
|
0766f788e latent_entropy: M... |
75 |
static void __latent_entropy irq_poll_softirq(struct softirq_action *h) |
5e605b64a block: add blk-io... |
76 |
{ |
170d800af block: Replace __... |
77 |
struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); |
511cbce2f irq_poll: make bl... |
78 |
int rearm = 0, budget = irq_poll_budget; |
5e605b64a block: add blk-io... |
79 |
unsigned long start_time = jiffies; |
5e605b64a block: add blk-io... |
80 81 82 83 |
local_irq_disable(); while (!list_empty(list)) { |
511cbce2f irq_poll: make bl... |
84 |
struct irq_poll *iop; |
5e605b64a block: add blk-io... |
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
int work, weight; /* * If softirq window is exhausted then punt. */ if (budget <= 0 || time_after(jiffies, start_time)) { rearm = 1; break; } local_irq_enable(); /* Even though interrupts have been re-enabled, this * access is safe because interrupts can only add new * entries to the tail of this list, and only ->poll() * calls can remove this head entry from the list. */ |
511cbce2f irq_poll: make bl... |
102 |
iop = list_entry(list->next, struct irq_poll, list); |
5e605b64a block: add blk-io... |
103 104 105 |
weight = iop->weight; work = 0; |
511cbce2f irq_poll: make bl... |
106 |
if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64a block: add blk-io... |
107 108 109 110 111 |
work = iop->poll(iop, weight); budget -= work; local_irq_disable(); |
fca51d64c block: fix commen... |
112 113 114 115 116 117 |
/* * Drivers must not modify the iopoll state, if they * consume their assigned weight (or more, some drivers can't * easily just stop processing, they have to complete an * entire mask of commands).In such cases this code * still "owns" the iopoll instance and therefore can |
5e605b64a block: add blk-io... |
118 119 120 |
* move the instance around on the list at-will. */ if (work >= weight) { |
0bc92ace5 irq_poll: fold ir... |
121 |
if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) |
511cbce2f irq_poll: make bl... |
122 |
__irq_poll_complete(iop); |
5e605b64a block: add blk-io... |
123 124 125 126 127 128 |
else list_move_tail(&iop->list, list); } } if (rearm) |
511cbce2f irq_poll: make bl... |
129 |
__raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); |
5e605b64a block: add blk-io... |
130 131 132 133 134 |
local_irq_enable(); } /** |
511cbce2f irq_poll: make bl... |
135 |
* irq_poll_disable - Disable iopoll on this @iop |
5e605b64a block: add blk-io... |
136 137 138 139 140 |
* @iop: The parent iopoll structure * * Description: * Disable io polling and wait for any pending callbacks to have completed. **/ |
511cbce2f irq_poll: make bl... |
141 |
void irq_poll_disable(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
142 |
{ |
511cbce2f irq_poll: make bl... |
143 144 |
set_bit(IRQ_POLL_F_DISABLE, &iop->state); while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) |
5e605b64a block: add blk-io... |
145 |
msleep(1); |
511cbce2f irq_poll: make bl... |
146 |
clear_bit(IRQ_POLL_F_DISABLE, &iop->state); |
5e605b64a block: add blk-io... |
147 |
} |
511cbce2f irq_poll: make bl... |
148 |
EXPORT_SYMBOL(irq_poll_disable); |
5e605b64a block: add blk-io... |
149 150 |
/** |
511cbce2f irq_poll: make bl... |
151 |
* irq_poll_enable - Enable iopoll on this @iop |
5e605b64a block: add blk-io... |
152 153 154 |
* @iop: The parent iopoll structure * * Description: |
1badcfbd7 block: fix long l... |
155 156 |
* Enable iopoll on this @iop. Note that the handler run will not be * scheduled, it will only mark it as active. |
5e605b64a block: add blk-io... |
157 |
**/ |
511cbce2f irq_poll: make bl... |
158 |
void irq_poll_enable(struct irq_poll *iop) |
5e605b64a block: add blk-io... |
159 |
{ |
511cbce2f irq_poll: make bl... |
160 |
BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); |
4e857c58e arch: Mass conver... |
161 |
smp_mb__before_atomic(); |
511cbce2f irq_poll: make bl... |
162 |
clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); |
5e605b64a block: add blk-io... |
163 |
} |
511cbce2f irq_poll: make bl... |
164 |
EXPORT_SYMBOL(irq_poll_enable); |
5e605b64a block: add blk-io... |
165 166 |
/** |
511cbce2f irq_poll: make bl... |
167 |
* irq_poll_init - Initialize this @iop |
5e605b64a block: add blk-io... |
168 169 170 171 172 |
* @iop: The parent iopoll structure * @weight: The default weight (or command completion budget) * @poll_fn: The handler to invoke * * Description: |
78d0264eb irq_poll: don't d... |
173 |
* Initialize and enable this irq_poll structure. |
5e605b64a block: add blk-io... |
174 |
**/ |
511cbce2f irq_poll: make bl... |
175 |
void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) |
5e605b64a block: add blk-io... |
176 177 178 179 180 |
{ memset(iop, 0, sizeof(*iop)); INIT_LIST_HEAD(&iop->list); iop->weight = weight; iop->poll = poll_fn; |
5e605b64a block: add blk-io... |
181 |
} |
511cbce2f irq_poll: make bl... |
182 |
EXPORT_SYMBOL(irq_poll_init); |
5e605b64a block: add blk-io... |
183 |
|
75e12ed65 lib/irq_poll: Con... |
184 |
static int irq_poll_cpu_dead(unsigned int cpu) |
5e605b64a block: add blk-io... |
185 186 187 188 189 |
{ /* * If a CPU goes away, splice its entries to the current CPU * and trigger a run of the softirq */ |
75e12ed65 lib/irq_poll: Con... |
190 191 192 193 194 |
local_irq_disable(); list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), this_cpu_ptr(&blk_cpu_iopoll)); __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); local_irq_enable(); |
5e605b64a block: add blk-io... |
195 |
|
75e12ed65 lib/irq_poll: Con... |
196 |
return 0; |
5e605b64a block: add blk-io... |
197 |
} |
511cbce2f irq_poll: make bl... |
198 |
static __init int irq_poll_setup(void) |
5e605b64a block: add blk-io... |
199 200 201 202 203 |
{ int i; for_each_possible_cpu(i) INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); |
511cbce2f irq_poll: make bl... |
204 |
open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq); |
75e12ed65 lib/irq_poll: Con... |
205 206 |
cpuhp_setup_state_nocalls(CPUHP_IRQ_POLL_DEAD, "irq_poll:dead", NULL, irq_poll_cpu_dead); |
5e605b64a block: add blk-io... |
207 208 |
return 0; } |
511cbce2f irq_poll: make bl... |
209 |
subsys_initcall(irq_poll_setup); |