Blame view
kernel/irq_work.c
3.13 KB
e360adbe2 irq_work: Add gen... |
1 2 3 4 5 6 |
/* * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> * * Provides a framework for enqueueing and running callbacks from hardirq * context. The enqueueing is NMI-safe. */ |
83e3fa6f0 irq_work: fix com... |
7 |
#include <linux/bug.h> |
e360adbe2 irq_work: Add gen... |
8 |
#include <linux/kernel.h> |
9984de1a5 kernel: Map most ... |
9 |
#include <linux/export.h> |
e360adbe2 irq_work: Add gen... |
10 |
#include <linux/irq_work.h> |
967d1f906 kernel: fix two i... |
11 |
#include <linux/percpu.h> |
e360adbe2 irq_work: Add gen... |
12 |
#include <linux/hardirq.h> |
ef1f09825 irq_work: fix com... |
13 |
#include <linux/irqflags.h> |
967d1f906 kernel: fix two i... |
14 |
#include <asm/processor.h> |
e360adbe2 irq_work: Add gen... |
15 16 17 18 19 20 21 22 |
/* * An entry can be in one of four states: * * free NULL, 0 -> {claimed} : free to be used * claimed NULL, 3 -> {pending} : claimed to be enqueued * pending next, 3 -> {busy} : queued, pending callback * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed |
e360adbe2 irq_work: Add gen... |
23 24 25 26 27 |
*/ #define IRQ_WORK_PENDING 1UL #define IRQ_WORK_BUSY 2UL #define IRQ_WORK_FLAGS 3UL |
38aaf8090 irq_work: Use lli... |
28 |
static DEFINE_PER_CPU(struct llist_head, irq_work_list); |
e360adbe2 irq_work: Add gen... |
29 30 31 32 |
/* * Claim the entry so that no one else will poke at it. */ |
38aaf8090 irq_work: Use lli... |
33 |
static bool irq_work_claim(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
34 |
{ |
38aaf8090 irq_work: Use lli... |
35 |
unsigned long flags, nflags; |
e360adbe2 irq_work: Add gen... |
36 |
|
38aaf8090 irq_work: Use lli... |
37 38 39 |
for (;;) { flags = work->flags; if (flags & IRQ_WORK_PENDING) |
e360adbe2 irq_work: Add gen... |
40 |
return false; |
38aaf8090 irq_work: Use lli... |
41 42 43 44 45 |
nflags = flags | IRQ_WORK_FLAGS; if (cmpxchg(&work->flags, flags, nflags) == flags) break; cpu_relax(); } |
e360adbe2 irq_work: Add gen... |
46 47 48 |
return true; } |
e360adbe2 irq_work: Add gen... |
49 50 51 52 53 54 55 56 57 58 |
void __weak arch_irq_work_raise(void) { /* * Lame architectures will get the timer tick callback */ } /* * Queue the entry and raise the IPI if needed. */ |
38aaf8090 irq_work: Use lli... |
59 |
static void __irq_work_queue(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
60 |
{ |
38aaf8090 irq_work: Use lli... |
61 |
bool empty; |
e360adbe2 irq_work: Add gen... |
62 |
|
20b876918 irq_work: Use per... |
63 |
preempt_disable(); |
e360adbe2 irq_work: Add gen... |
64 |
|
38aaf8090 irq_work: Use lli... |
65 |
empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); |
e360adbe2 irq_work: Add gen... |
66 |
/* The list was empty, raise self-interrupt to start processing. */ |
38aaf8090 irq_work: Use lli... |
67 |
if (empty) |
e360adbe2 irq_work: Add gen... |
68 |
arch_irq_work_raise(); |
20b876918 irq_work: Use per... |
69 |
preempt_enable(); |
e360adbe2 irq_work: Add gen... |
70 71 72 73 74 75 76 77 |
} /* * Enqueue the irq_work @entry, returns true on success, failure when the * @entry was already enqueued by someone else. * * Can be re-enqueued while the callback is still in progress. */ |
38aaf8090 irq_work: Use lli... |
78 |
bool irq_work_queue(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
79 |
{ |
38aaf8090 irq_work: Use lli... |
80 |
if (!irq_work_claim(work)) { |
e360adbe2 irq_work: Add gen... |
81 82 83 84 85 |
/* * Already enqueued, can't do! */ return false; } |
38aaf8090 irq_work: Use lli... |
86 |
__irq_work_queue(work); |
e360adbe2 irq_work: Add gen... |
87 88 89 90 91 92 93 94 95 96 |
return true; } EXPORT_SYMBOL_GPL(irq_work_queue); /* * Run the irq_work entries on this cpu. Requires to be ran from hardirq * context with local IRQs disabled. */ void irq_work_run(void) { |
38aaf8090 irq_work: Use lli... |
97 98 99 |
struct irq_work *work; struct llist_head *this_list; struct llist_node *llnode; |
e360adbe2 irq_work: Add gen... |
100 |
|
38aaf8090 irq_work: Use lli... |
101 102 |
this_list = &__get_cpu_var(irq_work_list); if (llist_empty(this_list)) |
e360adbe2 irq_work: Add gen... |
103 104 105 106 |
return; BUG_ON(!in_irq()); BUG_ON(!irqs_disabled()); |
38aaf8090 irq_work: Use lli... |
107 108 109 |
llnode = llist_del_all(this_list); while (llnode != NULL) { work = llist_entry(llnode, struct irq_work, llnode); |
e360adbe2 irq_work: Add gen... |
110 |
|
924f8f5af llist: Add llist_... |
111 |
llnode = llist_next(llnode); |
e360adbe2 irq_work: Add gen... |
112 113 |
/* |
38aaf8090 irq_work: Use lli... |
114 |
* Clear the PENDING bit, after this point the @work |
e360adbe2 irq_work: Add gen... |
115 116 |
* can be re-used. */ |
38aaf8090 irq_work: Use lli... |
117 118 |
work->flags = IRQ_WORK_BUSY; work->func(work); |
e360adbe2 irq_work: Add gen... |
119 120 121 122 |
/* * Clear the BUSY bit and return to the free state if * no-one else claimed it meanwhile. */ |
38aaf8090 irq_work: Use lli... |
123 |
(void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); |
e360adbe2 irq_work: Add gen... |
124 125 126 127 128 129 130 131 |
} } EXPORT_SYMBOL_GPL(irq_work_run); /* * Synchronize against the irq_work @entry, ensures the entry is not * currently in use. */ |
38aaf8090 irq_work: Use lli... |
132 |
void irq_work_sync(struct irq_work *work) |
e360adbe2 irq_work: Add gen... |
133 134 |
{ WARN_ON_ONCE(irqs_disabled()); |
38aaf8090 irq_work: Use lli... |
135 |
while (work->flags & IRQ_WORK_BUSY) |
e360adbe2 irq_work: Add gen... |
136 137 138 |
cpu_relax(); } EXPORT_SYMBOL_GPL(irq_work_sync); |