Blame view
kernel/task_work.c
2.17 KB
e73f8959a
|
1 2 3 |
#include <linux/spinlock.h> #include <linux/task_work.h> #include <linux/tracehook.h> |
9da33de62
|
4 |
static struct callback_head work_exited; /* all we need is ->next == NULL */ |
e73f8959a
|
5 |
int |
ac3d0da8f
|
6 |
task_work_add(struct task_struct *task, struct callback_head *work, bool notify) |
e73f8959a
|
7 |
{ |
ac3d0da8f
|
8 |
struct callback_head *head; |
9da33de62
|
9 |
|
ac3d0da8f
|
10 11 |
do { head = ACCESS_ONCE(task->task_works); |
9da33de62
|
12 13 |
if (unlikely(head == &work_exited)) return -ESRCH; |
ac3d0da8f
|
14 15 |
work->next = head; } while (cmpxchg(&task->task_works, head, work) != head); |
e73f8959a
|
16 |
|
ed3e694d7
|
17 |
if (notify) |
e73f8959a
|
18 |
set_notify_resume(task); |
ed3e694d7
|
19 |
return 0; |
e73f8959a
|
20 |
} |
67d121455
|
21 |
struct callback_head * |
e73f8959a
|
22 23 |
task_work_cancel(struct task_struct *task, task_work_func_t func) { |
ac3d0da8f
|
24 25 |
struct callback_head **pprev = &task->task_works; struct callback_head *work = NULL; |
e73f8959a
|
26 |
unsigned long flags; |
ac3d0da8f
|
27 28 29 30 |
/* * If cmpxchg() fails we continue without updating pprev. * Either we raced with task_work_add() which added the * new entry before this work, we will find it again. Or |
9da33de62
|
31 |
* we raced with task_work_run(), *pprev == NULL/exited. |
ac3d0da8f
|
32 |
*/ |
e73f8959a
|
33 |
raw_spin_lock_irqsave(&task->pi_lock, flags); |
ac3d0da8f
|
34 35 36 37 38 39 |
while ((work = ACCESS_ONCE(*pprev))) { read_barrier_depends(); if (work->func != func) pprev = &work->next; else if (cmpxchg(pprev, work, work->next) == work) break; |
e73f8959a
|
40 |
} |
e73f8959a
|
41 |
raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
ac3d0da8f
|
42 43 |
return work; |
e73f8959a
|
44 45 46 47 48 |
} void task_work_run(void) { struct task_struct *task = current; |
ac3d0da8f
|
49 |
struct callback_head *work, *head, *next; |
e73f8959a
|
50 |
|
ac3d0da8f
|
51 |
for (;;) { |
9da33de62
|
52 53 54 55 56 57 58 59 60 |
/* * work->func() can do task_work_add(), do not set * work_exited unless the list is empty. */ do { work = ACCESS_ONCE(task->task_works); head = !work && (task->flags & PF_EXITING) ? &work_exited : NULL; } while (cmpxchg(&task->task_works, work, head) != work); |
ac3d0da8f
|
61 62 63 64 65 66 67 68 69 |
if (!work) break; /* * Synchronize with task_work_cancel(). It can't remove * the first entry == work, cmpxchg(task_works) should * fail, but it can play with *work and other entries. */ raw_spin_unlock_wait(&task->pi_lock); smp_mb(); |
e73f8959a
|
70 |
|
ac3d0da8f
|
71 72 73 74 75 76 77 78 |
/* Reverse the list to run the works in fifo order */ head = NULL; do { next = work->next; work->next = head; head = work; work = next; } while (work); |
e73f8959a
|
79 |
|
ac3d0da8f
|
80 81 82 83 84 |
work = head; do { next = work->next; work->func(work); work = next; |
f341861fb
|
85 |
cond_resched(); |
ac3d0da8f
|
86 |
} while (work); |
e73f8959a
|
87 88 |
} } |