Blame view
kernel/stop_machine.c
4.54 KB
ffdb5976c
|
1 |
/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
e5582ca21
|
2 3 |
* GPL v2 and any later version. */ |
1da177e4c
|
4 5 |
#include <linux/cpu.h> #include <linux/err.h> |
ee527cd3a
|
6 7 8 9 |
#include <linux/kthread.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/stop_machine.h> |
1da177e4c
|
10 |
#include <linux/syscalls.h> |
a12bb4447
|
11 |
#include <linux/interrupt.h> |
1da177e4c
|
12 |
#include <asm/atomic.h> |
1da177e4c
|
13 |
#include <asm/uaccess.h> |
ffdb5976c
|
14 |
/* This controls the threads on each CPU. */ |
1da177e4c
|
15 |
enum stopmachine_state { |
ffdb5976c
|
16 17 18 |
/* Dummy starting state for thread. */ STOPMACHINE_NONE, /* Awaiting everyone to be scheduled. */ |
1da177e4c
|
19 |
STOPMACHINE_PREPARE, |
ffdb5976c
|
20 |
/* Disable interrupts. */ |
1da177e4c
|
21 |
STOPMACHINE_DISABLE_IRQ, |
ffdb5976c
|
22 |
/* Run the function */ |
5c2aed622
|
23 |
STOPMACHINE_RUN, |
ffdb5976c
|
24 |
/* Exit */ |
1da177e4c
|
25 26 |
STOPMACHINE_EXIT, }; |
ffdb5976c
|
27 |
static enum stopmachine_state state; |
1da177e4c
|
28 |
|
5c2aed622
|
29 30 31 |
struct stop_machine_data { int (*fn)(void *); void *data; |
ffdb5976c
|
32 33 |
int fnret; }; |
5c2aed622
|
34 |
|
ffdb5976c
|
35 36 37 |
/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ static unsigned int num_threads; static atomic_t thread_ack; |
ffdb5976c
|
38 |
static DEFINE_MUTEX(lock); |
9ea09af3b
|
39 40 41 42 |
/* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */ static DEFINE_MUTEX(setup_lock); /* Users of stop_machine. */ static int refcount; |
c9583e55f
|
43 44 45 46 |
static struct workqueue_struct *stop_machine_wq; static struct stop_machine_data active, idle; static const cpumask_t *active_cpus; static void *stop_machine_work; |
ffdb5976c
|
47 |
static void set_state(enum stopmachine_state newstate) |
1da177e4c
|
48 |
{ |
ffdb5976c
|
49 50 51 52 |
/* Reset ack counter. */ atomic_set(&thread_ack, num_threads); smp_wmb(); state = newstate; |
1da177e4c
|
53 |
} |
ffdb5976c
|
54 55 |
/* Last one to ack a state moves to the next state. */ static void ack_state(void) |
1da177e4c
|
56 |
{ |
c9583e55f
|
57 58 |
if (atomic_dec_and_test(&thread_ack)) set_state(state + 1); |
1da177e4c
|
59 |
} |
c9583e55f
|
60 61 62 |
/* This is the actual function which stops the CPU. It runs * in the context of a dedicated stopmachine workqueue. */ static void stop_cpu(struct work_struct *unused) |
1da177e4c
|
63 |
{ |
ffdb5976c
|
64 |
enum stopmachine_state curstate = STOPMACHINE_NONE; |
c9583e55f
|
65 66 |
struct stop_machine_data *smdata = &idle; int cpu = smp_processor_id(); |
8163bcac7
|
67 |
int err; |
c9583e55f
|
68 69 |
if (!active_cpus) { |
41c7bb958
|
70 |
if (cpu == cpumask_first(cpu_online_mask)) |
c9583e55f
|
71 72 |
smdata = &active; } else { |
41c7bb958
|
73 |
if (cpumask_test_cpu(cpu, active_cpus)) |
c9583e55f
|
74 75 |
smdata = &active; } |
ffdb5976c
|
76 77 78 |
/* Simple state machine */ do { /* Chill out and ensure we re-read stopmachine_state. */ |
3401a61e1
|
79 |
cpu_relax(); |
ffdb5976c
|
80 81 82 83 84 85 86 87 |
if (state != curstate) { curstate = state; switch (curstate) { case STOPMACHINE_DISABLE_IRQ: local_irq_disable(); hard_irq_disable(); break; case STOPMACHINE_RUN: |
8163bcac7
|
88 89 90 91 92 |
/* On multiple CPUs only a single error code * is needed to tell that something failed. */ err = smdata->fn(smdata->data); if (err) smdata->fnret = err; |
ffdb5976c
|
93 94 95 96 97 98 99 |
break; default: break; } ack_state(); } } while (curstate != STOPMACHINE_EXIT); |
1da177e4c
|
100 |
|
1da177e4c
|
101 102 |
local_irq_enable(); } |
ffdb5976c
|
103 104 |
/* Callback for CPUs which aren't supposed to do anything. */ static int chill(void *unused) |
5c2aed622
|
105 |
{ |
ffdb5976c
|
106 |
return 0; |
5c2aed622
|
107 |
} |
1da177e4c
|
108 |
|
9ea09af3b
|
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
int stop_machine_create(void) { mutex_lock(&setup_lock); if (refcount) goto done; stop_machine_wq = create_rt_workqueue("kstop"); if (!stop_machine_wq) goto err_out; stop_machine_work = alloc_percpu(struct work_struct); if (!stop_machine_work) goto err_out; done: refcount++; mutex_unlock(&setup_lock); return 0; err_out: if (stop_machine_wq) destroy_workqueue(stop_machine_wq); mutex_unlock(&setup_lock); return -ENOMEM; } EXPORT_SYMBOL_GPL(stop_machine_create); void stop_machine_destroy(void) { mutex_lock(&setup_lock); refcount--; if (refcount) goto done; destroy_workqueue(stop_machine_wq); free_percpu(stop_machine_work); done: mutex_unlock(&setup_lock); } EXPORT_SYMBOL_GPL(stop_machine_destroy); |
41c7bb958
|
145 |
int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) |
1da177e4c
|
146 |
{ |
c9583e55f
|
147 |
struct work_struct *sm_work; |
e14c8bf86
|
148 |
int i, ret; |
ffdb5976c
|
149 |
|
c9583e55f
|
150 151 152 153 |
/* Set up initial state. */ mutex_lock(&lock); num_threads = num_online_cpus(); active_cpus = cpus; |
ffdb5976c
|
154 155 156 157 158 |
active.fn = fn; active.data = data; active.fnret = 0; idle.fn = chill; idle.data = NULL; |
ffdb5976c
|
159 |
set_state(STOPMACHINE_PREPARE); |
1da177e4c
|
160 |
|
c9583e55f
|
161 |
/* Schedule the stop_cpu work on all cpus: hold this CPU so one |
ffdb5976c
|
162 |
* doesn't hit this CPU until we're ready. */ |
eeec4fad9
|
163 |
get_cpu(); |
c9583e55f
|
164 165 166 167 168 |
for_each_online_cpu(i) { sm_work = percpu_ptr(stop_machine_work, i); INIT_WORK(sm_work, stop_cpu); queue_work_on(i, stop_machine_wq, sm_work); } |
ffdb5976c
|
169 170 |
/* This will release the thread on our CPU. */ put_cpu(); |
c9583e55f
|
171 |
flush_workqueue(stop_machine_wq); |
e14c8bf86
|
172 |
ret = active.fnret; |
ffdb5976c
|
173 |
mutex_unlock(&lock); |
e14c8bf86
|
174 |
return ret; |
1da177e4c
|
175 |
} |
41c7bb958
|
176 |
int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) |
1da177e4c
|
177 |
{ |
1da177e4c
|
178 |
int ret; |
9ea09af3b
|
179 180 181 |
ret = stop_machine_create(); if (ret) return ret; |
1da177e4c
|
182 |
/* No CPUs can come up or down during this. */ |
86ef5c9a8
|
183 |
get_online_cpus(); |
eeec4fad9
|
184 |
ret = __stop_machine(fn, data, cpus); |
86ef5c9a8
|
185 |
put_online_cpus(); |
9ea09af3b
|
186 |
stop_machine_destroy(); |
1da177e4c
|
187 188 |
return ret; } |
eeec4fad9
|
189 |
EXPORT_SYMBOL_GPL(stop_machine); |