Blame view
kernel/padata.c
28.4 KB
08b21fbf4 padata: add SPDX ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
16295bec6 padata: Generic p... |
2 3 4 |
/* * padata.c - generic interface to process data streams in parallel * |
107f8bdac padata: Add a ref... |
5 6 |
* See Documentation/padata.txt for an api documentation. * |
16295bec6 padata: Generic p... |
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
* Copyright (C) 2008, 2009 secunet Security Networks AG * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com> * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ |
9984de1a5 kernel: Map most ... |
23 |
#include <linux/export.h> |
16295bec6 padata: Generic p... |
24 25 26 27 28 29 |
#include <linux/cpumask.h> #include <linux/err.h> #include <linux/cpu.h> #include <linux/padata.h> #include <linux/mutex.h> #include <linux/sched.h> |
5a0e3ad6a include cleanup: ... |
30 |
#include <linux/slab.h> |
5e017dc3f padata: Added sys... |
31 |
#include <linux/sysfs.h> |
16295bec6 padata: Generic p... |
32 |
#include <linux/rcupdate.h> |
30e92153b padata: Convert t... |
33 |
#include <linux/module.h> |
16295bec6 padata: Generic p... |
34 |
|
97e3d94aa padata: Dont scal... |
35 |
#define MAX_OBJ_NUM 1000 |
16295bec6 padata: Generic p... |
36 |
|
5fefc9b3e padata: Remove br... |
37 |
static void padata_free_pd(struct parallel_data *pd); |
16295bec6 padata: Generic p... |
38 39 40 |
static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) { int cpu, target_cpu; |
e15bacbeb padata: Make two ... |
41 |
target_cpu = cpumask_first(pd->cpumask.pcpu); |
16295bec6 padata: Generic p... |
42 |
for (cpu = 0; cpu < cpu_index; cpu++) |
e15bacbeb padata: Make two ... |
43 |
target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); |
16295bec6 padata: Generic p... |
44 45 46 |
return target_cpu; } |
bfde23ce2 padata: unbind pa... |
47 |
static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr) |
16295bec6 padata: Generic p... |
48 |
{ |
16295bec6 padata: Generic p... |
49 50 51 52 |
/* * Hash the sequence numbers to the cpus by taking * seq_nr mod. number of cpus in use. */ |
bfde23ce2 padata: unbind pa... |
53 |
int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu); |
16295bec6 padata: Generic p... |
54 55 56 |
return padata_index_to_cpu(pd, cpu_index); } |
e15bacbeb padata: Make two ... |
57 |
static void padata_parallel_worker(struct work_struct *parallel_work) |
16295bec6 padata: Generic p... |
58 |
{ |
e15bacbeb padata: Make two ... |
59 |
struct padata_parallel_queue *pqueue; |
16295bec6 padata: Generic p... |
60 61 62 |
LIST_HEAD(local_list); local_bh_disable(); |
e15bacbeb padata: Make two ... |
63 64 |
pqueue = container_of(parallel_work, struct padata_parallel_queue, work); |
16295bec6 padata: Generic p... |
65 |
|
e15bacbeb padata: Make two ... |
66 67 68 |
spin_lock(&pqueue->parallel.lock); list_replace_init(&pqueue->parallel.list, &local_list); spin_unlock(&pqueue->parallel.lock); |
16295bec6 padata: Generic p... |
69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
while (!list_empty(&local_list)) { struct padata_priv *padata; padata = list_entry(local_list.next, struct padata_priv, list); list_del_init(&padata->list); padata->parallel(padata); } local_bh_enable(); } |
0198ffd13 padata: Add some ... |
83 |
/** |
16295bec6 padata: Generic p... |
84 85 |
* padata_do_parallel - padata parallelization function * |
b4c8ed0bf crypto: pcrypt - ... |
86 |
* @ps: padatashell |
16295bec6 padata: Generic p... |
87 |
* @padata: object to be parallelized |
e6ce0e080 padata: make pada... |
88 89 90 91 |
* @cb_cpu: pointer to the CPU that the serialization callback function should * run on. If it's not in the serial cpumask of @pinst * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if * none found, returns -EINVAL. |
16295bec6 padata: Generic p... |
92 93 94 95 96 |
* * The parallelization callback function will run with BHs off. * Note: Every object which is parallelized by padata_do_parallel * must be seen by padata_do_serial. */ |
b4c8ed0bf crypto: pcrypt - ... |
97 |
int padata_do_parallel(struct padata_shell *ps, |
e6ce0e080 padata: make pada... |
98 |
struct padata_priv *padata, int *cb_cpu) |
16295bec6 padata: Generic p... |
99 |
{ |
b4c8ed0bf crypto: pcrypt - ... |
100 |
struct padata_instance *pinst = ps->pinst; |
e6ce0e080 padata: make pada... |
101 |
int i, cpu, cpu_index, target_cpu, err; |
e15bacbeb padata: Make two ... |
102 |
struct padata_parallel_queue *queue; |
16295bec6 padata: Generic p... |
103 104 105 |
struct parallel_data *pd; rcu_read_lock_bh(); |
b4c8ed0bf crypto: pcrypt - ... |
106 |
pd = rcu_dereference_bh(ps->pd); |
16295bec6 padata: Generic p... |
107 |
|
83f619f3c padata: make pada... |
108 |
err = -EINVAL; |
7424713b8 padata: Check for... |
109 |
if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) |
16295bec6 padata: Generic p... |
110 |
goto out; |
e6ce0e080 padata: make pada... |
111 112 113 114 115 116 117 118 119 120 121 122 123 |
if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) { if (!cpumask_weight(pd->cpumask.cbcpu)) goto out; /* Select an alternate fallback CPU and notify the caller. */ cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu); cpu = cpumask_first(pd->cpumask.cbcpu); for (i = 0; i < cpu_index; i++) cpu = cpumask_next(cpu, pd->cpumask.cbcpu); *cb_cpu = cpu; } |
16295bec6 padata: Generic p... |
124 125 126 127 128 129 130 |
err = -EBUSY; if ((pinst->flags & PADATA_RESET)) goto out; if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM) goto out; |
83f619f3c padata: make pada... |
131 |
err = 0; |
16295bec6 padata: Generic p... |
132 133 |
atomic_inc(&pd->refcnt); padata->pd = pd; |
e6ce0e080 padata: make pada... |
134 |
padata->cb_cpu = *cb_cpu; |
16295bec6 padata: Generic p... |
135 |
|
bfde23ce2 padata: unbind pa... |
136 137 |
padata->seq_nr = atomic_inc_return(&pd->seq_nr); target_cpu = padata_cpu_hash(pd, padata->seq_nr); |
350ef88e7 padata: ensure pa... |
138 |
padata->cpu = target_cpu; |
e15bacbeb padata: Make two ... |
139 |
queue = per_cpu_ptr(pd->pqueue, target_cpu); |
16295bec6 padata: Generic p... |
140 141 142 143 |
spin_lock(&queue->parallel.lock); list_add_tail(&padata->list, &queue->parallel.list); spin_unlock(&queue->parallel.lock); |
bfde23ce2 padata: unbind pa... |
144 |
queue_work(pinst->parallel_wq, &queue->work); |
16295bec6 padata: Generic p... |
145 146 147 148 149 150 151 |
out: rcu_read_unlock_bh(); return err; } EXPORT_SYMBOL(padata_do_parallel); |
0198ffd13 padata: Add some ... |
152 |
/* |
bfde23ce2 padata: unbind pa... |
153 |
* padata_find_next - Find the next object that needs serialization. |
0198ffd13 padata: Add some ... |
154 155 156 157 158 159 |
* * Return values are: * * A pointer to the control struct of the next object that needs * serialization, if present in one of the percpu reorder queues. * |
bfde23ce2 padata: unbind pa... |
160 |
* NULL, if the next object that needs serialization will |
0198ffd13 padata: Add some ... |
161 162 |
* be parallel processed by another cpu and is not yet present in * the cpu's reorder queue. |
0198ffd13 padata: Add some ... |
163 |
*/ |
bfde23ce2 padata: unbind pa... |
164 165 |
static struct padata_priv *padata_find_next(struct parallel_data *pd, bool remove_object) |
16295bec6 padata: Generic p... |
166 |
{ |
f0fcf2002 padata: use __thi... |
167 |
struct padata_parallel_queue *next_queue; |
16295bec6 padata: Generic p... |
168 169 |
struct padata_priv *padata; struct padata_list *reorder; |
6fc4dbcf0 padata: Replace d... |
170 |
int cpu = pd->cpu; |
16295bec6 padata: Generic p... |
171 |
|
e15bacbeb padata: Make two ... |
172 |
next_queue = per_cpu_ptr(pd->pqueue, cpu); |
16295bec6 padata: Generic p... |
173 |
reorder = &next_queue->reorder; |
de5540d08 padata: avoid rac... |
174 |
spin_lock(&reorder->lock); |
bfde23ce2 padata: unbind pa... |
175 176 177 178 |
if (list_empty(&reorder->list)) { spin_unlock(&reorder->lock); return NULL; } |
16295bec6 padata: Generic p... |
179 |
|
bfde23ce2 padata: unbind pa... |
180 |
padata = list_entry(reorder->list.next, struct padata_priv, list); |
16295bec6 padata: Generic p... |
181 |
|
bfde23ce2 padata: unbind pa... |
182 183 184 185 186 |
/* * Checks the rare case where two or more parallel jobs have hashed to * the same CPU and one of the later ones finishes first. */ if (padata->seq_nr != pd->processed) { |
de5540d08 padata: avoid rac... |
187 |
spin_unlock(&reorder->lock); |
bfde23ce2 padata: unbind pa... |
188 |
return NULL; |
16295bec6 padata: Generic p... |
189 |
} |
bfde23ce2 padata: unbind pa... |
190 191 192 193 194 |
if (remove_object) { list_del_init(&padata->list); atomic_dec(&pd->reorder_objects); ++pd->processed; pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false); |
16295bec6 padata: Generic p... |
195 |
} |
bfde23ce2 padata: unbind pa... |
196 |
spin_unlock(&reorder->lock); |
16295bec6 padata: Generic p... |
197 198 199 200 201 |
return padata; } static void padata_reorder(struct parallel_data *pd) { |
b4c8ed0bf crypto: pcrypt - ... |
202 |
struct padata_instance *pinst = pd->ps->pinst; |
3047817b8 padata: Fix race ... |
203 |
int cb_cpu; |
16295bec6 padata: Generic p... |
204 |
struct padata_priv *padata; |
e15bacbeb padata: Make two ... |
205 |
struct padata_serial_queue *squeue; |
6fc4dbcf0 padata: Replace d... |
206 |
struct padata_parallel_queue *next_queue; |
16295bec6 padata: Generic p... |
207 |
|
0198ffd13 padata: Add some ... |
208 209 210 211 212 213 214 215 216 217 |
/* * We need to ensure that only one cpu can work on dequeueing of * the reorder queue the time. Calculating in which percpu reorder * queue the next object will arrive takes some time. A spinlock * would be highly contended. Also it is not clear in which order * the objects arrive to the reorder queues. So a cpu could wait to * get the lock just to notice that there is nothing to do at the * moment. Therefore we use a trylock and let the holder of the lock * care for all the objects enqueued during the holdtime of the lock. */ |
16295bec6 padata: Generic p... |
218 |
if (!spin_trylock_bh(&pd->lock)) |
d46a5ac7a padata: Use a tim... |
219 |
return; |
16295bec6 padata: Generic p... |
220 221 |
while (1) { |
bfde23ce2 padata: unbind pa... |
222 |
padata = padata_find_next(pd, true); |
16295bec6 padata: Generic p... |
223 |
|
0198ffd13 padata: Add some ... |
224 |
/* |
69b348449 padata: get_next ... |
225 226 227 |
* If the next object that needs serialization is parallel * processed by another cpu and is still on it's way to the * cpu's reorder queue, nothing to do for now. |
0198ffd13 padata: Add some ... |
228 |
*/ |
bfde23ce2 padata: unbind pa... |
229 |
if (!padata) |
16295bec6 padata: Generic p... |
230 |
break; |
3047817b8 padata: Fix race ... |
231 232 |
cb_cpu = padata->cb_cpu; squeue = per_cpu_ptr(pd->squeue, cb_cpu); |
16295bec6 padata: Generic p... |
233 |
|
e15bacbeb padata: Make two ... |
234 235 236 |
spin_lock(&squeue->serial.lock); list_add_tail(&padata->list, &squeue->serial.list); spin_unlock(&squeue->serial.lock); |
16295bec6 padata: Generic p... |
237 |
|
45d153c08 padata: use separ... |
238 |
queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work); |
16295bec6 padata: Generic p... |
239 240 241 |
} spin_unlock_bh(&pd->lock); |
0198ffd13 padata: Add some ... |
242 243 |
/* * The next object that needs serialization might have arrived to |
6fc4dbcf0 padata: Replace d... |
244 |
* the reorder queues in the meantime. |
cf144f81a padata: use smp_m... |
245 |
* |
6fc4dbcf0 padata: Replace d... |
246 247 |
* Ensure reorder queue is read after pd->lock is dropped so we see * new objects from another task in padata_do_serial. Pairs with |
cf144f81a padata: use smp_m... |
248 |
* smp_mb__after_atomic in padata_do_serial. |
0198ffd13 padata: Add some ... |
249 |
*/ |
cf144f81a padata: use smp_m... |
250 |
smp_mb(); |
16295bec6 padata: Generic p... |
251 |
|
6fc4dbcf0 padata: Replace d... |
252 |
next_queue = per_cpu_ptr(pd->pqueue, pd->cpu); |
bfde23ce2 padata: unbind pa... |
253 254 |
if (!list_empty(&next_queue->reorder.list) && padata_find_next(pd, false)) |
45d153c08 padata: use separ... |
255 |
queue_work(pinst->serial_wq, &pd->reorder_work); |
16295bec6 padata: Generic p... |
256 |
} |
cf5868c8a padata: ensure th... |
257 258 |
static void invoke_padata_reorder(struct work_struct *work) { |
cf5868c8a padata: ensure th... |
259 260 261 |
struct parallel_data *pd; local_bh_disable(); |
6fc4dbcf0 padata: Replace d... |
262 |
pd = container_of(work, struct parallel_data, reorder_work); |
cf5868c8a padata: ensure th... |
263 264 265 |
padata_reorder(pd); local_bh_enable(); } |
e15bacbeb padata: Make two ... |
266 |
static void padata_serial_worker(struct work_struct *serial_work) |
16295bec6 padata: Generic p... |
267 |
{ |
e15bacbeb padata: Make two ... |
268 |
struct padata_serial_queue *squeue; |
16295bec6 padata: Generic p... |
269 270 |
struct parallel_data *pd; LIST_HEAD(local_list); |
5fefc9b3e padata: Remove br... |
271 |
int cnt; |
16295bec6 padata: Generic p... |
272 273 |
local_bh_disable(); |
e15bacbeb padata: Make two ... |
274 275 |
squeue = container_of(serial_work, struct padata_serial_queue, work); pd = squeue->pd; |
16295bec6 padata: Generic p... |
276 |
|
e15bacbeb padata: Make two ... |
277 278 279 |
spin_lock(&squeue->serial.lock); list_replace_init(&squeue->serial.list, &local_list); spin_unlock(&squeue->serial.lock); |
16295bec6 padata: Generic p... |
280 |
|
5fefc9b3e padata: Remove br... |
281 |
cnt = 0; |
16295bec6 padata: Generic p... |
282 283 284 285 286 287 288 289 290 |
while (!list_empty(&local_list)) { struct padata_priv *padata; padata = list_entry(local_list.next, struct padata_priv, list); list_del_init(&padata->list); padata->serial(padata); |
5fefc9b3e padata: Remove br... |
291 |
cnt++; |
16295bec6 padata: Generic p... |
292 293 |
} local_bh_enable(); |
5fefc9b3e padata: Remove br... |
294 295 296 |
if (atomic_sub_and_test(cnt, &pd->refcnt)) padata_free_pd(pd); |
16295bec6 padata: Generic p... |
297 |
} |
0198ffd13 padata: Add some ... |
298 |
/** |
16295bec6 padata: Generic p... |
299 300 301 302 303 304 305 306 307 |
* padata_do_serial - padata serialization function * * @padata: object to be serialized. * * padata_do_serial must be called for every parallelized object. * The serialization callback function will run with BHs off. */ void padata_do_serial(struct padata_priv *padata) { |
065cf5771 padata: purge get... |
308 309 310 |
struct parallel_data *pd = padata->pd; struct padata_parallel_queue *pqueue = per_cpu_ptr(pd->pqueue, padata->cpu); |
bfde23ce2 padata: unbind pa... |
311 |
struct padata_priv *cur; |
16295bec6 padata: Generic p... |
312 |
|
e15bacbeb padata: Make two ... |
313 |
spin_lock(&pqueue->reorder.lock); |
bfde23ce2 padata: unbind pa... |
314 315 316 317 318 |
/* Sort in ascending order of sequence number. */ list_for_each_entry_reverse(cur, &pqueue->reorder.list, list) if (cur->seq_nr < padata->seq_nr) break; list_add(&padata->list, &cur->list); |
6fc4dbcf0 padata: Replace d... |
319 |
atomic_inc(&pd->reorder_objects); |
e15bacbeb padata: Make two ... |
320 |
spin_unlock(&pqueue->reorder.lock); |
16295bec6 padata: Generic p... |
321 |
|
cf144f81a padata: use smp_m... |
322 |
/* |
6fc4dbcf0 padata: Replace d... |
323 |
* Ensure the addition to the reorder list is ordered correctly |
cf144f81a padata: use smp_m... |
324 325 326 327 |
* with the trylock of pd->lock in padata_reorder. Pairs with smp_mb * in padata_reorder. */ smp_mb__after_atomic(); |
6fc4dbcf0 padata: Replace d... |
328 |
padata_reorder(pd); |
16295bec6 padata: Generic p... |
329 330 |
} EXPORT_SYMBOL(padata_do_serial); |
b4c8ed0bf crypto: pcrypt - ... |
331 |
static int padata_setup_cpumasks(struct padata_instance *pinst) |
16295bec6 padata: Generic p... |
332 |
{ |
bfde23ce2 padata: unbind pa... |
333 |
struct workqueue_attrs *attrs; |
b4c8ed0bf crypto: pcrypt - ... |
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 |
int err; attrs = alloc_workqueue_attrs(); if (!attrs) return -ENOMEM; /* Restrict parallel_wq workers to pd->cpumask.pcpu. */ cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu); err = apply_workqueue_attrs(pinst->parallel_wq, attrs); free_workqueue_attrs(attrs); return err; } static int pd_setup_cpumasks(struct parallel_data *pd, const struct cpumask *pcpumask, const struct cpumask *cbcpumask) { |
bfde23ce2 padata: unbind pa... |
352 |
int err = -ENOMEM; |
16295bec6 padata: Generic p... |
353 |
|
bfde23ce2 padata: unbind pa... |
354 355 |
if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) goto out; |
bfde23ce2 padata: unbind pa... |
356 357 |
if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) goto free_pcpu_mask; |
bfde23ce2 padata: unbind pa... |
358 |
|
b4c8ed0bf crypto: pcrypt - ... |
359 360 |
cpumask_copy(pd->cpumask.pcpu, pcpumask); cpumask_copy(pd->cpumask.cbcpu, cbcpumask); |
bfde23ce2 padata: unbind pa... |
361 |
|
e15bacbeb padata: Make two ... |
362 |
return 0; |
bfde23ce2 padata: unbind pa... |
363 |
|
bfde23ce2 padata: unbind pa... |
364 365 366 367 |
free_pcpu_mask: free_cpumask_var(pd->cpumask.pcpu); out: return err; |
e15bacbeb padata: Make two ... |
368 |
} |
16295bec6 padata: Generic p... |
369 |
|
e15bacbeb padata: Make two ... |
370 371 372 373 374 |
static void __padata_list_init(struct padata_list *pd_list) { INIT_LIST_HEAD(&pd_list->list); spin_lock_init(&pd_list->lock); } |
16295bec6 padata: Generic p... |
375 |
|
e15bacbeb padata: Make two ... |
376 377 378 379 380 |
/* Initialize all percpu queues used by serial workers */ static void padata_init_squeues(struct parallel_data *pd) { int cpu; struct padata_serial_queue *squeue; |
7b389b2cc padata: Initializ... |
381 |
|
e15bacbeb padata: Make two ... |
382 383 384 385 386 387 388 |
for_each_cpu(cpu, pd->cpumask.cbcpu) { squeue = per_cpu_ptr(pd->squeue, cpu); squeue->pd = pd; __padata_list_init(&squeue->serial); INIT_WORK(&squeue->work, padata_serial_worker); } } |
16295bec6 padata: Generic p... |
389 |
|
e15bacbeb padata: Make two ... |
390 391 392 |
/* Initialize all percpu queues used by parallel workers */ static void padata_init_pqueues(struct parallel_data *pd) { |
c51636a30 padata: remove cp... |
393 |
int cpu; |
e15bacbeb padata: Make two ... |
394 |
struct padata_parallel_queue *pqueue; |
16295bec6 padata: Generic p... |
395 |
|
c51636a30 padata: remove cp... |
396 |
for_each_cpu(cpu, pd->cpumask.pcpu) { |
e15bacbeb padata: Make two ... |
397 |
pqueue = per_cpu_ptr(pd->pqueue, cpu); |
1bd845bcb padata: set cpu_i... |
398 |
|
e15bacbeb padata: Make two ... |
399 400 401 402 |
__padata_list_init(&pqueue->reorder); __padata_list_init(&pqueue->parallel); INIT_WORK(&pqueue->work, padata_parallel_worker); atomic_set(&pqueue->num_obj, 0); |
16295bec6 padata: Generic p... |
403 |
} |
e15bacbeb padata: Make two ... |
404 |
} |
16295bec6 padata: Generic p... |
405 |
|
e15bacbeb padata: Make two ... |
406 |
/* Allocate and initialize the internal cpumask dependend resources. */ |
b4c8ed0bf crypto: pcrypt - ... |
407 |
static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) |
e15bacbeb padata: Make two ... |
408 |
{ |
b4c8ed0bf crypto: pcrypt - ... |
409 410 411 |
struct padata_instance *pinst = ps->pinst; const struct cpumask *cbcpumask; const struct cpumask *pcpumask; |
e15bacbeb padata: Make two ... |
412 |
struct parallel_data *pd; |
16295bec6 padata: Generic p... |
413 |
|
b4c8ed0bf crypto: pcrypt - ... |
414 415 |
cbcpumask = pinst->rcpumask.cbcpu; pcpumask = pinst->rcpumask.pcpu; |
e15bacbeb padata: Make two ... |
416 417 418 |
pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); if (!pd) goto err; |
16295bec6 padata: Generic p... |
419 |
|
e15bacbeb padata: Make two ... |
420 421 422 423 424 425 426 |
pd->pqueue = alloc_percpu(struct padata_parallel_queue); if (!pd->pqueue) goto err_free_pd; pd->squeue = alloc_percpu(struct padata_serial_queue); if (!pd->squeue) goto err_free_pqueue; |
bfde23ce2 padata: unbind pa... |
427 |
|
b4c8ed0bf crypto: pcrypt - ... |
428 429 |
pd->ps = ps; if (pd_setup_cpumasks(pd, pcpumask, cbcpumask)) |
e15bacbeb padata: Make two ... |
430 |
goto err_free_squeue; |
16295bec6 padata: Generic p... |
431 |
|
e15bacbeb padata: Make two ... |
432 433 |
padata_init_pqueues(pd); padata_init_squeues(pd); |
0b6b098ef padata: make the ... |
434 |
atomic_set(&pd->seq_nr, -1); |
16295bec6 padata: Generic p... |
435 |
atomic_set(&pd->reorder_objects, 0); |
5fefc9b3e padata: Remove br... |
436 |
atomic_set(&pd->refcnt, 1); |
16295bec6 padata: Generic p... |
437 |
spin_lock_init(&pd->lock); |
ec9c7d193 padata: initializ... |
438 |
pd->cpu = cpumask_first(pd->cpumask.pcpu); |
6fc4dbcf0 padata: Replace d... |
439 |
INIT_WORK(&pd->reorder_work, invoke_padata_reorder); |
16295bec6 padata: Generic p... |
440 441 |
return pd; |
e15bacbeb padata: Make two ... |
442 443 444 445 |
err_free_squeue: free_percpu(pd->squeue); err_free_pqueue: free_percpu(pd->pqueue); |
16295bec6 padata: Generic p... |
446 447 448 449 450 451 452 453 |
err_free_pd: kfree(pd); err: return NULL; } static void padata_free_pd(struct parallel_data *pd) { |
e15bacbeb padata: Make two ... |
454 455 456 457 |
free_cpumask_var(pd->cpumask.pcpu); free_cpumask_var(pd->cpumask.cbcpu); free_percpu(pd->pqueue); free_percpu(pd->squeue); |
16295bec6 padata: Generic p... |
458 459 |
kfree(pd); } |
4c8791702 padata: Check for... |
460 461 462 463 |
static void __padata_start(struct padata_instance *pinst) { pinst->flags |= PADATA_INIT; } |
ee8365551 padata: Block unt... |
464 465 466 467 468 469 470 471 |
static void __padata_stop(struct padata_instance *pinst) { if (!(pinst->flags & PADATA_INIT)) return; pinst->flags &= ~PADATA_INIT; synchronize_rcu(); |
ee8365551 padata: Block unt... |
472 |
} |
25985edce Fix common misspe... |
473 |
/* Replace the internal control structure with a new one. */ |
b4c8ed0bf crypto: pcrypt - ... |
474 |
static int padata_replace_one(struct padata_shell *ps) |
16295bec6 padata: Generic p... |
475 |
{ |
b4c8ed0bf crypto: pcrypt - ... |
476 |
struct parallel_data *pd_new; |
16295bec6 padata: Generic p... |
477 |
|
b4c8ed0bf crypto: pcrypt - ... |
478 479 480 |
pd_new = padata_alloc_pd(ps); if (!pd_new) return -ENOMEM; |
16295bec6 padata: Generic p... |
481 |
|
b4c8ed0bf crypto: pcrypt - ... |
482 483 |
ps->opd = rcu_dereference_protected(ps->pd, 1); rcu_assign_pointer(ps->pd, pd_new); |
16295bec6 padata: Generic p... |
484 |
|
b4c8ed0bf crypto: pcrypt - ... |
485 486 |
return 0; } |
0685dfa0a padata: validate ... |
487 |
static int padata_replace(struct padata_instance *pinst) |
b4c8ed0bf crypto: pcrypt - ... |
488 489 490 491 492 493 |
{ int notification_mask = 0; struct padata_shell *ps; int err; pinst->flags |= PADATA_RESET; |
16295bec6 padata: Generic p... |
494 |
|
b4c8ed0bf crypto: pcrypt - ... |
495 496 497 |
cpumask_copy(pinst->omask, pinst->rcpumask.pcpu); cpumask_and(pinst->rcpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask); |
b4c8ed0bf crypto: pcrypt - ... |
498 |
if (!cpumask_equal(pinst->omask, pinst->rcpumask.pcpu)) |
e15bacbeb padata: Make two ... |
499 |
notification_mask |= PADATA_CPU_PARALLEL; |
b4c8ed0bf crypto: pcrypt - ... |
500 501 502 503 |
cpumask_copy(pinst->omask, pinst->rcpumask.cbcpu); cpumask_and(pinst->rcpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask); |
b4c8ed0bf crypto: pcrypt - ... |
504 |
if (!cpumask_equal(pinst->omask, pinst->rcpumask.cbcpu)) |
e15bacbeb padata: Make two ... |
505 |
notification_mask |= PADATA_CPU_SERIAL; |
b4c8ed0bf crypto: pcrypt - ... |
506 507 508 509 510 511 512 513 514 515 516 |
list_for_each_entry(ps, &pinst->pslist, list) { err = padata_replace_one(ps); if (err) break; } synchronize_rcu(); list_for_each_entry_continue_reverse(ps, &pinst->pslist, list) if (atomic_dec_and_test(&ps->opd->refcnt)) padata_free_pd(ps->opd); |
16295bec6 padata: Generic p... |
517 |
|
e15bacbeb padata: Make two ... |
518 519 |
if (notification_mask) blocking_notifier_call_chain(&pinst->cpumask_change_notifier, |
c635696c7 padata: Pass the ... |
520 |
notification_mask, |
b4c8ed0bf crypto: pcrypt - ... |
521 |
&pinst->cpumask); |
16295bec6 padata: Generic p... |
522 523 |
pinst->flags &= ~PADATA_RESET; |
b4c8ed0bf crypto: pcrypt - ... |
524 525 |
return err; |
16295bec6 padata: Generic p... |
526 |
} |
0198ffd13 padata: Add some ... |
527 |
/** |
e15bacbeb padata: Make two ... |
528 529 |
* padata_register_cpumask_notifier - Registers a notifier that will be called * if either pcpu or cbcpu or both cpumasks change. |
16295bec6 padata: Generic p... |
530 |
* |
e15bacbeb padata: Make two ... |
531 532 |
* @pinst: A poineter to padata instance * @nblock: A pointer to notifier block. |
16295bec6 padata: Generic p... |
533 |
*/ |
e15bacbeb padata: Make two ... |
534 535 |
int padata_register_cpumask_notifier(struct padata_instance *pinst, struct notifier_block *nblock) |
16295bec6 padata: Generic p... |
536 |
{ |
e15bacbeb padata: Make two ... |
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 |
return blocking_notifier_chain_register(&pinst->cpumask_change_notifier, nblock); } EXPORT_SYMBOL(padata_register_cpumask_notifier); /** * padata_unregister_cpumask_notifier - Unregisters cpumask notifier * registered earlier using padata_register_cpumask_notifier * * @pinst: A pointer to data instance. * @nlock: A pointer to notifier block. */ int padata_unregister_cpumask_notifier(struct padata_instance *pinst, struct notifier_block *nblock) { return blocking_notifier_chain_unregister( &pinst->cpumask_change_notifier, nblock); } EXPORT_SYMBOL(padata_unregister_cpumask_notifier); |
33e544506 padata: Handle em... |
557 558 559 560 |
/* If cpumask contains no active cpu, we mark the instance as invalid. */ static bool padata_validate_cpumask(struct padata_instance *pinst, const struct cpumask *cpumask) { |
13614e0fb padata: Use the o... |
561 |
if (!cpumask_intersects(cpumask, cpu_online_mask)) { |
33e544506 padata: Handle em... |
562 563 564 565 566 567 568 |
pinst->flags |= PADATA_INVALID; return false; } pinst->flags &= ~PADATA_INVALID; return true; } |
65ff577e6 padata: Rearrange... |
569 570 571 572 573 |
static int __padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, cpumask_var_t cbcpumask) { int valid; |
b4c8ed0bf crypto: pcrypt - ... |
574 |
int err; |
65ff577e6 padata: Rearrange... |
575 576 577 578 579 580 581 582 583 584 585 586 |
valid = padata_validate_cpumask(pinst, pcpumask); if (!valid) { __padata_stop(pinst); goto out_replace; } valid = padata_validate_cpumask(pinst, cbcpumask); if (!valid) __padata_stop(pinst); out_replace: |
65ff577e6 padata: Rearrange... |
587 588 |
cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); |
0685dfa0a padata: validate ... |
589 |
err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst); |
65ff577e6 padata: Rearrange... |
590 591 592 |
if (valid) __padata_start(pinst); |
b4c8ed0bf crypto: pcrypt - ... |
593 |
return err; |
65ff577e6 padata: Rearrange... |
594 595 596 |
} /** |
e15bacbeb padata: Make two ... |
597 598 |
* padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value * equivalent to @cpumask. |
16295bec6 padata: Generic p... |
599 600 |
* * @pinst: padata instance |
e15bacbeb padata: Make two ... |
601 602 |
* @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding * to parallel and serial cpumasks respectively. |
16295bec6 padata: Generic p... |
603 604 |
* @cpumask: the cpumask to use */ |
e15bacbeb padata: Make two ... |
605 606 607 608 |
int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask) { struct cpumask *serial_mask, *parallel_mask; |
65ff577e6 padata: Rearrange... |
609 610 611 |
int err = -EINVAL; mutex_lock(&pinst->lock); |
6751fb3c0 padata: Use get_o... |
612 |
get_online_cpus(); |
e15bacbeb padata: Make two ... |
613 614 615 616 617 618 619 620 621 622 |
switch (cpumask_type) { case PADATA_CPU_PARALLEL: serial_mask = pinst->cpumask.cbcpu; parallel_mask = cpumask; break; case PADATA_CPU_SERIAL: parallel_mask = pinst->cpumask.pcpu; serial_mask = cpumask; break; default: |
65ff577e6 padata: Rearrange... |
623 |
goto out; |
16295bec6 padata: Generic p... |
624 |
} |
65ff577e6 padata: Rearrange... |
625 |
err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask); |
16295bec6 padata: Generic p... |
626 627 |
out: |
6751fb3c0 padata: Use get_o... |
628 |
put_online_cpus(); |
16295bec6 padata: Generic p... |
629 630 631 632 633 |
mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_set_cpumask); |
19d795b67 kernel/padata.c: ... |
634 635 636 637 638 639 640 641 642 643 644 645 646 |
/** * padata_start - start the parallel processing * * @pinst: padata instance to start */ int padata_start(struct padata_instance *pinst) { int err = 0; mutex_lock(&pinst->lock); if (pinst->flags & PADATA_INVALID) err = -EINVAL; |
8ddab4287 padata: clean an ... |
647 |
__padata_start(pinst); |
19d795b67 kernel/padata.c: ... |
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 |
mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_start); /** * padata_stop - stop the parallel processing * * @pinst: padata instance to stop */ void padata_stop(struct padata_instance *pinst) { mutex_lock(&pinst->lock); __padata_stop(pinst); mutex_unlock(&pinst->lock); } EXPORT_SYMBOL(padata_stop); #ifdef CONFIG_HOTPLUG_CPU |
16295bec6 padata: Generic p... |
669 670 |
static int __padata_add_cpu(struct padata_instance *pinst, int cpu) { |
b4c8ed0bf crypto: pcrypt - ... |
671 |
int err = 0; |
16295bec6 padata: Generic p... |
672 |
|
13614e0fb padata: Use the o... |
673 |
if (cpumask_test_cpu(cpu, cpu_online_mask)) { |
0685dfa0a padata: validate ... |
674 |
err = padata_replace(pinst); |
33e544506 padata: Handle em... |
675 |
|
e15bacbeb padata: Make two ... |
676 677 |
if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) |
33e544506 padata: Handle em... |
678 |
__padata_start(pinst); |
16295bec6 padata: Generic p... |
679 |
} |
b4c8ed0bf crypto: pcrypt - ... |
680 |
return err; |
16295bec6 padata: Generic p... |
681 |
} |
16295bec6 padata: Generic p... |
682 683 |
static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) { |
b4c8ed0bf crypto: pcrypt - ... |
684 |
int err = 0; |
16295bec6 padata: Generic p... |
685 |
|
0685dfa0a padata: validate ... |
686 |
if (!cpumask_test_cpu(cpu, cpu_online_mask)) { |
e15bacbeb padata: Make two ... |
687 |
if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || |
b89661dff padata: Allocate ... |
688 |
!padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) |
33e544506 padata: Handle em... |
689 |
__padata_stop(pinst); |
33e544506 padata: Handle em... |
690 |
|
0685dfa0a padata: validate ... |
691 |
err = padata_replace(pinst); |
16295bec6 padata: Generic p... |
692 |
} |
b4c8ed0bf crypto: pcrypt - ... |
693 |
return err; |
16295bec6 padata: Generic p... |
694 |
} |
e15bacbeb padata: Make two ... |
695 |
/** |
25985edce Fix common misspe... |
696 |
* padata_remove_cpu - remove a cpu from the one or both(serial and parallel) |
e15bacbeb padata: Make two ... |
697 |
* padata cpumasks. |
16295bec6 padata: Generic p... |
698 699 700 |
* * @pinst: padata instance * @cpu: cpu to remove |
e15bacbeb padata: Make two ... |
701 702 703 704 |
* @mask: bitmask specifying from which cpumask @cpu should be removed * The @mask may be any combination of the following flags: * PADATA_CPU_SERIAL - serial cpumask * PADATA_CPU_PARALLEL - parallel cpumask |
16295bec6 padata: Generic p... |
705 |
*/ |
e15bacbeb padata: Make two ... |
706 |
int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask) |
16295bec6 padata: Generic p... |
707 708 |
{ int err; |
e15bacbeb padata: Make two ... |
709 710 |
if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL))) return -EINVAL; |
16295bec6 padata: Generic p... |
711 |
mutex_lock(&pinst->lock); |
6751fb3c0 padata: Use get_o... |
712 |
get_online_cpus(); |
e15bacbeb padata: Make two ... |
713 714 715 716 |
if (mask & PADATA_CPU_SERIAL) cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu); if (mask & PADATA_CPU_PARALLEL) cpumask_clear_cpu(cpu, pinst->cpumask.pcpu); |
16295bec6 padata: Generic p... |
717 |
err = __padata_remove_cpu(pinst, cpu); |
6751fb3c0 padata: Use get_o... |
718 |
put_online_cpus(); |
16295bec6 padata: Generic p... |
719 720 721 722 723 724 |
mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_remove_cpu); |
e15bacbeb padata: Make two ... |
725 726 727 728 729 |
static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) { return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || cpumask_test_cpu(cpu, pinst->cpumask.cbcpu); } |
30e92153b padata: Convert t... |
730 |
static int padata_cpu_online(unsigned int cpu, struct hlist_node *node) |
16295bec6 padata: Generic p... |
731 |
{ |
16295bec6 padata: Generic p... |
732 |
struct padata_instance *pinst; |
30e92153b padata: Convert t... |
733 |
int ret; |
16295bec6 padata: Generic p... |
734 |
|
30e92153b padata: Convert t... |
735 736 737 |
pinst = hlist_entry_safe(node, struct padata_instance, node); if (!pinst_has_cpu(pinst, cpu)) return 0; |
16295bec6 padata: Generic p... |
738 |
|
30e92153b padata: Convert t... |
739 740 741 742 743 |
mutex_lock(&pinst->lock); ret = __padata_add_cpu(pinst, cpu); mutex_unlock(&pinst->lock); return ret; } |
16295bec6 padata: Generic p... |
744 |
|
0685dfa0a padata: validate ... |
745 |
static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node) |
30e92153b padata: Convert t... |
746 747 748 749 750 751 752 |
{ struct padata_instance *pinst; int ret; pinst = hlist_entry_safe(node, struct padata_instance, node); if (!pinst_has_cpu(pinst, cpu)) return 0; |
16295bec6 padata: Generic p... |
753 |
|
30e92153b padata: Convert t... |
754 755 756 757 |
mutex_lock(&pinst->lock); ret = __padata_remove_cpu(pinst, cpu); mutex_unlock(&pinst->lock); return ret; |
16295bec6 padata: Generic p... |
758 |
} |
30e92153b padata: Convert t... |
759 760 |
static enum cpuhp_state hp_online; |
e2cb2f1c2 padata: cpu hotpl... |
761 |
#endif |
16295bec6 padata: Generic p... |
762 |
|
5e017dc3f padata: Added sys... |
763 764 765 |
static void __padata_free(struct padata_instance *pinst) { #ifdef CONFIG_HOTPLUG_CPU |
0685dfa0a padata: validate ... |
766 |
cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD, &pinst->node); |
30e92153b padata: Convert t... |
767 |
cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node); |
5e017dc3f padata: Added sys... |
768 |
#endif |
b4c8ed0bf crypto: pcrypt - ... |
769 |
WARN_ON(!list_empty(&pinst->pslist)); |
5e017dc3f padata: Added sys... |
770 |
padata_stop(pinst); |
b4c8ed0bf crypto: pcrypt - ... |
771 772 773 |
free_cpumask_var(pinst->omask); free_cpumask_var(pinst->rcpumask.cbcpu); free_cpumask_var(pinst->rcpumask.pcpu); |
5e017dc3f padata: Added sys... |
774 775 |
free_cpumask_var(pinst->cpumask.pcpu); free_cpumask_var(pinst->cpumask.cbcpu); |
45d153c08 padata: use separ... |
776 777 |
destroy_workqueue(pinst->serial_wq); destroy_workqueue(pinst->parallel_wq); |
5e017dc3f padata: Added sys... |
778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 |
kfree(pinst); } #define kobj2pinst(_kobj) \ container_of(_kobj, struct padata_instance, kobj) #define attr2pentry(_attr) \ container_of(_attr, struct padata_sysfs_entry, attr) static void padata_sysfs_release(struct kobject *kobj) { struct padata_instance *pinst = kobj2pinst(kobj); __padata_free(pinst); } struct padata_sysfs_entry { struct attribute attr; ssize_t (*show)(struct padata_instance *, struct attribute *, char *); ssize_t (*store)(struct padata_instance *, struct attribute *, const char *, size_t); }; static ssize_t show_cpumask(struct padata_instance *pinst, struct attribute *attr, char *buf) { struct cpumask *cpumask; ssize_t len; mutex_lock(&pinst->lock); if (!strcmp(attr->name, "serial_cpumask")) cpumask = pinst->cpumask.cbcpu; else cpumask = pinst->cpumask.pcpu; |
4497da6f9 padata: use %*pb[... |
810 811 812 |
len = snprintf(buf, PAGE_SIZE, "%*pb ", nr_cpu_ids, cpumask_bits(cpumask)); |
5e017dc3f padata: Added sys... |
813 |
mutex_unlock(&pinst->lock); |
4497da6f9 padata: use %*pb[... |
814 |
return len < PAGE_SIZE ? len : -EINVAL; |
5e017dc3f padata: Added sys... |
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 |
} static ssize_t store_cpumask(struct padata_instance *pinst, struct attribute *attr, const char *buf, size_t count) { cpumask_var_t new_cpumask; ssize_t ret; int mask_type; if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL)) return -ENOMEM; ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask), nr_cpumask_bits); if (ret < 0) goto out; mask_type = !strcmp(attr->name, "serial_cpumask") ? PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL; ret = padata_set_cpumask(pinst, mask_type, new_cpumask); if (!ret) ret = count; out: free_cpumask_var(new_cpumask); return ret; } #define PADATA_ATTR_RW(_name, _show_name, _store_name) \ static struct padata_sysfs_entry _name##_attr = \ __ATTR(_name, 0644, _show_name, _store_name) #define PADATA_ATTR_RO(_name, _show_name) \ static struct padata_sysfs_entry _name##_attr = \ __ATTR(_name, 0400, _show_name, NULL) PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask); PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask); /* * Padata sysfs provides the following objects: * serial_cpumask [RW] - cpumask for serial workers * parallel_cpumask [RW] - cpumask for parallel workers */ static struct attribute *padata_default_attrs[] = { &serial_cpumask_attr.attr, ¶llel_cpumask_attr.attr, NULL, }; |
2064fbc77 padata: Replace p... |
864 |
ATTRIBUTE_GROUPS(padata_default); |
5e017dc3f padata: Added sys... |
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 |
static ssize_t padata_sysfs_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct padata_instance *pinst; struct padata_sysfs_entry *pentry; ssize_t ret = -EIO; pinst = kobj2pinst(kobj); pentry = attr2pentry(attr); if (pentry->show) ret = pentry->show(pinst, attr, buf); return ret; } static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { struct padata_instance *pinst; struct padata_sysfs_entry *pentry; ssize_t ret = -EIO; pinst = kobj2pinst(kobj); pentry = attr2pentry(attr); if (pentry->show) ret = pentry->store(pinst, attr, buf, count); return ret; } static const struct sysfs_ops padata_sysfs_ops = { .show = padata_sysfs_show, .store = padata_sysfs_store, }; static struct kobj_type padata_attr_type = { .sysfs_ops = &padata_sysfs_ops, |
2064fbc77 padata: Replace p... |
903 |
.default_groups = padata_default_groups, |
5e017dc3f padata: Added sys... |
904 905 |
.release = padata_sysfs_release, }; |
0198ffd13 padata: Add some ... |
906 |
/** |
e6cc11707 padata: Rename pa... |
907 908 |
* padata_alloc - allocate and initialize a padata instance and specify * cpumasks for serial and parallel workers. |
16295bec6 padata: Generic p... |
909 |
* |
b128a3040 padata: allocate ... |
910 |
* @name: used to identify the instance |
e15bacbeb padata: Make two ... |
911 912 |
* @pcpumask: cpumask that will be used for padata parallelization * @cbcpumask: cpumask that will be used for padata serialization |
16295bec6 padata: Generic p... |
913 |
*/ |
b128a3040 padata: allocate ... |
914 |
static struct padata_instance *padata_alloc(const char *name, |
9596695ee padata: Make pada... |
915 916 |
const struct cpumask *pcpumask, const struct cpumask *cbcpumask) |
16295bec6 padata: Generic p... |
917 |
{ |
16295bec6 padata: Generic p... |
918 |
struct padata_instance *pinst; |
16295bec6 padata: Generic p... |
919 920 921 922 |
pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); if (!pinst) goto err; |
bfde23ce2 padata: unbind pa... |
923 924 |
pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0, name); |
45d153c08 padata: use separ... |
925 |
if (!pinst->parallel_wq) |
16295bec6 padata: Generic p... |
926 |
goto err_free_inst; |
b128a3040 padata: allocate ... |
927 |
|
cc491d8e6 padata, pcrypt: t... |
928 |
get_online_cpus(); |
45d153c08 padata: use separ... |
929 930 931 |
pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1, name); if (!pinst->serial_wq) |
cc491d8e6 padata, pcrypt: t... |
932 |
goto err_put_cpus; |
45d153c08 padata: use separ... |
933 934 935 |
if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) goto err_free_serial_wq; |
e15bacbeb padata: Make two ... |
936 937 |
if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { free_cpumask_var(pinst->cpumask.pcpu); |
45d153c08 padata: use separ... |
938 |
goto err_free_serial_wq; |
33e544506 padata: Handle em... |
939 |
} |
e15bacbeb padata: Make two ... |
940 941 942 |
if (!padata_validate_cpumask(pinst, pcpumask) || !padata_validate_cpumask(pinst, cbcpumask)) goto err_free_masks; |
16295bec6 padata: Generic p... |
943 |
|
b4c8ed0bf crypto: pcrypt - ... |
944 |
if (!alloc_cpumask_var(&pinst->rcpumask.pcpu, GFP_KERNEL)) |
e15bacbeb padata: Make two ... |
945 |
goto err_free_masks; |
b4c8ed0bf crypto: pcrypt - ... |
946 947 948 949 |
if (!alloc_cpumask_var(&pinst->rcpumask.cbcpu, GFP_KERNEL)) goto err_free_rcpumask_pcpu; if (!alloc_cpumask_var(&pinst->omask, GFP_KERNEL)) goto err_free_rcpumask_cbcpu; |
747813878 padata: Allocate ... |
950 |
|
b4c8ed0bf crypto: pcrypt - ... |
951 |
INIT_LIST_HEAD(&pinst->pslist); |
16295bec6 padata: Generic p... |
952 |
|
e15bacbeb padata: Make two ... |
953 954 |
cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); |
b4c8ed0bf crypto: pcrypt - ... |
955 956 957 958 959 |
cpumask_and(pinst->rcpumask.pcpu, pcpumask, cpu_online_mask); cpumask_and(pinst->rcpumask.cbcpu, cbcpumask, cpu_online_mask); if (padata_setup_cpumasks(pinst)) goto err_free_omask; |
16295bec6 padata: Generic p... |
960 961 |
pinst->flags = 0; |
e15bacbeb padata: Make two ... |
962 |
BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier); |
5e017dc3f padata: Added sys... |
963 |
kobject_init(&pinst->kobj, &padata_attr_type); |
16295bec6 padata: Generic p... |
964 |
mutex_init(&pinst->lock); |
b8b4a4166 padata - Register... |
965 |
#ifdef CONFIG_HOTPLUG_CPU |
c5a81c8ff padata: Avoid nes... |
966 |
cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, &pinst->node); |
0685dfa0a padata: validate ... |
967 968 |
cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD, &pinst->node); |
b8b4a4166 padata - Register... |
969 |
#endif |
cc491d8e6 padata, pcrypt: t... |
970 971 |
put_online_cpus(); |
16295bec6 padata: Generic p... |
972 |
return pinst; |
b4c8ed0bf crypto: pcrypt - ... |
973 974 975 976 977 978 |
err_free_omask: free_cpumask_var(pinst->omask); err_free_rcpumask_cbcpu: free_cpumask_var(pinst->rcpumask.cbcpu); err_free_rcpumask_pcpu: free_cpumask_var(pinst->rcpumask.pcpu); |
e15bacbeb padata: Make two ... |
979 980 981 |
err_free_masks: free_cpumask_var(pinst->cpumask.pcpu); free_cpumask_var(pinst->cpumask.cbcpu); |
45d153c08 padata: use separ... |
982 983 |
err_free_serial_wq: destroy_workqueue(pinst->serial_wq); |
cc491d8e6 padata, pcrypt: t... |
984 985 |
err_put_cpus: put_online_cpus(); |
45d153c08 padata: use separ... |
986 |
destroy_workqueue(pinst->parallel_wq); |
16295bec6 padata: Generic p... |
987 988 989 990 991 |
err_free_inst: kfree(pinst); err: return NULL; } |
16295bec6 padata: Generic p... |
992 |
|
0198ffd13 padata: Add some ... |
993 |
/** |
9596695ee padata: Make pada... |
994 995 996 997 |
* padata_alloc_possible - Allocate and initialize padata instance. * Use the cpu_possible_mask for serial and * parallel workers. * |
b128a3040 padata: allocate ... |
998 |
* @name: used to identify the instance |
9596695ee padata: Make pada... |
999 |
*/ |
b128a3040 padata: allocate ... |
1000 |
struct padata_instance *padata_alloc_possible(const char *name) |
9596695ee padata: Make pada... |
1001 |
{ |
b128a3040 padata: allocate ... |
1002 |
return padata_alloc(name, cpu_possible_mask, cpu_possible_mask); |
9596695ee padata: Make pada... |
1003 1004 1005 1006 |
} EXPORT_SYMBOL(padata_alloc_possible); /** |
16295bec6 padata: Generic p... |
1007 1008 |
* padata_free - free a padata instance * |
0198ffd13 padata: Add some ... |
1009 |
* @padata_inst: padata instance to free |
16295bec6 padata: Generic p... |
1010 1011 1012 |
*/ void padata_free(struct padata_instance *pinst) { |
5e017dc3f padata: Added sys... |
1013 |
kobject_put(&pinst->kobj); |
16295bec6 padata: Generic p... |
1014 1015 |
} EXPORT_SYMBOL(padata_free); |
30e92153b padata: Convert t... |
1016 |
|
b4c8ed0bf crypto: pcrypt - ... |
1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 |
/** * padata_alloc_shell - Allocate and initialize padata shell. * * @pinst: Parent padata_instance object. */ struct padata_shell *padata_alloc_shell(struct padata_instance *pinst) { struct parallel_data *pd; struct padata_shell *ps; ps = kzalloc(sizeof(*ps), GFP_KERNEL); if (!ps) goto out; ps->pinst = pinst; get_online_cpus(); pd = padata_alloc_pd(ps); put_online_cpus(); if (!pd) goto out_free_ps; mutex_lock(&pinst->lock); RCU_INIT_POINTER(ps->pd, pd); list_add(&ps->list, &pinst->pslist); mutex_unlock(&pinst->lock); return ps; out_free_ps: kfree(ps); out: return NULL; } EXPORT_SYMBOL(padata_alloc_shell); /** * padata_free_shell - free a padata shell * * @ps: padata shell to free */ void padata_free_shell(struct padata_shell *ps) { struct padata_instance *pinst = ps->pinst; mutex_lock(&pinst->lock); list_del(&ps->list); padata_free_pd(rcu_dereference_protected(ps->pd, 1)); mutex_unlock(&pinst->lock); kfree(ps); } EXPORT_SYMBOL(padata_free_shell); |
30e92153b padata: Convert t... |
1071 1072 1073 1074 1075 1076 1077 |
#ifdef CONFIG_HOTPLUG_CPU static __init int padata_driver_init(void) { int ret; ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online", |
0685dfa0a padata: validate ... |
1078 |
padata_cpu_online, NULL); |
30e92153b padata: Convert t... |
1079 1080 1081 |
if (ret < 0) return ret; hp_online = ret; |
0685dfa0a padata: validate ... |
1082 1083 1084 1085 1086 1087 1088 |
ret = cpuhp_setup_state_multi(CPUHP_PADATA_DEAD, "padata:dead", NULL, padata_cpu_dead); if (ret < 0) { cpuhp_remove_multi_state(hp_online); return ret; } |
30e92153b padata: Convert t... |
1089 1090 1091 1092 1093 1094 |
return 0; } module_init(padata_driver_init); static __exit void padata_driver_exit(void) { |
0685dfa0a padata: validate ... |
1095 |
cpuhp_remove_multi_state(CPUHP_PADATA_DEAD); |
30e92153b padata: Convert t... |
1096 1097 1098 1099 |
cpuhp_remove_multi_state(hp_online); } module_exit(padata_driver_exit); #endif |