Blame view

include/linux/cpumask.h 26.6 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
  #ifndef __LINUX_CPUMASK_H
  #define __LINUX_CPUMASK_H
  
  /*
   * Cpumasks provide a bitmap suitable for representing the
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
6
7
   * set of CPU's in a system, one bit position per CPU number.  In general,
   * only nr_cpu_ids (<= NR_CPUS) bits are valid.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
8
   */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
9
10
11
  #include <linux/kernel.h>
  #include <linux/threads.h>
  #include <linux/bitmap.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
12

2d3854a37   Rusty Russell   cpumask: introduc...
13
  typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
14

ae7a47e72   Rusty Russell   cpumask: make cpu...
15
  /**
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
16
17
   * cpumask_bits - get the bits in a cpumask
   * @maskp: the struct cpumask *
ae7a47e72   Rusty Russell   cpumask: make cpu...
18
   *
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
19
20
   * You should only assume nr_cpu_ids bits of this mask are valid.  This is
   * a macro so it's const-correct.
ae7a47e72   Rusty Russell   cpumask: make cpu...
21
   */
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
22
  #define cpumask_bits(maskp) ((maskp)->bits)
7ea931c9f   Paul Jackson   mempolicy: add bi...
23

41df0d61c   Mike Travis   x86: Add performa...
24
  #if NR_CPUS == 1
41df0d61c   Mike Travis   x86: Add performa...
25
  #define nr_cpu_ids		1
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
26
  #else
41df0d61c   Mike Travis   x86: Add performa...
27
  extern int nr_cpu_ids;
41df0d61c   Mike Travis   x86: Add performa...
28
  #endif
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
29
30
31
32
33
34
35
  #ifdef CONFIG_CPUMASK_OFFSTACK
  /* Assuming NR_CPUS is huge, a runtime limit is more efficient.  Also,
   * not all bits may be allocated. */
  #define nr_cpumask_bits	nr_cpu_ids
  #else
  #define nr_cpumask_bits	NR_CPUS
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
36
37
38
  
  /*
   * The following particular system cpumasks and operations manage
b3199c025   Rusty Russell   cpumask: switch o...
39
   * possible, present, active and online cpus.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
40
   *
b3199c025   Rusty Russell   cpumask: switch o...
41
42
43
44
   *     cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
   *     cpu_present_mask - has bit 'cpu' set iff cpu is populated
   *     cpu_online_mask  - has bit 'cpu' set iff cpu available to scheduler
   *     cpu_active_mask  - has bit 'cpu' set iff cpu available to migration
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
45
   *
b3199c025   Rusty Russell   cpumask: switch o...
46
   *  If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
47
   *
b3199c025   Rusty Russell   cpumask: switch o...
48
49
50
51
52
53
54
55
   *  The cpu_possible_mask is fixed at boot time, as the set of CPU id's
   *  that it is possible might ever be plugged in at anytime during the
   *  life of that system boot.  The cpu_present_mask is dynamic(*),
   *  representing which CPUs are currently plugged in.  And
   *  cpu_online_mask is the dynamic subset of cpu_present_mask,
   *  indicating those CPUs available for scheduling.
   *
   *  If HOTPLUG is enabled, then cpu_possible_mask is forced to have
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
56
57
58
   *  all NR_CPUS bits set, otherwise it is just the set of CPUs that
   *  ACPI reports present at boot.
   *
b3199c025   Rusty Russell   cpumask: switch o...
59
   *  If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
60
   *  depending on what ACPI reports as currently plugged in, otherwise
b3199c025   Rusty Russell   cpumask: switch o...
61
   *  cpu_present_mask is just a copy of cpu_possible_mask.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
62
   *
b3199c025   Rusty Russell   cpumask: switch o...
63
64
   *  (*) Well, cpu_present_mask is dynamic in the hotplug case.  If not
   *      hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
65
66
67
68
   *
   * Subtleties:
   * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
   *    assumption that their single CPU is online.  The UP
b3199c025   Rusty Russell   cpumask: switch o...
69
   *    cpu_{online,possible,present}_masks are placebos.  Changing them
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
70
71
72
73
74
   *    will have no useful affect on the following num_*_cpus()
   *    and cpu_*() macros in the UP case.  This ugliness is a UP
   *    optimization - don't waste any instructions or memory references
   *    asking if you're online or how many CPUs there are if there is
   *    only one CPU.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
75
   */
b3199c025   Rusty Russell   cpumask: switch o...
76
77
78
79
  extern const struct cpumask *const cpu_possible_mask;
  extern const struct cpumask *const cpu_online_mask;
  extern const struct cpumask *const cpu_present_mask;
  extern const struct cpumask *const cpu_active_mask;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
80
  #if NR_CPUS > 1
ae7a47e72   Rusty Russell   cpumask: make cpu...
81
82
83
  #define num_online_cpus()	cpumask_weight(cpu_online_mask)
  #define num_possible_cpus()	cpumask_weight(cpu_possible_mask)
  #define num_present_cpus()	cpumask_weight(cpu_present_mask)
6ad4c1888   Peter Zijlstra   sched: Fix balanc...
84
  #define num_active_cpus()	cpumask_weight(cpu_active_mask)
ae7a47e72   Rusty Russell   cpumask: make cpu...
85
86
87
88
  #define cpu_online(cpu)		cpumask_test_cpu((cpu), cpu_online_mask)
  #define cpu_possible(cpu)	cpumask_test_cpu((cpu), cpu_possible_mask)
  #define cpu_present(cpu)	cpumask_test_cpu((cpu), cpu_present_mask)
  #define cpu_active(cpu)		cpumask_test_cpu((cpu), cpu_active_mask)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
89
  #else
221e3ebf6   Heiko Carstens   cpumask: let num_...
90
91
92
93
  #define num_online_cpus()	1U
  #define num_possible_cpus()	1U
  #define num_present_cpus()	1U
  #define num_active_cpus()	1U
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
94
95
96
  #define cpu_online(cpu)		((cpu) == 0)
  #define cpu_possible(cpu)	((cpu) == 0)
  #define cpu_present(cpu)	((cpu) == 0)
e761b7725   Max Krasnyansky   cpu hotplug, sche...
97
  #define cpu_active(cpu)		((cpu) == 0)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
98
  #endif
2d3854a37   Rusty Russell   cpumask: introduc...
99
100
101
102
103
104
105
106
107
108
  /* verify cpu argument to cpumask_* operators */
  static inline unsigned int cpumask_check(unsigned int cpu)
  {
  #ifdef CONFIG_DEBUG_PER_CPU_MAPS
  	WARN_ON_ONCE(cpu >= nr_cpumask_bits);
  #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
  	return cpu;
  }
  
  #if NR_CPUS == 1
984f2f377   Rusty Russell   cpumask: introduc...
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
  /* Uniprocessor.  Assume all masks are "1". */
  static inline unsigned int cpumask_first(const struct cpumask *srcp)
  {
  	return 0;
  }
  
  /* Valid inputs for n are -1 and 0. */
  static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
  {
  	return n+1;
  }
  
  static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  {
  	return n+1;
  }
  
  static inline unsigned int cpumask_next_and(int n,
  					    const struct cpumask *srcp,
  					    const struct cpumask *andp)
  {
  	return n+1;
  }
  
  /* cpu must be a valid cpu, ie 0, so there's no other choice. */
  static inline unsigned int cpumask_any_but(const struct cpumask *mask,
  					   unsigned int cpu)
  {
  	return 1;
  }
2d3854a37   Rusty Russell   cpumask: introduc...
139
140
141
  
  #define for_each_cpu(cpu, mask)			\
  	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
8bd93a2c5   Paul E. McKenney   rcu: Accelerate g...
142
143
  #define for_each_cpu_not(cpu, mask)		\
  	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
2d3854a37   Rusty Russell   cpumask: introduc...
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  #define for_each_cpu_and(cpu, mask, and)	\
  	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
  #else
  /**
   * cpumask_first - get the first cpu in a cpumask
   * @srcp: the cpumask pointer
   *
   * Returns >= nr_cpu_ids if no cpus set.
   */
  static inline unsigned int cpumask_first(const struct cpumask *srcp)
  {
  	return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_next - get the next cpu in a cpumask
   * @n: the cpu prior to the place to search (ie. return will be > @n)
   * @srcp: the cpumask pointer
   *
   * Returns >= nr_cpu_ids if no further cpus set.
   */
  static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
  {
  	/* -1 is a legal arg here. */
  	if (n != -1)
  		cpumask_check(n);
  	return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
  }
  
  /**
   * cpumask_next_zero - get the next unset cpu in a cpumask
   * @n: the cpu prior to the place to search (ie. return will be > @n)
   * @srcp: the cpumask pointer
   *
   * Returns >= nr_cpu_ids if no further cpus unset.
   */
  static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
  {
  	/* -1 is a legal arg here. */
  	if (n != -1)
  		cpumask_check(n);
  	return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
  }
  
  int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
  int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
984f2f377   Rusty Russell   cpumask: introduc...
190
191
192
193
194
195
196
  /**
   * for_each_cpu - iterate over every cpu in a mask
   * @cpu: the (optionally unsigned) integer iterator
   * @mask: the cpumask pointer
   *
   * After the loop, cpu is >= nr_cpu_ids.
   */
2d3854a37   Rusty Russell   cpumask: introduc...
197
198
199
200
  #define for_each_cpu(cpu, mask)				\
  	for ((cpu) = -1;				\
  		(cpu) = cpumask_next((cpu), (mask)),	\
  		(cpu) < nr_cpu_ids;)
984f2f377   Rusty Russell   cpumask: introduc...
201
202
  
  /**
8bd93a2c5   Paul E. McKenney   rcu: Accelerate g...
203
204
205
206
207
208
209
210
211
212
213
214
   * for_each_cpu_not - iterate over every cpu in a complemented mask
   * @cpu: the (optionally unsigned) integer iterator
   * @mask: the cpumask pointer
   *
   * After the loop, cpu is >= nr_cpu_ids.
   */
  #define for_each_cpu_not(cpu, mask)				\
  	for ((cpu) = -1;					\
  		(cpu) = cpumask_next_zero((cpu), (mask)),	\
  		(cpu) < nr_cpu_ids;)
  
  /**
984f2f377   Rusty Russell   cpumask: introduc...
215
216
217
218
219
220
221
222
223
224
225
226
227
   * for_each_cpu_and - iterate over every cpu in both masks
   * @cpu: the (optionally unsigned) integer iterator
   * @mask: the first cpumask pointer
   * @and: the second cpumask pointer
   *
   * This saves a temporary CPU mask in many places.  It is equivalent to:
   *	struct cpumask tmp;
   *	cpumask_and(&tmp, &mask, &and);
   *	for_each_cpu(cpu, &tmp)
   *		...
   *
   * After the loop, cpu is >= nr_cpu_ids.
   */
2d3854a37   Rusty Russell   cpumask: introduc...
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
  #define for_each_cpu_and(cpu, mask, and)				\
  	for ((cpu) = -1;						\
  		(cpu) = cpumask_next_and((cpu), (mask), (and)),		\
  		(cpu) < nr_cpu_ids;)
  #endif /* SMP */
  
  #define CPU_BITS_NONE						\
  {								\
  	[0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL			\
  }
  
  #define CPU_BITS_CPU0						\
  {								\
  	[0] =  1UL						\
  }
  
  /**
   * cpumask_set_cpu - set a cpu in a cpumask
   * @cpu: cpu number (< nr_cpu_ids)
   * @dstp: the cpumask pointer
   */
  static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
  {
  	set_bit(cpumask_check(cpu), cpumask_bits(dstp));
  }
  
  /**
   * cpumask_clear_cpu - clear a cpu in a cpumask
   * @cpu: cpu number (< nr_cpu_ids)
   * @dstp: the cpumask pointer
   */
  static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
  {
  	clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
  }
  
  /**
   * cpumask_test_cpu - test for a cpu in a cpumask
   * @cpu: cpu number (< nr_cpu_ids)
   * @cpumask: the cpumask pointer
   *
   * No static inline type checking - see Subtlety (1) above.
   */
  #define cpumask_test_cpu(cpu, cpumask) \
ae7a47e72   Rusty Russell   cpumask: make cpu...
272
  	test_bit(cpumask_check(cpu), cpumask_bits((cpumask)))
2d3854a37   Rusty Russell   cpumask: introduc...
273
274
275
276
277
278
279
280
281
282
283
284
285
286
  
  /**
   * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
   * @cpu: cpu number (< nr_cpu_ids)
   * @cpumask: the cpumask pointer
   *
   * test_and_set_bit wrapper for cpumasks.
   */
  static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
  {
  	return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  }
  
  /**
54fdade1c   Xiao Guangrong   generic-ipi: make...
287
288
289
290
291
292
293
294
295
296
297
298
   * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
   * @cpu: cpu number (< nr_cpu_ids)
   * @cpumask: the cpumask pointer
   *
   * test_and_clear_bit wrapper for cpumasks.
   */
  static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
  {
  	return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
  }
  
  /**
2d3854a37   Rusty Russell   cpumask: introduc...
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
   * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
   * @dstp: the cpumask pointer
   */
  static inline void cpumask_setall(struct cpumask *dstp)
  {
  	bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
   * @dstp: the cpumask pointer
   */
  static inline void cpumask_clear(struct cpumask *dstp)
  {
  	bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_and - *dstp = *src1p & *src2p
   * @dstp: the cpumask result
   * @src1p: the first input
   * @src2p: the second input
   */
f4b0373b2   Linus Torvalds   Make bitmask 'and...
322
  static inline int cpumask_and(struct cpumask *dstp,
2d3854a37   Rusty Russell   cpumask: introduc...
323
324
325
  			       const struct cpumask *src1p,
  			       const struct cpumask *src2p)
  {
f4b0373b2   Linus Torvalds   Make bitmask 'and...
326
  	return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
2d3854a37   Rusty Russell   cpumask: introduc...
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
  				       cpumask_bits(src2p), nr_cpumask_bits);
  }
  
  /**
   * cpumask_or - *dstp = *src1p | *src2p
   * @dstp: the cpumask result
   * @src1p: the first input
   * @src2p: the second input
   */
  static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
  			      const struct cpumask *src2p)
  {
  	bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
  				      cpumask_bits(src2p), nr_cpumask_bits);
  }
  
  /**
   * cpumask_xor - *dstp = *src1p ^ *src2p
   * @dstp: the cpumask result
   * @src1p: the first input
   * @src2p: the second input
   */
  static inline void cpumask_xor(struct cpumask *dstp,
  			       const struct cpumask *src1p,
  			       const struct cpumask *src2p)
  {
  	bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
  				       cpumask_bits(src2p), nr_cpumask_bits);
  }
  
  /**
   * cpumask_andnot - *dstp = *src1p & ~*src2p
   * @dstp: the cpumask result
   * @src1p: the first input
   * @src2p: the second input
   */
f4b0373b2   Linus Torvalds   Make bitmask 'and...
363
  static inline int cpumask_andnot(struct cpumask *dstp,
2d3854a37   Rusty Russell   cpumask: introduc...
364
365
366
  				  const struct cpumask *src1p,
  				  const struct cpumask *src2p)
  {
f4b0373b2   Linus Torvalds   Make bitmask 'and...
367
  	return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
2d3854a37   Rusty Russell   cpumask: introduc...
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
  					  cpumask_bits(src2p), nr_cpumask_bits);
  }
  
  /**
   * cpumask_complement - *dstp = ~*srcp
   * @dstp: the cpumask result
   * @srcp: the input to invert
   */
  static inline void cpumask_complement(struct cpumask *dstp,
  				      const struct cpumask *srcp)
  {
  	bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
  					      nr_cpumask_bits);
  }
  
  /**
   * cpumask_equal - *src1p == *src2p
   * @src1p: the first input
   * @src2p: the second input
   */
  static inline bool cpumask_equal(const struct cpumask *src1p,
  				const struct cpumask *src2p)
  {
  	return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
  						 nr_cpumask_bits);
  }
  
  /**
   * cpumask_intersects - (*src1p & *src2p) != 0
   * @src1p: the first input
   * @src2p: the second input
   */
  static inline bool cpumask_intersects(const struct cpumask *src1p,
  				     const struct cpumask *src2p)
  {
  	return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
  						      nr_cpumask_bits);
  }
  
  /**
   * cpumask_subset - (*src1p & ~*src2p) == 0
   * @src1p: the first input
   * @src2p: the second input
   */
  static inline int cpumask_subset(const struct cpumask *src1p,
  				 const struct cpumask *src2p)
  {
  	return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
  						  nr_cpumask_bits);
  }
  
  /**
   * cpumask_empty - *srcp == 0
   * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
   */
  static inline bool cpumask_empty(const struct cpumask *srcp)
  {
  	return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_full - *srcp == 0xFFFFFFFF...
   * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
   */
  static inline bool cpumask_full(const struct cpumask *srcp)
  {
  	return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_weight - Count of bits in *srcp
   * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
   */
  static inline unsigned int cpumask_weight(const struct cpumask *srcp)
  {
  	return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_shift_right - *dstp = *srcp >> n
   * @dstp: the cpumask result
   * @srcp: the input to shift
   * @n: the number of bits to shift by
   */
  static inline void cpumask_shift_right(struct cpumask *dstp,
  				       const struct cpumask *srcp, int n)
  {
  	bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
  					       nr_cpumask_bits);
  }
  
  /**
   * cpumask_shift_left - *dstp = *srcp << n
   * @dstp: the cpumask result
   * @srcp: the input to shift
   * @n: the number of bits to shift by
   */
  static inline void cpumask_shift_left(struct cpumask *dstp,
  				      const struct cpumask *srcp, int n)
  {
  	bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
  					      nr_cpumask_bits);
  }
  
  /**
   * cpumask_copy - *dstp = *srcp
   * @dstp: the result
   * @srcp: the input cpumask
   */
  static inline void cpumask_copy(struct cpumask *dstp,
  				const struct cpumask *srcp)
  {
  	bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
  }
  
  /**
   * cpumask_any - pick a "random" cpu from *srcp
   * @srcp: the input cpumask
   *
   * Returns >= nr_cpu_ids if no cpus set.
   */
  #define cpumask_any(srcp) cpumask_first(srcp)
  
  /**
   * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
   * @src1p: the first input
   * @src2p: the second input
   *
   * Returns >= nr_cpu_ids if no cpus set in both.  See also cpumask_next_and().
   */
  #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
  
  /**
   * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
   * @mask1: the first input cpumask
   * @mask2: the second input cpumask
   *
   * Returns >= nr_cpu_ids if no cpus set.
   */
  #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
  
  /**
cd83e42c6   Rusty Russell   cpumask: new API, v2
510
511
512
513
514
515
   * cpumask_of - the cpumask containing just a given cpu
   * @cpu: the cpu (<= nr_cpu_ids)
   */
  #define cpumask_of(cpu) (get_cpu_mask(cpu))
  
  /**
29c0177e6   Rusty Russell   cpumask: change c...
516
517
518
519
520
521
522
523
524
525
526
   * cpumask_scnprintf - print a cpumask into a string as comma-separated hex
   * @buf: the buffer to sprintf into
   * @len: the length of the buffer
   * @srcp: the cpumask to print
   *
   * If len is zero, returns zero.  Otherwise returns the length of the
   * (nul-terminated) @buf string.
   */
  static inline int cpumask_scnprintf(char *buf, int len,
  				    const struct cpumask *srcp)
  {
ae7a47e72   Rusty Russell   cpumask: make cpu...
527
  	return bitmap_scnprintf(buf, len, cpumask_bits(srcp), nr_cpumask_bits);
29c0177e6   Rusty Russell   cpumask: change c...
528
529
530
531
532
533
534
535
536
537
538
539
540
  }
  
  /**
   * cpumask_parse_user - extract a cpumask from a user string
   * @buf: the buffer to extract from
   * @len: the length of the buffer
   * @dstp: the cpumask to set.
   *
   * Returns -errno, or 0 for success.
   */
  static inline int cpumask_parse_user(const char __user *buf, int len,
  				     struct cpumask *dstp)
  {
ae7a47e72   Rusty Russell   cpumask: make cpu...
541
  	return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
29c0177e6   Rusty Russell   cpumask: change c...
542
543
544
  }
  
  /**
4b060420a   Mike Travis   bitmap, irq: add ...
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
   * cpumask_parselist_user - extract a cpumask from a user string
   * @buf: the buffer to extract from
   * @len: the length of the buffer
   * @dstp: the cpumask to set.
   *
   * Returns -errno, or 0 for success.
   */
  static inline int cpumask_parselist_user(const char __user *buf, int len,
  				     struct cpumask *dstp)
  {
  	return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
  							nr_cpumask_bits);
  }
  
  /**
29c0177e6   Rusty Russell   cpumask: change c...
560
561
562
563
564
565
566
567
568
569
570
   * cpulist_scnprintf - print a cpumask into a string as comma-separated list
   * @buf: the buffer to sprintf into
   * @len: the length of the buffer
   * @srcp: the cpumask to print
   *
   * If len is zero, returns zero.  Otherwise returns the length of the
   * (nul-terminated) @buf string.
   */
  static inline int cpulist_scnprintf(char *buf, int len,
  				    const struct cpumask *srcp)
  {
ae7a47e72   Rusty Russell   cpumask: make cpu...
571
572
  	return bitmap_scnlistprintf(buf, len, cpumask_bits(srcp),
  				    nr_cpumask_bits);
29c0177e6   Rusty Russell   cpumask: change c...
573
574
575
576
577
578
579
580
581
582
583
584
  }
  
  /**
   * cpulist_parse_user - extract a cpumask from a user string of ranges
   * @buf: the buffer to extract from
   * @len: the length of the buffer
   * @dstp: the cpumask to set.
   *
   * Returns -errno, or 0 for success.
   */
  static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
  {
ae7a47e72   Rusty Russell   cpumask: make cpu...
585
  	return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
2d3854a37   Rusty Russell   cpumask: introduc...
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
  }
  
  /**
   * cpumask_size - size to allocate for a 'struct cpumask' in bytes
   *
   * This will eventually be a runtime variable, depending on nr_cpu_ids.
   */
  static inline size_t cpumask_size(void)
  {
  	/* FIXME: Once all cpumask assignments are eliminated, this
  	 * can be nr_cpumask_bits */
  	return BITS_TO_LONGS(NR_CPUS) * sizeof(long);
  }
  
  /*
   * cpumask_var_t: struct cpumask for stack usage.
   *
   * Oh, the wicked games we play!  In order to make kernel coding a
   * little more difficult, we typedef cpumask_var_t to an array or a
   * pointer: doing &mask on an array is a noop, so it still works.
   *
   * ie.
   *	cpumask_var_t tmpmask;
   *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
   *		return -ENOMEM;
   *
   *	  ... use 'tmpmask' like a normal struct cpumask * ...
   *
   *	free_cpumask_var(tmpmask);
a64a26e82   KOSAKI Motohiro   cpumask: add cpum...
615
616
617
618
619
620
621
622
623
624
625
626
627
628
   *
   *
   * However, one notable exception is there. alloc_cpumask_var() allocates
   * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
   * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
   *
   *	cpumask_var_t tmpmask;
   *	if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
   *		return -ENOMEM;
   *
   *	var = *tmpmask;
   *
   * This code makes NR_CPUS length memcopy and brings to a memory corruption.
   * cpumask_copy() provide safe copy functionality.
2d3854a37   Rusty Russell   cpumask: introduc...
629
630
631
   */
  #ifdef CONFIG_CPUMASK_OFFSTACK
  typedef struct cpumask *cpumask_var_t;
7b4967c53   Mike Travis   cpumask: Add allo...
632
  bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
2d3854a37   Rusty Russell   cpumask: introduc...
633
  bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
0281b5dc0   Yinghai Lu   cpumask: introduc...
634
635
  bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
  bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
2d3854a37   Rusty Russell   cpumask: introduc...
636
637
  void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
  void free_cpumask_var(cpumask_var_t mask);
cd83e42c6   Rusty Russell   cpumask: new API, v2
638
  void free_bootmem_cpumask_var(cpumask_var_t mask);
2d3854a37   Rusty Russell   cpumask: introduc...
639
640
641
642
643
644
645
646
  
  #else
  typedef struct cpumask cpumask_var_t[1];
  
  static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  {
  	return true;
  }
7b4967c53   Mike Travis   cpumask: Add allo...
647
648
649
650
651
  static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  					  int node)
  {
  	return true;
  }
0281b5dc0   Yinghai Lu   cpumask: introduc...
652
653
654
655
656
657
658
659
660
661
662
663
  static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
  {
  	cpumask_clear(*mask);
  	return true;
  }
  
  static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
  					  int node)
  {
  	cpumask_clear(*mask);
  	return true;
  }
2d3854a37   Rusty Russell   cpumask: introduc...
664
665
666
667
668
669
670
  static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
  {
  }
  
  static inline void free_cpumask_var(cpumask_var_t mask)
  {
  }
cd83e42c6   Rusty Russell   cpumask: new API, v2
671
672
673
674
  
  static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
  {
  }
2d3854a37   Rusty Russell   cpumask: introduc...
675
  #endif /* CONFIG_CPUMASK_OFFSTACK */
2d3854a37   Rusty Russell   cpumask: introduc...
676
677
678
679
680
681
682
  /* It's common to want to use cpu_all_mask in struct member initializers,
   * so it has to refer to an address rather than a pointer. */
  extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
  #define cpu_all_mask to_cpumask(cpu_all_bits)
  
  /* First bits of cpu_bit_bitmap are in fact unset. */
  #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
ae7a47e72   Rusty Russell   cpumask: make cpu...
683
684
685
  #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
  #define for_each_online_cpu(cpu)   for_each_cpu((cpu), cpu_online_mask)
  #define for_each_present_cpu(cpu)  for_each_cpu((cpu), cpu_present_mask)
2d3854a37   Rusty Russell   cpumask: introduc...
686
  /* Wrappers for arch boot code to manipulate normally-constant masks */
3fa415206   Rusty Russell   cpumask: make set...
687
688
689
690
691
692
693
  void set_cpu_possible(unsigned int cpu, bool possible);
  void set_cpu_present(unsigned int cpu, bool present);
  void set_cpu_online(unsigned int cpu, bool online);
  void set_cpu_active(unsigned int cpu, bool active);
  void init_cpu_present(const struct cpumask *src);
  void init_cpu_possible(const struct cpumask *src);
  void init_cpu_online(const struct cpumask *src);
6ba2ef7ba   Rusty Russell   cpumask: Move dep...
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
  
  /**
   * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
   * @bitmap: the bitmap
   *
   * There are a few places where cpumask_var_t isn't appropriate and
   * static cpumasks must be used (eg. very early boot), yet we don't
   * expose the definition of 'struct cpumask'.
   *
   * This does the conversion, and can be used as a constant initializer.
   */
  #define to_cpumask(bitmap)						\
  	((struct cpumask *)(1 ? (bitmap)				\
  			    : (void *)sizeof(__check_is_bitmap(bitmap))))
  
  static inline int __check_is_bitmap(const unsigned long *bitmap)
  {
  	return 1;
  }
  
  /*
   * Special-case data structure for "single bit set only" constant CPU masks.
   *
   * We pre-generate all the 64 (or 32) possible bit positions, with enough
   * padding to the left and the right, and return the constant pointer
   * appropriately offset.
   */
  extern const unsigned long
  	cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
  
  static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
  {
  	const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
  	p -= cpu / BITS_PER_LONG;
  	return to_cpumask(p);
  }
  
  #define cpu_is_offline(cpu)	unlikely(!cpu_online(cpu))
  
  #if NR_CPUS <= BITS_PER_LONG
  #define CPU_BITS_ALL						\
  {								\
  	[BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD	\
  }
  
  #else /* NR_CPUS > BITS_PER_LONG */
  
  #define CPU_BITS_ALL						\
  {								\
  	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,		\
  	[BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD		\
  }
  #endif /* NR_CPUS > BITS_PER_LONG */
  
  /*
   *
   * From here down, all obsolete.  Use cpumask_ variants!
   *
   */
  #ifndef CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
  /* These strip const, as traditionally they weren't const. */
  #define cpu_possible_map	(*(cpumask_t *)cpu_possible_mask)
  #define cpu_online_map		(*(cpumask_t *)cpu_online_mask)
  #define cpu_present_map		(*(cpumask_t *)cpu_present_mask)
  #define cpu_active_map		(*(cpumask_t *)cpu_active_mask)
  
  #define cpumask_of_cpu(cpu) (*get_cpu_mask(cpu))
  
  #define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS)
  
  #if NR_CPUS <= BITS_PER_LONG
  
  #define CPU_MASK_ALL							\
  (cpumask_t) { {								\
  	[BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD			\
  } }
  
  #else
  
  #define CPU_MASK_ALL							\
  (cpumask_t) { {								\
  	[0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL,			\
  	[BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD			\
  } }
  
  #endif
  
  #define CPU_MASK_NONE							\
  (cpumask_t) { {								\
  	[0 ... BITS_TO_LONGS(NR_CPUS)-1] =  0UL				\
  } }
  
  #define CPU_MASK_CPU0							\
  (cpumask_t) { {								\
  	[0] =  1UL							\
  } }
  
  #if NR_CPUS == 1
  #define first_cpu(src)		({ (void)(src); 0; })
  #define next_cpu(n, src)	({ (void)(src); 1; })
  #define any_online_cpu(mask)	0
  #define for_each_cpu_mask(cpu, mask)	\
  	for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
  #else /* NR_CPUS > 1 */
  int __first_cpu(const cpumask_t *srcp);
  int __next_cpu(int n, const cpumask_t *srcp);
  int __any_online_cpu(const cpumask_t *mask);
  
  #define first_cpu(src)		__first_cpu(&(src))
  #define next_cpu(n, src)	__next_cpu((n), &(src))
  #define any_online_cpu(mask) __any_online_cpu(&(mask))
  #define for_each_cpu_mask(cpu, mask)			\
  	for ((cpu) = -1;				\
  		(cpu) = next_cpu((cpu), (mask)),	\
  		(cpu) < NR_CPUS; )
  #endif /* SMP */
  
  #if NR_CPUS <= 64
  
  #define for_each_cpu_mask_nr(cpu, mask)	for_each_cpu_mask(cpu, mask)
  
  #else /* NR_CPUS > 64 */
  
  int __next_cpu_nr(int n, const cpumask_t *srcp);
  #define for_each_cpu_mask_nr(cpu, mask)			\
  	for ((cpu) = -1;				\
  		(cpu) = __next_cpu_nr((cpu), &(mask)),	\
  		(cpu) < nr_cpu_ids; )
  
  #endif /* NR_CPUS > 64 */
  
  #define cpus_addr(src) ((src).bits)
  
  #define cpu_set(cpu, dst) __cpu_set((cpu), &(dst))
  static inline void __cpu_set(int cpu, volatile cpumask_t *dstp)
  {
  	set_bit(cpu, dstp->bits);
  }
  
  #define cpu_clear(cpu, dst) __cpu_clear((cpu), &(dst))
  static inline void __cpu_clear(int cpu, volatile cpumask_t *dstp)
  {
  	clear_bit(cpu, dstp->bits);
  }
  
  #define cpus_setall(dst) __cpus_setall(&(dst), NR_CPUS)
  static inline void __cpus_setall(cpumask_t *dstp, int nbits)
  {
  	bitmap_fill(dstp->bits, nbits);
  }
  
  #define cpus_clear(dst) __cpus_clear(&(dst), NR_CPUS)
  static inline void __cpus_clear(cpumask_t *dstp, int nbits)
  {
  	bitmap_zero(dstp->bits, nbits);
  }
  
  /* No static inline type checking - see Subtlety (1) above. */
  #define cpu_isset(cpu, cpumask) test_bit((cpu), (cpumask).bits)
  
  #define cpu_test_and_set(cpu, cpumask) __cpu_test_and_set((cpu), &(cpumask))
  static inline int __cpu_test_and_set(int cpu, cpumask_t *addr)
  {
  	return test_and_set_bit(cpu, addr->bits);
  }
  
  #define cpus_and(dst, src1, src2) __cpus_and(&(dst), &(src1), &(src2), NR_CPUS)
  static inline int __cpus_and(cpumask_t *dstp, const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_or(dst, src1, src2) __cpus_or(&(dst), &(src1), &(src2), NR_CPUS)
  static inline void __cpus_or(cpumask_t *dstp, const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_xor(dst, src1, src2) __cpus_xor(&(dst), &(src1), &(src2), NR_CPUS)
  static inline void __cpus_xor(cpumask_t *dstp, const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_andnot(dst, src1, src2) \
  				__cpus_andnot(&(dst), &(src1), &(src2), NR_CPUS)
  static inline int __cpus_andnot(cpumask_t *dstp, const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_equal(src1, src2) __cpus_equal(&(src1), &(src2), NR_CPUS)
  static inline int __cpus_equal(const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	return bitmap_equal(src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_intersects(src1, src2) __cpus_intersects(&(src1), &(src2), NR_CPUS)
  static inline int __cpus_intersects(const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	return bitmap_intersects(src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_subset(src1, src2) __cpus_subset(&(src1), &(src2), NR_CPUS)
  static inline int __cpus_subset(const cpumask_t *src1p,
  					const cpumask_t *src2p, int nbits)
  {
  	return bitmap_subset(src1p->bits, src2p->bits, nbits);
  }
  
  #define cpus_empty(src) __cpus_empty(&(src), NR_CPUS)
  static inline int __cpus_empty(const cpumask_t *srcp, int nbits)
  {
  	return bitmap_empty(srcp->bits, nbits);
  }
  
  #define cpus_weight(cpumask) __cpus_weight(&(cpumask), NR_CPUS)
  static inline int __cpus_weight(const cpumask_t *srcp, int nbits)
  {
  	return bitmap_weight(srcp->bits, nbits);
  }
  
  #define cpus_shift_left(dst, src, n) \
  			__cpus_shift_left(&(dst), &(src), (n), NR_CPUS)
  static inline void __cpus_shift_left(cpumask_t *dstp,
  					const cpumask_t *srcp, int n, int nbits)
  {
  	bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
  }
  #endif /* !CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
930
  #endif /* __LINUX_CPUMASK_H */