Blame view

include/asm-generic/percpu.h 3.17 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
  #ifndef _ASM_GENERIC_PERCPU_H_
  #define _ASM_GENERIC_PERCPU_H_
5028eaa97   David Howells   PERCPU: Collect t...
3

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
4
  #include <linux/compiler.h>
ae1ee11be   Rusty Russell   [PATCH] i386: Use...
5
  #include <linux/threads.h>
5028eaa97   David Howells   PERCPU: Collect t...
6
  #include <linux/percpu-defs.h>
acdac8720   travis@sgi.com   percpu: make the ...
7

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
8
  #ifdef CONFIG_SMP
acdac8720   travis@sgi.com   percpu: make the ...
9
10
11
12
13
14
15
16
  /*
   * per_cpu_offset() is the offset that has to be added to a
   * percpu variable to get to the instance for a certain processor.
   *
   * Most arches use the __per_cpu_offset array for those offsets but
   * some arches have their own ways of determining the offset (x86_64, s390).
   */
  #ifndef __per_cpu_offset
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
17
  extern unsigned long __per_cpu_offset[NR_CPUS];
a875a69f8   Ingo Molnar   [PATCH] lockdep: ...
18
  #define per_cpu_offset(x) (__per_cpu_offset[x])
acdac8720   travis@sgi.com   percpu: make the ...
19
20
21
22
23
24
25
26
27
28
  #endif
  
  /*
   * Determine the offset for the currently active processor.
   * An arch may define __my_cpu_offset to provide a more effective
   * means of obtaining the offset to the per cpu variables of the
   * current processor.
   */
  #ifndef __my_cpu_offset
  #define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
1e8352784   Hugh Dickins   percpu: fix DEBUG...
29
30
  #endif
  #ifdef CONFIG_DEBUG_PREEMPT
acdac8720   travis@sgi.com   percpu: make the ...
31
32
33
34
35
36
37
38
39
40
41
  #define my_cpu_offset per_cpu_offset(smp_processor_id())
  #else
  #define my_cpu_offset __my_cpu_offset
  #endif
  
  /*
   * Add a offset to a pointer but keep the pointer as is.
   *
   * Only S390 provides its own means of moving the pointer.
   */
  #ifndef SHIFT_PERCPU_PTR
e0fdb0e05   Rusty Russell   percpu: add __per...
42
  /* Weird cast keeps both GCC and sparse happy. */
545695fb4   Tejun Heo   percpu: make acce...
43
44
45
46
  #define SHIFT_PERCPU_PTR(__p, __offset)	({				\
  	__verify_pcpu_ptr((__p));					\
  	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
  })
acdac8720   travis@sgi.com   percpu: make the ...
47
  #endif
a875a69f8   Ingo Molnar   [PATCH] lockdep: ...
48

acdac8720   travis@sgi.com   percpu: make the ...
49
  /*
dd5af90a7   Mike Travis   x86/non-x86: perc...
50
   * A percpu variable may point to a discarded regions. The following are
acdac8720   travis@sgi.com   percpu: make the ...
51
52
53
54
   * established ways to produce a usable pointer from the percpu variable
   * offset.
   */
  #define per_cpu(var, cpu) \
dd17c8f72   Rusty Russell   percpu: remove pe...
55
  	(*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
acdac8720   travis@sgi.com   percpu: make the ...
56

db7829c6c   Brian Gerst   x86, percpu: Opti...
57
  #ifndef __this_cpu_ptr
7340a0b15   Christoph Lameter   this_cpu: Introdu...
58
  #define __this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
db7829c6c   Brian Gerst   x86, percpu: Opti...
59
60
61
62
63
64
  #endif
  #ifdef CONFIG_DEBUG_PREEMPT
  #define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
  #else
  #define this_cpu_ptr(ptr) __this_cpu_ptr(ptr)
  #endif
acdac8720   travis@sgi.com   percpu: make the ...
65

677243d74   Brian Gerst   percpu: Optimize ...
66
67
  #define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
  #define __raw_get_cpu_var(var) (*__this_cpu_ptr(&(var)))
dd5af90a7   Mike Travis   x86/non-x86: perc...
68
  #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
acdac8720   travis@sgi.com   percpu: make the ...
69
70
  extern void setup_per_cpu_areas(void);
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
71

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
72
  #else /* ! SMP */
18cb2aef9   Namhyung Kim   percpu: handle __...
73
74
75
76
77
78
79
80
81
82
  #define VERIFY_PERCPU_PTR(__p) ({			\
  	__verify_pcpu_ptr((__p));			\
  	(typeof(*(__p)) __kernel __force *)(__p);	\
  })
  
  #define per_cpu(var, cpu)	(*((void)(cpu), VERIFY_PERCPU_PTR(&(var))))
  #define __get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
  #define __raw_get_cpu_var(var)	(*VERIFY_PERCPU_PTR(&(var)))
  #define this_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
  #define __this_cpu_ptr(ptr)	this_cpu_ptr(ptr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
83
84
  
  #endif	/* SMP */
9b8de7479   David Howells   FRV: Fix the sect...
85
86
  #ifndef PER_CPU_BASE_SECTION
  #ifdef CONFIG_SMP
3d9a854c2   Denys Vlasenko   Rename .data[.per...
87
  #define PER_CPU_BASE_SECTION ".data..percpu"
9b8de7479   David Howells   FRV: Fix the sect...
88
89
90
91
92
93
94
95
96
  #else
  #define PER_CPU_BASE_SECTION ".data"
  #endif
  #endif
  
  #ifdef CONFIG_SMP
  
  #ifdef MODULE
  #define PER_CPU_SHARED_ALIGNED_SECTION ""
53f824520   Jeremy Fitzhardinge   x86/i386: Put ali...
97
  #define PER_CPU_ALIGNED_SECTION ""
9b8de7479   David Howells   FRV: Fix the sect...
98
  #else
3d9a854c2   Denys Vlasenko   Rename .data[.per...
99
100
  #define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
  #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
9b8de7479   David Howells   FRV: Fix the sect...
101
  #endif
3d9a854c2   Denys Vlasenko   Rename .data[.per...
102
  #define PER_CPU_FIRST_SECTION "..first"
9b8de7479   David Howells   FRV: Fix the sect...
103
104
105
106
  
  #else
  
  #define PER_CPU_SHARED_ALIGNED_SECTION ""
3d9a854c2   Denys Vlasenko   Rename .data[.per...
107
  #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
9b8de7479   David Howells   FRV: Fix the sect...
108
109
110
  #define PER_CPU_FIRST_SECTION ""
  
  #endif
acdac8720   travis@sgi.com   percpu: make the ...
111
112
113
  #ifndef PER_CPU_ATTRIBUTES
  #define PER_CPU_ATTRIBUTES
  #endif
b01e8dc34   Tejun Heo   alpha: fix percpu...
114
115
116
  #ifndef PER_CPU_DEF_ATTRIBUTES
  #define PER_CPU_DEF_ATTRIBUTES
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
117
  #endif /* _ASM_GENERIC_PERCPU_H_ */