Blame view
kernel/rcu/tree.h
19.4 KB
22e409253 rcu/tree: Convert... |
1 |
/* SPDX-License-Identifier: GPL-2.0+ */ |
9f77da9f4 rcu: Move private... |
2 3 4 5 |
/* * Read-Copy Update mechanism for mutual exclusion (tree-based version) * Internal non-public definitions. * |
9f77da9f4 rcu: Move private... |
6 7 8 |
* Copyright IBM Corporation, 2008 * * Author: Ingo Molnar <mingo@elte.hu> |
22e409253 rcu/tree: Convert... |
9 |
* Paul E. McKenney <paulmck@linux.ibm.com> |
9f77da9f4 rcu: Move private... |
10 11 12 13 |
*/ #include <linux/cache.h> #include <linux/spinlock.h> |
037741a6d sched/headers: Pr... |
14 |
#include <linux/rtmutex.h> |
9f77da9f4 rcu: Move private... |
15 16 17 |
#include <linux/threads.h> #include <linux/cpumask.h> #include <linux/seqlock.h> |
abedf8e24 rcu: Use simple w... |
18 |
#include <linux/swait.h> |
f2425b4ef srcu: Move combin... |
19 |
#include <linux/rcu_node_tree.h> |
f885b7f2b rcu: Control RCU_... |
20 |
|
45753c5f3 srcu: Debloat the... |
21 |
#include "rcu_segcblist.h" |
25f3d7eff rcu: Parallelize ... |
22 23 |
/* Communicate arguments to a workqueue handler. */ struct rcu_exp_work { |
25f3d7eff rcu: Parallelize ... |
24 25 26 |
unsigned long rew_s; struct work_struct rew_work; }; |
d71df90ea rcu: add tracing ... |
27 28 29 30 |
/* RCU's kthread states for tracing. */ #define RCU_KTHREAD_STOPPED 0 #define RCU_KTHREAD_RUNNING 1 #define RCU_KTHREAD_WAITING 2 |
15ba0ba86 rcu: add grace-pe... |
31 32 33 |
#define RCU_KTHREAD_OFFCPU 3 #define RCU_KTHREAD_YIELDING 4 #define RCU_KTHREAD_MAX 4 |
d71df90ea rcu: add tracing ... |
34 |
|
9f77da9f4 rcu: Move private... |
35 36 37 38 |
/* * Definition for node within the RCU grace-period-detection hierarchy. */ struct rcu_node { |
67c583a7d RCU: Privatize rc... |
39 40 41 |
raw_spinlock_t __private lock; /* Root rcu_node's lock protects */ /* some rcu_state fields as well as */ /* following. */ |
360fbbb48 rcu: Update comme... |
42 |
unsigned long gp_seq; /* Track rsp->gp_seq. */ |
adbccddb4 rcu: Fix rcu_{nod... |
43 |
unsigned long gp_seq_needed; /* Track furthest future GP request. */ |
4bc8d5557 rcu: Add debuggin... |
44 |
unsigned long completedqs; /* All QSes done for this node. */ |
9f77da9f4 rcu: Move private... |
45 46 |
unsigned long qsmask; /* CPUs or groups that need to switch in */ /* order for current grace period to proceed.*/ |
1eba8f843 rcu: Clean up cod... |
47 48 49 50 |
/* In leaf rcu_node, each bit corresponds to */ /* an rcu_data structure, otherwise, each */ /* bit corresponds to a child rcu_node */ /* structure. */ |
f2e2df597 rcu: Add diagnost... |
51 |
unsigned long rcu_gp_init_mask; /* Mask of offline CPUs at GP init. */ |
9f77da9f4 rcu: Move private... |
52 |
unsigned long qsmaskinit; |
b9585e940 rcu: Consolidate ... |
53 |
/* Per-GP initial value for qsmask. */ |
0aa04b055 rcu: Process offl... |
54 55 56 57 |
/* Initialized from ->qsmaskinitnext at the */ /* beginning of each grace period. */ unsigned long qsmaskinitnext; /* Online CPUs for next grace period. */ |
b9585e940 rcu: Consolidate ... |
58 59 60 61 62 63 64 65 66 |
unsigned long expmask; /* CPUs or groups that need to check in */ /* to allow the current expedited GP */ /* to complete. */ unsigned long expmaskinit; /* Per-GP initial values for expmask. */ /* Initialized from ->expmaskinitnext at the */ /* beginning of each expedited GP. */ unsigned long expmaskinitnext; /* Online CPUs for next expedited GP. */ |
1de6e56dd rcu: Clarify role... |
67 68 |
/* Any CPU that has ever been online will */ /* have its bit set. */ |
b2b00ddf1 rcu: React to cal... |
69 70 |
unsigned long cbovldmask; /* CPUs experiencing callback overload. */ |
9b9500da8 rcu: Make RCU CPU... |
71 |
unsigned long ffmask; /* Fully functional CPUs. */ |
9f77da9f4 rcu: Move private... |
72 |
unsigned long grpmask; /* Mask to apply to parent qsmask. */ |
1eba8f843 rcu: Clean up cod... |
73 |
/* Only one bit will be set in this mask. */ |
a2dae4308 rcu: grplo/grphi ... |
74 75 |
int grplo; /* lowest-numbered CPU here. */ int grphi; /* highest-numbered CPU here. */ |
7a0c2b094 rcu: grpnum just ... |
76 |
u8 grpnum; /* group number for next level up. */ |
9f77da9f4 rcu: Move private... |
77 |
u8 level; /* root is at level 0. */ |
0aa04b055 rcu: Process offl... |
78 79 80 81 |
bool wait_blkd_tasks;/* Necessary to wait for blocked tasks to */ /* exit RCU read-side critical sections */ /* before propagating offline up the */ /* rcu_node tree? */ |
9f77da9f4 rcu: Move private... |
82 |
struct rcu_node *parent; |
12f5f524c rcu: merge TREE_P... |
83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
struct list_head blkd_tasks; /* Tasks blocked in RCU read-side critical */ /* section. Tasks are placed at the head */ /* of this list and age towards the tail. */ struct list_head *gp_tasks; /* Pointer to the first task blocking the */ /* current grace period, or NULL if there */ /* is no such task. */ struct list_head *exp_tasks; /* Pointer to the first task blocking the */ /* current expedited grace period, or NULL */ /* if there is no such task. If there */ /* is no current expedited grace period, */ /* then there can cannot be any such task. */ |
27f4d2805 rcu: priority boo... |
97 98 99 100 101 102 103 104 |
struct list_head *boost_tasks; /* Pointer to first task that needs to be */ /* priority boosted, or NULL if no priority */ /* boosting is needed for this rcu_node */ /* structure. If there are no tasks */ /* queued on this rcu_node structure that */ /* are blocking the current grace period, */ /* there can be no such task. */ |
abaa93d9e rcu: Simplify pri... |
105 106 107 |
struct rt_mutex boost_mtx; /* Used only for the priority-boosting */ /* side effect, not as a lock. */ |
27f4d2805 rcu: priority boo... |
108 109 110 111 112 |
unsigned long boost_time; /* When to start boosting (jiffies). */ struct task_struct *boost_kthread_task; /* kthread that takes care of priority */ /* boosting for this rcu_node structure. */ |
d71df90ea rcu: add tracing ... |
113 114 |
unsigned int boost_kthread_status; /* State of boost_kthread_task for tracing. */ |
dae6e64d2 rcu: Introduce pr... |
115 |
#ifdef CONFIG_RCU_NOCB_CPU |
abedf8e24 rcu: Use simple w... |
116 |
struct swait_queue_head nocb_gp_wq[2]; |
dae6e64d2 rcu: Introduce pr... |
117 |
/* Place for rcu_nocb_kthread() to wait GP. */ |
dae6e64d2 rcu: Introduce pr... |
118 |
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
394f2769a rcu: Prevent forc... |
119 |
raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; |
385b73c06 rcu: Get rid of s... |
120 |
|
f6a12f34a rcu: Enforce expe... |
121 122 |
spinlock_t exp_lock ____cacheline_internodealigned_in_smp; unsigned long exp_seq_rq; |
3b5f668e7 rcu: Overlap wake... |
123 |
wait_queue_head_t exp_wq[4]; |
25f3d7eff rcu: Parallelize ... |
124 125 |
struct rcu_exp_work rew; bool exp_need_flush; /* Need to flush workitem? */ |
9f77da9f4 rcu: Move private... |
126 |
} ____cacheline_internodealigned_in_smp; |
a0b6c9a78 rcu: Clean up cod... |
127 |
/* |
bc75e9998 rcu: Correctly ha... |
128 129 130 131 |
* Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and * are indexed relative to this interval rather than the global CPU ID space. * This generates the bit for a CPU in node-local masks. */ |
df63fa5bc rcu: Convert "1UL... |
132 |
#define leaf_node_cpu_bit(rnp, cpu) (BIT((cpu) - (rnp)->grplo)) |
bc75e9998 rcu: Correctly ha... |
133 134 |
/* |
5b74c4589 rcu: Make ->cpu_n... |
135 136 137 138 139 140 141 142 143 144 |
* Union to allow "aggregate OR" operation on the need for a quiescent * state by the normal and expedited grace periods. */ union rcu_noqs { struct { u8 norm; u8 exp; } b; /* Bits. */ u16 s; /* Set of bits, aggregate OR here. */ }; |
9f77da9f4 rcu: Move private... |
145 146 147 |
/* Per-CPU data for read-copy update. */ struct rcu_data { /* 1) quiescent-state and grace-period handling : */ |
360fbbb48 rcu: Update comme... |
148 |
unsigned long gp_seq; /* Track rsp->gp_seq counter. */ |
adbccddb4 rcu: Fix rcu_{nod... |
149 |
unsigned long gp_seq_needed; /* Track furthest future GP request. */ |
5b74c4589 rcu: Make ->cpu_n... |
150 |
union rcu_noqs cpu_no_qs; /* No QSes yet for this CPU. */ |
97c668b8e rcu: Rename qs_pe... |
151 |
bool core_needs_qs; /* Core waits for quiesc state. */ |
9f77da9f4 rcu: Move private... |
152 |
bool beenonline; /* CPU online at least once. */ |
ff3bb6f4d rcu: Remove ->gpn... |
153 |
bool gpwrap; /* Possible ->gp_seq wrap. */ |
1bb336443 rcu: Rename rcu_d... |
154 |
bool exp_deferred_qs; /* This CPU awaiting a deferred QS? */ |
c0f97f20e rcu: Move rcu_cpu... |
155 |
bool cpu_started; /* RCU watching this onlining CPU. */ |
9f77da9f4 rcu: Move private... |
156 157 |
struct rcu_node *mynode; /* This CPU's leaf of hierarchy */ unsigned long grpmask; /* Mask to apply to leaf qsmask. */ |
a858af287 rcu: Print schedu... |
158 159 160 161 |
unsigned long ticks_this_gp; /* The number of scheduling-clock */ /* ticks this CPU has handled */ /* during and after the last grace */ /* period it is aware of. */ |
0864f057b rcu: Use irq_work... |
162 163 |
struct irq_work defer_qs_iw; /* Obtain later scheduler attention. */ bool defer_qs_iw_pending; /* Scheduler attention pending? */ |
a657f2617 rcu: Execute RCU ... |
164 |
struct work_struct strict_work; /* Schedule readers for strict GPs. */ |
9f77da9f4 rcu: Move private... |
165 166 |
/* 2) batch handling */ |
15fecf89e srcu: Abstract mu... |
167 168 169 |
struct rcu_segcblist cblist; /* Segmented callback list, with */ /* different callbacks waiting for */ /* different grace periods. */ |
37c72e56f rcu: Prevent RCU ... |
170 171 |
long qlen_last_fqs_check; /* qlen at last check for QS forcing */ |
e816d56fa rcu: Add callback... |
172 |
unsigned long n_cbs_invoked; /* # callbacks invoked since boot. */ |
37c72e56f rcu: Prevent RCU ... |
173 174 |
unsigned long n_force_qs_snap; /* did other CPU force QS recently? */ |
9f77da9f4 rcu: Move private... |
175 |
long blimit; /* Upper limit on a processed batch */ |
9f77da9f4 rcu: Move private... |
176 |
/* 3) dynticks interface. */ |
9f77da9f4 rcu: Move private... |
177 |
int dynticks_snap; /* Per-GP tracking for dynticks. */ |
dc5a4f293 rcu: Switch ->dyn... |
178 179 180 181 182 |
long dynticks_nesting; /* Track process nesting level. */ long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */ atomic_t dynticks; /* Even value for idle, else odd. */ bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */ bool rcu_urgent_qs; /* GP old need light quiescent state. */ |
66e4c33b5 rcu: Force tick o... |
183 |
bool rcu_forced_tick; /* Forced tick to provide QS. */ |
df1e849ae rcu: Enable tick ... |
184 |
bool rcu_forced_tick_exp; /* ... provide QS to expedited GP. */ |
cc72046cc rcu: Merge rcu_dy... |
185 |
#ifdef CONFIG_RCU_FAST_NO_HZ |
dc5a4f293 rcu: Switch ->dyn... |
186 187 188 |
unsigned long last_accelerate; /* Last jiffy CBs were accelerated. */ unsigned long last_advance_all; /* Last jiffy CBs were all advanced. */ int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */ |
cc72046cc rcu: Merge rcu_dy... |
189 |
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ |
9f77da9f4 rcu: Move private... |
190 |
|
8d8a9d0e7 rcu: Remove obsol... |
191 |
/* 4) rcu_barrier(), OOM callbacks, and expediting. */ |
06668efa9 rcu: Move _rcu_ba... |
192 |
struct rcu_head barrier_head; |
0742ac3e2 rcu: Make expedit... |
193 |
int exp_dynticks_snap; /* Double-check need for IPI. */ |
06668efa9 rcu: Move _rcu_ba... |
194 |
|
8d8a9d0e7 rcu: Remove obsol... |
195 |
/* 5) Callback offloading. */ |
3fbfbf7a3 rcu: Add callback... |
196 |
#ifdef CONFIG_RCU_NOCB_CPU |
12f54c3a8 rcu/nocb: Provide... |
197 198 |
struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */ struct task_struct *nocb_gp_kthread; |
8be6e1b15 rcu: Use timer as... |
199 |
raw_spinlock_t nocb_lock; /* Guard following pair of fields. */ |
81c0b3d72 rcu/nocb: Avoid -... |
200 |
atomic_t nocb_lock_contended; /* Contention experienced. */ |
9fdd3bc90 rcu: Break more c... |
201 |
int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ |
8be6e1b15 rcu: Use timer as... |
202 |
struct timer_list nocb_timer; /* Enforce finite deferral. */ |
d1b222c6b rcu/nocb: Add byp... |
203 204 205 206 207 208 209 210 |
unsigned long nocb_gp_adv_time; /* Last call_rcu() CB adv (jiffies). */ /* The following fields are used by call_rcu, hence own cacheline. */ raw_spinlock_t nocb_bypass_lock ____cacheline_internodealigned_in_smp; struct rcu_cblist nocb_bypass; /* Lock-contention-bypass CB list. */ unsigned long nocb_bypass_first; /* Time (jiffies) of first enqueue. */ unsigned long nocb_nobypass_last; /* Last ->cblist enqueue (jiffies). */ int nocb_nobypass_count; /* # ->cblist enqueues at ^^^ time. */ |
fbce7497e rcu: Parallelize ... |
211 |
|
6484fe54b rcu/nocb: Update ... |
212 |
/* The following fields are used by GP kthread, hence own cacheline. */ |
4fd8c5f15 rcu/nocb: Reduce ... |
213 |
raw_spinlock_t nocb_gp_lock ____cacheline_internodealigned_in_smp; |
d1b222c6b rcu/nocb: Add byp... |
214 |
struct timer_list nocb_bypass_timer; /* Force nocb_bypass flush. */ |
f7a81b12d rcu/nocb: Print n... |
215 216 217 218 219 |
u8 nocb_gp_sleep; /* Is the nocb GP thread asleep? */ u8 nocb_gp_bypass; /* Found a bypass on last scan? */ u8 nocb_gp_gp; /* GP to wait for on last scan? */ unsigned long nocb_gp_seq; /* If so, ->gp_seq to wait for. */ unsigned long nocb_gp_loops; /* # passes through wait code. */ |
12f54c3a8 rcu/nocb: Provide... |
220 |
struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ |
5d6742b37 rcu/nocb: Use rcu... |
221 |
bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ |
12f54c3a8 rcu/nocb: Provide... |
222 |
struct task_struct *nocb_cb_kthread; |
58bf6f77c rcu/nocb: Rename ... |
223 224 |
struct rcu_data *nocb_next_cb_rdp; /* Next rcu_data in wakeup chain. */ |
fbce7497e rcu: Parallelize ... |
225 |
|
d1b222c6b rcu/nocb: Add byp... |
226 |
/* The following fields are used by CB kthread, hence new cacheline. */ |
58bf6f77c rcu/nocb: Rename ... |
227 |
struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp; |
6484fe54b rcu/nocb: Update ... |
228 |
/* GP rdp takes GP-end wakeups. */ |
3fbfbf7a3 rcu: Add callback... |
229 |
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
37f62d7cf rcu: Move rcu_cpu... |
230 231 232 |
/* 6) RCU priority boosting. */ struct task_struct *rcu_cpu_kthread_task; /* rcuc per-CPU kthread or NULL. */ |
6ffdde28b rcu: Move rcu_cpu... |
233 |
unsigned int rcu_cpu_kthread_status; |
f7e972ee1 rcu: Move rcu_cpu... |
234 |
char rcu_cpu_has_work; |
37f62d7cf rcu: Move rcu_cpu... |
235 236 |
/* 7) Diagnostic data, including RCU CPU stall warnings. */ |
6231069bd rcu: Add softirq-... |
237 |
unsigned int softirq_snap; /* Snapshot of softirq activity. */ |
9b9500da8 rcu: Make RCU CPU... |
238 239 240 |
/* ->rcu_iw* fields protected by leaf rcu_node ->lock. */ struct irq_work rcu_iw; /* Check for non-irq activity. */ bool rcu_iw_pending; /* Is ->rcu_iw pending? */ |
8aa670cda rcu: Convert ->rc... |
241 |
unsigned long rcu_iw_gp_seq; /* ->gp_seq associated with ->rcu_iw. */ |
577389423 rcu: Add CPU onli... |
242 243 244 245 |
unsigned long rcu_ofl_gp_seq; /* ->gp_seq at last offline. */ short rcu_ofl_gp_flags; /* ->gp_flags at last offline. */ unsigned long rcu_onl_gp_seq; /* ->gp_seq at last online. */ short rcu_onl_gp_flags; /* ->gp_flags at last online. */ |
d3052109c rcu: More aggress... |
246 |
unsigned long last_fqs_resched; /* Time of last rcu_resched(). */ |
6231069bd rcu: Add softirq-... |
247 |
|
9f77da9f4 rcu: Move private... |
248 249 |
int cpu; }; |
9fdd3bc90 rcu: Break more c... |
250 |
/* Values for nocb_defer_wakeup field in struct rcu_data. */ |
511324e46 rcu: Use RCU_NOCB... |
251 252 253 |
#define RCU_NOCB_WAKE_NOT 0 #define RCU_NOCB_WAKE 1 #define RCU_NOCB_WAKE_FORCE 2 |
9fdd3bc90 rcu: Break more c... |
254 |
|
026ad2835 rcu: Drive quiesc... |
255 256 257 |
#define RCU_JIFFIES_TILL_FORCE_QS (1 + (HZ > 250) + (HZ > 500)) /* For jiffies_till_first_fqs and */ /* and jiffies_till_next_fqs. */ |
007b09243 rcu: Increase RCU... |
258 |
|
026ad2835 rcu: Drive quiesc... |
259 260 261 262 263 264 265 |
#define RCU_JIFFIES_FQS_DIV 256 /* Very large systems need more */ /* delay between bouts of */ /* quiescent-state forcing. */ #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time to take */ /* at least one scheduling clock */ /* irq before ratting on them. */ |
9f77da9f4 rcu: Move private... |
266 |
|
08bca60a6 rcu: Remove waitq... |
267 268 269 270 271 272 273 274 275 276 |
#define rcu_wait(cond) \ do { \ for (;;) { \ set_current_state(TASK_INTERRUPTIBLE); \ if (cond) \ break; \ schedule(); \ } \ __set_current_state(TASK_RUNNING); \ } while (0) |
9f77da9f4 rcu: Move private... |
277 278 279 280 281 282 283 284 285 286 287 288 289 |
/* * RCU global state, including node hierarchy. This hierarchy is * represented in "heap" form in a dense array. The root (first level) * of the hierarchy is in ->node[0] (referenced by ->level[0]), the second * level in ->node[1] through ->node[m] (->node[1] referenced by ->level[1]), * and the third level in ->node[m+1] and following (->node[m+1] referenced * by ->level[2]). The number of levels is determined by the number of * CPUs and by CONFIG_RCU_FANOUT. Small systems will have a "hierarchy" * consisting of a single rcu_node. */ struct rcu_state { struct rcu_node node[NUM_RCU_NODES]; /* Hierarchy. */ |
032dfc872 rcu: Shut up bogu... |
290 291 292 |
struct rcu_node *level[RCU_NUM_LVLS + 1]; /* Hierarchy levels (+1 to */ /* shut bogus gcc warning) */ |
b9585e940 rcu: Consolidate ... |
293 |
int ncpus; /* # CPUs seen so far. */ |
9f77da9f4 rcu: Move private... |
294 295 |
/* The following fields are guarded by the root rcu_node's lock. */ |
77f81fe08 rcu: Finish foldi... |
296 297 |
u8 boost ____cacheline_internodealigned_in_smp; /* Subject to priority boost. */ |
de30ad512 rcu: Introduce gr... |
298 |
unsigned long gp_seq; /* Grace-period sequence #. */ |
00943a609 rcu: gp_max is pr... |
299 300 |
unsigned long gp_max; /* Maximum GP duration in */ /* jiffies. */ |
b3dbec76e rcu: Move RCU gra... |
301 |
struct task_struct *gp_kthread; /* Task for grace periods. */ |
abedf8e24 rcu: Use simple w... |
302 |
struct swait_queue_head gp_wq; /* Where GP task waits. */ |
afea227fd rcutorture: Expor... |
303 304 |
short gp_flags; /* Commands for GP task. */ short gp_state; /* GP kthread sleep state. */ |
fd897573f rcu: Improve diag... |
305 306 |
unsigned long gp_wake_time; /* Last GP kthread wake. */ unsigned long gp_wake_seq; /* ->gp_seq at ^^^. */ |
1eba8f843 rcu: Clean up cod... |
307 |
|
d9a3da069 rcu: Add expedite... |
308 |
/* End of fields guarded by root rcu_node's lock. */ |
1eba8f843 rcu: Clean up cod... |
309 |
|
7be7f0be9 rcu: Move rcu_bar... |
310 |
struct mutex barrier_mutex; /* Guards barrier fields. */ |
24ebbca8e rcu: Move rcu_bar... |
311 |
atomic_t barrier_cpu_count; /* # CPUs waiting on. */ |
7db74df88 rcu: Move rcu_bar... |
312 |
struct completion barrier_completion; /* Wake at barrier end. */ |
4f525a528 rcu: Apply rcu_se... |
313 |
unsigned long barrier_sequence; /* ++ at start and end of */ |
dd46a7882 rcu: Inline _rcu_... |
314 |
/* rcu_barrier(). */ |
a4fbe35a1 rcu: Grace-period... |
315 |
/* End of fields guarded by barrier_mutex. */ |
f6a12f34a rcu: Enforce expe... |
316 |
struct mutex exp_mutex; /* Serialize expedited GP. */ |
3b5f668e7 rcu: Overlap wake... |
317 |
struct mutex exp_wake_mutex; /* Serialize wakeup. */ |
d6ada2cf2 rcu: Rework synch... |
318 |
unsigned long expedited_sequence; /* Take a ticket. */ |
3a6d7c64d rcu: Make expedit... |
319 |
atomic_t expedited_need_qs; /* # CPUs left to check in. */ |
abedf8e24 rcu: Use simple w... |
320 |
struct swait_queue_head expedited_wq; /* Wait for check-ins. */ |
b9585e940 rcu: Consolidate ... |
321 |
int ncpus_snap; /* # CPUs seen last time. */ |
b2b00ddf1 rcu: React to cal... |
322 323 |
u8 cbovld; /* Callback overload now? */ u8 cbovldnext; /* ^ ^ next time? */ |
40694d664 rcu: Move synchro... |
324 |
|
9f77da9f4 rcu: Move private... |
325 326 |
unsigned long jiffies_force_qs; /* Time at which to invoke */ /* force_quiescent_state(). */ |
8c7c4829a rcu: Awaken grace... |
327 328 |
unsigned long jiffies_kick_kthreads; /* Time at which to kick */ /* kthreads, if configured. */ |
9f77da9f4 rcu: Move private... |
329 330 |
unsigned long n_force_qs; /* Number of calls to */ /* force_quiescent_state(). */ |
9f77da9f4 rcu: Move private... |
331 332 |
unsigned long gp_start; /* Time at which GP started, */ /* but in jiffies. */ |
c51d7b5e6 rcutorture: Print... |
333 334 |
unsigned long gp_end; /* Time last GP ended, again */ /* in jiffies. */ |
6ccd2ecd4 rcu: Improve diag... |
335 336 |
unsigned long gp_activity; /* Time of last GP kthread */ /* activity in jiffies. */ |
26d950a94 rcu: Diagnostics ... |
337 338 |
unsigned long gp_req_activity; /* Time of last GP request */ /* in jiffies. */ |
9f77da9f4 rcu: Move private... |
339 340 |
unsigned long jiffies_stall; /* Time at which to check */ /* for CPU stalls. */ |
6193c76ab rcu: Kick CPU hal... |
341 342 |
unsigned long jiffies_resched; /* Time at which to resched */ /* a reluctant CPU. */ |
fc908ed33 rcu: Make RCU_CPU... |
343 344 |
unsigned long n_force_qs_gpstart; /* Snapshot of n_force_qs at */ /* GP start. */ |
e66c33d57 rcu: Add const an... |
345 |
const char *name; /* Name of structure. */ |
a48898585 rcu: Distinguish ... |
346 |
char abbr; /* Abbreviated name. */ |
1e64b15a4 rcu: Fix grace-pe... |
347 |
|
894d45bbf rcu: Convert rcu_... |
348 |
raw_spinlock_t ofl_lock ____cacheline_internodealigned_in_smp; |
1e64b15a4 rcu: Fix grace-pe... |
349 350 |
/* Synchronize offline with */ /* GP pre-initialization. */ |
9f77da9f4 rcu: Move private... |
351 |
}; |
4cdfc175c rcu: Move quiesce... |
352 353 354 |
/* Values for rcu_state structure's gp_flags field. */ #define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */ #define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */ |
1fca4d12f rcu: Expedite fir... |
355 |
#define RCU_GP_FLAG_OVLD 0x4 /* Experiencing callback overload. */ |
4cdfc175c rcu: Move quiesce... |
356 |
|
c34d2f418 rcu: Correct comm... |
357 |
/* Values for rcu_state structure's gp_state field. */ |
77f81fe08 rcu: Finish foldi... |
358 |
#define RCU_GP_IDLE 0 /* Initial state and no GP in progress. */ |
afea227fd rcutorture: Expor... |
359 |
#define RCU_GP_WAIT_GPS 1 /* Wait for grace-period start. */ |
319362c90 rcu: Provide more... |
360 |
#define RCU_GP_DONE_GPS 2 /* Wait done for grace-period start. */ |
fea3f222d rcu: Record ->gp_... |
361 362 363 364 365 366 |
#define RCU_GP_ONOFF 3 /* Grace-period initialization hotplug. */ #define RCU_GP_INIT 4 /* Grace-period initialization. */ #define RCU_GP_WAIT_FQS 5 /* Wait for force-quiescent-state time. */ #define RCU_GP_DOING_FQS 6 /* Wait done for force-quiescent-state time. */ #define RCU_GP_CLEANUP 7 /* Grace-period cleanup started. */ #define RCU_GP_CLEANED 8 /* Grace-period cleanup complete. */ |
afea227fd rcutorture: Expor... |
367 |
|
358be2d36 rcu: Remove RCU_S... |
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 |
/* * In order to export the rcu_state name to the tracing tools, it * needs to be added in the __tracepoint_string section. * This requires defining a separate variable tp_<sname>_varname * that points to the string being used, and this will allow * the tracing userspace tools to be able to decipher the string * address to the matching string. */ #ifdef CONFIG_PREEMPT_RCU #define RCU_ABBR 'p' #define RCU_NAME_RAW "rcu_preempt" #else /* #ifdef CONFIG_PREEMPT_RCU */ #define RCU_ABBR 's' #define RCU_NAME_RAW "rcu_sched" #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ #ifndef CONFIG_TRACING #define RCU_NAME RCU_NAME_RAW #else /* #ifdef CONFIG_TRACING */ static char rcu_name[] = RCU_NAME_RAW; static const char *tp_rcu_varname __used __tracepoint_string = rcu_name; #define RCU_NAME rcu_name #endif /* #else #ifdef CONFIG_TRACING */ |
6b50e119c rcutorture: Print... |
390 |
|
32255d51b rcu: Move RCU CPU... |
391 |
/* Forward declarations for tree_plugin.h */ |
dbe01350f rcu: Remove inlin... |
392 |
static void rcu_bootup_announce(void); |
45975c7d2 rcu: Define RCU-s... |
393 |
static void rcu_qs(void); |
27f4d2805 rcu: priority boo... |
394 |
static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); |
b668c9cf3 rcu: Fix grace-pe... |
395 |
#ifdef CONFIG_HOTPLUG_CPU |
8af3a5e78 rcu: Abstract rcu... |
396 |
static bool rcu_preempt_has_tasks(struct rcu_node *rnp); |
b668c9cf3 rcu: Fix grace-pe... |
397 |
#endif /* #ifdef CONFIG_HOTPLUG_CPU */ |
74611ecb0 rcu: Add online/o... |
398 |
static int rcu_print_task_exp_stall(struct rcu_node *rnp); |
81ab59a3a rcu: Remove rsp p... |
399 |
static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); |
c98cac603 rcu: Rename rcu_c... |
400 |
static void rcu_flavor_sched_clock_irq(int user); |
81ab59a3a rcu: Remove rsp p... |
401 |
static void dump_blkd_tasks(struct rcu_node *rnp, int ncheck); |
1217ed1ba rcu: permit rcu_r... |
402 |
static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); |
a46e0899e rcu: use softirq ... |
403 |
static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); |
dff1672d9 rcu: Keep invokin... |
404 |
static bool rcu_is_callbacks_kthread(void); |
48d07c04b rcu: Enable elimi... |
405 |
static void rcu_cpu_kthread_setup(unsigned int cpu); |
9386c0b75 rcu: Rationalize ... |
406 |
static void __init rcu_spawn_boost_kthreads(void); |
49fb4c629 rcu: delete __cpu... |
407 |
static void rcu_prepare_kthreads(int cpu); |
8fa7845df rcu: Remove "cpu"... |
408 |
static void rcu_cleanup_after_idle(void); |
198bbf812 rcu: Remove "cpu"... |
409 |
static void rcu_prepare_for_idle(void); |
0aa04b055 rcu: Process offl... |
410 |
static bool rcu_preempt_has_tasks(struct rcu_node *rnp); |
3e3100989 rcu: Defer report... |
411 412 |
static bool rcu_preempt_need_deferred_qs(struct task_struct *t); static void rcu_preempt_deferred_qs(struct task_struct *t); |
a858af287 rcu: Print schedu... |
413 |
static void zero_cpu_stall_ticks(struct rcu_data *rdp); |
abedf8e24 rcu: Use simple w... |
414 415 |
static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); |
dae6e64d2 rcu: Introduce pr... |
416 |
static void rcu_init_one_nocb(struct rcu_node *rnp); |
d1b222c6b rcu/nocb: Add byp... |
417 418 419 420 |
static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j); static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool *was_alldone, unsigned long flags); |
5d6742b37 rcu/nocb: Use rcu... |
421 422 |
static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); |
9fdd3bc90 rcu: Break more c... |
423 |
static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp); |
96d3fd0d3 rcu: Break call_r... |
424 |
static void do_nocb_deferred_wakeup(struct rcu_data *rdp); |
3fbfbf7a3 rcu: Add callback... |
425 |
static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp); |
ad368d15b rcu: Rename and c... |
426 |
static void rcu_spawn_cpu_nocb_kthread(int cpu); |
35ce7f29a rcu: Create rcuo ... |
427 |
static void __init rcu_spawn_nocb_kthreads(void); |
f7a81b12d rcu/nocb: Print n... |
428 |
static void show_rcu_nocb_state(struct rcu_data *rdp); |
5d6742b37 rcu/nocb: Use rcu... |
429 430 431 432 |
static void rcu_nocb_lock(struct rcu_data *rdp); static void rcu_nocb_unlock(struct rcu_data *rdp); static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, unsigned long flags); |
d1b222c6b rcu/nocb: Add byp... |
433 |
static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp); |
35ce7f29a rcu: Create rcuo ... |
434 |
#ifdef CONFIG_RCU_NOCB_CPU |
4580b0541 rcu: Remove rsp p... |
435 |
static void __init rcu_organize_nocb_kthreads(void); |
81c0b3d72 rcu/nocb: Avoid -... |
436 437 |
#define rcu_nocb_lock_irqsave(rdp, flags) \ do { \ |
d1b222c6b rcu/nocb: Add byp... |
438 |
if (!rcu_segcblist_is_offloaded(&(rdp)->cblist)) \ |
81c0b3d72 rcu/nocb: Avoid -... |
439 |
local_irq_save(flags); \ |
d1b222c6b rcu/nocb: Add byp... |
440 |
else \ |
81c0b3d72 rcu/nocb: Avoid -... |
441 |
raw_spin_lock_irqsave(&(rdp)->nocb_lock, (flags)); \ |
81c0b3d72 rcu/nocb: Avoid -... |
442 443 444 445 |
} while (0) #else /* #ifdef CONFIG_RCU_NOCB_CPU */ #define rcu_nocb_lock_irqsave(rdp, flags) local_irq_save(flags) #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ |
eb75767be nohz_full: Force ... |
446 |
static void rcu_bind_gp_kthread(void); |
4580b0541 rcu: Remove rsp p... |
447 |
static bool rcu_nohz_full_cpu(void); |
176f8f7a5 rcu: Make TASKS_R... |
448 449 |
static void rcu_dynticks_task_enter(void); static void rcu_dynticks_task_exit(void); |
7d0c9c50c rcu-tasks: Avoid ... |
450 451 |
static void rcu_dynticks_task_trace_enter(void); static void rcu_dynticks_task_trace_exit(void); |
32255d51b rcu: Move RCU CPU... |
452 453 |
/* Forward declarations for tree_stall.h */ |
32255d51b rcu: Move RCU CPU... |
454 |
static void record_gp_stall_check_time(void); |
7ac1907c9 rcu: Move irq-dis... |
455 |
static void rcu_iw_handler(struct irq_work *iwp); |
32255d51b rcu: Move RCU CPU... |
456 |
static void check_cpu_stall(struct rcu_data *rdp); |
b51bcbbf1 rcu: Move forward... |
457 458 |
static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, const unsigned long gpssdelay); |