Blame view
kernel/rcutiny_plugin.h
33.2 KB
bbad93798 rcu: slim down rc... |
1 |
/* |
a57eb940d rcu: Add a TINY_P... |
2 |
* Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition |
bbad93798 rcu: slim down rc... |
3 |
* Internal non-public definitions that provide either classic |
a57eb940d rcu: Add a TINY_P... |
4 |
* or preemptible semantics. |
bbad93798 rcu: slim down rc... |
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
* * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * |
a57eb940d rcu: Add a TINY_P... |
20 |
* Copyright (c) 2010 Linaro |
bbad93798 rcu: slim down rc... |
21 22 23 |
* * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> */ |
b2c0710c4 rcu: move TINY_RC... |
24 |
#include <linux/kthread.h> |
bdfa97bf7 kernel: fix up mo... |
25 |
#include <linux/module.h> |
9e571a82f rcu: add tracing ... |
26 27 |
#include <linux/debugfs.h> #include <linux/seq_file.h> |
24278d148 rcu: priority boo... |
28 29 30 31 32 |
/* Global control variables for rcupdate callback mechanism. */ struct rcu_ctrlblk { struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ struct rcu_head **curtail; /* ->next pointer of last CB. */ |
9e571a82f rcu: add tracing ... |
33 |
RCU_TRACE(long qlen); /* Number of pending CBs. */ |
6bfc09e23 rcu: Provide RCU ... |
34 35 36 |
RCU_TRACE(unsigned long gp_start); /* Start time for stalls. */ RCU_TRACE(unsigned long ticks_this_gp); /* Statistic for stalls. */ RCU_TRACE(unsigned long jiffies_stall); /* Jiffies at next stall. */ |
e99033c5c rcu: Put names in... |
37 |
RCU_TRACE(char *name); /* Name of RCU type. */ |
24278d148 rcu: priority boo... |
38 39 40 41 42 43 |
}; /* Definition for rcupdate control block. */ static struct rcu_ctrlblk rcu_sched_ctrlblk = { .donetail = &rcu_sched_ctrlblk.rcucblist, .curtail = &rcu_sched_ctrlblk.rcucblist, |
e99033c5c rcu: Put names in... |
44 |
RCU_TRACE(.name = "rcu_sched") |
24278d148 rcu: priority boo... |
45 46 47 48 49 |
}; static struct rcu_ctrlblk rcu_bh_ctrlblk = { .donetail = &rcu_bh_ctrlblk.rcucblist, .curtail = &rcu_bh_ctrlblk.rcucblist, |
e99033c5c rcu: Put names in... |
50 |
RCU_TRACE(.name = "rcu_bh") |
24278d148 rcu: priority boo... |
51 52 53 54 55 56 |
}; #ifdef CONFIG_DEBUG_LOCK_ALLOC int rcu_scheduler_active __read_mostly; EXPORT_SYMBOL_GPL(rcu_scheduler_active); #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
6bfc09e23 rcu: Provide RCU ... |
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
#ifdef CONFIG_RCU_TRACE static void check_cpu_stall(struct rcu_ctrlblk *rcp) { unsigned long j; unsigned long js; if (rcu_cpu_stall_suppress) return; rcp->ticks_this_gp++; j = jiffies; js = rcp->jiffies_stall; if (*rcp->curtail && ULONG_CMP_GE(j, js)) { pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld) ", rcp->name, rcp->ticks_this_gp, rcu_dynticks_nesting, jiffies - rcp->gp_start, rcp->qlen); dump_stack(); } if (*rcp->curtail && ULONG_CMP_GE(j, js)) rcp->jiffies_stall = jiffies + 3 * rcu_jiffies_till_stall_check() + 3; else if (ULONG_CMP_GE(j, js)) rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check(); } static void check_cpu_stall_preempt(void); #endif /* #ifdef CONFIG_RCU_TRACE */ static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp) { #ifdef CONFIG_RCU_TRACE rcp->ticks_this_gp = 0; rcp->gp_start = jiffies; rcp->jiffies_stall = jiffies + rcu_jiffies_till_stall_check(); #endif /* #ifdef CONFIG_RCU_TRACE */ } static void check_cpu_stalls(void) { RCU_TRACE(check_cpu_stall(&rcu_bh_ctrlblk)); RCU_TRACE(check_cpu_stall(&rcu_sched_ctrlblk)); RCU_TRACE(check_cpu_stall_preempt()); } |
a57eb940d rcu: Add a TINY_P... |
102 103 104 |
#ifdef CONFIG_TINY_PREEMPT_RCU #include <linux/delay.h> |
a57eb940d rcu: Add a TINY_P... |
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
/* Global control variables for preemptible RCU. */ struct rcu_preempt_ctrlblk { struct rcu_ctrlblk rcb; /* curtail: ->next ptr of last CB for GP. */ struct rcu_head **nexttail; /* Tasks blocked in a preemptible RCU */ /* read-side critical section while an */ /* preemptible-RCU grace period is in */ /* progress must wait for a later grace */ /* period. This pointer points to the */ /* ->next pointer of the last task that */ /* must wait for a later grace period, or */ /* to &->rcb.rcucblist if there is no */ /* such task. */ struct list_head blkd_tasks; /* Tasks blocked in RCU read-side critical */ /* section. Tasks are placed at the head */ /* of this list and age towards the tail. */ struct list_head *gp_tasks; /* Pointer to the first task blocking the */ /* current grace period, or NULL if there */ |
24278d148 rcu: priority boo... |
125 |
/* is no such task. */ |
a57eb940d rcu: Add a TINY_P... |
126 127 128 129 130 131 |
struct list_head *exp_tasks; /* Pointer to first task blocking the */ /* current expedited grace period, or NULL */ /* if there is no such task. If there */ /* is no current expedited grace period, */ /* then there cannot be any such task. */ |
24278d148 rcu: priority boo... |
132 133 134 135 136 137 138 139 |
#ifdef CONFIG_RCU_BOOST struct list_head *boost_tasks; /* Pointer to first task that needs to be */ /* priority-boosted, or NULL if no priority */ /* boosting is needed. If there is no */ /* current or expedited grace period, there */ /* can be no such task. */ #endif /* #ifdef CONFIG_RCU_BOOST */ |
a57eb940d rcu: Add a TINY_P... |
140 141 142 143 |
u8 gpnum; /* Current grace period. */ u8 gpcpu; /* Last grace period blocked by the CPU. */ u8 completed; /* Last grace period completed. */ /* If all three are equal, RCU is idle. */ |
9e571a82f rcu: add tracing ... |
144 |
#ifdef CONFIG_RCU_BOOST |
24278d148 rcu: priority boo... |
145 |
unsigned long boost_time; /* When to start boosting (jiffies) */ |
9e571a82f rcu: add tracing ... |
146 147 148 149 150 |
#endif /* #ifdef CONFIG_RCU_BOOST */ #ifdef CONFIG_RCU_TRACE unsigned long n_grace_periods; #ifdef CONFIG_RCU_BOOST unsigned long n_tasks_boosted; |
7e8b4c723 rcu: Converge TIN... |
151 |
/* Total number of tasks boosted. */ |
9e571a82f rcu: add tracing ... |
152 |
unsigned long n_exp_boosts; |
7e8b4c723 rcu: Converge TIN... |
153 |
/* Number of tasks boosted for expedited GP. */ |
9e571a82f rcu: add tracing ... |
154 |
unsigned long n_normal_boosts; |
7e8b4c723 rcu: Converge TIN... |
155 156 157 158 159 160 161 162 163 164 165 166 |
/* Number of tasks boosted for normal GP. */ unsigned long n_balk_blkd_tasks; /* Refused to boost: no blocked tasks. */ unsigned long n_balk_exp_gp_tasks; /* Refused to boost: nothing blocking GP. */ unsigned long n_balk_boost_tasks; /* Refused to boost: already boosting. */ unsigned long n_balk_notyet; /* Refused to boost: not yet time. */ unsigned long n_balk_nos; /* Refused to boost: not sure why, though. */ /* This can happen due to race conditions. */ |
9e571a82f rcu: add tracing ... |
167 168 |
#endif /* #ifdef CONFIG_RCU_BOOST */ #endif /* #ifdef CONFIG_RCU_TRACE */ |
a57eb940d rcu: Add a TINY_P... |
169 170 171 172 173 174 175 |
}; static struct rcu_preempt_ctrlblk rcu_preempt_ctrlblk = { .rcb.donetail = &rcu_preempt_ctrlblk.rcb.rcucblist, .rcb.curtail = &rcu_preempt_ctrlblk.rcb.rcucblist, .nexttail = &rcu_preempt_ctrlblk.rcb.rcucblist, .blkd_tasks = LIST_HEAD_INIT(rcu_preempt_ctrlblk.blkd_tasks), |
e99033c5c rcu: Put names in... |
176 |
RCU_TRACE(.rcb.name = "rcu_preempt") |
a57eb940d rcu: Add a TINY_P... |
177 178 179 180 181 182 183 184 |
}; static int rcu_preempted_readers_exp(void); static void rcu_report_exp_done(void); /* * Return true if the CPU has not yet responded to the current grace period. */ |
dd7c4d897 rcu: performance ... |
185 |
static int rcu_cpu_blocking_cur_gp(void) |
a57eb940d rcu: Add a TINY_P... |
186 187 188 189 190 191 192 |
{ return rcu_preempt_ctrlblk.gpcpu != rcu_preempt_ctrlblk.gpnum; } /* * Check for a running RCU reader. Because there is only one CPU, * there can be but one running RCU reader at a time. ;-) |
26861faf8 rcu: Protect __rc... |
193 194 195 196 197 198 199 200 201 202 |
* * Returns zero if there are no running readers. Returns a positive * number if there is at least one reader within its RCU read-side * critical section. Returns a negative number if an outermost reader * is in the midst of exiting from its RCU read-side critical section * * Returns zero if there are no running readers. Returns a positive * number if there is at least one reader within its RCU read-side * critical section. Returns a negative number if an outermost reader * is in the midst of exiting from its RCU read-side critical section. |
a57eb940d rcu: Add a TINY_P... |
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 |
*/ static int rcu_preempt_running_reader(void) { return current->rcu_read_lock_nesting; } /* * Check for preempted RCU readers blocking any grace period. * If the caller needs a reliable answer, it must disable hard irqs. */ static int rcu_preempt_blocked_readers_any(void) { return !list_empty(&rcu_preempt_ctrlblk.blkd_tasks); } /* * Check for preempted RCU readers blocking the current grace period. * If the caller needs a reliable answer, it must disable hard irqs. */ static int rcu_preempt_blocked_readers_cgp(void) { return rcu_preempt_ctrlblk.gp_tasks != NULL; } /* * Return true if another preemptible-RCU grace period is needed. */ static int rcu_preempt_needs_another_gp(void) { return *rcu_preempt_ctrlblk.rcb.curtail != NULL; } /* * Return true if a preemptible-RCU grace period is in progress. * The caller must disable hardirqs. */ static int rcu_preempt_gp_in_progress(void) { return rcu_preempt_ctrlblk.completed != rcu_preempt_ctrlblk.gpnum; } /* |
24278d148 rcu: priority boo... |
245 246 247 248 249 250 251 252 253 254 255 256 |
* Advance a ->blkd_tasks-list pointer to the next entry, instead * returning NULL if at the end of the list. */ static struct list_head *rcu_next_node_entry(struct task_struct *t) { struct list_head *np; np = t->rcu_node_entry.next; if (np == &rcu_preempt_ctrlblk.blkd_tasks) np = NULL; return np; } |
9e571a82f rcu: add tracing ... |
257 258 259 260 |
#ifdef CONFIG_RCU_TRACE #ifdef CONFIG_RCU_BOOST static void rcu_initiate_boost_trace(void); |
9e571a82f rcu: add tracing ... |
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
#endif /* #ifdef CONFIG_RCU_BOOST */ /* * Dump additional statistice for TINY_PREEMPT_RCU. */ static void show_tiny_preempt_stats(struct seq_file *m) { seq_printf(m, "rcu_preempt: qlen=%ld gp=%lu g%u/p%u/c%u tasks=%c%c%c ", rcu_preempt_ctrlblk.rcb.qlen, rcu_preempt_ctrlblk.n_grace_periods, rcu_preempt_ctrlblk.gpnum, rcu_preempt_ctrlblk.gpcpu, rcu_preempt_ctrlblk.completed, "T."[list_empty(&rcu_preempt_ctrlblk.blkd_tasks)], "N."[!rcu_preempt_ctrlblk.gp_tasks], "E."[!rcu_preempt_ctrlblk.exp_tasks]); #ifdef CONFIG_RCU_BOOST |
203373c81 rcu: remove usele... |
279 280 281 282 |
seq_printf(m, "%sttb=%c ntb=%lu neb=%lu nnb=%lu j=%04x bt=%04x ", " ", "B."[!rcu_preempt_ctrlblk.boost_tasks], |
9e571a82f rcu: add tracing ... |
283 284 285 286 287 |
rcu_preempt_ctrlblk.n_tasks_boosted, rcu_preempt_ctrlblk.n_exp_boosts, rcu_preempt_ctrlblk.n_normal_boosts, (int)(jiffies & 0xffff), (int)(rcu_preempt_ctrlblk.boost_time & 0xffff)); |
7e8b4c723 rcu: Converge TIN... |
288 289 290 291 292 293 294 295 |
seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu ny=%lu nos=%lu ", " balk", rcu_preempt_ctrlblk.n_balk_blkd_tasks, rcu_preempt_ctrlblk.n_balk_exp_gp_tasks, rcu_preempt_ctrlblk.n_balk_boost_tasks, rcu_preempt_ctrlblk.n_balk_notyet, rcu_preempt_ctrlblk.n_balk_nos); |
9e571a82f rcu: add tracing ... |
296 297 298 299 |
#endif /* #ifdef CONFIG_RCU_BOOST */ } #endif /* #ifdef CONFIG_RCU_TRACE */ |
24278d148 rcu: priority boo... |
300 301 302 |
#ifdef CONFIG_RCU_BOOST #include "rtmutex_common.h" |
965a002b4 rcu: Make TINY_RC... |
303 304 305 306 307 308 |
#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO /* Controls for rcu_kthread() kthread. */ static struct task_struct *rcu_kthread_task; static DECLARE_WAIT_QUEUE_HEAD(rcu_kthread_wq); static unsigned long have_rcu_kthread_work; |
24278d148 rcu: priority boo... |
309 310 311 312 313 314 315 316 |
/* * Carry out RCU priority boosting on the task indicated by ->boost_tasks, * and advance ->boost_tasks to the next task in the ->blkd_tasks list. */ static int rcu_boost(void) { unsigned long flags; struct rt_mutex mtx; |
24278d148 rcu: priority boo... |
317 |
struct task_struct *t; |
7e8b4c723 rcu: Converge TIN... |
318 |
struct list_head *tb; |
24278d148 rcu: priority boo... |
319 |
|
7e8b4c723 rcu: Converge TIN... |
320 321 |
if (rcu_preempt_ctrlblk.boost_tasks == NULL && rcu_preempt_ctrlblk.exp_tasks == NULL) |
24278d148 rcu: priority boo... |
322 |
return 0; /* Nothing to boost. */ |
7e8b4c723 rcu: Converge TIN... |
323 |
|
7a11e2058 rcu: Move TINY_PR... |
324 |
local_irq_save(flags); |
7e8b4c723 rcu: Converge TIN... |
325 326 327 328 329 330 331 332 |
/* * Recheck with irqs disabled: all tasks in need of boosting * might exit their RCU read-side critical sections on their own * if we are preempted just before disabling irqs. */ if (rcu_preempt_ctrlblk.boost_tasks == NULL && rcu_preempt_ctrlblk.exp_tasks == NULL) { |
7a11e2058 rcu: Move TINY_PR... |
333 |
local_irq_restore(flags); |
7e8b4c723 rcu: Converge TIN... |
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 |
return 0; } /* * Preferentially boost tasks blocking expedited grace periods. * This cannot starve the normal grace periods because a second * expedited grace period must boost all blocked tasks, including * those blocking the pre-existing normal grace period. */ if (rcu_preempt_ctrlblk.exp_tasks != NULL) { tb = rcu_preempt_ctrlblk.exp_tasks; RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++); } else { tb = rcu_preempt_ctrlblk.boost_tasks; RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++); } RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++); /* * We boost task t by manufacturing an rt_mutex that appears to * be held by task t. We leave a pointer to that rt_mutex where * task t can find it, and task t will release the mutex when it * exits its outermost RCU read-side critical section. Then * simply acquiring this artificial rt_mutex will boost task * t's priority. (Thanks to tglx for suggesting this approach!) */ t = container_of(tb, struct task_struct, rcu_node_entry); |
24278d148 rcu: priority boo... |
361 362 |
rt_mutex_init_proxy_locked(&mtx, t); t->rcu_boost_mutex = &mtx; |
7a11e2058 rcu: Move TINY_PR... |
363 |
local_irq_restore(flags); |
24278d148 rcu: priority boo... |
364 |
rt_mutex_lock(&mtx); |
7e8b4c723 rcu: Converge TIN... |
365 |
rt_mutex_unlock(&mtx); /* Keep lockdep happy. */ |
4f89b336f rcu: Apply ACCESS... |
366 367 |
return ACCESS_ONCE(rcu_preempt_ctrlblk.boost_tasks) != NULL || ACCESS_ONCE(rcu_preempt_ctrlblk.exp_tasks) != NULL; |
24278d148 rcu: priority boo... |
368 369 370 371 372 373 374 |
} /* * Check to see if it is now time to start boosting RCU readers blocking * the current grace period, and, if so, tell the rcu_kthread_task to * start boosting them. If there is an expedited boost in progress, * we wait for it to complete. |
9e571a82f rcu: add tracing ... |
375 376 377 378 |
* * If there are no blocked readers blocking the current grace period, * return 0 to let the caller know, otherwise return 1. Note that this * return value is independent of whether or not boosting was done. |
24278d148 rcu: priority boo... |
379 |
*/ |
9e571a82f rcu: add tracing ... |
380 |
static int rcu_initiate_boost(void) |
24278d148 rcu: priority boo... |
381 |
{ |
7e8b4c723 rcu: Converge TIN... |
382 383 384 |
if (!rcu_preempt_blocked_readers_cgp() && rcu_preempt_ctrlblk.exp_tasks == NULL) { RCU_TRACE(rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++); |
9e571a82f rcu: add tracing ... |
385 386 |
return 0; } |
7e8b4c723 rcu: Converge TIN... |
387 388 389 390 391 392 393 |
if (rcu_preempt_ctrlblk.exp_tasks != NULL || (rcu_preempt_ctrlblk.gp_tasks != NULL && rcu_preempt_ctrlblk.boost_tasks == NULL && ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time))) { if (rcu_preempt_ctrlblk.exp_tasks == NULL) rcu_preempt_ctrlblk.boost_tasks = rcu_preempt_ctrlblk.gp_tasks; |
965a002b4 rcu: Make TINY_RC... |
394 |
invoke_rcu_callbacks(); |
c701d5d9b rcu: Fix code-sty... |
395 |
} else { |
9e571a82f rcu: add tracing ... |
396 |
RCU_TRACE(rcu_initiate_boost_trace()); |
c701d5d9b rcu: Fix code-sty... |
397 |
} |
9e571a82f rcu: add tracing ... |
398 |
return 1; |
24278d148 rcu: priority boo... |
399 |
} |
ddeb75814 rcu: code cleanup... |
400 |
#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000) |
24278d148 rcu: priority boo... |
401 402 403 404 405 406 407 |
/* * Do priority-boost accounting for the start of a new grace period. */ static void rcu_preempt_boost_start_gp(void) { rcu_preempt_ctrlblk.boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES; |
24278d148 rcu: priority boo... |
408 409 410 411 412 |
} #else /* #ifdef CONFIG_RCU_BOOST */ /* |
9e571a82f rcu: add tracing ... |
413 414 415 |
* If there is no RCU priority boosting, we don't initiate boosting, * but we do indicate whether there are blocked readers blocking the * current grace period. |
24278d148 rcu: priority boo... |
416 |
*/ |
9e571a82f rcu: add tracing ... |
417 |
static int rcu_initiate_boost(void) |
24278d148 rcu: priority boo... |
418 |
{ |
9e571a82f rcu: add tracing ... |
419 |
return rcu_preempt_blocked_readers_cgp(); |
24278d148 rcu: priority boo... |
420 421 422 |
} /* |
24278d148 rcu: priority boo... |
423 424 425 426 427 428 429 430 431 |
* If there is no RCU priority boosting, nothing to do at grace-period start. */ static void rcu_preempt_boost_start_gp(void) { } #endif /* else #ifdef CONFIG_RCU_BOOST */ /* |
a57eb940d rcu: Add a TINY_P... |
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 |
* Record a preemptible-RCU quiescent state for the specified CPU. Note * that this just means that the task currently running on the CPU is * in a quiescent state. There might be any number of tasks blocked * while in an RCU read-side critical section. * * Unlike the other rcu_*_qs() functions, callers to this function * must disable irqs in order to protect the assignment to * ->rcu_read_unlock_special. * * Because this is a single-CPU implementation, the only way a grace * period can end is if the CPU is in a quiescent state. The reason is * that a blocked preemptible-RCU reader can exit its critical section * only if the CPU is running it at the time. Therefore, when the * last task blocking the current grace period exits its RCU read-side * critical section, neither the CPU nor blocked tasks will be stopping * the current grace period. (In contrast, SMP implementations * might have CPUs running in RCU read-side critical sections that * block later grace periods -- but this is not possible given only * one CPU.) */ static void rcu_preempt_cpu_qs(void) { /* Record both CPU and task as having responded to current GP. */ rcu_preempt_ctrlblk.gpcpu = rcu_preempt_ctrlblk.gpnum; current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS; |
24278d148 rcu: priority boo... |
457 |
/* If there is no GP then there is nothing more to do. */ |
9e571a82f rcu: add tracing ... |
458 |
if (!rcu_preempt_gp_in_progress()) |
a57eb940d rcu: Add a TINY_P... |
459 |
return; |
9e571a82f rcu: add tracing ... |
460 |
/* |
ddeb75814 rcu: code cleanup... |
461 |
* Check up on boosting. If there are readers blocking the |
9e571a82f rcu: add tracing ... |
462 463 464 |
* current grace period, leave. */ if (rcu_initiate_boost()) |
24278d148 rcu: priority boo... |
465 |
return; |
a57eb940d rcu: Add a TINY_P... |
466 467 468 469 470 471 472 473 474 |
/* Advance callbacks. */ rcu_preempt_ctrlblk.completed = rcu_preempt_ctrlblk.gpnum; rcu_preempt_ctrlblk.rcb.donetail = rcu_preempt_ctrlblk.rcb.curtail; rcu_preempt_ctrlblk.rcb.curtail = rcu_preempt_ctrlblk.nexttail; /* If there are no blocked readers, next GP is done instantly. */ if (!rcu_preempt_blocked_readers_any()) rcu_preempt_ctrlblk.rcb.donetail = rcu_preempt_ctrlblk.nexttail; |
b2c0710c4 rcu: move TINY_RC... |
475 |
/* If there are done callbacks, cause them to be invoked. */ |
a57eb940d rcu: Add a TINY_P... |
476 |
if (*rcu_preempt_ctrlblk.rcb.donetail != NULL) |
965a002b4 rcu: Make TINY_RC... |
477 |
invoke_rcu_callbacks(); |
a57eb940d rcu: Add a TINY_P... |
478 479 480 481 482 483 484 485 486 487 488 |
} /* * Start a new RCU grace period if warranted. Hard irqs must be disabled. */ static void rcu_preempt_start_gp(void) { if (!rcu_preempt_gp_in_progress() && rcu_preempt_needs_another_gp()) { /* Official start of GP. */ rcu_preempt_ctrlblk.gpnum++; |
9e571a82f rcu: add tracing ... |
489 |
RCU_TRACE(rcu_preempt_ctrlblk.n_grace_periods++); |
6bfc09e23 rcu: Provide RCU ... |
490 |
reset_cpu_stall_ticks(&rcu_preempt_ctrlblk.rcb); |
a57eb940d rcu: Add a TINY_P... |
491 492 493 494 495 |
/* Any blocked RCU readers block new GP. */ if (rcu_preempt_blocked_readers_any()) rcu_preempt_ctrlblk.gp_tasks = rcu_preempt_ctrlblk.blkd_tasks.next; |
24278d148 rcu: priority boo... |
496 497 |
/* Set up for RCU priority boosting. */ rcu_preempt_boost_start_gp(); |
a57eb940d rcu: Add a TINY_P... |
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 |
/* If there is no running reader, CPU is done with GP. */ if (!rcu_preempt_running_reader()) rcu_preempt_cpu_qs(); } } /* * We have entered the scheduler, and the current task might soon be * context-switched away from. If this task is in an RCU read-side * critical section, we will no longer be able to rely on the CPU to * record that fact, so we enqueue the task on the blkd_tasks list. * If the task started after the current grace period began, as recorded * by ->gpcpu, we enqueue at the beginning of the list. Otherwise * before the element referenced by ->gp_tasks (or at the tail if * ->gp_tasks is NULL) and point ->gp_tasks at the newly added element. * The task will dequeue itself when it exits the outermost enclosing * RCU read-side critical section. Therefore, the current grace period * cannot be permitted to complete until the ->gp_tasks pointer becomes * NULL. * * Caller must disable preemption. */ void rcu_preempt_note_context_switch(void) { struct task_struct *t = current; unsigned long flags; local_irq_save(flags); /* must exclude scheduler_tick(). */ |
26861faf8 rcu: Protect __rc... |
526 |
if (rcu_preempt_running_reader() > 0 && |
a57eb940d rcu: Add a TINY_P... |
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 |
(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) { /* Possibly blocking in an RCU read-side critical section. */ t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED; /* * If this CPU has already checked in, then this task * will hold up the next grace period rather than the * current grace period. Queue the task accordingly. * If the task is queued for the current grace period * (i.e., this CPU has not yet passed through a quiescent * state for the current grace period), then as long * as that task remains queued, the current grace period * cannot end. */ list_add(&t->rcu_node_entry, &rcu_preempt_ctrlblk.blkd_tasks); |
dd7c4d897 rcu: performance ... |
543 |
if (rcu_cpu_blocking_cur_gp()) |
a57eb940d rcu: Add a TINY_P... |
544 |
rcu_preempt_ctrlblk.gp_tasks = &t->rcu_node_entry; |
26861faf8 rcu: Protect __rc... |
545 546 547 548 549 550 551 |
} else if (rcu_preempt_running_reader() < 0 && t->rcu_read_unlock_special) { /* * Complete exit from RCU read-side critical section on * behalf of preempted instance of __rcu_read_unlock(). */ rcu_read_unlock_special(t); |
a57eb940d rcu: Add a TINY_P... |
552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 |
} /* * Either we were not in an RCU read-side critical section to * begin with, or we have now recorded that critical section * globally. Either way, we can now note a quiescent state * for this CPU. Again, if we were in an RCU read-side critical * section, and if that critical section was blocking the current * grace period, then the fact that the task has been enqueued * means that current grace period continues to be blocked. */ rcu_preempt_cpu_qs(); local_irq_restore(flags); } /* |
a57eb940d rcu: Add a TINY_P... |
568 569 570 571 |
* Handle special cases during rcu_read_unlock(), such as needing to * notify RCU core processing or task having blocked during the RCU * read-side critical section. */ |
2a3fa843b rcu: Consolidate ... |
572 |
void rcu_read_unlock_special(struct task_struct *t) |
a57eb940d rcu: Add a TINY_P... |
573 574 575 576 577 |
{ int empty; int empty_exp; unsigned long flags; struct list_head *np; |
1aa03f118 rcu: Simplify unb... |
578 579 580 |
#ifdef CONFIG_RCU_BOOST struct rt_mutex *rbmp = NULL; #endif /* #ifdef CONFIG_RCU_BOOST */ |
a57eb940d rcu: Add a TINY_P... |
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 |
int special; /* * NMI handlers cannot block and cannot safely manipulate state. * They therefore cannot possibly be special, so just leave. */ if (in_nmi()) return; local_irq_save(flags); /* * If RCU core is waiting for this CPU to exit critical section, * let it know that we have done so. */ special = t->rcu_read_unlock_special; if (special & RCU_READ_UNLOCK_NEED_QS) rcu_preempt_cpu_qs(); /* Hardware IRQ handlers cannot block. */ |
8762705ad rcu: Inform RCU o... |
601 |
if (in_irq() || in_serving_softirq()) { |
a57eb940d rcu: Add a TINY_P... |
602 603 604 605 606 607 608 609 610 611 612 613 614 615 |
local_irq_restore(flags); return; } /* Clean up if blocked during RCU read-side critical section. */ if (special & RCU_READ_UNLOCK_BLOCKED) { t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED; /* * Remove this task from the ->blkd_tasks list and adjust * any pointers that might have been referencing it. */ empty = !rcu_preempt_blocked_readers_cgp(); empty_exp = rcu_preempt_ctrlblk.exp_tasks == NULL; |
24278d148 rcu: priority boo... |
616 |
np = rcu_next_node_entry(t); |
ddeb75814 rcu: code cleanup... |
617 |
list_del_init(&t->rcu_node_entry); |
a57eb940d rcu: Add a TINY_P... |
618 619 620 621 |
if (&t->rcu_node_entry == rcu_preempt_ctrlblk.gp_tasks) rcu_preempt_ctrlblk.gp_tasks = np; if (&t->rcu_node_entry == rcu_preempt_ctrlblk.exp_tasks) rcu_preempt_ctrlblk.exp_tasks = np; |
24278d148 rcu: priority boo... |
622 623 624 625 |
#ifdef CONFIG_RCU_BOOST if (&t->rcu_node_entry == rcu_preempt_ctrlblk.boost_tasks) rcu_preempt_ctrlblk.boost_tasks = np; #endif /* #ifdef CONFIG_RCU_BOOST */ |
a57eb940d rcu: Add a TINY_P... |
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 |
/* * If this was the last task on the current list, and if * we aren't waiting on the CPU, report the quiescent state * and start a new grace period if needed. */ if (!empty && !rcu_preempt_blocked_readers_cgp()) { rcu_preempt_cpu_qs(); rcu_preempt_start_gp(); } /* * If this was the last task on the expedited lists, * then we need wake up the waiting task. */ if (!empty_exp && rcu_preempt_ctrlblk.exp_tasks == NULL) rcu_report_exp_done(); } |
24278d148 rcu: priority boo... |
644 645 |
#ifdef CONFIG_RCU_BOOST /* Unboost self if was boosted. */ |
1aa03f118 rcu: Simplify unb... |
646 647 |
if (t->rcu_boost_mutex != NULL) { rbmp = t->rcu_boost_mutex; |
24278d148 rcu: priority boo... |
648 |
t->rcu_boost_mutex = NULL; |
1aa03f118 rcu: Simplify unb... |
649 |
rt_mutex_unlock(rbmp); |
24278d148 rcu: priority boo... |
650 651 |
} #endif /* #ifdef CONFIG_RCU_BOOST */ |
a57eb940d rcu: Add a TINY_P... |
652 653 654 655 |
local_irq_restore(flags); } /* |
a57eb940d rcu: Add a TINY_P... |
656 657 658 659 660 661 662 663 664 |
* Check for a quiescent state from the current CPU. When a task blocks, * the task is recorded in the rcu_preempt_ctrlblk structure, which is * checked elsewhere. This is called from the scheduling-clock interrupt. * * Caller must disable hard irqs. */ static void rcu_preempt_check_callbacks(void) { struct task_struct *t = current; |
dd7c4d897 rcu: performance ... |
665 666 667 |
if (rcu_preempt_gp_in_progress() && (!rcu_preempt_running_reader() || !rcu_cpu_blocking_cur_gp())) |
a57eb940d rcu: Add a TINY_P... |
668 669 670 |
rcu_preempt_cpu_qs(); if (&rcu_preempt_ctrlblk.rcb.rcucblist != rcu_preempt_ctrlblk.rcb.donetail) |
965a002b4 rcu: Make TINY_RC... |
671 |
invoke_rcu_callbacks(); |
dd7c4d897 rcu: performance ... |
672 673 |
if (rcu_preempt_gp_in_progress() && rcu_cpu_blocking_cur_gp() && |
26861faf8 rcu: Protect __rc... |
674 |
rcu_preempt_running_reader() > 0) |
a57eb940d rcu: Add a TINY_P... |
675 676 677 678 679 |
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS; } /* * TINY_PREEMPT_RCU has an extra callback-list tail pointer to |
b2c0710c4 rcu: move TINY_RC... |
680 |
* update, so this is invoked from rcu_process_callbacks() to |
a57eb940d rcu: Add a TINY_P... |
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 |
* handle that case. Of course, it is invoked for all flavors of * RCU, but RCU callbacks can appear only on one of the lists, and * neither ->nexttail nor ->donetail can possibly be NULL, so there * is no need for an explicit check. */ static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk *rcp) { if (rcu_preempt_ctrlblk.nexttail == rcp->donetail) rcu_preempt_ctrlblk.nexttail = &rcp->rcucblist; } /* * Process callbacks for preemptible RCU. */ static void rcu_preempt_process_callbacks(void) { |
965a002b4 rcu: Make TINY_RC... |
697 |
__rcu_process_callbacks(&rcu_preempt_ctrlblk.rcb); |
a57eb940d rcu: Add a TINY_P... |
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 |
} /* * Queue a preemptible -RCU callback for invocation after a grace period. */ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) { unsigned long flags; debug_rcu_head_queue(head); head->func = func; head->next = NULL; local_irq_save(flags); *rcu_preempt_ctrlblk.nexttail = head; rcu_preempt_ctrlblk.nexttail = &head->next; |
9e571a82f rcu: add tracing ... |
714 |
RCU_TRACE(rcu_preempt_ctrlblk.rcb.qlen++); |
a57eb940d rcu: Add a TINY_P... |
715 716 717 718 |
rcu_preempt_start_gp(); /* checks to see if GP needed. */ local_irq_restore(flags); } EXPORT_SYMBOL_GPL(call_rcu); |
a57eb940d rcu: Add a TINY_P... |
719 720 721 722 723 724 725 726 727 728 729 |
/* * synchronize_rcu - wait until a grace period has elapsed. * * Control will return to the caller some time after a full grace * period has elapsed, in other words after all currently executing RCU * read-side critical sections have completed. RCU read-side critical * sections are delimited by rcu_read_lock() and rcu_read_unlock(), * and may be nested. */ void synchronize_rcu(void) { |
fe15d706c rcu: Add lockdep-... |
730 731 732 733 |
rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) && !lock_is_held(&rcu_lock_map) && !lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_rcu() in RCU read-side critical section"); |
a57eb940d rcu: Add a TINY_P... |
734 735 736 737 738 739 740 741 742 743 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC if (!rcu_scheduler_active) return; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ WARN_ON_ONCE(rcu_preempt_running_reader()); if (!rcu_preempt_blocked_readers_any()) return; /* Once we get past the fastpath checks, same code as rcu_barrier(). */ |
3705b88db rcu: Add a module... |
744 745 746 747 |
if (rcu_expedited) synchronize_rcu_expedited(); else rcu_barrier(); |
a57eb940d rcu: Add a TINY_P... |
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 |
} EXPORT_SYMBOL_GPL(synchronize_rcu); static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq); static unsigned long sync_rcu_preempt_exp_count; static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex); /* * Return non-zero if there are any tasks in RCU read-side critical * sections blocking the current preemptible-RCU expedited grace period. * If there is no preemptible-RCU expedited grace period currently in * progress, returns zero unconditionally. */ static int rcu_preempted_readers_exp(void) { return rcu_preempt_ctrlblk.exp_tasks != NULL; } /* * Report the exit from RCU read-side critical section for the last task * that queued itself during or before the current expedited preemptible-RCU * grace period. */ static void rcu_report_exp_done(void) { wake_up(&sync_rcu_preempt_exp_wq); } /* * Wait for an rcu-preempt grace period, but expedite it. The basic idea * is to rely in the fact that there is but one CPU, and that it is * illegal for a task to invoke synchronize_rcu_expedited() while in a * preemptible-RCU read-side critical section. Therefore, any such * critical sections must correspond to blocked tasks, which must therefore * be on the ->blkd_tasks list. So just record the current head of the * list in the ->exp_tasks pointer, and wait for all tasks including and * after the task pointed to by ->exp_tasks to drain. */ void synchronize_rcu_expedited(void) { unsigned long flags; struct rcu_preempt_ctrlblk *rpcp = &rcu_preempt_ctrlblk; unsigned long snap; barrier(); /* ensure prior action seen before grace period. */ WARN_ON_ONCE(rcu_preempt_running_reader()); /* * Acquire lock so that there is only one preemptible RCU grace * period in flight. Of course, if someone does the expedited * grace period for us while we are acquiring the lock, just leave. */ snap = sync_rcu_preempt_exp_count + 1; mutex_lock(&sync_rcu_preempt_exp_mutex); if (ULONG_CMP_LT(snap, sync_rcu_preempt_exp_count)) goto unlock_mb_ret; /* Others did our work for us. */ local_irq_save(flags); /* * All RCU readers have to already be on blkd_tasks because * we cannot legally be executing in an RCU read-side critical * section. */ /* Snapshot current head of ->blkd_tasks list. */ rpcp->exp_tasks = rpcp->blkd_tasks.next; if (rpcp->exp_tasks == &rpcp->blkd_tasks) rpcp->exp_tasks = NULL; |
a57eb940d rcu: Add a TINY_P... |
818 819 |
/* Wait for tail of ->blkd_tasks list to drain. */ |
c701d5d9b rcu: Fix code-sty... |
820 |
if (!rcu_preempted_readers_exp()) { |
7e8b4c723 rcu: Converge TIN... |
821 |
local_irq_restore(flags); |
c701d5d9b rcu: Fix code-sty... |
822 |
} else { |
7e8b4c723 rcu: Converge TIN... |
823 824 |
rcu_initiate_boost(); local_irq_restore(flags); |
a57eb940d rcu: Add a TINY_P... |
825 826 |
wait_event(sync_rcu_preempt_exp_wq, !rcu_preempted_readers_exp()); |
7e8b4c723 rcu: Converge TIN... |
827 |
} |
a57eb940d rcu: Add a TINY_P... |
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 |
/* Clean up and exit. */ barrier(); /* ensure expedited GP seen before counter increment. */ sync_rcu_preempt_exp_count++; unlock_mb_ret: mutex_unlock(&sync_rcu_preempt_exp_mutex); barrier(); /* ensure subsequent action seen after grace period. */ } EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); /* * Does preemptible RCU need the CPU to stay out of dynticks mode? */ int rcu_preempt_needs_cpu(void) { |
a57eb940d rcu: Add a TINY_P... |
843 844 |
return rcu_preempt_ctrlblk.rcb.rcucblist != NULL; } |
a57eb940d rcu: Add a TINY_P... |
845 |
#else /* #ifdef CONFIG_TINY_PREEMPT_RCU */ |
9e571a82f rcu: add tracing ... |
846 847 848 849 850 851 852 853 854 855 856 |
#ifdef CONFIG_RCU_TRACE /* * Because preemptible RCU does not exist, it is not necessary to * dump out its statistics. */ static void show_tiny_preempt_stats(struct seq_file *m) { } #endif /* #ifdef CONFIG_RCU_TRACE */ |
a57eb940d rcu: Add a TINY_P... |
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 |
/* * Because preemptible RCU does not exist, it never has any callbacks * to check. */ static void rcu_preempt_check_callbacks(void) { } /* * Because preemptible RCU does not exist, it never has any callbacks * to remove. */ static void rcu_preempt_remove_callbacks(struct rcu_ctrlblk *rcp) { } /* * Because preemptible RCU does not exist, it never has any callbacks * to process. */ static void rcu_preempt_process_callbacks(void) { } #endif /* #else #ifdef CONFIG_TINY_PREEMPT_RCU */ |
965a002b4 rcu: Make TINY_RC... |
882 883 884 885 886 887 888 889 890 |
#ifdef CONFIG_RCU_BOOST /* * Wake up rcu_kthread() to process callbacks now eligible for invocation * or to boost readers. */ static void invoke_rcu_callbacks(void) { have_rcu_kthread_work = 1; |
768dfffdf rcu: Prevent RCU ... |
891 892 |
if (rcu_kthread_task != NULL) wake_up(&rcu_kthread_wq); |
965a002b4 rcu: Make TINY_RC... |
893 |
} |
4968c300e rcu: Augment rcu_... |
894 895 896 897 898 899 900 901 902 903 904 905 |
#ifdef CONFIG_RCU_TRACE /* * Is the current CPU running the RCU-callbacks kthread? * Caller must have preemption disabled. */ static bool rcu_is_callbacks_kthread(void) { return rcu_kthread_task == current; } #endif /* #ifdef CONFIG_RCU_TRACE */ |
965a002b4 rcu: Make TINY_RC... |
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 |
/* * This kthread invokes RCU callbacks whose grace periods have * elapsed. It is awakened as needed, and takes the place of the * RCU_SOFTIRQ that is used for this purpose when boosting is disabled. * This is a kthread, but it is never stopped, at least not until * the system goes down. */ static int rcu_kthread(void *arg) { unsigned long work; unsigned long morework; unsigned long flags; for (;;) { wait_event_interruptible(rcu_kthread_wq, have_rcu_kthread_work != 0); morework = rcu_boost(); local_irq_save(flags); work = have_rcu_kthread_work; have_rcu_kthread_work = morework; local_irq_restore(flags); if (work) rcu_process_callbacks(NULL); schedule_timeout_interruptible(1); /* Leave CPU for others. */ } return 0; /* Not reached, but needed to shut gcc up. */ } /* * Spawn the kthread that invokes RCU callbacks. */ static int __init rcu_spawn_kthreads(void) { struct sched_param sp; rcu_kthread_task = kthread_run(rcu_kthread, NULL, "rcu_kthread"); sp.sched_priority = RCU_BOOST_PRIO; sched_setscheduler_nocheck(rcu_kthread_task, SCHED_FIFO, &sp); return 0; } early_initcall(rcu_spawn_kthreads); #else /* #ifdef CONFIG_RCU_BOOST */ |
768dfffdf rcu: Prevent RCU ... |
950 951 |
/* Hold off callback invocation until early_initcall() time. */ static int rcu_scheduler_fully_active __read_mostly; |
965a002b4 rcu: Make TINY_RC... |
952 953 954 955 956 |
/* * Start up softirq processing of callbacks. */ void invoke_rcu_callbacks(void) { |
768dfffdf rcu: Prevent RCU ... |
957 958 |
if (rcu_scheduler_fully_active) raise_softirq(RCU_SOFTIRQ); |
965a002b4 rcu: Make TINY_RC... |
959 |
} |
4968c300e rcu: Augment rcu_... |
960 961 962 963 964 965 966 967 968 969 970 |
#ifdef CONFIG_RCU_TRACE /* * There is no callback kthread, so this thread is never it. */ static bool rcu_is_callbacks_kthread(void) { return false; } #endif /* #ifdef CONFIG_RCU_TRACE */ |
768dfffdf rcu: Prevent RCU ... |
971 |
static int __init rcu_scheduler_really_started(void) |
965a002b4 rcu: Make TINY_RC... |
972 |
{ |
768dfffdf rcu: Prevent RCU ... |
973 |
rcu_scheduler_fully_active = 1; |
965a002b4 rcu: Make TINY_RC... |
974 |
open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
768dfffdf rcu: Prevent RCU ... |
975 976 |
raise_softirq(RCU_SOFTIRQ); /* Invoke any callbacks from early boot. */ return 0; |
965a002b4 rcu: Make TINY_RC... |
977 |
} |
768dfffdf rcu: Prevent RCU ... |
978 |
early_initcall(rcu_scheduler_really_started); |
965a002b4 rcu: Make TINY_RC... |
979 980 |
#endif /* #else #ifdef CONFIG_RCU_BOOST */ |
bbad93798 rcu: slim down rc... |
981 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC |
bbad93798 rcu: slim down rc... |
982 983 984 985 986 987 |
#include <linux/kernel_stat.h> /* * During boot, we forgive RCU lockdep issues. After this function is * invoked, we start taking RCU lockdep issues seriously. */ |
b2c0710c4 rcu: move TINY_RC... |
988 |
void __init rcu_scheduler_starting(void) |
bbad93798 rcu: slim down rc... |
989 990 991 992 993 994 |
{ WARN_ON(nr_context_switches() > 0); rcu_scheduler_active = 1; } #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
24278d148 rcu: priority boo... |
995 |
|
9e571a82f rcu: add tracing ... |
996 997 998 999 1000 1001 |
#ifdef CONFIG_RCU_TRACE #ifdef CONFIG_RCU_BOOST static void rcu_initiate_boost_trace(void) { |
7e8b4c723 rcu: Converge TIN... |
1002 1003 1004 1005 1006 |
if (list_empty(&rcu_preempt_ctrlblk.blkd_tasks)) rcu_preempt_ctrlblk.n_balk_blkd_tasks++; else if (rcu_preempt_ctrlblk.gp_tasks == NULL && rcu_preempt_ctrlblk.exp_tasks == NULL) rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++; |
9e571a82f rcu: add tracing ... |
1007 |
else if (rcu_preempt_ctrlblk.boost_tasks != NULL) |
7e8b4c723 rcu: Converge TIN... |
1008 |
rcu_preempt_ctrlblk.n_balk_boost_tasks++; |
9e571a82f rcu: add tracing ... |
1009 |
else if (!ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time)) |
7e8b4c723 rcu: Converge TIN... |
1010 |
rcu_preempt_ctrlblk.n_balk_notyet++; |
9e571a82f rcu: add tracing ... |
1011 |
else |
7e8b4c723 rcu: Converge TIN... |
1012 |
rcu_preempt_ctrlblk.n_balk_nos++; |
9e571a82f rcu: add tracing ... |
1013 1014 1015 1016 1017 1018 1019 |
} #endif /* #ifdef CONFIG_RCU_BOOST */ static void rcu_trace_sub_qlen(struct rcu_ctrlblk *rcp, int n) { unsigned long flags; |
7a11e2058 rcu: Move TINY_PR... |
1020 |
local_irq_save(flags); |
9e571a82f rcu: add tracing ... |
1021 |
rcp->qlen -= n; |
7a11e2058 rcu: Move TINY_PR... |
1022 |
local_irq_restore(flags); |
9e571a82f rcu: add tracing ... |
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 |
} /* * Dump statistics for TINY_RCU, such as they are. */ static int show_tiny_stats(struct seq_file *m, void *unused) { show_tiny_preempt_stats(m); seq_printf(m, "rcu_sched: qlen: %ld ", rcu_sched_ctrlblk.qlen); seq_printf(m, "rcu_bh: qlen: %ld ", rcu_bh_ctrlblk.qlen); return 0; } static int show_tiny_stats_open(struct inode *inode, struct file *file) { return single_open(file, show_tiny_stats, NULL); } static const struct file_operations show_tiny_stats_fops = { .owner = THIS_MODULE, .open = show_tiny_stats_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; static struct dentry *rcudir; static int __init rcutiny_trace_init(void) { struct dentry *retval; rcudir = debugfs_create_dir("rcu", NULL); if (!rcudir) goto free_out; retval = debugfs_create_file("rcudata", 0444, rcudir, NULL, &show_tiny_stats_fops); if (!retval) goto free_out; return 0; free_out: debugfs_remove_recursive(rcudir); return 1; } static void __exit rcutiny_trace_cleanup(void) { debugfs_remove_recursive(rcudir); } module_init(rcutiny_trace_init); module_exit(rcutiny_trace_cleanup); MODULE_AUTHOR("Paul E. McKenney"); MODULE_DESCRIPTION("Read-Copy Update tracing for tiny implementation"); MODULE_LICENSE("GPL"); |
6bfc09e23 rcu: Provide RCU ... |
1081 1082 1083 1084 1085 1086 |
static void check_cpu_stall_preempt(void) { #ifdef CONFIG_TINY_PREEMPT_RCU check_cpu_stall(&rcu_preempt_ctrlblk.rcb); #endif /* #ifdef CONFIG_TINY_PREEMPT_RCU */ } |
9e571a82f rcu: add tracing ... |
1087 |
#endif /* #ifdef CONFIG_RCU_TRACE */ |