Blame view
kernel/delayacct.c
4.69 KB
7170066ec treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
ca74e92b4 [PATCH] per-task-... |
2 3 4 |
/* delayacct.c - per-task delay accounting * * Copyright (C) Shailabh Nagar, IBM Corp. 2006 |
ca74e92b4 [PATCH] per-task-... |
5 6 7 |
*/ #include <linux/sched.h> |
9164bb4a1 sched/headers: Pr... |
8 |
#include <linux/sched/task.h> |
32ef5517c sched/headers: Pr... |
9 |
#include <linux/sched/cputime.h> |
ca74e92b4 [PATCH] per-task-... |
10 |
#include <linux/slab.h> |
6952b61de headers: taskstat... |
11 |
#include <linux/taskstats.h> |
ca74e92b4 [PATCH] per-task-... |
12 13 14 |
#include <linux/time.h> #include <linux/sysctl.h> #include <linux/delayacct.h> |
c9aaa8957 KVM: Steal time i... |
15 |
#include <linux/module.h> |
ca74e92b4 [PATCH] per-task-... |
16 |
|
163ecdff0 [PATCH] delay acc... |
17 |
int delayacct_on __read_mostly = 1; /* Delay accounting turned on/off */ |
c9aaa8957 KVM: Steal time i... |
18 |
EXPORT_SYMBOL_GPL(delayacct_on); |
e18b890bb [PATCH] slab: rem... |
19 |
struct kmem_cache *delayacct_cache; |
ca74e92b4 [PATCH] per-task-... |
20 |
|
163ecdff0 [PATCH] delay acc... |
21 |
static int __init delayacct_setup_disable(char *str) |
ca74e92b4 [PATCH] per-task-... |
22 |
{ |
163ecdff0 [PATCH] delay acc... |
23 |
delayacct_on = 0; |
ca74e92b4 [PATCH] per-task-... |
24 25 |
return 1; } |
163ecdff0 [PATCH] delay acc... |
26 |
__setup("nodelayacct", delayacct_setup_disable); |
ca74e92b4 [PATCH] per-task-... |
27 28 29 |
void delayacct_init(void) { |
5d097056c kmemcg: account c... |
30 |
delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT); |
ca74e92b4 [PATCH] per-task-... |
31 32 33 34 35 |
delayacct_tsk_init(&init_task); } void __delayacct_tsk_init(struct task_struct *tsk) { |
e94b17660 [PATCH] slab: rem... |
36 |
tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL); |
ca74e92b4 [PATCH] per-task-... |
37 |
if (tsk->delays) |
02acc80d1 delayacct: Use ra... |
38 |
raw_spin_lock_init(&tsk->delays->lock); |
ca74e92b4 [PATCH] per-task-... |
39 |
} |
ca74e92b4 [PATCH] per-task-... |
40 |
/* |
9667a23db delayacct: Make a... |
41 42 |
* Finish delay accounting for a statistic using its timestamps (@start), * accumalator (@total) and @count |
ca74e92b4 [PATCH] per-task-... |
43 |
*/ |
02acc80d1 delayacct: Use ra... |
44 45 |
static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count) |
ca74e92b4 [PATCH] per-task-... |
46 |
{ |
9667a23db delayacct: Make a... |
47 |
s64 ns = ktime_get_ns() - *start; |
64efade11 [PATCH] lockdep: ... |
48 |
unsigned long flags; |
ca74e92b4 [PATCH] per-task-... |
49 |
|
9667a23db delayacct: Make a... |
50 |
if (ns > 0) { |
02acc80d1 delayacct: Use ra... |
51 |
raw_spin_lock_irqsave(lock, flags); |
9667a23db delayacct: Make a... |
52 53 |
*total += ns; (*count)++; |
02acc80d1 delayacct: Use ra... |
54 |
raw_spin_unlock_irqrestore(lock, flags); |
9667a23db delayacct: Make a... |
55 |
} |
ca74e92b4 [PATCH] per-task-... |
56 |
} |
0ff922452 [PATCH] per-task-... |
57 58 |
void __delayacct_blkio_start(void) { |
9667a23db delayacct: Make a... |
59 |
current->delays->blkio_start = ktime_get_ns(); |
0ff922452 [PATCH] per-task-... |
60 |
} |
c96f5471c delayacct: Accoun... |
61 62 63 64 65 |
/* * We cannot rely on the `current` macro, as we haven't yet switched back to * the process being woken. */ void __delayacct_blkio_end(struct task_struct *p) |
0ff922452 [PATCH] per-task-... |
66 |
{ |
c96f5471c delayacct: Accoun... |
67 68 69 70 71 72 73 74 75 76 77 78 79 |
struct task_delay_info *delays = p->delays; u64 *total; u32 *count; if (p->delays->flags & DELAYACCT_PF_SWAPIN) { total = &delays->swapin_delay; count = &delays->swapin_count; } else { total = &delays->blkio_delay; count = &delays->blkio_count; } delayacct_end(&delays->lock, &delays->blkio_start, total, count); |
0ff922452 [PATCH] per-task-... |
80 |
} |
6f44993fe [PATCH] per-task-... |
81 82 83 |
int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) { |
dbf3da1c2 delaycct: Convert... |
84 |
u64 utime, stime, stimescaled, utimescaled; |
68f6783d2 delayacct: Remove... |
85 86 87 |
unsigned long long t2, t3; unsigned long flags, t1; s64 tmp; |
6f44993fe [PATCH] per-task-... |
88 |
|
dbf3da1c2 delaycct: Convert... |
89 |
task_cputime(tsk, &utime, &stime); |
68f6783d2 delayacct: Remove... |
90 |
tmp = (s64)d->cpu_run_real_total; |
dbf3da1c2 delaycct: Convert... |
91 |
tmp += utime + stime; |
6f44993fe [PATCH] per-task-... |
92 |
d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp; |
dbf3da1c2 delaycct: Convert... |
93 |
task_cputime_scaled(tsk, &utimescaled, &stimescaled); |
68f6783d2 delayacct: Remove... |
94 |
tmp = (s64)d->cpu_scaled_run_real_total; |
dbf3da1c2 delaycct: Convert... |
95 |
tmp += utimescaled + stimescaled; |
c66f08be7 Add scaled time t... |
96 97 |
d->cpu_scaled_run_real_total = (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp; |
6f44993fe [PATCH] per-task-... |
98 99 100 101 |
/* * No locking available for sched_info (and too expensive to add one) * Mitigate by taking snapshot of values */ |
2d72376b3 sched: clean up s... |
102 |
t1 = tsk->sched_info.pcount; |
6f44993fe [PATCH] per-task-... |
103 |
t2 = tsk->sched_info.run_delay; |
9c2c48020 schedstat: consol... |
104 |
t3 = tsk->se.sum_exec_runtime; |
6f44993fe [PATCH] per-task-... |
105 106 |
d->cpu_count += t1; |
172ba844a sched: update del... |
107 |
tmp = (s64)d->cpu_delay_total + t2; |
6f44993fe [PATCH] per-task-... |
108 |
d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp; |
172ba844a sched: update del... |
109 |
tmp = (s64)d->cpu_run_virtual_total + t3; |
6f44993fe [PATCH] per-task-... |
110 111 112 113 |
d->cpu_run_virtual_total = (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp; /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */ |
02acc80d1 delayacct: Use ra... |
114 |
raw_spin_lock_irqsave(&tsk->delays->lock, flags); |
6f44993fe [PATCH] per-task-... |
115 116 117 118 |
tmp = d->blkio_delay_total + tsk->delays->blkio_delay; d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp; tmp = d->swapin_delay_total + tsk->delays->swapin_delay; d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp; |
016ae219b per-task-delay-ac... |
119 120 |
tmp = d->freepages_delay_total + tsk->delays->freepages_delay; d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp; |
b1d29ba82 delayacct: track ... |
121 122 |
tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay; d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp; |
6f44993fe [PATCH] per-task-... |
123 124 |
d->blkio_count += tsk->delays->blkio_count; d->swapin_count += tsk->delays->swapin_count; |
016ae219b per-task-delay-ac... |
125 |
d->freepages_count += tsk->delays->freepages_count; |
b1d29ba82 delayacct: track ... |
126 |
d->thrashing_count += tsk->delays->thrashing_count; |
02acc80d1 delayacct: Use ra... |
127 |
raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); |
6f44993fe [PATCH] per-task-... |
128 |
|
6f44993fe [PATCH] per-task-... |
129 130 |
return 0; } |
258904546 [PATCH] per-task-... |
131 132 133 134 |
__u64 __delayacct_blkio_ticks(struct task_struct *tsk) { __u64 ret; |
64efade11 [PATCH] lockdep: ... |
135 |
unsigned long flags; |
258904546 [PATCH] per-task-... |
136 |
|
02acc80d1 delayacct: Use ra... |
137 |
raw_spin_lock_irqsave(&tsk->delays->lock, flags); |
258904546 [PATCH] per-task-... |
138 139 |
ret = nsec_to_clock_t(tsk->delays->blkio_delay + tsk->delays->swapin_delay); |
02acc80d1 delayacct: Use ra... |
140 |
raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); |
258904546 [PATCH] per-task-... |
141 142 |
return ret; } |
873b47717 per-task-delay-ac... |
143 144 |
void __delayacct_freepages_start(void) { |
9667a23db delayacct: Make a... |
145 |
current->delays->freepages_start = ktime_get_ns(); |
873b47717 per-task-delay-ac... |
146 147 148 149 |
} void __delayacct_freepages_end(void) { |
c96f5471c delayacct: Accoun... |
150 151 152 153 154 |
delayacct_end( ¤t->delays->lock, ¤t->delays->freepages_start, ¤t->delays->freepages_delay, ¤t->delays->freepages_count); |
873b47717 per-task-delay-ac... |
155 |
} |
b1d29ba82 delayacct: track ... |
156 157 158 159 160 161 162 163 164 165 166 167 |
void __delayacct_thrashing_start(void) { current->delays->thrashing_start = ktime_get_ns(); } void __delayacct_thrashing_end(void) { delayacct_end(¤t->delays->lock, ¤t->delays->thrashing_start, ¤t->delays->thrashing_delay, ¤t->delays->thrashing_count); } |