Blame view
kernel/delayacct.c
4.69 KB
7170066ec
|
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
ca74e92b4
|
2 3 4 |
/* delayacct.c - per-task delay accounting * * Copyright (C) Shailabh Nagar, IBM Corp. 2006 |
ca74e92b4
|
5 6 7 |
*/ #include <linux/sched.h> |
9164bb4a1
|
8 |
#include <linux/sched/task.h> |
32ef5517c
|
9 |
#include <linux/sched/cputime.h> |
ca74e92b4
|
10 |
#include <linux/slab.h> |
6952b61de
|
11 |
#include <linux/taskstats.h> |
ca74e92b4
|
12 13 14 |
#include <linux/time.h> #include <linux/sysctl.h> #include <linux/delayacct.h> |
c9aaa8957
|
15 |
#include <linux/module.h> |
ca74e92b4
|
16 |
|
163ecdff0
|
17 |
int delayacct_on __read_mostly = 1; /* Delay accounting turned on/off */ |
c9aaa8957
|
18 |
EXPORT_SYMBOL_GPL(delayacct_on); |
e18b890bb
|
19 |
struct kmem_cache *delayacct_cache; |
ca74e92b4
|
20 |
|
163ecdff0
|
21 |
static int __init delayacct_setup_disable(char *str) |
ca74e92b4
|
22 |
{ |
163ecdff0
|
23 |
delayacct_on = 0; |
ca74e92b4
|
24 25 |
return 1; } |
163ecdff0
|
26 |
__setup("nodelayacct", delayacct_setup_disable); |
ca74e92b4
|
27 28 29 |
void delayacct_init(void) { |
5d097056c
|
30 |
delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT); |
ca74e92b4
|
31 32 33 34 35 |
delayacct_tsk_init(&init_task); } void __delayacct_tsk_init(struct task_struct *tsk) { |
e94b17660
|
36 |
tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL); |
ca74e92b4
|
37 |
if (tsk->delays) |
02acc80d1
|
38 |
raw_spin_lock_init(&tsk->delays->lock); |
ca74e92b4
|
39 |
} |
ca74e92b4
|
40 |
/* |
9667a23db
|
41 42 |
* Finish delay accounting for a statistic using its timestamps (@start), * accumalator (@total) and @count |
ca74e92b4
|
43 |
*/ |
02acc80d1
|
44 45 |
static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count) |
ca74e92b4
|
46 |
{ |
9667a23db
|
47 |
s64 ns = ktime_get_ns() - *start; |
64efade11
|
48 |
unsigned long flags; |
ca74e92b4
|
49 |
|
9667a23db
|
50 |
if (ns > 0) { |
02acc80d1
|
51 |
raw_spin_lock_irqsave(lock, flags); |
9667a23db
|
52 53 |
*total += ns; (*count)++; |
02acc80d1
|
54 |
raw_spin_unlock_irqrestore(lock, flags); |
9667a23db
|
55 |
} |
ca74e92b4
|
56 |
} |
0ff922452
|
57 58 |
void __delayacct_blkio_start(void) { |
9667a23db
|
59 |
current->delays->blkio_start = ktime_get_ns(); |
0ff922452
|
60 |
} |
c96f5471c
|
61 62 63 64 65 |
/* * We cannot rely on the `current` macro, as we haven't yet switched back to * the process being woken. */ void __delayacct_blkio_end(struct task_struct *p) |
0ff922452
|
66 |
{ |
c96f5471c
|
67 68 69 70 71 72 73 74 75 76 77 78 79 |
struct task_delay_info *delays = p->delays; u64 *total; u32 *count; if (p->delays->flags & DELAYACCT_PF_SWAPIN) { total = &delays->swapin_delay; count = &delays->swapin_count; } else { total = &delays->blkio_delay; count = &delays->blkio_count; } delayacct_end(&delays->lock, &delays->blkio_start, total, count); |
0ff922452
|
80 |
} |
6f44993fe
|
81 82 83 |
int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk) { |
dbf3da1c2
|
84 |
u64 utime, stime, stimescaled, utimescaled; |
68f6783d2
|
85 86 87 |
unsigned long long t2, t3; unsigned long flags, t1; s64 tmp; |
6f44993fe
|
88 |
|
dbf3da1c2
|
89 |
task_cputime(tsk, &utime, &stime); |
68f6783d2
|
90 |
tmp = (s64)d->cpu_run_real_total; |
dbf3da1c2
|
91 |
tmp += utime + stime; |
6f44993fe
|
92 |
d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp; |
dbf3da1c2
|
93 |
task_cputime_scaled(tsk, &utimescaled, &stimescaled); |
68f6783d2
|
94 |
tmp = (s64)d->cpu_scaled_run_real_total; |
dbf3da1c2
|
95 |
tmp += utimescaled + stimescaled; |
c66f08be7
|
96 97 |
d->cpu_scaled_run_real_total = (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp; |
6f44993fe
|
98 99 100 101 |
/* * No locking available for sched_info (and too expensive to add one) * Mitigate by taking snapshot of values */ |
2d72376b3
|
102 |
t1 = tsk->sched_info.pcount; |
6f44993fe
|
103 |
t2 = tsk->sched_info.run_delay; |
9c2c48020
|
104 |
t3 = tsk->se.sum_exec_runtime; |
6f44993fe
|
105 106 |
d->cpu_count += t1; |
172ba844a
|
107 |
tmp = (s64)d->cpu_delay_total + t2; |
6f44993fe
|
108 |
d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp; |
172ba844a
|
109 |
tmp = (s64)d->cpu_run_virtual_total + t3; |
6f44993fe
|
110 111 112 113 |
d->cpu_run_virtual_total = (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp; /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */ |
02acc80d1
|
114 |
raw_spin_lock_irqsave(&tsk->delays->lock, flags); |
6f44993fe
|
115 116 117 118 |
tmp = d->blkio_delay_total + tsk->delays->blkio_delay; d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp; tmp = d->swapin_delay_total + tsk->delays->swapin_delay; d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp; |
016ae219b
|
119 120 |
tmp = d->freepages_delay_total + tsk->delays->freepages_delay; d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp; |
b1d29ba82
|
121 122 |
tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay; d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp; |
6f44993fe
|
123 124 |
d->blkio_count += tsk->delays->blkio_count; d->swapin_count += tsk->delays->swapin_count; |
016ae219b
|
125 |
d->freepages_count += tsk->delays->freepages_count; |
b1d29ba82
|
126 |
d->thrashing_count += tsk->delays->thrashing_count; |
02acc80d1
|
127 |
raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); |
6f44993fe
|
128 |
|
6f44993fe
|
129 130 |
return 0; } |
258904546
|
131 132 133 134 |
__u64 __delayacct_blkio_ticks(struct task_struct *tsk) { __u64 ret; |
64efade11
|
135 |
unsigned long flags; |
258904546
|
136 |
|
02acc80d1
|
137 |
raw_spin_lock_irqsave(&tsk->delays->lock, flags); |
258904546
|
138 139 |
ret = nsec_to_clock_t(tsk->delays->blkio_delay + tsk->delays->swapin_delay); |
02acc80d1
|
140 |
raw_spin_unlock_irqrestore(&tsk->delays->lock, flags); |
258904546
|
141 142 |
return ret; } |
873b47717
|
143 144 |
void __delayacct_freepages_start(void) { |
9667a23db
|
145 |
current->delays->freepages_start = ktime_get_ns(); |
873b47717
|
146 147 148 149 |
} void __delayacct_freepages_end(void) { |
c96f5471c
|
150 151 152 153 154 |
delayacct_end( ¤t->delays->lock, ¤t->delays->freepages_start, ¤t->delays->freepages_delay, ¤t->delays->freepages_count); |
873b47717
|
155 |
} |
b1d29ba82
|
156 157 158 159 160 161 162 163 164 165 166 167 |
void __delayacct_thrashing_start(void) { current->delays->thrashing_start = ktime_get_ns(); } void __delayacct_thrashing_end(void) { delayacct_end(¤t->delays->lock, ¤t->delays->thrashing_start, ¤t->delays->thrashing_delay, ¤t->delays->thrashing_count); } |