Blame view
drivers/perf/arm_pmu.c
21 KB
1b8873a0c ARM: 5902/4: arm/... |
1 2 3 4 5 6 |
#undef DEBUG /* * ARM performance counter support. * * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles |
43eab8782 ARM: perf: separa... |
7 |
* Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com> |
796d12959 ARM: 5903/1: arm/... |
8 |
* |
1b8873a0c ARM: 5902/4: arm/... |
9 |
* This code is based on the sparc64 perf event code, which is in turn based |
d39976f0f arm: perf: factor... |
10 |
* on the x86 code. |
1b8873a0c ARM: 5902/4: arm/... |
11 12 |
*/ #define pr_fmt(fmt) "hw perfevents: " fmt |
74cf0bc75 arm: perf: unify ... |
13 |
#include <linux/bitmap.h> |
cc88116da arm: perf: treat ... |
14 |
#include <linux/cpumask.h> |
da4e4f18a drivers/perf: arm... |
15 |
#include <linux/cpu_pm.h> |
74cf0bc75 arm: perf: unify ... |
16 |
#include <linux/export.h> |
1b8873a0c ARM: 5902/4: arm/... |
17 |
#include <linux/kernel.h> |
fa8ad7889 arm: perf: factor... |
18 |
#include <linux/perf/arm_pmu.h> |
74cf0bc75 arm: perf: unify ... |
19 |
#include <linux/slab.h> |
e60175710 sched/headers: Pr... |
20 |
#include <linux/sched/clock.h> |
74cf0bc75 arm: perf: unify ... |
21 |
#include <linux/spinlock.h> |
bbd645593 ARM: perf: suppor... |
22 23 |
#include <linux/irq.h> #include <linux/irqdesc.h> |
1b8873a0c ARM: 5902/4: arm/... |
24 |
|
1b8873a0c ARM: 5902/4: arm/... |
25 |
#include <asm/irq_regs.h> |
1b8873a0c ARM: 5902/4: arm/... |
26 |
|
84b4be57a arm_pmu: note IRQ... |
27 28 |
static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu); static DEFINE_PER_CPU(int, cpu_irq); |
e2da97d32 arm_pmu: Add supp... |
29 |
static inline u64 arm_pmu_event_max_period(struct perf_event *event) |
8d3e99424 arm_pmu: Clean up... |
30 |
{ |
e2da97d32 arm_pmu: Add supp... |
31 32 33 34 |
if (event->hw.flags & ARMPMU_EVT_64BIT) return GENMASK_ULL(63, 0); else return GENMASK_ULL(31, 0); |
8d3e99424 arm_pmu: Clean up... |
35 |
} |
1b8873a0c ARM: 5902/4: arm/... |
36 |
static int |
e1f431b57 ARM: perf: refact... |
37 38 39 40 41 |
armpmu_map_cache_event(const unsigned (*cache_map) [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX], u64 config) |
1b8873a0c ARM: 5902/4: arm/... |
42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
{ unsigned int cache_type, cache_op, cache_result, ret; cache_type = (config >> 0) & 0xff; if (cache_type >= PERF_COUNT_HW_CACHE_MAX) return -EINVAL; cache_op = (config >> 8) & 0xff; if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) return -EINVAL; cache_result = (config >> 16) & 0xff; if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) return -EINVAL; |
6c833bb92 arm64: perf: Allo... |
56 57 |
if (!cache_map) return -ENOENT; |
e1f431b57 ARM: perf: refact... |
58 |
ret = (int)(*cache_map)[cache_type][cache_op][cache_result]; |
1b8873a0c ARM: 5902/4: arm/... |
59 60 61 62 63 64 65 66 |
if (ret == CACHE_OP_UNSUPPORTED) return -ENOENT; return ret; } static int |
6dbc00297 ARM: perf: prepar... |
67 |
armpmu_map_hw_event(const unsigned (*event_map)[PERF_COUNT_HW_MAX], u64 config) |
84fee97a0 ARM: perf: consol... |
68 |
{ |
d9f966357 ARM: 7810/1: perf... |
69 70 71 72 |
int mapping; if (config >= PERF_COUNT_HW_MAX) return -EINVAL; |
6c833bb92 arm64: perf: Allo... |
73 74 |
if (!event_map) return -ENOENT; |
d9f966357 ARM: 7810/1: perf... |
75 |
mapping = (*event_map)[config]; |
e1f431b57 ARM: perf: refact... |
76 |
return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping; |
84fee97a0 ARM: perf: consol... |
77 78 79 |
} static int |
e1f431b57 ARM: perf: refact... |
80 |
armpmu_map_raw_event(u32 raw_event_mask, u64 config) |
84fee97a0 ARM: perf: consol... |
81 |
{ |
e1f431b57 ARM: perf: refact... |
82 83 |
return (int)(config & raw_event_mask); } |
6dbc00297 ARM: perf: prepar... |
84 85 86 87 88 89 90 91 |
int armpmu_map_event(struct perf_event *event, const unsigned (*event_map)[PERF_COUNT_HW_MAX], const unsigned (*cache_map) [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask) |
e1f431b57 ARM: perf: refact... |
92 93 |
{ u64 config = event->attr.config; |
67b4305aa arm: perf: use ID... |
94 |
int type = event->attr.type; |
e1f431b57 ARM: perf: refact... |
95 |
|
67b4305aa arm: perf: use ID... |
96 97 98 99 |
if (type == event->pmu->type) return armpmu_map_raw_event(raw_event_mask, config); switch (type) { |
e1f431b57 ARM: perf: refact... |
100 |
case PERF_TYPE_HARDWARE: |
6dbc00297 ARM: perf: prepar... |
101 |
return armpmu_map_hw_event(event_map, config); |
e1f431b57 ARM: perf: refact... |
102 103 104 105 106 107 108 |
case PERF_TYPE_HW_CACHE: return armpmu_map_cache_event(cache_map, config); case PERF_TYPE_RAW: return armpmu_map_raw_event(raw_event_mask, config); } return -ENOENT; |
84fee97a0 ARM: perf: consol... |
109 |
} |
ed6f2a522 ARM: perf: consis... |
110 |
int armpmu_event_set_period(struct perf_event *event) |
1b8873a0c ARM: 5902/4: arm/... |
111 |
{ |
8a16b34e2 ARM: perf: add su... |
112 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
ed6f2a522 ARM: perf: consis... |
113 |
struct hw_perf_event *hwc = &event->hw; |
e78505958 perf: Convert per... |
114 |
s64 left = local64_read(&hwc->period_left); |
1b8873a0c ARM: 5902/4: arm/... |
115 |
s64 period = hwc->sample_period; |
8d3e99424 arm_pmu: Clean up... |
116 |
u64 max_period; |
1b8873a0c ARM: 5902/4: arm/... |
117 |
int ret = 0; |
e2da97d32 arm_pmu: Add supp... |
118 |
max_period = arm_pmu_event_max_period(event); |
1b8873a0c ARM: 5902/4: arm/... |
119 120 |
if (unlikely(left <= -period)) { left = period; |
e78505958 perf: Convert per... |
121 |
local64_set(&hwc->period_left, left); |
1b8873a0c ARM: 5902/4: arm/... |
122 123 124 125 126 127 |
hwc->last_period = period; ret = 1; } if (unlikely(left <= 0)) { left += period; |
e78505958 perf: Convert per... |
128 |
local64_set(&hwc->period_left, left); |
1b8873a0c ARM: 5902/4: arm/... |
129 130 131 |
hwc->last_period = period; ret = 1; } |
2d9ed7406 ARM: 8255/1: perf... |
132 133 134 135 136 137 |
/* * Limit the maximum period to prevent the counter value * from overtaking the one we are about to program. In * effect we are reducing max_period to account for * interrupt latency (and we are being very conservative). */ |
8d3e99424 arm_pmu: Clean up... |
138 139 |
if (left > (max_period >> 1)) left = (max_period >> 1); |
1b8873a0c ARM: 5902/4: arm/... |
140 |
|
e78505958 perf: Convert per... |
141 |
local64_set(&hwc->prev_count, (u64)-left); |
1b8873a0c ARM: 5902/4: arm/... |
142 |
|
e2da97d32 arm_pmu: Add supp... |
143 |
armpmu->write_counter(event, (u64)(-left) & max_period); |
1b8873a0c ARM: 5902/4: arm/... |
144 145 146 147 148 |
perf_event_update_userpage(event); return ret; } |
ed6f2a522 ARM: perf: consis... |
149 |
u64 armpmu_event_update(struct perf_event *event) |
1b8873a0c ARM: 5902/4: arm/... |
150 |
{ |
8a16b34e2 ARM: perf: add su... |
151 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
ed6f2a522 ARM: perf: consis... |
152 |
struct hw_perf_event *hwc = &event->hw; |
a737823d3 ARM: 6835/1: perf... |
153 |
u64 delta, prev_raw_count, new_raw_count; |
e2da97d32 arm_pmu: Add supp... |
154 |
u64 max_period = arm_pmu_event_max_period(event); |
1b8873a0c ARM: 5902/4: arm/... |
155 156 |
again: |
e78505958 perf: Convert per... |
157 |
prev_raw_count = local64_read(&hwc->prev_count); |
ed6f2a522 ARM: perf: consis... |
158 |
new_raw_count = armpmu->read_counter(event); |
1b8873a0c ARM: 5902/4: arm/... |
159 |
|
e78505958 perf: Convert per... |
160 |
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, |
1b8873a0c ARM: 5902/4: arm/... |
161 162 |
new_raw_count) != prev_raw_count) goto again; |
8d3e99424 arm_pmu: Clean up... |
163 |
delta = (new_raw_count - prev_raw_count) & max_period; |
1b8873a0c ARM: 5902/4: arm/... |
164 |
|
e78505958 perf: Convert per... |
165 166 |
local64_add(delta, &event->count); local64_sub(delta, &hwc->period_left); |
1b8873a0c ARM: 5902/4: arm/... |
167 168 169 170 171 |
return new_raw_count; } static void |
a4eaf7f14 perf: Rework the ... |
172 |
armpmu_read(struct perf_event *event) |
1b8873a0c ARM: 5902/4: arm/... |
173 |
{ |
ed6f2a522 ARM: perf: consis... |
174 |
armpmu_event_update(event); |
1b8873a0c ARM: 5902/4: arm/... |
175 176 177 |
} static void |
a4eaf7f14 perf: Rework the ... |
178 |
armpmu_stop(struct perf_event *event, int flags) |
1b8873a0c ARM: 5902/4: arm/... |
179 |
{ |
8a16b34e2 ARM: perf: add su... |
180 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
1b8873a0c ARM: 5902/4: arm/... |
181 |
struct hw_perf_event *hwc = &event->hw; |
a4eaf7f14 perf: Rework the ... |
182 183 184 185 186 |
/* * ARM pmu always has to update the counter, so ignore * PERF_EF_UPDATE, see comments in armpmu_start(). */ if (!(hwc->state & PERF_HES_STOPPED)) { |
ed6f2a522 ARM: perf: consis... |
187 188 |
armpmu->disable(event); armpmu_event_update(event); |
a4eaf7f14 perf: Rework the ... |
189 190 |
hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } |
1b8873a0c ARM: 5902/4: arm/... |
191 |
} |
ed6f2a522 ARM: perf: consis... |
192 |
static void armpmu_start(struct perf_event *event, int flags) |
1b8873a0c ARM: 5902/4: arm/... |
193 |
{ |
8a16b34e2 ARM: perf: add su... |
194 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
1b8873a0c ARM: 5902/4: arm/... |
195 |
struct hw_perf_event *hwc = &event->hw; |
a4eaf7f14 perf: Rework the ... |
196 197 198 199 200 201 202 203 |
/* * ARM pmu always has to reprogram the period, so ignore * PERF_EF_RELOAD, see the comment below. */ if (flags & PERF_EF_RELOAD) WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); hwc->state = 0; |
1b8873a0c ARM: 5902/4: arm/... |
204 205 |
/* * Set the period again. Some counters can't be stopped, so when we |
a4eaf7f14 perf: Rework the ... |
206 |
* were stopped we simply disabled the IRQ source and the counter |
1b8873a0c ARM: 5902/4: arm/... |
207 208 209 210 |
* may have been left counting. If we don't do this step then we may * get an interrupt too soon or *way* too late if the overflow has * happened since disabling. */ |
ed6f2a522 ARM: perf: consis... |
211 212 |
armpmu_event_set_period(event); armpmu->enable(event); |
1b8873a0c ARM: 5902/4: arm/... |
213 |
} |
a4eaf7f14 perf: Rework the ... |
214 215 216 |
static void armpmu_del(struct perf_event *event, int flags) { |
8a16b34e2 ARM: perf: add su... |
217 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
116792508 arm: perf: kill g... |
218 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
a4eaf7f14 perf: Rework the ... |
219 220 |
struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; |
a4eaf7f14 perf: Rework the ... |
221 |
armpmu_stop(event, PERF_EF_UPDATE); |
8be3f9a23 ARM: perf: remove... |
222 |
hw_events->events[idx] = NULL; |
7dfc8db1d arm_pmu: Tidy up ... |
223 |
armpmu->clear_event_idx(hw_events, event); |
a4eaf7f14 perf: Rework the ... |
224 |
perf_event_update_userpage(event); |
7dfc8db1d arm_pmu: Tidy up ... |
225 226 |
/* Clear the allocated counter */ hwc->idx = -1; |
a4eaf7f14 perf: Rework the ... |
227 |
} |
1b8873a0c ARM: 5902/4: arm/... |
228 |
static int |
a4eaf7f14 perf: Rework the ... |
229 |
armpmu_add(struct perf_event *event, int flags) |
1b8873a0c ARM: 5902/4: arm/... |
230 |
{ |
8a16b34e2 ARM: perf: add su... |
231 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
116792508 arm: perf: kill g... |
232 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
1b8873a0c ARM: 5902/4: arm/... |
233 234 |
struct hw_perf_event *hwc = &event->hw; int idx; |
1b8873a0c ARM: 5902/4: arm/... |
235 |
|
cc88116da arm: perf: treat ... |
236 237 238 |
/* An event following a process won't be stopped earlier */ if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) return -ENOENT; |
1b8873a0c ARM: 5902/4: arm/... |
239 |
/* If we don't have a space for the counter then finish early. */ |
ed6f2a522 ARM: perf: consis... |
240 |
idx = armpmu->get_event_idx(hw_events, event); |
a9e469d1c drivers/perf: arm... |
241 242 |
if (idx < 0) return idx; |
1b8873a0c ARM: 5902/4: arm/... |
243 244 245 246 247 248 |
/* * If there is an event in the counter we are going to use then make * sure it is disabled. */ event->hw.idx = idx; |
ed6f2a522 ARM: perf: consis... |
249 |
armpmu->disable(event); |
8be3f9a23 ARM: perf: remove... |
250 |
hw_events->events[idx] = event; |
1b8873a0c ARM: 5902/4: arm/... |
251 |
|
a4eaf7f14 perf: Rework the ... |
252 253 254 |
hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; if (flags & PERF_EF_START) armpmu_start(event, PERF_EF_RELOAD); |
1b8873a0c ARM: 5902/4: arm/... |
255 256 257 |
/* Propagate our changes to the userspace mapping. */ perf_event_update_userpage(event); |
a9e469d1c drivers/perf: arm... |
258 |
return 0; |
1b8873a0c ARM: 5902/4: arm/... |
259 |
} |
1b8873a0c ARM: 5902/4: arm/... |
260 |
static int |
e429817b4 ARM: perf: reject... |
261 262 |
validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, struct perf_event *event) |
1b8873a0c ARM: 5902/4: arm/... |
263 |
{ |
e429817b4 ARM: perf: reject... |
264 |
struct arm_pmu *armpmu; |
1b8873a0c ARM: 5902/4: arm/... |
265 |
|
c95eb3184 ARM: 7809/1: perf... |
266 267 |
if (is_software_event(event)) return 1; |
e429817b4 ARM: perf: reject... |
268 269 270 271 272 273 274 |
/* * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The * core perf code won't check that the pmu->ctx == leader->ctx * until after pmu->event_init(event). */ if (event->pmu != pmu) return 0; |
2dfcb802d ARM: perf: fix gr... |
275 |
if (event->state < PERF_EVENT_STATE_OFF) |
cb2d8b342 ARM: 7698/1: perf... |
276 277 278 |
return 1; if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) |
65b4711ff ARM: 6352/1: perf... |
279 |
return 1; |
1b8873a0c ARM: 5902/4: arm/... |
280 |
|
e429817b4 ARM: perf: reject... |
281 |
armpmu = to_arm_pmu(event->pmu); |
ed6f2a522 ARM: perf: consis... |
282 |
return armpmu->get_event_idx(hw_events, event) >= 0; |
1b8873a0c ARM: 5902/4: arm/... |
283 284 285 286 287 288 |
} static int validate_group(struct perf_event *event) { struct perf_event *sibling, *leader = event->group_leader; |
8be3f9a23 ARM: perf: remove... |
289 |
struct pmu_hw_events fake_pmu; |
1b8873a0c ARM: 5902/4: arm/... |
290 |
|
bce34d144 ARM: perf: initia... |
291 292 293 294 |
/* * Initialise the fake PMU. We only need to populate the * used_mask for the purposes of validation. */ |
a4560846e arm: perf: limit ... |
295 |
memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); |
1b8873a0c ARM: 5902/4: arm/... |
296 |
|
e429817b4 ARM: perf: reject... |
297 |
if (!validate_event(event->pmu, &fake_pmu, leader)) |
aa2bc1ade perf: Don't use -... |
298 |
return -EINVAL; |
1b8873a0c ARM: 5902/4: arm/... |
299 |
|
edb39592a perf: Fix sibling... |
300 |
for_each_sibling_event(sibling, leader) { |
e429817b4 ARM: perf: reject... |
301 |
if (!validate_event(event->pmu, &fake_pmu, sibling)) |
aa2bc1ade perf: Don't use -... |
302 |
return -EINVAL; |
1b8873a0c ARM: 5902/4: arm/... |
303 |
} |
e429817b4 ARM: perf: reject... |
304 |
if (!validate_event(event->pmu, &fake_pmu, event)) |
aa2bc1ade perf: Don't use -... |
305 |
return -EINVAL; |
1b8873a0c ARM: 5902/4: arm/... |
306 307 308 |
return 0; } |
051f1b131 ARM: perf: move i... |
309 |
static irqreturn_t armpmu_dispatch_irq(int irq, void *dev) |
0e25a5c98 ARM: perf_event: ... |
310 |
{ |
bbd645593 ARM: perf: suppor... |
311 |
struct arm_pmu *armpmu; |
5f5092e72 ARM: perf: hook u... |
312 313 |
int ret; u64 start_clock, finish_clock; |
bbd645593 ARM: perf: suppor... |
314 |
|
5ebd92003 arm: perf: fold p... |
315 316 317 318 319 320 321 |
/* * we request the IRQ with a (possibly percpu) struct arm_pmu**, but * the handlers expect a struct arm_pmu*. The percpu_irq framework will * do any necessary shifting, we just need to perform the first * dereference. */ armpmu = *(void **)dev; |
84b4be57a arm_pmu: note IRQ... |
322 323 |
if (WARN_ON_ONCE(!armpmu)) return IRQ_NONE; |
765413707 drivers/perf: arm... |
324 |
|
5f5092e72 ARM: perf: hook u... |
325 |
start_clock = sched_clock(); |
0788f1e97 arm_pmu: simplify... |
326 |
ret = armpmu->handle_irq(armpmu); |
5f5092e72 ARM: perf: hook u... |
327 328 329 330 |
finish_clock = sched_clock(); perf_sample_event_took(finish_clock - start_clock); return ret; |
0e25a5c98 ARM: perf_event: ... |
331 |
} |
1b8873a0c ARM: 5902/4: arm/... |
332 |
static int |
05d22fde3 ARM: perf: allow ... |
333 334 335 336 337 338 339 |
event_requires_mode_exclusion(struct perf_event_attr *attr) { return attr->exclude_idle || attr->exclude_user || attr->exclude_kernel || attr->exclude_hv; } static int |
1b8873a0c ARM: 5902/4: arm/... |
340 341 |
__hw_perf_event_init(struct perf_event *event) { |
8a16b34e2 ARM: perf: add su... |
342 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
1b8873a0c ARM: 5902/4: arm/... |
343 |
struct hw_perf_event *hwc = &event->hw; |
9dcbf4665 ARM: perf: simpli... |
344 |
int mapping; |
1b8873a0c ARM: 5902/4: arm/... |
345 |
|
e2da97d32 arm_pmu: Add supp... |
346 |
hwc->flags = 0; |
e1f431b57 ARM: perf: refact... |
347 |
mapping = armpmu->map_event(event); |
1b8873a0c ARM: 5902/4: arm/... |
348 349 350 351 352 353 354 355 356 |
if (mapping < 0) { pr_debug("event %x:%llx not supported ", event->attr.type, event->attr.config); return mapping; } /* |
05d22fde3 ARM: perf: allow ... |
357 358 359 360 361 362 363 364 365 366 367 |
* We don't assign an index until we actually place the event onto * hardware. Use -1 to signify that we haven't decided where to put it * yet. For SMP systems, each core has it's own PMU so we can't do any * clever allocation or constraints checking at this point. */ hwc->idx = -1; hwc->config_base = 0; hwc->config = 0; hwc->event_base = 0; /* |
1b8873a0c ARM: 5902/4: arm/... |
368 |
* Check whether we need to exclude the counter from certain modes. |
1b8873a0c ARM: 5902/4: arm/... |
369 |
*/ |
05d22fde3 ARM: perf: allow ... |
370 371 372 |
if ((!armpmu->set_event_filter || armpmu->set_event_filter(hwc, &event->attr)) && event_requires_mode_exclusion(&event->attr)) { |
1b8873a0c ARM: 5902/4: arm/... |
373 374 375 |
pr_debug("ARM performance counters do not support " "mode exclusion "); |
fdeb8e35f ARM: 7441/1: perf... |
376 |
return -EOPNOTSUPP; |
1b8873a0c ARM: 5902/4: arm/... |
377 378 379 |
} /* |
05d22fde3 ARM: perf: allow ... |
380 |
* Store the event encoding into the config_base field. |
1b8873a0c ARM: 5902/4: arm/... |
381 |
*/ |
05d22fde3 ARM: perf: allow ... |
382 |
hwc->config_base |= (unsigned long)mapping; |
1b8873a0c ARM: 5902/4: arm/... |
383 |
|
edcb4d3c3 perf/ARM: Use com... |
384 |
if (!is_sampling_event(event)) { |
572734718 ARM: 7354/1: perf... |
385 386 387 388 389 390 |
/* * For non-sampling runs, limit the sample_period to half * of the counter width. That way, the new counter value * is far less likely to overtake the previous one unless * you have some serious IRQ latency issues. */ |
e2da97d32 arm_pmu: Add supp... |
391 |
hwc->sample_period = arm_pmu_event_max_period(event) >> 1; |
1b8873a0c ARM: 5902/4: arm/... |
392 |
hwc->last_period = hwc->sample_period; |
e78505958 perf: Convert per... |
393 |
local64_set(&hwc->period_left, hwc->sample_period); |
1b8873a0c ARM: 5902/4: arm/... |
394 |
} |
1b8873a0c ARM: 5902/4: arm/... |
395 |
if (event->group_leader != event) { |
e595ede60 ARM: 7664/1: perf... |
396 |
if (validate_group(event) != 0) |
1b8873a0c ARM: 5902/4: arm/... |
397 398 |
return -EINVAL; } |
9dcbf4665 ARM: perf: simpli... |
399 |
return 0; |
1b8873a0c ARM: 5902/4: arm/... |
400 |
} |
b0a873ebb perf: Register PM... |
401 |
static int armpmu_event_init(struct perf_event *event) |
1b8873a0c ARM: 5902/4: arm/... |
402 |
{ |
8a16b34e2 ARM: perf: add su... |
403 |
struct arm_pmu *armpmu = to_arm_pmu(event->pmu); |
1b8873a0c ARM: 5902/4: arm/... |
404 |
|
cc88116da arm: perf: treat ... |
405 406 407 408 409 410 411 412 413 414 |
/* * Reject CPU-affine events for CPUs that are of a different class to * that which this PMU handles. Process-following events (where * event->cpu == -1) can be migrated between CPUs, and thus we have to * reject them later (in armpmu_add) if they're scheduled on a * different class of CPU. */ if (event->cpu != -1 && !cpumask_test_cpu(event->cpu, &armpmu->supported_cpus)) return -ENOENT; |
2481c5fa6 perf: Disable PER... |
415 416 417 |
/* does not support taken branch sampling */ if (has_branch_stack(event)) return -EOPNOTSUPP; |
e1f431b57 ARM: perf: refact... |
418 |
if (armpmu->map_event(event) == -ENOENT) |
b0a873ebb perf: Register PM... |
419 |
return -ENOENT; |
b0a873ebb perf: Register PM... |
420 |
|
c09adab01 drivers/perf: arm... |
421 |
return __hw_perf_event_init(event); |
1b8873a0c ARM: 5902/4: arm/... |
422 |
} |
a4eaf7f14 perf: Rework the ... |
423 |
static void armpmu_enable(struct pmu *pmu) |
1b8873a0c ARM: 5902/4: arm/... |
424 |
{ |
8be3f9a23 ARM: perf: remove... |
425 |
struct arm_pmu *armpmu = to_arm_pmu(pmu); |
116792508 arm: perf: kill g... |
426 |
struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); |
7325eaec4 ARM: perf: Remove... |
427 |
int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events); |
1b8873a0c ARM: 5902/4: arm/... |
428 |
|
cc88116da arm: perf: treat ... |
429 430 431 |
/* For task-bound events we may be called on other CPUs */ if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) return; |
f4f38430c ARM: 6989/1: perf... |
432 |
if (enabled) |
ed6f2a522 ARM: perf: consis... |
433 |
armpmu->start(armpmu); |
1b8873a0c ARM: 5902/4: arm/... |
434 |
} |
a4eaf7f14 perf: Rework the ... |
435 |
static void armpmu_disable(struct pmu *pmu) |
1b8873a0c ARM: 5902/4: arm/... |
436 |
{ |
8a16b34e2 ARM: perf: add su... |
437 |
struct arm_pmu *armpmu = to_arm_pmu(pmu); |
cc88116da arm: perf: treat ... |
438 439 440 441 |
/* For task-bound events we may be called on other CPUs */ if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) return; |
ed6f2a522 ARM: perf: consis... |
442 |
armpmu->stop(armpmu); |
1b8873a0c ARM: 5902/4: arm/... |
443 |
} |
c904e32a6 arm: perf: filter... |
444 445 446 447 448 449 450 451 452 |
/* * In heterogeneous systems, events are specific to a particular * microarchitecture, and aren't suitable for another. Thus, only match CPUs of * the same microarchitecture. */ static int armpmu_filter_match(struct perf_event *event) { struct arm_pmu *armpmu = to_arm_pmu(event->pmu); unsigned int cpu = smp_processor_id(); |
ca2b49725 arm64: perf: Reje... |
453 454 455 456 457 458 459 |
int ret; ret = cpumask_test_cpu(cpu, &armpmu->supported_cpus); if (ret && armpmu->filter_match) return armpmu->filter_match(event); return ret; |
c904e32a6 arm: perf: filter... |
460 |
} |
48538b586 drivers/perf: arm... |
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 |
static ssize_t armpmu_cpumask_show(struct device *dev, struct device_attribute *attr, char *buf) { struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev)); return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus); } static DEVICE_ATTR(cpus, S_IRUGO, armpmu_cpumask_show, NULL); static struct attribute *armpmu_common_attrs[] = { &dev_attr_cpus.attr, NULL, }; static struct attribute_group armpmu_common_attr_group = { .attrs = armpmu_common_attrs, }; |
74cf0bc75 arm: perf: unify ... |
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 |
/* Set at runtime when we know what CPU type we are. */ static struct arm_pmu *__oprofile_cpu_pmu; /* * Despite the names, these two functions are CPU-specific and are used * by the OProfile/perf code. */ const char *perf_pmu_name(void) { if (!__oprofile_cpu_pmu) return NULL; return __oprofile_cpu_pmu->name; } EXPORT_SYMBOL_GPL(perf_pmu_name); int perf_num_counters(void) { int max_events = 0; if (__oprofile_cpu_pmu != NULL) max_events = __oprofile_cpu_pmu->num_events; return max_events; } EXPORT_SYMBOL_GPL(perf_num_counters); |
84b4be57a arm_pmu: note IRQ... |
504 |
static int armpmu_count_irq_users(const int irq) |
74cf0bc75 arm: perf: unify ... |
505 |
{ |
84b4be57a arm_pmu: note IRQ... |
506 |
int cpu, count = 0; |
74cf0bc75 arm: perf: unify ... |
507 |
|
84b4be57a arm_pmu: note IRQ... |
508 509 510 511 512 513 514 |
for_each_possible_cpu(cpu) { if (per_cpu(cpu_irq, cpu) == irq) count++; } return count; } |
7ed98e016 drivers/perf: arm... |
515 |
|
167e61438 arm_pmu: acpi: re... |
516 |
void armpmu_free_irq(int irq, int cpu) |
84b4be57a arm_pmu: note IRQ... |
517 518 519 520 |
{ if (per_cpu(cpu_irq, cpu) == 0) return; if (WARN_ON(irq != per_cpu(cpu_irq, cpu))) |
0e2663d92 drivers/perf: arm... |
521 |
return; |
7ed98e016 drivers/perf: arm... |
522 |
|
84b4be57a arm_pmu: note IRQ... |
523 524 525 526 527 528 |
if (!irq_is_percpu_devid(irq)) free_irq(irq, per_cpu_ptr(&cpu_armpmu, cpu)); else if (armpmu_count_irq_users(irq) == 1) free_percpu_irq(irq, &cpu_armpmu); per_cpu(cpu_irq, cpu) = 0; |
0e2663d92 drivers/perf: arm... |
529 |
} |
7ed98e016 drivers/perf: arm... |
530 |
|
167e61438 arm_pmu: acpi: re... |
531 |
int armpmu_request_irq(int irq, int cpu) |
84b4be57a arm_pmu: note IRQ... |
532 533 534 |
{ int err = 0; const irq_handler_t handler = armpmu_dispatch_irq; |
0e2663d92 drivers/perf: arm... |
535 536 |
if (!irq) return 0; |
74cf0bc75 arm: perf: unify ... |
537 |
|
43fc9a2fe arm_pmu: acpi: ch... |
538 |
if (!irq_is_percpu_devid(irq)) { |
a3287c41f drivers/perf: arm... |
539 540 541 542 543 544 545 546 547 548 |
unsigned long irq_flags; err = irq_force_affinity(irq, cpumask_of(cpu)); if (err && num_possible_cpus() > 1) { pr_warn("unable to set irq affinity (irq=%d, cpu=%u) ", irq, cpu); goto err_out; } |
c0248c966 arm_pmu: kill arm... |
549 550 551 |
irq_flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD; |
a3287c41f drivers/perf: arm... |
552 |
|
6de3f7911 arm_pmu: explicit... |
553 |
irq_set_status_flags(irq, IRQ_NOAUTOEN); |
a3287c41f drivers/perf: arm... |
554 |
err = request_irq(irq, handler, irq_flags, "arm-pmu", |
84b4be57a arm_pmu: note IRQ... |
555 556 |
per_cpu_ptr(&cpu_armpmu, cpu)); } else if (armpmu_count_irq_users(irq) == 0) { |
43fc9a2fe arm_pmu: acpi: ch... |
557 |
err = request_percpu_irq(irq, handler, "arm-pmu", |
84b4be57a arm_pmu: note IRQ... |
558 |
&cpu_armpmu); |
0e2663d92 drivers/perf: arm... |
559 |
} |
7ed98e016 drivers/perf: arm... |
560 |
|
a3287c41f drivers/perf: arm... |
561 562 |
if (err) goto err_out; |
74cf0bc75 arm: perf: unify ... |
563 |
|
84b4be57a arm_pmu: note IRQ... |
564 |
per_cpu(cpu_irq, cpu) = irq; |
74cf0bc75 arm: perf: unify ... |
565 |
return 0; |
a3287c41f drivers/perf: arm... |
566 567 568 569 570 |
err_out: pr_err("unable to request IRQ%d for ARM PMU counters ", irq); return err; |
74cf0bc75 arm: perf: unify ... |
571 |
} |
c09adab01 drivers/perf: arm... |
572 573 574 575 576 |
static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu) { struct pmu_hw_events __percpu *hw_events = pmu->hw_events; return per_cpu(hw_events->irq, cpu); } |
74cf0bc75 arm: perf: unify ... |
577 578 579 580 581 582 |
/* * PMU hardware loses all context when a CPU goes offline. * When a CPU is hotplugged back in, since some hardware registers are * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading * junk values out of them. */ |
6e103c0cf arm/perf: Use mul... |
583 |
static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) |
74cf0bc75 arm: perf: unify ... |
584 |
{ |
6e103c0cf arm/perf: Use mul... |
585 |
struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); |
c09adab01 drivers/perf: arm... |
586 |
int irq; |
74cf0bc75 arm: perf: unify ... |
587 |
|
6e103c0cf arm/perf: Use mul... |
588 589 590 591 |
if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) return 0; if (pmu->reset) pmu->reset(pmu); |
c09adab01 drivers/perf: arm... |
592 |
|
84b4be57a arm_pmu: note IRQ... |
593 |
per_cpu(cpu_armpmu, cpu) = pmu; |
c09adab01 drivers/perf: arm... |
594 595 |
irq = armpmu_get_cpu_irq(pmu, cpu); if (irq) { |
6de3f7911 arm_pmu: explicit... |
596 |
if (irq_is_percpu_devid(irq)) |
c09adab01 drivers/perf: arm... |
597 |
enable_percpu_irq(irq, IRQ_TYPE_NONE); |
6de3f7911 arm_pmu: explicit... |
598 599 |
else enable_irq(irq); |
c09adab01 drivers/perf: arm... |
600 601 602 603 604 605 606 607 608 609 610 611 612 613 |
} return 0; } static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) { struct arm_pmu *pmu = hlist_entry_safe(node, struct arm_pmu, node); int irq; if (!cpumask_test_cpu(cpu, &pmu->supported_cpus)) return 0; irq = armpmu_get_cpu_irq(pmu, cpu); |
6de3f7911 arm_pmu: explicit... |
614 615 616 617 |
if (irq) { if (irq_is_percpu_devid(irq)) disable_percpu_irq(irq); else |
b08e5fd90 arm_pmu: Use disa... |
618 |
disable_irq_nosync(irq); |
6de3f7911 arm_pmu: explicit... |
619 |
} |
c09adab01 drivers/perf: arm... |
620 |
|
84b4be57a arm_pmu: note IRQ... |
621 |
per_cpu(cpu_armpmu, cpu) = NULL; |
7d88eb695 arm/perf: Convert... |
622 |
return 0; |
74cf0bc75 arm: perf: unify ... |
623 |
} |
da4e4f18a drivers/perf: arm... |
624 625 626 627 628 629 630 631 |
#ifdef CONFIG_CPU_PM static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) { struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); struct perf_event *event; int idx; for (idx = 0; idx < armpmu->num_events; idx++) { |
da4e4f18a drivers/perf: arm... |
632 |
event = hw_events->events[idx]; |
c13207905 arm64: perf: Add ... |
633 634 |
if (!event) continue; |
da4e4f18a drivers/perf: arm... |
635 636 637 638 639 640 641 642 643 644 |
switch (cmd) { case CPU_PM_ENTER: /* * Stop and update the counter */ armpmu_stop(event, PERF_EF_UPDATE); break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: |
cbcc72e03 drivers/perf: arm... |
645 646 647 648 649 650 651 652 653 654 655 656 657 |
/* * Restore and enable the counter. * armpmu_start() indirectly calls * * perf_event_update_userpage() * * that requires RCU read locking to be functional, * wrap the call within RCU_NONIDLE to make the * RCU subsystem aware this cpu is not idle from * an RCU perspective for the armpmu_start() call * duration. */ RCU_NONIDLE(armpmu_start(event, PERF_EF_RELOAD)); |
da4e4f18a drivers/perf: arm... |
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 |
break; default: break; } } } static int cpu_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, void *v) { struct arm_pmu *armpmu = container_of(b, struct arm_pmu, cpu_pm_nb); struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); int enabled = bitmap_weight(hw_events->used_mask, armpmu->num_events); if (!cpumask_test_cpu(smp_processor_id(), &armpmu->supported_cpus)) return NOTIFY_DONE; /* * Always reset the PMU registers on power-up even if * there are no events running. */ if (cmd == CPU_PM_EXIT && armpmu->reset) armpmu->reset(armpmu); if (!enabled) return NOTIFY_OK; switch (cmd) { case CPU_PM_ENTER: armpmu->stop(armpmu); cpu_pm_pmu_setup(armpmu, cmd); break; case CPU_PM_EXIT: cpu_pm_pmu_setup(armpmu, cmd); case CPU_PM_ENTER_FAILED: armpmu->start(armpmu); break; default: return NOTIFY_DONE; } return NOTIFY_OK; } static int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) { cpu_pmu->cpu_pm_nb.notifier_call = cpu_pm_pmu_notify; return cpu_pm_register_notifier(&cpu_pmu->cpu_pm_nb); } static void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { cpu_pm_unregister_notifier(&cpu_pmu->cpu_pm_nb); } #else static inline int cpu_pm_pmu_register(struct arm_pmu *cpu_pmu) { return 0; } static inline void cpu_pm_pmu_unregister(struct arm_pmu *cpu_pmu) { } #endif |
74cf0bc75 arm: perf: unify ... |
716 717 718 |
static int cpu_pmu_init(struct arm_pmu *cpu_pmu) { int err; |
74cf0bc75 arm: perf: unify ... |
719 |
|
c09adab01 drivers/perf: arm... |
720 721 |
err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_STARTING, &cpu_pmu->node); |
6e103c0cf arm/perf: Use mul... |
722 |
if (err) |
2681f0184 drivers/perf: arm... |
723 |
goto out; |
74cf0bc75 arm: perf: unify ... |
724 |
|
da4e4f18a drivers/perf: arm... |
725 726 727 |
err = cpu_pm_pmu_register(cpu_pmu); if (err) goto out_unregister; |
74cf0bc75 arm: perf: unify ... |
728 |
return 0; |
da4e4f18a drivers/perf: arm... |
729 |
out_unregister: |
6e103c0cf arm/perf: Use mul... |
730 731 |
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, &cpu_pmu->node); |
2681f0184 drivers/perf: arm... |
732 |
out: |
74cf0bc75 arm: perf: unify ... |
733 734 735 736 737 |
return err; } static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu) { |
da4e4f18a drivers/perf: arm... |
738 |
cpu_pm_pmu_unregister(cpu_pmu); |
6e103c0cf arm/perf: Use mul... |
739 740 |
cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_STARTING, &cpu_pmu->node); |
74cf0bc75 arm: perf: unify ... |
741 |
} |
0dc1a1851 arm_pmu: add armp... |
742 |
static struct arm_pmu *__armpmu_alloc(gfp_t flags) |
2681f0184 drivers/perf: arm... |
743 744 745 |
{ struct arm_pmu *pmu; int cpu; |
0dc1a1851 arm_pmu: add armp... |
746 |
pmu = kzalloc(sizeof(*pmu), flags); |
2681f0184 drivers/perf: arm... |
747 748 749 750 751 |
if (!pmu) { pr_info("failed to allocate PMU device! "); goto out; } |
0dc1a1851 arm_pmu: add armp... |
752 |
pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags); |
2681f0184 drivers/perf: arm... |
753 754 755 756 757 |
if (!pmu->hw_events) { pr_info("failed to allocate per-cpu PMU data. "); goto out_free_pmu; } |
70cd908a1 drivers/perf: arm... |
758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 |
pmu->pmu = (struct pmu) { .pmu_enable = armpmu_enable, .pmu_disable = armpmu_disable, .event_init = armpmu_event_init, .add = armpmu_add, .del = armpmu_del, .start = armpmu_start, .stop = armpmu_stop, .read = armpmu_read, .filter_match = armpmu_filter_match, .attr_groups = pmu->attr_groups, /* * This is a CPU PMU potentially in a heterogeneous * configuration (e.g. big.LITTLE). This is not an uncore PMU, * and we have taken ctx sharing into account (e.g. with our * pmu::filter_match callback and pmu::event_init group * validation). */ .capabilities = PERF_PMU_CAP_HETEROGENEOUS_CPUS, }; pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] = &armpmu_common_attr_group; |
2681f0184 drivers/perf: arm... |
781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 |
for_each_possible_cpu(cpu) { struct pmu_hw_events *events; events = per_cpu_ptr(pmu->hw_events, cpu); raw_spin_lock_init(&events->pmu_lock); events->percpu_pmu = pmu; } return pmu; out_free_pmu: kfree(pmu); out: return NULL; } |
0dc1a1851 arm_pmu: add armp... |
796 797 798 799 800 801 802 803 804 |
struct arm_pmu *armpmu_alloc(void) { return __armpmu_alloc(GFP_KERNEL); } struct arm_pmu *armpmu_alloc_atomic(void) { return __armpmu_alloc(GFP_ATOMIC); } |
18bfcfe51 drivers/perf: arm... |
805 |
void armpmu_free(struct arm_pmu *pmu) |
2681f0184 drivers/perf: arm... |
806 807 808 809 |
{ free_percpu(pmu->hw_events); kfree(pmu); } |
74a2b3ea2 drivers/perf: arm... |
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 |
int armpmu_register(struct arm_pmu *pmu) { int ret; ret = cpu_pmu_init(pmu); if (ret) return ret; ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (ret) goto out_destroy; if (!__oprofile_cpu_pmu) __oprofile_cpu_pmu = pmu; pr_info("enabled with %s PMU driver, %d counters available ", pmu->name, pmu->num_events); return 0; out_destroy: cpu_pmu_destroy(pmu); return ret; } |
37b502f12 arm/perf: Fix hot... |
835 836 837 |
static int arm_pmu_hp_init(void) { int ret; |
6e103c0cf arm/perf: Use mul... |
838 |
ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_STARTING, |
73c1b41e6 cpu/hotplug: Clea... |
839 |
"perf/arm/pmu:starting", |
c09adab01 drivers/perf: arm... |
840 841 |
arm_perf_starting_cpu, arm_perf_teardown_cpu); |
37b502f12 arm/perf: Fix hot... |
842 843 844 845 846 847 848 |
if (ret) pr_err("CPU hotplug notifier for ARM PMU could not be registered: %d ", ret); return ret; } subsys_initcall(arm_pmu_hp_init); |