Blame view
arch/x86/oprofile/nmi_int.c
16.9 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 |
/** * @file nmi_int.c * |
4d4036e0e oprofile: Impleme... |
4 |
* @remark Copyright 2002-2009 OProfile authors |
1da177e4c Linux-2.6.12-rc2 |
5 6 7 |
* @remark Read the file COPYING * * @author John Levon <levon@movementarian.org> |
adf5ec0bc x86/oprofile: int... |
8 |
* @author Robert Richter <robert.richter@amd.com> |
4d4036e0e oprofile: Impleme... |
9 10 11 |
* @author Barry Kasindorf <barry.kasindorf@amd.com> * @author Jason Yeh <jason.yeh@amd.com> * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 16 17 |
*/ #include <linux/init.h> #include <linux/notifier.h> #include <linux/smp.h> #include <linux/oprofile.h> |
f3c6ea1b0 x86: Use syscore_... |
18 |
#include <linux/syscore_ops.h> |
1da177e4c Linux-2.6.12-rc2 |
19 |
#include <linux/slab.h> |
1cfcea1b2 [PATCH] x86_64: A... |
20 |
#include <linux/moduleparam.h> |
1eeb66a1b move die notifier... |
21 |
#include <linux/kdebug.h> |
80a8c9fff x86: fix oprofile... |
22 |
#include <linux/cpu.h> |
1da177e4c Linux-2.6.12-rc2 |
23 24 25 |
#include <asm/nmi.h> #include <asm/msr.h> #include <asm/apic.h> |
b75f53dba x86: fix style er... |
26 |
|
1da177e4c Linux-2.6.12-rc2 |
27 28 |
#include "op_counter.h" #include "op_x86_model.h" |
2fbe7b25c [PATCH] i386/x86-... |
29 |
|
259a83a8a x86/oprofile: Rem... |
30 |
static struct op_x86_model_spec *model; |
d18d00f5d x86: oprofile: re... |
31 32 |
static DEFINE_PER_CPU(struct op_msrs, cpu_msrs); static DEFINE_PER_CPU(unsigned long, saved_lvtpc); |
2fbe7b25c [PATCH] i386/x86-... |
33 |
|
6ae56b55b oprofile/x86: pro... |
34 35 36 |
/* must be protected with get_online_cpus()/put_online_cpus(): */ static int nmi_enabled; static int ctr_running; |
1da177e4c Linux-2.6.12-rc2 |
37 |
|
4d4036e0e oprofile: Impleme... |
38 |
struct op_counter_config counter_config[OP_MAX_COUNTER]; |
3370d3585 x86/oprofile: rep... |
39 40 41 42 43 44 45 46 47 48 49 50 |
/* common functions */ u64 op_x86_get_ctrl(struct op_x86_model_spec const *model, struct op_counter_config *counter_config) { u64 val = 0; u16 event = (u16)counter_config->event; val |= ARCH_PERFMON_EVENTSEL_INT; val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0; val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0; val |= (counter_config->unit_mask & 0xFF) << 8; |
914a76ca5 oprofile, x86: Al... |
51 52 53 54 |
counter_config->extra &= (ARCH_PERFMON_EVENTSEL_INV | ARCH_PERFMON_EVENTSEL_EDGE | ARCH_PERFMON_EVENTSEL_CMASK); val |= counter_config->extra; |
3370d3585 x86/oprofile: rep... |
55 56 57 58 59 60 |
event &= model->event_mask ? model->event_mask : 0xFF; val |= event & 0xFF; val |= (event & 0x0F00) << 24; return val; } |
9c48f1c62 x86, nmi: Wire up... |
61 62 63 64 65 66 67 68 69 |
static int profile_exceptions_notify(unsigned int val, struct pt_regs *regs) { if (ctr_running) model->check_ctrs(regs, &__get_cpu_var(cpu_msrs)); else if (!nmi_enabled) return NMI_DONE; else model->stop(&__get_cpu_var(cpu_msrs)); return NMI_HANDLED; |
1da177e4c Linux-2.6.12-rc2 |
70 |
} |
2fbe7b25c [PATCH] i386/x86-... |
71 |
|
b75f53dba x86: fix style er... |
72 |
static void nmi_cpu_save_registers(struct op_msrs *msrs) |
1da177e4c Linux-2.6.12-rc2 |
73 |
{ |
b75f53dba x86: fix style er... |
74 75 |
struct op_msr *counters = msrs->counters; struct op_msr *controls = msrs->controls; |
1da177e4c Linux-2.6.12-rc2 |
76 |
unsigned int i; |
1a245c453 x86/oprofile: rem... |
77 |
for (i = 0; i < model->num_counters; ++i) { |
95e74e62c x86/oprofile: use... |
78 79 |
if (counters[i].addr) rdmsrl(counters[i].addr, counters[i].saved); |
1da177e4c Linux-2.6.12-rc2 |
80 |
} |
b75f53dba x86: fix style er... |
81 |
|
1a245c453 x86/oprofile: rem... |
82 |
for (i = 0; i < model->num_controls; ++i) { |
95e74e62c x86/oprofile: use... |
83 84 |
if (controls[i].addr) rdmsrl(controls[i].addr, controls[i].saved); |
1da177e4c Linux-2.6.12-rc2 |
85 86 |
} } |
b28d1b923 x86/oprofile: Mov... |
87 88 89 |
static void nmi_cpu_start(void *dummy) { struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs); |
2623a1d55 oprofile/x86: fix... |
90 91 92 93 |
if (!msrs->controls) WARN_ON_ONCE(1); else model->start(msrs); |
b28d1b923 x86/oprofile: Mov... |
94 95 96 97 |
} static int nmi_start(void) { |
6ae56b55b oprofile/x86: pro... |
98 |
get_online_cpus(); |
6ae56b55b oprofile/x86: pro... |
99 |
ctr_running = 1; |
8fe7e94eb oprofile, x86: Fi... |
100 101 102 |
/* make ctr_running visible to the nmi handler: */ smp_mb(); on_each_cpu(nmi_cpu_start, NULL, 1); |
6ae56b55b oprofile/x86: pro... |
103 |
put_online_cpus(); |
b28d1b923 x86/oprofile: Mov... |
104 105 106 107 108 109 |
return 0; } static void nmi_cpu_stop(void *dummy) { struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs); |
2623a1d55 oprofile/x86: fix... |
110 111 112 113 |
if (!msrs->controls) WARN_ON_ONCE(1); else model->stop(msrs); |
b28d1b923 x86/oprofile: Mov... |
114 115 116 117 |
} static void nmi_stop(void) { |
6ae56b55b oprofile/x86: pro... |
118 |
get_online_cpus(); |
b28d1b923 x86/oprofile: Mov... |
119 |
on_each_cpu(nmi_cpu_stop, NULL, 1); |
6ae56b55b oprofile/x86: pro... |
120 121 |
ctr_running = 0; put_online_cpus(); |
b28d1b923 x86/oprofile: Mov... |
122 |
} |
d8471ad3a oprofile: Introdu... |
123 124 125 |
#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX static DEFINE_PER_CPU(int, switch_index); |
39e97f40c x86/oprofile: Add... |
126 127 128 129 |
static inline int has_mux(void) { return !!model->switch_ctrl; } |
d8471ad3a oprofile: Introdu... |
130 131 |
inline int op_x86_phys_to_virt(int phys) { |
0a3aee0da x86: Use this_cpu... |
132 |
return __this_cpu_read(switch_index) + phys; |
d8471ad3a oprofile: Introdu... |
133 |
} |
61d149d52 x86/oprofile: Imp... |
134 135 136 137 |
inline int op_x86_virt_to_phys(int virt) { return virt % model->num_counters; } |
6ab82f958 x86/oprofile: Imp... |
138 139 140 |
static void nmi_shutdown_mux(void) { int i; |
39e97f40c x86/oprofile: Add... |
141 142 143 |
if (!has_mux()) return; |
6ab82f958 x86/oprofile: Imp... |
144 145 146 147 148 149 150 151 152 153 154 155 |
for_each_possible_cpu(i) { kfree(per_cpu(cpu_msrs, i).multiplex); per_cpu(cpu_msrs, i).multiplex = NULL; per_cpu(switch_index, i) = 0; } } static int nmi_setup_mux(void) { size_t multiplex_size = sizeof(struct op_msr) * model->num_virt_counters; int i; |
39e97f40c x86/oprofile: Add... |
156 157 158 |
if (!has_mux()) return 1; |
6ab82f958 x86/oprofile: Imp... |
159 160 |
for_each_possible_cpu(i) { per_cpu(cpu_msrs, i).multiplex = |
c17c8fbf3 oprofile/x86: use... |
161 |
kzalloc(multiplex_size, GFP_KERNEL); |
6ab82f958 x86/oprofile: Imp... |
162 163 164 |
if (!per_cpu(cpu_msrs, i).multiplex) return 0; } |
39e97f40c x86/oprofile: Add... |
165 |
|
6ab82f958 x86/oprofile: Imp... |
166 167 |
return 1; } |
48fb4b467 x86/oprofile: Mov... |
168 169 170 171 |
static void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { int i; struct op_msr *multiplex = msrs->multiplex; |
39e97f40c x86/oprofile: Add... |
172 173 |
if (!has_mux()) return; |
48fb4b467 x86/oprofile: Mov... |
174 175 176 177 |
for (i = 0; i < model->num_virt_counters; ++i) { if (counter_config[i].enabled) { multiplex[i].saved = -(u64)counter_config[i].count; } else { |
48fb4b467 x86/oprofile: Mov... |
178 179 180 181 182 183 |
multiplex[i].saved = 0; } } per_cpu(switch_index, cpu) = 0; } |
d0f585dd2 x86/oprofile: Mov... |
184 185 |
static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs) { |
68dc819ce oprofile/x86: fix... |
186 |
struct op_msr *counters = msrs->counters; |
d0f585dd2 x86/oprofile: Mov... |
187 188 189 190 191 |
struct op_msr *multiplex = msrs->multiplex; int i; for (i = 0; i < model->num_counters; ++i) { int virt = op_x86_phys_to_virt(i); |
68dc819ce oprofile/x86: fix... |
192 193 |
if (counters[i].addr) rdmsrl(counters[i].addr, multiplex[virt].saved); |
d0f585dd2 x86/oprofile: Mov... |
194 195 196 197 198 |
} } static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs) { |
68dc819ce oprofile/x86: fix... |
199 |
struct op_msr *counters = msrs->counters; |
d0f585dd2 x86/oprofile: Mov... |
200 201 202 203 204 |
struct op_msr *multiplex = msrs->multiplex; int i; for (i = 0; i < model->num_counters; ++i) { int virt = op_x86_phys_to_virt(i); |
68dc819ce oprofile/x86: fix... |
205 206 |
if (counters[i].addr) wrmsrl(counters[i].addr, multiplex[virt].saved); |
d0f585dd2 x86/oprofile: Mov... |
207 208 |
} } |
b28d1b923 x86/oprofile: Mov... |
209 210 211 212 213 214 215 216 217 218 219 |
static void nmi_cpu_switch(void *dummy) { int cpu = smp_processor_id(); int si = per_cpu(switch_index, cpu); struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); nmi_cpu_stop(NULL); nmi_cpu_save_mpx_registers(msrs); /* move to next set */ si += model->num_counters; |
d8cc108f4 oprofile/x86: fix... |
220 |
if ((si >= model->num_virt_counters) || (counter_config[si].count == 0)) |
b28d1b923 x86/oprofile: Mov... |
221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 |
per_cpu(switch_index, cpu) = 0; else per_cpu(switch_index, cpu) = si; model->switch_ctrl(model, msrs); nmi_cpu_restore_mpx_registers(msrs); nmi_cpu_start(NULL); } /* * Quick check to see if multiplexing is necessary. * The check should be sufficient since counters are used * in ordre. */ static int nmi_multiplex_on(void) { return counter_config[model->num_counters].count ? 0 : -EINVAL; } static int nmi_switch_event(void) { |
39e97f40c x86/oprofile: Add... |
244 |
if (!has_mux()) |
b28d1b923 x86/oprofile: Mov... |
245 246 247 |
return -ENOSYS; /* not implemented */ if (nmi_multiplex_on() < 0) return -EINVAL; /* not necessary */ |
6ae56b55b oprofile/x86: pro... |
248 249 250 251 |
get_online_cpus(); if (ctr_running) on_each_cpu(nmi_cpu_switch, NULL, 1); put_online_cpus(); |
b28d1b923 x86/oprofile: Mov... |
252 |
|
b28d1b923 x86/oprofile: Mov... |
253 254 |
return 0; } |
528051447 x86/oprofile: Ena... |
255 256 257 258 259 |
static inline void mux_init(struct oprofile_operations *ops) { if (has_mux()) ops->switch_events = nmi_switch_event; } |
4d015f79e x86/oprofile: Imp... |
260 261 262 263 264 265 266 267 268 |
static void mux_clone(int cpu) { if (!has_mux()) return; memcpy(per_cpu(cpu_msrs, cpu).multiplex, per_cpu(cpu_msrs, 0).multiplex, sizeof(struct op_msr) * model->num_virt_counters); } |
d8471ad3a oprofile: Introdu... |
269 270 271 |
#else inline int op_x86_phys_to_virt(int phys) { return phys; } |
61d149d52 x86/oprofile: Imp... |
272 |
inline int op_x86_virt_to_phys(int virt) { return virt; } |
6ab82f958 x86/oprofile: Imp... |
273 274 |
static inline void nmi_shutdown_mux(void) { } static inline int nmi_setup_mux(void) { return 1; } |
48fb4b467 x86/oprofile: Mov... |
275 276 |
static inline void nmi_cpu_setup_mux(int cpu, struct op_msrs const * const msrs) { } |
528051447 x86/oprofile: Ena... |
277 |
static inline void mux_init(struct oprofile_operations *ops) { } |
4d015f79e x86/oprofile: Imp... |
278 |
static void mux_clone(int cpu) { } |
d8471ad3a oprofile: Introdu... |
279 280 |
#endif |
1da177e4c Linux-2.6.12-rc2 |
281 282 283 |
static void free_msrs(void) { int i; |
c8912599c [PATCH] for_each_... |
284 |
for_each_possible_cpu(i) { |
d18d00f5d x86: oprofile: re... |
285 286 287 288 |
kfree(per_cpu(cpu_msrs, i).counters); per_cpu(cpu_msrs, i).counters = NULL; kfree(per_cpu(cpu_msrs, i).controls); per_cpu(cpu_msrs, i).controls = NULL; |
1da177e4c Linux-2.6.12-rc2 |
289 |
} |
8f5a2dd83 oprofile/x86: rew... |
290 |
nmi_shutdown_mux(); |
1da177e4c Linux-2.6.12-rc2 |
291 |
} |
1da177e4c Linux-2.6.12-rc2 |
292 293 |
static int allocate_msrs(void) { |
1da177e4c Linux-2.6.12-rc2 |
294 295 |
size_t controls_size = sizeof(struct op_msr) * model->num_controls; size_t counters_size = sizeof(struct op_msr) * model->num_counters; |
4c168eaf7 Revert "Oprofile ... |
296 |
int i; |
0939c17c7 x86: fix oprofile... |
297 |
for_each_possible_cpu(i) { |
c17c8fbf3 oprofile/x86: use... |
298 |
per_cpu(cpu_msrs, i).counters = kzalloc(counters_size, |
6ab82f958 x86/oprofile: Imp... |
299 300 |
GFP_KERNEL); if (!per_cpu(cpu_msrs, i).counters) |
8f5a2dd83 oprofile/x86: rew... |
301 |
goto fail; |
c17c8fbf3 oprofile/x86: use... |
302 |
per_cpu(cpu_msrs, i).controls = kzalloc(controls_size, |
6ab82f958 x86/oprofile: Imp... |
303 304 |
GFP_KERNEL); if (!per_cpu(cpu_msrs, i).controls) |
8f5a2dd83 oprofile/x86: rew... |
305 |
goto fail; |
1da177e4c Linux-2.6.12-rc2 |
306 |
} |
8f5a2dd83 oprofile/x86: rew... |
307 308 |
if (!nmi_setup_mux()) goto fail; |
6ab82f958 x86/oprofile: Imp... |
309 |
return 1; |
8f5a2dd83 oprofile/x86: rew... |
310 311 312 313 |
fail: free_msrs(); return 0; |
1da177e4c Linux-2.6.12-rc2 |
314 |
} |
b75f53dba x86: fix style er... |
315 |
static void nmi_cpu_setup(void *dummy) |
1da177e4c Linux-2.6.12-rc2 |
316 317 |
{ int cpu = smp_processor_id(); |
d18d00f5d x86: oprofile: re... |
318 |
struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); |
44ab9a6b0 x86/oprofile: Rew... |
319 |
nmi_cpu_save_registers(msrs); |
2d21a29fb locking, oprofile... |
320 |
raw_spin_lock(&oprofilefs_lock); |
ef8828ddf x86/oprofile: pas... |
321 |
model->setup_ctrs(model, msrs); |
6bfccd099 x86/oprofile: Fix... |
322 |
nmi_cpu_setup_mux(cpu, msrs); |
2d21a29fb locking, oprofile... |
323 |
raw_spin_unlock(&oprofilefs_lock); |
d18d00f5d x86: oprofile: re... |
324 |
per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC); |
1da177e4c Linux-2.6.12-rc2 |
325 326 |
apic_write(APIC_LVTPC, APIC_DM_NMI); } |
44ab9a6b0 x86/oprofile: Rew... |
327 |
static void nmi_cpu_restore_registers(struct op_msrs *msrs) |
1da177e4c Linux-2.6.12-rc2 |
328 |
{ |
b75f53dba x86: fix style er... |
329 330 |
struct op_msr *counters = msrs->counters; struct op_msr *controls = msrs->controls; |
1da177e4c Linux-2.6.12-rc2 |
331 |
unsigned int i; |
1a245c453 x86/oprofile: rem... |
332 |
for (i = 0; i < model->num_controls; ++i) { |
95e74e62c x86/oprofile: use... |
333 334 |
if (controls[i].addr) wrmsrl(controls[i].addr, controls[i].saved); |
1da177e4c Linux-2.6.12-rc2 |
335 |
} |
b75f53dba x86: fix style er... |
336 |
|
1a245c453 x86/oprofile: rem... |
337 |
for (i = 0; i < model->num_counters; ++i) { |
95e74e62c x86/oprofile: use... |
338 339 |
if (counters[i].addr) wrmsrl(counters[i].addr, counters[i].saved); |
1da177e4c Linux-2.6.12-rc2 |
340 341 |
} } |
1da177e4c Linux-2.6.12-rc2 |
342 |
|
b75f53dba x86: fix style er... |
343 |
static void nmi_cpu_shutdown(void *dummy) |
1da177e4c Linux-2.6.12-rc2 |
344 345 346 |
{ unsigned int v; int cpu = smp_processor_id(); |
82a225283 x86/oprofile: Use... |
347 |
struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); |
b75f53dba x86: fix style er... |
348 |
|
1da177e4c Linux-2.6.12-rc2 |
349 350 351 352 353 354 355 |
/* restoring APIC_LVTPC can trigger an apic error because the delivery * mode and vector nr combination can be illegal. That's by design: on * power on apic lvt contain a zero vector nr which are legal only for * NMI delivery mode. So inhibit apic err before restoring lvtpc */ v = apic_read(APIC_LVTERR); apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); |
d18d00f5d x86: oprofile: re... |
356 |
apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu)); |
1da177e4c Linux-2.6.12-rc2 |
357 |
apic_write(APIC_LVTERR, v); |
44ab9a6b0 x86/oprofile: Rew... |
358 |
nmi_cpu_restore_registers(msrs); |
1da177e4c Linux-2.6.12-rc2 |
359 |
} |
6ae56b55b oprofile/x86: pro... |
360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 |
static void nmi_cpu_up(void *dummy) { if (nmi_enabled) nmi_cpu_setup(dummy); if (ctr_running) nmi_cpu_start(dummy); } static void nmi_cpu_down(void *dummy) { if (ctr_running) nmi_cpu_stop(dummy); if (nmi_enabled) nmi_cpu_shutdown(dummy); } |
b75f53dba x86: fix style er... |
375 |
static int nmi_create_files(struct super_block *sb, struct dentry *root) |
1da177e4c Linux-2.6.12-rc2 |
376 377 |
{ unsigned int i; |
4d4036e0e oprofile: Impleme... |
378 |
for (i = 0; i < model->num_virt_counters; ++i) { |
b75f53dba x86: fix style er... |
379 |
struct dentry *dir; |
0c6856f70 [PATCH] oprofile:... |
380 |
char buf[4]; |
b75f53dba x86: fix style er... |
381 382 |
/* quick little hack to _not_ expose a counter if it is not |
cb9c448c6 [PATCH] i386: Uti... |
383 384 385 386 |
* available for use. This should protect userspace app. * NOTE: assumes 1:1 mapping here (that counters are organized * sequentially in their struct assignment). */ |
11be1a7b5 x86/oprofile: Add... |
387 |
if (!avail_to_resrv_perfctr_nmi_bit(op_x86_virt_to_phys(i))) |
cb9c448c6 [PATCH] i386: Uti... |
388 |
continue; |
0c6856f70 [PATCH] oprofile:... |
389 |
snprintf(buf, sizeof(buf), "%d", i); |
1da177e4c Linux-2.6.12-rc2 |
390 |
dir = oprofilefs_mkdir(sb, root, buf); |
b75f53dba x86: fix style er... |
391 392 393 394 395 396 |
oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled); oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event); oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count); oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask); oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel); oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user); |
914a76ca5 oprofile, x86: Al... |
397 |
oprofilefs_create_ulong(sb, dir, "extra", &counter_config[i].extra); |
1da177e4c Linux-2.6.12-rc2 |
398 399 400 401 |
} return 0; } |
b75f53dba x86: fix style er... |
402 |
|
69046d430 x86/oprofile: reo... |
403 404 405 406 407 408 409 |
static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action, void *data) { int cpu = (unsigned long)data; switch (action) { case CPU_DOWN_FAILED: case CPU_ONLINE: |
6ae56b55b oprofile/x86: pro... |
410 |
smp_call_function_single(cpu, nmi_cpu_up, NULL, 0); |
69046d430 x86/oprofile: reo... |
411 412 |
break; case CPU_DOWN_PREPARE: |
6ae56b55b oprofile/x86: pro... |
413 |
smp_call_function_single(cpu, nmi_cpu_down, NULL, 1); |
69046d430 x86/oprofile: reo... |
414 415 416 417 418 419 420 421 |
break; } return NOTIFY_DONE; } static struct notifier_block oprofile_cpu_nb = { .notifier_call = oprofile_cpu_notifier }; |
69046d430 x86/oprofile: reo... |
422 |
|
d30d64c6d oprofile/x86: reo... |
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 |
static int nmi_setup(void) { int err = 0; int cpu; if (!allocate_msrs()) return -ENOMEM; /* We need to serialize save and setup for HT because the subset * of msrs are distinct for save and setup operations */ /* Assume saved/restored counters are the same on all CPUs */ err = model->fill_in_addresses(&per_cpu(cpu_msrs, 0)); if (err) goto fail; for_each_possible_cpu(cpu) { if (!cpu) continue; memcpy(per_cpu(cpu_msrs, cpu).counters, per_cpu(cpu_msrs, 0).counters, sizeof(struct op_msr) * model->num_counters); memcpy(per_cpu(cpu_msrs, cpu).controls, per_cpu(cpu_msrs, 0).controls, sizeof(struct op_msr) * model->num_controls); mux_clone(cpu); } nmi_enabled = 0; ctr_running = 0; |
8fe7e94eb oprofile, x86: Fi... |
457 458 |
/* make variables visible to the nmi handler: */ smp_mb(); |
9c48f1c62 x86, nmi: Wire up... |
459 460 |
err = register_nmi_handler(NMI_LOCAL, profile_exceptions_notify, 0, "oprofile"); |
d30d64c6d oprofile/x86: reo... |
461 462 463 464 |
if (err) goto fail; get_online_cpus(); |
3de668ee8 oprofile/x86: not... |
465 |
register_cpu_notifier(&oprofile_cpu_nb); |
d30d64c6d oprofile/x86: reo... |
466 |
nmi_enabled = 1; |
8fe7e94eb oprofile, x86: Fi... |
467 468 469 |
/* make nmi_enabled visible to the nmi handler: */ smp_mb(); on_each_cpu(nmi_cpu_setup, NULL, 1); |
d30d64c6d oprofile/x86: reo... |
470 471 472 473 474 475 476 477 478 479 480 481 482 |
put_online_cpus(); return 0; fail: free_msrs(); return err; } static void nmi_shutdown(void) { struct op_msrs *msrs; get_online_cpus(); |
3de668ee8 oprofile/x86: not... |
483 |
unregister_cpu_notifier(&oprofile_cpu_nb); |
d30d64c6d oprofile/x86: reo... |
484 485 486 487 |
on_each_cpu(nmi_cpu_shutdown, NULL, 1); nmi_enabled = 0; ctr_running = 0; put_online_cpus(); |
8fe7e94eb oprofile, x86: Fi... |
488 489 |
/* make variables visible to the nmi handler: */ smp_mb(); |
9c48f1c62 x86, nmi: Wire up... |
490 |
unregister_nmi_handler(NMI_LOCAL, "oprofile"); |
d30d64c6d oprofile/x86: reo... |
491 492 493 494 495 |
msrs = &get_cpu_var(cpu_msrs); model->shutdown(msrs); free_msrs(); put_cpu_var(cpu_msrs); } |
69046d430 x86/oprofile: reo... |
496 |
#ifdef CONFIG_PM |
f3c6ea1b0 x86: Use syscore_... |
497 |
static int nmi_suspend(void) |
69046d430 x86/oprofile: reo... |
498 499 500 501 502 503 |
{ /* Only one CPU left, just stop that one */ if (nmi_enabled == 1) nmi_cpu_stop(NULL); return 0; } |
f3c6ea1b0 x86: Use syscore_... |
504 |
static void nmi_resume(void) |
69046d430 x86/oprofile: reo... |
505 506 507 |
{ if (nmi_enabled == 1) nmi_cpu_start(NULL); |
69046d430 x86/oprofile: reo... |
508 |
} |
f3c6ea1b0 x86: Use syscore_... |
509 |
static struct syscore_ops oprofile_syscore_ops = { |
69046d430 x86/oprofile: reo... |
510 511 512 |
.resume = nmi_resume, .suspend = nmi_suspend, }; |
f3c6ea1b0 x86: Use syscore_... |
513 |
static void __init init_suspend_resume(void) |
69046d430 x86/oprofile: reo... |
514 |
{ |
f3c6ea1b0 x86: Use syscore_... |
515 |
register_syscore_ops(&oprofile_syscore_ops); |
69046d430 x86/oprofile: reo... |
516 |
} |
f3c6ea1b0 x86: Use syscore_... |
517 |
static void exit_suspend_resume(void) |
69046d430 x86/oprofile: reo... |
518 |
{ |
f3c6ea1b0 x86: Use syscore_... |
519 |
unregister_syscore_ops(&oprofile_syscore_ops); |
69046d430 x86/oprofile: reo... |
520 521 522 |
} #else |
269f45c25 oprofile, x86: fi... |
523 |
|
f3c6ea1b0 x86: Use syscore_... |
524 525 |
static inline void init_suspend_resume(void) { } static inline void exit_suspend_resume(void) { } |
269f45c25 oprofile, x86: fi... |
526 |
|
69046d430 x86/oprofile: reo... |
527 |
#endif /* CONFIG_PM */ |
b75f53dba x86: fix style er... |
528 |
static int __init p4_init(char **cpu_type) |
1da177e4c Linux-2.6.12-rc2 |
529 530 |
{ __u8 cpu_model = boot_cpu_data.x86_model; |
1f3d7b606 oprofile: remove ... |
531 |
if (cpu_model > 6 || cpu_model == 5) |
1da177e4c Linux-2.6.12-rc2 |
532 533 534 535 536 537 538 539 |
return 0; #ifndef CONFIG_SMP *cpu_type = "i386/p4"; model = &op_p4_spec; return 1; #else switch (smp_num_siblings) { |
b75f53dba x86: fix style er... |
540 541 542 543 544 545 546 547 548 |
case 1: *cpu_type = "i386/p4"; model = &op_p4_spec; return 1; case 2: *cpu_type = "i386/p4-ht"; model = &op_p4_ht2_spec; return 1; |
1da177e4c Linux-2.6.12-rc2 |
549 550 551 552 553 554 555 556 557 |
} #endif printk(KERN_INFO "oprofile: P4 HyperThreading detected with > 2 threads "); printk(KERN_INFO "oprofile: Reverting to timer mode. "); return 0; } |
159a80b21 oprofile, x86: Ad... |
558 559 560 561 562 563 564 565 566 |
enum __force_cpu_type { reserved = 0, /* do not force */ timer, arch_perfmon, }; static int force_cpu_type; static int set_cpu_type(const char *str, struct kernel_param *kp) |
7e4e0bd50 oprofile: introdu... |
567 |
{ |
159a80b21 oprofile, x86: Ad... |
568 569 570 571 572 573 |
if (!strcmp(str, "timer")) { force_cpu_type = timer; printk(KERN_INFO "oprofile: forcing NMI timer mode "); } else if (!strcmp(str, "arch_perfmon")) { force_cpu_type = arch_perfmon; |
7e4e0bd50 oprofile: introdu... |
574 575 |
printk(KERN_INFO "oprofile: forcing architectural perfmon "); |
159a80b21 oprofile, x86: Ad... |
576 577 |
} else { force_cpu_type = 0; |
7e4e0bd50 oprofile: introdu... |
578 579 580 581 |
} return 0; } |
159a80b21 oprofile, x86: Ad... |
582 |
module_param_call(cpu_type, set_cpu_type, NULL, NULL, 0); |
1dcdb5a9e oprofile: re-add ... |
583 |
|
b75f53dba x86: fix style er... |
584 |
static int __init ppro_init(char **cpu_type) |
1da177e4c Linux-2.6.12-rc2 |
585 586 |
{ __u8 cpu_model = boot_cpu_data.x86_model; |
259a83a8a x86/oprofile: Rem... |
587 |
struct op_x86_model_spec *spec = &op_ppro_spec; /* default */ |
1da177e4c Linux-2.6.12-rc2 |
588 |
|
159a80b21 oprofile, x86: Ad... |
589 |
if (force_cpu_type == arch_perfmon && cpu_has_arch_perfmon) |
1dcdb5a9e oprofile: re-add ... |
590 |
return 0; |
45c34e05c Oprofile: Change ... |
591 592 593 594 595 596 597 598 599 600 601 602 |
/* * Documentation on identifying Intel processors by CPU family * and model can be found in the Intel Software Developer's * Manuals (SDM): * * http://www.intel.com/products/processor/manuals/ * * As of May 2010 the documentation for this was in the: * "Intel 64 and IA-32 Architectures Software Developer's * Manual Volume 3B: System Programming Guide", "Table B-1 * CPUID Signature Values of DisplayFamily_DisplayModel". */ |
4b9f12a37 x86/oprofile/nmi_... |
603 604 605 606 607 608 609 610 |
switch (cpu_model) { case 0 ... 2: *cpu_type = "i386/ppro"; break; case 3 ... 5: *cpu_type = "i386/pii"; break; case 6 ... 8: |
3d337c653 x86/oprofile: fix... |
611 |
case 10 ... 11: |
4b9f12a37 x86/oprofile/nmi_... |
612 613 614 |
*cpu_type = "i386/piii"; break; case 9: |
3d337c653 x86/oprofile: fix... |
615 |
case 13: |
4b9f12a37 x86/oprofile/nmi_... |
616 617 |
*cpu_type = "i386/p6_mobile"; break; |
4b9f12a37 x86/oprofile/nmi_... |
618 |
case 14: |
64471ebe5 [PATCH] Add Core ... |
619 |
*cpu_type = "i386/core"; |
4b9f12a37 x86/oprofile/nmi_... |
620 |
break; |
c33f543d3 oprofile: Add Sup... |
621 622 623 |
case 0x0f: case 0x16: case 0x17: |
bb7ab785a oprofile: Add Sup... |
624 |
case 0x1d: |
4b9f12a37 x86/oprofile/nmi_... |
625 626 |
*cpu_type = "i386/core_2"; break; |
45c34e05c Oprofile: Change ... |
627 |
case 0x1a: |
a7c55cbee oprofile: add sup... |
628 |
case 0x1e: |
e83e452b0 oprofile/x86: add... |
629 |
case 0x2e: |
802070f54 x86/oprofile: fix... |
630 |
spec = &op_arch_perfmon_spec; |
6adf406f0 oprofile: add sup... |
631 632 |
*cpu_type = "i386/core_i7"; break; |
45c34e05c Oprofile: Change ... |
633 |
case 0x1c: |
6adf406f0 oprofile: add sup... |
634 635 |
*cpu_type = "i386/atom"; break; |
4b9f12a37 x86/oprofile/nmi_... |
636 637 |
default: /* Unknown */ |
1da177e4c Linux-2.6.12-rc2 |
638 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
639 |
} |
802070f54 x86/oprofile: fix... |
640 |
model = spec; |
1da177e4c Linux-2.6.12-rc2 |
641 642 |
return 1; } |
96d0821ca [PATCH] Fix funct... |
643 |
int __init op_nmi_init(struct oprofile_operations *ops) |
1da177e4c Linux-2.6.12-rc2 |
644 645 646 |
{ __u8 vendor = boot_cpu_data.x86_vendor; __u8 family = boot_cpu_data.x86; |
b99170288 oprofile: Impleme... |
647 |
char *cpu_type = NULL; |
adf5ec0bc x86/oprofile: int... |
648 |
int ret = 0; |
1da177e4c Linux-2.6.12-rc2 |
649 650 651 |
if (!cpu_has_apic) return -ENODEV; |
b75f53dba x86: fix style er... |
652 |
|
159a80b21 oprofile, x86: Ad... |
653 654 |
if (force_cpu_type == timer) return -ENODEV; |
1da177e4c Linux-2.6.12-rc2 |
655 |
switch (vendor) { |
b75f53dba x86: fix style er... |
656 657 |
case X86_VENDOR_AMD: /* Needs to be at least an Athlon (or hammer in 32bit mode) */ |
1da177e4c Linux-2.6.12-rc2 |
658 |
|
b75f53dba x86: fix style er... |
659 |
switch (family) { |
b75f53dba x86: fix style er... |
660 |
case 6: |
b75f53dba x86: fix style er... |
661 662 663 |
cpu_type = "i386/athlon"; break; case 0xf: |
d20f24c66 x86/oprofile: sim... |
664 665 666 667 |
/* * Actually it could be i386/hammer too, but * give user space an consistent name. */ |
b75f53dba x86: fix style er... |
668 669 670 |
cpu_type = "x86-64/hammer"; break; case 0x10: |
b75f53dba x86: fix style er... |
671 672 |
cpu_type = "x86-64/family10"; break; |
12f2b2610 oprofile: Add sup... |
673 |
case 0x11: |
12f2b2610 oprofile: Add sup... |
674 675 |
cpu_type = "x86-64/family11h"; break; |
3acbf0849 oprofile, x86: Ad... |
676 677 678 |
case 0x12: cpu_type = "x86-64/family12h"; break; |
e63414740 oprofile, x86: Ad... |
679 680 681 |
case 0x14: cpu_type = "x86-64/family14h"; break; |
30570bced oprofile, x86: Ad... |
682 683 684 |
case 0x15: cpu_type = "x86-64/family15h"; break; |
d20f24c66 x86/oprofile: sim... |
685 686 |
default: return -ENODEV; |
b75f53dba x86: fix style er... |
687 |
} |
d20f24c66 x86/oprofile: sim... |
688 |
model = &op_amd_spec; |
b75f53dba x86: fix style er... |
689 690 691 692 693 694 |
break; case X86_VENDOR_INTEL: switch (family) { /* Pentium IV */ case 0xf: |
b99170288 oprofile: Impleme... |
695 |
p4_init(&cpu_type); |
1da177e4c Linux-2.6.12-rc2 |
696 |
break; |
b75f53dba x86: fix style er... |
697 698 699 |
/* A P6-class processor */ case 6: |
b99170288 oprofile: Impleme... |
700 |
ppro_init(&cpu_type); |
1da177e4c Linux-2.6.12-rc2 |
701 702 703 |
break; default: |
b99170288 oprofile: Impleme... |
704 |
break; |
b75f53dba x86: fix style er... |
705 |
} |
b99170288 oprofile: Impleme... |
706 |
|
e419294ed x86/oprofile: mov... |
707 708 709 710 |
if (cpu_type) break; if (!cpu_has_arch_perfmon) |
b99170288 oprofile: Impleme... |
711 |
return -ENODEV; |
e419294ed x86/oprofile: mov... |
712 713 714 715 |
/* use arch perfmon as fallback */ cpu_type = "i386/arch_perfmon"; model = &op_arch_perfmon_spec; |
b75f53dba x86: fix style er... |
716 717 718 719 |
break; default: return -ENODEV; |
1da177e4c Linux-2.6.12-rc2 |
720 |
} |
270d3e1a1 OProfile: enable ... |
721 |
/* default values, can be overwritten by model */ |
6e63ea4b0 x86/oprofile: Whi... |
722 723 724 725 726 727 |
ops->create_files = nmi_create_files; ops->setup = nmi_setup; ops->shutdown = nmi_shutdown; ops->start = nmi_start; ops->stop = nmi_stop; ops->cpu_type = cpu_type; |
270d3e1a1 OProfile: enable ... |
728 |
|
adf5ec0bc x86/oprofile: int... |
729 730 731 732 |
if (model->init) ret = model->init(ops); if (ret) return ret; |
52471c67e x86/oprofile: Mod... |
733 734 |
if (!model->num_virt_counters) model->num_virt_counters = model->num_counters; |
528051447 x86/oprofile: Ena... |
735 |
mux_init(ops); |
f3c6ea1b0 x86: Use syscore_... |
736 |
init_suspend_resume(); |
10f0412f5 oprofile, x86: fi... |
737 |
|
1da177e4c Linux-2.6.12-rc2 |
738 739 740 741 |
printk(KERN_INFO "oprofile: using NMI interrupt. "); return 0; } |
96d0821ca [PATCH] Fix funct... |
742 |
void op_nmi_exit(void) |
1da177e4c Linux-2.6.12-rc2 |
743 |
{ |
f3c6ea1b0 x86: Use syscore_... |
744 |
exit_suspend_resume(); |
1da177e4c Linux-2.6.12-rc2 |
745 |
} |