oprofile: Implement performance counter multiplexing

The number of hardware counters is limited. The multiplexing feature enables OProfile to gather more events than counters are provided by the hardware. This is realized by switching between events at an user specified time interval. A new file (/dev/oprofile/time_slice) is added for the user to specify the timer interval in ms. If the number of events to profile is higher than the number of hardware counters available, the patch will schedule a work queue that switches the event counter and re-writes the different sets of values into it. The switching mechanism needs to be implemented for each architecture to support multiplexing. This patch only implements AMD CPU support, but multiplexing can be easily extended for other models and architectures. There are follow-on patches that rework parts of this patch. Signed-off-by: Jason Yeh <jason.yeh@amd.com> Signed-off-by: Robert Richter <robert.richter@amd.com>

oprofile: Implement performance counter multiplexing
The number of hardware counters is limited. The multiplexing feature enables OProfile to gather more events than counters are provided by the hardware. This is realized by switching between events at an user specified time interval. A new file (/dev/oprofile/time_slice) is added for the user to specify the timer interval in ms. If the number of events to profile is higher than the number of hardware counters available, the patch will schedule a work queue that switches the event counter and re-writes the different sets of values into it. The switching mechanism needs to be implemented for each architecture to support multiplexing. This patch only implements AMD CPU support, but multiplexing can be easily extended for other models and architectures. There are follow-on patches that rework parts of this patch. Signed-off-by: Jason Yeh <jason.yeh@amd.com> Signed-off-by: Robert Richter <robert.richter@amd.com>
Jason Yeh · Robert Richter
1 parent 6e63ea4b0b
Showing 12 changed files with 415 additions and 20 deletions Side-by-side Diff
arch/Kconfig
arch/x86/oprofile/nmi_int.c
arch/x86/oprofile/op_counter.h
arch/x86/oprofile/op_model_amd.c
arch/x86/oprofile/op_model_p4.c
arch/x86/oprofile/op_model_ppro.c
arch/x86/oprofile/op_x86_model.h
drivers/oprofile/oprof.c
drivers/oprofile/oprof.h
drivers/oprofile/oprofile_files.c
drivers/oprofile/oprofile_stats.c
include/linux/oprofile.h
@@ -30,6 +30,18 @@
  
 	  If unsure, say N.
  
+config OPROFILE_EVENT_MULTIPLEX
+	bool "OProfile multiplexing support (EXPERIMENTAL)"
+	default n
+	depends on OPROFILE && X86
+	help
+	  The number of hardware counters is limited. The multiplexing
+	  feature enables OProfile to gather more events than counters
+	  are provided by the hardware. This is realized by switching
+	  between events at an user specified time interval.
+
+	  If unsure, say N.
+
 config HAVE_OPROFILE
 	bool
  
 /**
  * @file nmi_int.c
  *
- * @remark Copyright 2002-2008 OProfile authors
+ * @remark Copyright 2002-2009 OProfile authors
  * @remark Read the file COPYING
  *
  * @author John Levon <levon@movementarian.org>
  * @author Robert Richter <robert.richter@amd.com>
+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
+ * @author Jason Yeh <jason.yeh@amd.com>
+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  */
  
 #include <linux/init.h>
@@ -24,6 +27,12 @@
 #include "op_counter.h"
 #include "op_x86_model.h"
  
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+DEFINE_PER_CPU(int, switch_index);
+#endif
+
+
 static struct op_x86_model_spec const *model;
 static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
 static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
@@ -31,6 +40,13 @@
 /* 0 == registered but off, 1 == registered and on */
 static int nmi_enabled = 0;
  
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+extern atomic_t multiplex_counter;
+#endif
+
+struct op_counter_config counter_config[OP_MAX_COUNTER];
+
 /* common functions */
  
 u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
@@ -95,6 +111,11 @@
 		per_cpu(cpu_msrs, i).counters = NULL;
 		kfree(per_cpu(cpu_msrs, i).controls);
 		per_cpu(cpu_msrs, i).controls = NULL;
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		kfree(per_cpu(cpu_msrs, i).multiplex);
+		per_cpu(cpu_msrs, i).multiplex = NULL;
+#endif
 	}
 }
  
@@ -103,6 +124,9 @@
 	int success = 1;
 	size_t controls_size = sizeof(struct op_msr) * model->num_controls;
 	size_t counters_size = sizeof(struct op_msr) * model->num_counters;
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	size_t multiplex_size = sizeof(struct op_msr) * model->num_virt_counters;
+#endif
  
 	int i;
 	for_each_possible_cpu(i) {
@@ -118,6 +142,14 @@
 			success = 0;
 			break;
 		}
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		per_cpu(cpu_msrs, i).multiplex =
+				kmalloc(multiplex_size, GFP_KERNEL);
+		if (!per_cpu(cpu_msrs, i).multiplex) {
+			success = 0;
+			break;
+		}
+#endif
 	}
  
 	if (!success)
@@ -126,6 +158,25 @@
 	return success;
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void nmi_setup_cpu_mux(struct op_msrs const * const msrs)
+{
+	int i;
+	struct op_msr *multiplex = msrs->multiplex;
+
+	for (i = 0; i < model->num_virt_counters; ++i) {
+		if (counter_config[i].enabled) {
+			multiplex[i].saved = -(u64)counter_config[i].count;
+		} else {
+			multiplex[i].addr  = 0;
+			multiplex[i].saved = 0;
+		}
+	}
+}
+
+#endif
+
 static void nmi_cpu_setup(void *dummy)
 {
 	int cpu = smp_processor_id();
@@ -133,6 +184,9 @@
 	nmi_cpu_save_registers(msrs);
 	spin_lock(&oprofilefs_lock);
 	model->setup_ctrs(model, msrs);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	nmi_setup_cpu_mux(msrs);
+#endif
 	spin_unlock(&oprofilefs_lock);
 	per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
 	apic_write(APIC_LVTPC, APIC_DM_NMI);
  
  
@@ -173,14 +227,52 @@
 			memcpy(per_cpu(cpu_msrs, cpu).controls,
 				per_cpu(cpu_msrs, 0).controls,
 				sizeof(struct op_msr) * model->num_controls);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+			memcpy(per_cpu(cpu_msrs, cpu).multiplex,
+				per_cpu(cpu_msrs, 0).multiplex,
+				sizeof(struct op_msr) * model->num_virt_counters);
+#endif
 		}
-
 	}
 	on_each_cpu(nmi_cpu_setup, NULL, 1);
 	nmi_enabled = 1;
 	return 0;
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
+{
+	unsigned int si = __get_cpu_var(switch_index);
+	struct op_msr *multiplex = msrs->multiplex;
+	unsigned int i;
+
+	for (i = 0; i < model->num_counters; ++i) {
+		int offset = i + si;
+		if (multiplex[offset].addr) {
+			rdmsrl(multiplex[offset].addr,
+			       multiplex[offset].saved);
+		}
+	}
+}
+
+static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
+{
+	unsigned int si = __get_cpu_var(switch_index);
+	struct op_msr *multiplex = msrs->multiplex;
+	unsigned int i;
+
+	for (i = 0; i < model->num_counters; ++i) {
+		int offset = i + si;
+		if (multiplex[offset].addr) {
+			wrmsrl(multiplex[offset].addr,
+			       multiplex[offset].saved);
+		}
+	}
+}
+
+#endif
+
 static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 {
 	struct op_msr *counters = msrs->counters;
@@ -214,6 +306,9 @@
 	apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
 	apic_write(APIC_LVTERR, v);
 	nmi_cpu_restore_registers(msrs);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	__get_cpu_var(switch_index) = 0;
+#endif
 }
  
 static void nmi_shutdown(void)
  
  
@@ -252,16 +347,15 @@
 	on_each_cpu(nmi_cpu_stop, NULL, 1);
 }
  
-struct op_counter_config counter_config[OP_MAX_COUNTER];
-
 static int nmi_create_files(struct super_block *sb, struct dentry *root)
 {
 	unsigned int i;
  
-	for (i = 0; i < model->num_counters; ++i) {
+	for (i = 0; i < model->num_virt_counters; ++i) {
 		struct dentry *dir;
 		char buf[4];
  
+#ifndef CONFIG_OPROFILE_EVENT_MULTIPLEX
 		/* quick little hack to _not_ expose a counter if it is not
 		 * available for use.  This should protect userspace app.
 		 * NOTE:  assumes 1:1 mapping here (that counters are organized
@@ -269,6 +363,7 @@
 		 */
 		if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
 			continue;
+#endif /* CONFIG_OPROFILE_EVENT_MULTIPLEX */
  
 		snprintf(buf,  sizeof(buf), "%d", i);
 		dir = oprofilefs_mkdir(sb, root, buf);
@@ -283,6 +378,57 @@
 	return 0;
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void nmi_cpu_switch(void *dummy)
+{
+	int cpu = smp_processor_id();
+	int si = per_cpu(switch_index, cpu);
+	struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
+
+	nmi_cpu_stop(NULL);
+	nmi_cpu_save_mpx_registers(msrs);
+
+	/* move to next set */
+	si += model->num_counters;
+	if ((si > model->num_virt_counters) || (counter_config[si].count == 0))
+		per_cpu(switch_index, cpu) = 0;
+	else
+		per_cpu(switch_index, cpu) = si;
+
+	model->switch_ctrl(model, msrs);
+	nmi_cpu_restore_mpx_registers(msrs);
+
+	nmi_cpu_start(NULL);
+}
+
+
+/*
+ * Quick check to see if multiplexing is necessary.
+ * The check should be sufficient since counters are used
+ * in ordre.
+ */
+static int nmi_multiplex_on(void)
+{
+	return counter_config[model->num_counters].count ? 0 : -EINVAL;
+}
+
+static int nmi_switch_event(void)
+{
+	if (!model->switch_ctrl)
+		return -ENOSYS;		/* not implemented */
+	if (nmi_multiplex_on() < 0)
+		return -EINVAL;		/* not necessary */
+
+	on_each_cpu(nmi_cpu_switch, NULL, 1);
+
+	atomic_inc(&multiplex_counter);
+
+	return 0;
+}
+
+#endif
+
 #ifdef CONFIG_SMP
 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 				 void *data)
  
@@ -516,12 +662,18 @@
 	register_cpu_notifier(&oprofile_cpu_nb);
 #endif
 	/* default values, can be overwritten by model */
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	__raw_get_cpu_var(switch_index) = 0;
+#endif
 	ops->create_files	= nmi_create_files;
 	ops->setup		= nmi_setup;
 	ops->shutdown		= nmi_shutdown;
 	ops->start		= nmi_start;
 	ops->stop		= nmi_stop;
 	ops->cpu_type		= cpu_type;
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	ops->switch_events	= nmi_switch_event;
+#endif
  
 	if (model->init)
 		ret = model->init(ops);
@@ -10,7 +10,7 @@
 #ifndef OP_COUNTER_H
 #define OP_COUNTER_H
  
-#define OP_MAX_COUNTER 8
+#define OP_MAX_COUNTER 32
  
 /* Per-perfctr configuration as set via
  * oprofilefs.
@@ -9,12 +9,15 @@
  * @author Philippe Elie
  * @author Graydon Hoare
  * @author Robert Richter <robert.richter@amd.com>
- * @author Barry Kasindorf
+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
+ * @author Jason Yeh <jason.yeh@amd.com>
+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  */
  
 #include <linux/oprofile.h>
 #include <linux/device.h>
 #include <linux/pci.h>
+#include <linux/percpu.h>
  
 #include <asm/ptrace.h>
 #include <asm/msr.h>
  
@@ -25,12 +28,23 @@
  
 #define NUM_COUNTERS 4
 #define NUM_CONTROLS 4
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+#define NUM_VIRT_COUNTERS 32
+#define NUM_VIRT_CONTROLS 32
+#else
+#define NUM_VIRT_COUNTERS NUM_COUNTERS
+#define NUM_VIRT_CONTROLS NUM_CONTROLS
+#endif
+
 #define OP_EVENT_MASK			0x0FFF
 #define OP_CTR_OVERFLOW			(1ULL<<31)
  
 #define MSR_AMD_EVENTSEL_RESERVED	((0xFFFFFCF0ULL<<32)|(1ULL<<21))
  
-static unsigned long reset_value[NUM_COUNTERS];
+static unsigned long reset_value[NUM_VIRT_COUNTERS];
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+DECLARE_PER_CPU(int, switch_index);
+#endif
  
 #ifdef CONFIG_OPROFILE_IBS
  
@@ -82,6 +96,16 @@
 		else
 			msrs->controls[i].addr = 0;
 	}
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	for (i = 0; i < NUM_VIRT_COUNTERS; i++) {
+		int hw_counter = i % NUM_CONTROLS;
+		if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
+			msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter;
+		else
+			msrs->multiplex[i].addr = 0;
+	}
+#endif
 }
  
 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
@@ -90,6 +114,15 @@
 	u64 val;
 	int i;
  
+	/* setup reset_value */
+	for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
+		if (counter_config[i].enabled) {
+			reset_value[i] = counter_config[i].count;
+		} else {
+			reset_value[i] = 0;
+		}
+	}
+
 	/* clear all counters */
 	for (i = 0; i < NUM_CONTROLS; ++i) {
 		if (unlikely(!msrs->controls[i].addr))
  
  
  
@@ -108,20 +141,49 @@
  
 	/* enable active counters */
 	for (i = 0; i < NUM_COUNTERS; ++i) {
-		if (counter_config[i].enabled && msrs->counters[i].addr) {
-			reset_value[i] = counter_config[i].count;
-			wrmsrl(msrs->counters[i].addr,
-			       -(u64)counter_config[i].count);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		int offset = i + __get_cpu_var(switch_index);
+#else
+		int offset = i;
+#endif
+		if (counter_config[offset].enabled && msrs->counters[i].addr) {
+			/* setup counter registers */
+			wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
+
+			/* setup control registers */
 			rdmsrl(msrs->controls[i].addr, val);
 			val &= model->reserved;
-			val |= op_x86_get_ctrl(model, &counter_config[i]);
+			val |= op_x86_get_ctrl(model, &counter_config[offset]);
 			wrmsrl(msrs->controls[i].addr, val);
-		} else {
-			reset_value[i] = 0;
 		}
 	}
 }
  
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void op_amd_switch_ctrl(struct op_x86_model_spec const *model,
+			       struct op_msrs const * const msrs)
+{
+	u64 val;
+	int i;
+
+	/* enable active counters */
+	for (i = 0; i < NUM_COUNTERS; ++i) {
+		int offset = i + __get_cpu_var(switch_index);
+		if (counter_config[offset].enabled) {
+			/* setup control registers */
+			rdmsrl(msrs->controls[i].addr, val);
+			val &= model->reserved;
+			val |= op_x86_get_ctrl(model, &counter_config[offset]);
+			wrmsrl(msrs->controls[i].addr, val);
+		}
+	}
+}
+
+#endif
+
+
 #ifdef CONFIG_OPROFILE_IBS
  
 static inline int
  
@@ -230,14 +292,19 @@
 	int i;
  
 	for (i = 0; i < NUM_COUNTERS; ++i) {
-		if (!reset_value[i])
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		int offset = i + __get_cpu_var(switch_index);
+#else
+		int offset = i;
+#endif
+		if (!reset_value[offset])
 			continue;
 		rdmsrl(msrs->counters[i].addr, val);
 		/* bit is clear if overflowed: */
 		if (val & OP_CTR_OVERFLOW)
 			continue;
-		oprofile_add_sample(regs, i);
-		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[i]);
+		oprofile_add_sample(regs, offset);
+		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
 	}
  
 	op_amd_handle_ibs(regs, msrs);
  
@@ -250,8 +317,14 @@
 {
 	u64 val;
 	int i;
+
 	for (i = 0; i < NUM_COUNTERS; ++i) {
-		if (reset_value[i]) {
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		int offset = i + __get_cpu_var(switch_index);
+#else
+		int offset = i;
+#endif
+		if (reset_value[offset]) {
 			rdmsrl(msrs->controls[i].addr, val);
 			val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 			wrmsrl(msrs->controls[i].addr, val);
  
@@ -271,7 +344,11 @@
 	 * pm callback
 	 */
 	for (i = 0; i < NUM_COUNTERS; ++i) {
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+		if (!reset_value[i + per_cpu(switch_index, smp_processor_id())])
+#else
 		if (!reset_value[i])
+#endif
 			continue;
 		rdmsrl(msrs->controls[i].addr, val);
 		val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
@@ -289,7 +366,7 @@
 		if (msrs->counters[i].addr)
 			release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 	}
-	for (i = 0; i < NUM_CONTROLS; ++i) {
+	for (i = 0; i < NUM_COUNTERS; ++i) {
 		if (msrs->controls[i].addr)
 			release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 	}
@@ -463,6 +540,8 @@
 struct op_x86_model_spec const op_amd_spec = {
 	.num_counters		= NUM_COUNTERS,
 	.num_controls		= NUM_CONTROLS,
+	.num_virt_counters	= NUM_VIRT_COUNTERS,
+	.num_virt_controls	= NUM_VIRT_CONTROLS,
 	.reserved		= MSR_AMD_EVENTSEL_RESERVED,
 	.event_mask		= OP_EVENT_MASK,
 	.init			= op_amd_init,
@@ -473,5 +552,8 @@
 	.start			= &op_amd_start,
 	.stop			= &op_amd_stop,
 	.shutdown		= &op_amd_shutdown,
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	.switch_ctrl		= &op_amd_switch_ctrl,
+#endif
 };
@@ -698,6 +698,8 @@
 struct op_x86_model_spec const op_p4_ht2_spec = {
 	.num_counters		= NUM_COUNTERS_HT2,
 	.num_controls		= NUM_CONTROLS_HT2,
+	.num_virt_counters	= NUM_COUNTERS_HT2,
+	.num_virt_controls	= NUM_CONTROLS_HT2,
 	.fill_in_addresses	= &p4_fill_in_addresses,
 	.setup_ctrs		= &p4_setup_ctrs,
 	.check_ctrs		= &p4_check_ctrs,
@@ -710,6 +712,8 @@
 struct op_x86_model_spec const op_p4_spec = {
 	.num_counters		= NUM_COUNTERS_NON_HT,
 	.num_controls		= NUM_CONTROLS_NON_HT,
+	.num_virt_counters	= NUM_COUNTERS_NON_HT,
+	.num_virt_controls	= NUM_CONTROLS_NON_HT,
 	.fill_in_addresses	= &p4_fill_in_addresses,
 	.setup_ctrs		= &p4_setup_ctrs,
 	.check_ctrs		= &p4_check_ctrs,
@@ -206,6 +206,8 @@
 struct op_x86_model_spec const op_ppro_spec = {
 	.num_counters		= 2,
 	.num_controls		= 2,
+	.num_virt_counters	= 2,
+	.num_virt_controls	= 2,
 	.reserved		= MSR_PPRO_EVENTSEL_RESERVED,
 	.fill_in_addresses	= &ppro_fill_in_addresses,
 	.setup_ctrs		= &ppro_setup_ctrs,
@@ -23,6 +23,7 @@
 struct op_msrs {
 	struct op_msr *counters;
 	struct op_msr *controls;
+	struct op_msr *multiplex;
 };
  
 struct pt_regs;
@@ -35,6 +36,8 @@
 struct op_x86_model_spec {
 	unsigned int	num_counters;
 	unsigned int	num_controls;
+	unsigned int	num_virt_counters;
+	unsigned int	num_virt_controls;
 	u64		reserved;
 	u16		event_mask;
 	int		(*init)(struct oprofile_operations *ops);
@@ -47,6 +50,10 @@
 	void		(*start)(struct op_msrs const * const msrs);
 	void		(*stop)(struct op_msrs const * const msrs);
 	void		(*shutdown)(struct op_msrs const * const msrs);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	void		(*switch_ctrl)(struct op_x86_model_spec const *model,
+				       struct op_msrs const * const msrs);
+#endif
 };
  
 struct op_counter_config;
@@ -12,6 +12,8 @@
 #include <linux/init.h>
 #include <linux/oprofile.h>
 #include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/time.h>
 #include <asm/mutex.h>
  
 #include "oprof.h"
@@ -27,6 +29,15 @@
 static unsigned long is_setup;
 static DEFINE_MUTEX(start_mutex);
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void switch_worker(struct work_struct *work);
+static DECLARE_DELAYED_WORK(switch_work, switch_worker);
+unsigned long timeout_jiffies;
+#define MULTIPLEXING_TIMER_DEFAULT 1
+
+#endif
+
 /* timer
    0 - use performance monitoring hardware if available
    1 - use the timer int mechanism regardless
  
@@ -87,7 +98,21 @@
 	return err;
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  
+static void start_switch_worker(void)
+{
+	schedule_delayed_work(&switch_work, timeout_jiffies);
+}
+
+static void switch_worker(struct work_struct *work)
+{
+	if (!oprofile_ops.switch_events())
+		start_switch_worker();
+}
+
+#endif
+
 /* Actually start profiling (echo 1>/dev/oprofile/enable) */
 int oprofile_start(void)
 {
@@ -108,6 +133,11 @@
 	if ((err = oprofile_ops.start()))
 		goto out;
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	if (oprofile_ops.switch_events)
+		start_switch_worker();
+#endif
+
 	oprofile_started = 1;
 out:
 	mutex_unlock(&start_mutex);
@@ -123,6 +153,11 @@
 		goto out;
 	oprofile_ops.stop();
 	oprofile_started = 0;
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	cancel_delayed_work_sync(&switch_work);
+#endif
+
 	/* wake up the daemon to read what remains */
 	wake_up_buffer_waiter();
 out:
  
@@ -155,7 +190,37 @@
 	mutex_unlock(&start_mutex);
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  
+/* User inputs in ms, converts to jiffies */
+int oprofile_set_timeout(unsigned long val_msec)
+{
+	int err = 0;
+
+	mutex_lock(&start_mutex);
+
+	if (oprofile_started) {
+		err = -EBUSY;
+		goto out;
+	}
+
+	if (!oprofile_ops.switch_events) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	timeout_jiffies = msecs_to_jiffies(val_msec);
+	if (timeout_jiffies == MAX_JIFFY_OFFSET)
+		timeout_jiffies = msecs_to_jiffies(MULTIPLEXING_TIMER_DEFAULT);
+
+out:
+	mutex_unlock(&start_mutex);
+	return err;
+
+}
+
+#endif
+
 int oprofile_set_backtrace(unsigned long val)
 {
 	int err = 0;
  
@@ -179,9 +244,22 @@
 	return err;
 }
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static void __init oprofile_multiplexing_init(void)
+{
+	timeout_jiffies = msecs_to_jiffies(MULTIPLEXING_TIMER_DEFAULT);
+}
+
+#endif
+
 static int __init oprofile_init(void)
 {
 	int err;
+
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	oprofile_multiplexing_init();
+#endif
  
 	err = oprofile_arch_init(&oprofile_ops);
  
@@ -27,6 +27,7 @@
 extern struct oprofile_operations oprofile_ops;
 extern unsigned long oprofile_started;
 extern unsigned long oprofile_backtrace_depth;
+extern unsigned long timeout_jiffies;
  
 struct super_block;
 struct dentry;
@@ -35,6 +36,7 @@
 void oprofile_timer_init(struct oprofile_operations *ops);
  
 int oprofile_set_backtrace(unsigned long depth);
+int oprofile_set_timeout(unsigned long time);
  
 #endif /* OPROF_H */
@@ -9,6 +9,7 @@
  
 #include <linux/fs.h>
 #include <linux/oprofile.h>
+#include <linux/jiffies.h>
  
 #include "event_buffer.h"
 #include "oprofile_stats.h"
@@ -22,6 +23,45 @@
 unsigned long oprofile_cpu_buffer_size;
 unsigned long oprofile_buffer_watershed;
  
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+
+static ssize_t timeout_read(struct file *file, char __user *buf,
+		size_t count, loff_t *offset)
+{
+	return oprofilefs_ulong_to_user(jiffies_to_msecs(timeout_jiffies),
+				buf, count, offset);
+}
+
+
+static ssize_t timeout_write(struct file *file, char const __user *buf,
+		size_t count, loff_t *offset)
+{
+	unsigned long val;
+	int retval;
+
+	if (*offset)
+		return -EINVAL;
+
+	retval = oprofilefs_ulong_from_user(&val, buf, count);
+	if (retval)
+		return retval;
+
+	retval = oprofile_set_timeout(val);
+
+	if (retval)
+		return retval;
+	return count;
+}
+
+
+static const struct file_operations timeout_fops = {
+	.read		= timeout_read,
+	.write		= timeout_write,
+};
+
+#endif
+
+
 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
 {
 	return oprofilefs_ulong_to_user(oprofile_backtrace_depth, buf, count,
@@ -139,6 +179,9 @@
 	oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops);
 	oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops);
 	oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	oprofilefs_create_file(sb, root, "time_slice", &timeout_fops);
+#endif
 	oprofile_create_stats_files(sb, root);
 	if (oprofile_ops.create_files)
 		oprofile_ops.create_files(sb, root);
@@ -16,6 +16,9 @@
 #include "cpu_buffer.h"
  
 struct oprofile_stat_struct oprofile_stats;
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+atomic_t multiplex_counter;
+#endif
  
 void oprofile_reset_stats(void)
 {
@@ -34,6 +37,9 @@
 	atomic_set(&oprofile_stats.sample_lost_no_mapping, 0);
 	atomic_set(&oprofile_stats.event_lost_overflow, 0);
 	atomic_set(&oprofile_stats.bt_lost_no_mapping, 0);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	atomic_set(&multiplex_counter, 0);
+#endif
 }
  
  
@@ -76,5 +82,9 @@
 		&oprofile_stats.event_lost_overflow);
 	oprofilefs_create_ro_atomic(sb, dir, "bt_lost_no_mapping",
 		&oprofile_stats.bt_lost_no_mapping);
+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
+	oprofilefs_create_ro_atomic(sb, dir, "multiplex_counter",
+		&multiplex_counter);
+#endif
 }
@@ -67,6 +67,9 @@
  
 	/* Initiate a stack backtrace. Optional. */
 	void (*backtrace)(struct pt_regs * const regs, unsigned int depth);
+
+	/* Multiplex between different events. Optional. */
+	int (*switch_events)(void);
 	/* CPU identification string. */
 	char * cpu_type;
 };
...	...	@@ -30,6 +30,18 @@
30	30
31	31	If unsure, say N.
32	32
	33	+config OPROFILE_EVENT_MULTIPLEX
	34	+ bool "OProfile multiplexing support (EXPERIMENTAL)"
	35	+ default n
	36	+ depends on OPROFILE && X86
	37	+ help
	38	+ The number of hardware counters is limited. The multiplexing
	39	+ feature enables OProfile to gather more events than counters
	40	+ are provided by the hardware. This is realized by switching
	41	+ between events at an user specified time interval.
	42	+
	43	+ If unsure, say N.
	44	+
33	45	config HAVE_OPROFILE
34	46	bool
35	47
1	1	/**
2	2	* @file nmi_int.c
3	3	*
4		- * @remark Copyright 2002-2008 OProfile authors
	4	+ * @remark Copyright 2002-2009 OProfile authors
5	5	* @remark Read the file COPYING
6	6	*
7	7	* @author John Levon <levon@movementarian.org>
8	8	* @author Robert Richter <robert.richter@amd.com>
	9	+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
	10	+ * @author Jason Yeh <jason.yeh@amd.com>
	11	+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
9	12	*/
10	13
11	14	#include <linux/init.h>
...	...	@@ -24,6 +27,12 @@
24	27	#include "op_counter.h"
25	28	#include "op_x86_model.h"
26	29
	30	+
	31	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	32	+DEFINE_PER_CPU(int, switch_index);
	33	+#endif
	34	+
	35	+
27	36	static struct op_x86_model_spec const *model;
28	37	static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
29	38	static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
...	...	@@ -31,6 +40,13 @@
31	40	/* 0 == registered but off, 1 == registered and on */
32	41	static int nmi_enabled = 0;
33	42
	43	+
	44	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	45	+extern atomic_t multiplex_counter;
	46	+#endif
	47	+
	48	+struct op_counter_config counter_config[OP_MAX_COUNTER];
	49	+
34	50	/* common functions */
35	51
36	52	u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
...	...	@@ -95,6 +111,11 @@
95	111	per_cpu(cpu_msrs, i).counters = NULL;
96	112	kfree(per_cpu(cpu_msrs, i).controls);
97	113	per_cpu(cpu_msrs, i).controls = NULL;
	114	+
	115	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	116	+ kfree(per_cpu(cpu_msrs, i).multiplex);
	117	+ per_cpu(cpu_msrs, i).multiplex = NULL;
	118	+#endif
98	119	}
99	120	}
100	121
...	...	@@ -103,6 +124,9 @@
103	124	int success = 1;
104	125	size_t controls_size = sizeof(struct op_msr) * model->num_controls;
105	126	size_t counters_size = sizeof(struct op_msr) * model->num_counters;
	127	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	128	+ size_t multiplex_size = sizeof(struct op_msr) * model->num_virt_counters;
	129	+#endif
106	130
107	131	int i;
108	132	for_each_possible_cpu(i) {
...	...	@@ -118,6 +142,14 @@
118	142	success = 0;
119	143	break;
120	144	}
	145	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	146	+ per_cpu(cpu_msrs, i).multiplex =
	147	+ kmalloc(multiplex_size, GFP_KERNEL);
	148	+ if (!per_cpu(cpu_msrs, i).multiplex) {
	149	+ success = 0;
	150	+ break;
	151	+ }
	152	+#endif
121	153	}
122	154
123	155	if (!success)
...	...	@@ -126,6 +158,25 @@
126	158	return success;
127	159	}
128	160
	161	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	162	+
	163	+static void nmi_setup_cpu_mux(struct op_msrs const * const msrs)
	164	+{
	165	+ int i;
	166	+ struct op_msr *multiplex = msrs->multiplex;
	167	+
	168	+ for (i = 0; i < model->num_virt_counters; ++i) {
	169	+ if (counter_config[i].enabled) {
	170	+ multiplex[i].saved = -(u64)counter_config[i].count;
	171	+ } else {
	172	+ multiplex[i].addr = 0;
	173	+ multiplex[i].saved = 0;
	174	+ }
	175	+ }
	176	+}
	177	+
	178	+#endif
	179	+
129	180	static void nmi_cpu_setup(void *dummy)
130	181	{
131	182	int cpu = smp_processor_id();
...	...	@@ -133,6 +184,9 @@
133	184	nmi_cpu_save_registers(msrs);
134	185	spin_lock(&oprofilefs_lock);
135	186	model->setup_ctrs(model, msrs);
	187	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	188	+ nmi_setup_cpu_mux(msrs);
	189	+#endif
136	190	spin_unlock(&oprofilefs_lock);
137	191	per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
138	192	apic_write(APIC_LVTPC, APIC_DM_NMI);
139	193
140	194
...	...	@@ -173,14 +227,52 @@
173	227	memcpy(per_cpu(cpu_msrs, cpu).controls,
174	228	per_cpu(cpu_msrs, 0).controls,
175	229	sizeof(struct op_msr) * model->num_controls);
	230	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	231	+ memcpy(per_cpu(cpu_msrs, cpu).multiplex,
	232	+ per_cpu(cpu_msrs, 0).multiplex,
	233	+ sizeof(struct op_msr) * model->num_virt_counters);
	234	+#endif
176	235	}
177		-
178	236	}
179	237	on_each_cpu(nmi_cpu_setup, NULL, 1);
180	238	nmi_enabled = 1;
181	239	return 0;
182	240	}
183	241
	242	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	243	+
	244	+static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
	245	+{
	246	+ unsigned int si = __get_cpu_var(switch_index);
	247	+ struct op_msr *multiplex = msrs->multiplex;
	248	+ unsigned int i;
	249	+
	250	+ for (i = 0; i < model->num_counters; ++i) {
	251	+ int offset = i + si;
	252	+ if (multiplex[offset].addr) {
	253	+ rdmsrl(multiplex[offset].addr,
	254	+ multiplex[offset].saved);
	255	+ }
	256	+ }
	257	+}
	258	+
	259	+static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
	260	+{
	261	+ unsigned int si = __get_cpu_var(switch_index);
	262	+ struct op_msr *multiplex = msrs->multiplex;
	263	+ unsigned int i;
	264	+
	265	+ for (i = 0; i < model->num_counters; ++i) {
	266	+ int offset = i + si;
	267	+ if (multiplex[offset].addr) {
	268	+ wrmsrl(multiplex[offset].addr,
	269	+ multiplex[offset].saved);
	270	+ }
	271	+ }
	272	+}
	273	+
	274	+#endif
	275	+
184	276	static void nmi_cpu_restore_registers(struct op_msrs *msrs)
185	277	{
186	278	struct op_msr *counters = msrs->counters;
...	...	@@ -214,6 +306,9 @@
214	306	apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
215	307	apic_write(APIC_LVTERR, v);
216	308	nmi_cpu_restore_registers(msrs);
	309	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	310	+ __get_cpu_var(switch_index) = 0;
	311	+#endif
217	312	}
218	313
219	314	static void nmi_shutdown(void)
220	315
221	316
...	...	@@ -252,16 +347,15 @@
252	347	on_each_cpu(nmi_cpu_stop, NULL, 1);
253	348	}
254	349
255		-struct op_counter_config counter_config[OP_MAX_COUNTER];
256		-
257	350	static int nmi_create_files(struct super_block sb, struct dentry root)
258	351	{
259	352	unsigned int i;
260	353
261		- for (i = 0; i < model->num_counters; ++i) {
	354	+ for (i = 0; i < model->num_virt_counters; ++i) {
262	355	struct dentry *dir;
263	356	char buf[4];
264	357
	358	+#ifndef CONFIG_OPROFILE_EVENT_MULTIPLEX
265	359	/* quick little hack to _not_ expose a counter if it is not
266	360	* available for use. This should protect userspace app.
267	361	* NOTE: assumes 1:1 mapping here (that counters are organized
...	...	@@ -269,6 +363,7 @@
269	363	*/
270	364	if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
271	365	continue;
	366	+#endif /* CONFIG_OPROFILE_EVENT_MULTIPLEX */
272	367
273	368	snprintf(buf, sizeof(buf), "%d", i);
274	369	dir = oprofilefs_mkdir(sb, root, buf);
...	...	@@ -283,6 +378,57 @@
283	378	return 0;
284	379	}
285	380
	381	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	382	+
	383	+static void nmi_cpu_switch(void *dummy)
	384	+{
	385	+ int cpu = smp_processor_id();
	386	+ int si = per_cpu(switch_index, cpu);
	387	+ struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
	388	+
	389	+ nmi_cpu_stop(NULL);
	390	+ nmi_cpu_save_mpx_registers(msrs);
	391	+
	392	+ /* move to next set */
	393	+ si += model->num_counters;
	394	+ if ((si > model->num_virt_counters) \|\| (counter_config[si].count == 0))
	395	+ per_cpu(switch_index, cpu) = 0;
	396	+ else
	397	+ per_cpu(switch_index, cpu) = si;
	398	+
	399	+ model->switch_ctrl(model, msrs);
	400	+ nmi_cpu_restore_mpx_registers(msrs);
	401	+
	402	+ nmi_cpu_start(NULL);
	403	+}
	404	+
	405	+
	406	+/*
	407	+ * Quick check to see if multiplexing is necessary.
	408	+ * The check should be sufficient since counters are used
	409	+ * in ordre.
	410	+ */
	411	+static int nmi_multiplex_on(void)
	412	+{
	413	+ return counter_config[model->num_counters].count ? 0 : -EINVAL;
	414	+}
	415	+
	416	+static int nmi_switch_event(void)
	417	+{
	418	+ if (!model->switch_ctrl)
	419	+ return -ENOSYS; /* not implemented */
	420	+ if (nmi_multiplex_on() < 0)
	421	+ return -EINVAL; /* not necessary */
	422	+
	423	+ on_each_cpu(nmi_cpu_switch, NULL, 1);
	424	+
	425	+ atomic_inc(&multiplex_counter);
	426	+
	427	+ return 0;
	428	+}
	429	+
	430	+#endif
	431	+
286	432	#ifdef CONFIG_SMP
287	433	static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
288	434	void *data)
289	435
...	...	@@ -516,12 +662,18 @@
516	662	register_cpu_notifier(&oprofile_cpu_nb);
517	663	#endif
518	664	/* default values, can be overwritten by model */
	665	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	666	+ __raw_get_cpu_var(switch_index) = 0;
	667	+#endif
519	668	ops->create_files = nmi_create_files;
520	669	ops->setup = nmi_setup;
521	670	ops->shutdown = nmi_shutdown;
522	671	ops->start = nmi_start;
523	672	ops->stop = nmi_stop;
524	673	ops->cpu_type = cpu_type;
	674	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	675	+ ops->switch_events = nmi_switch_event;
	676	+#endif
525	677
526	678	if (model->init)
527	679	ret = model->init(ops);
...	...	@@ -10,7 +10,7 @@
10	10	#ifndef OP_COUNTER_H
11	11	#define OP_COUNTER_H
12	12
13		-#define OP_MAX_COUNTER 8
	13	+#define OP_MAX_COUNTER 32
14	14
15	15	/* Per-perfctr configuration as set via
16	16	* oprofilefs.
...	...	@@ -9,12 +9,15 @@
9	9	* @author Philippe Elie
10	10	* @author Graydon Hoare
11	11	* @author Robert Richter <robert.richter@amd.com>
12		- * @author Barry Kasindorf
	12	+ * @author Barry Kasindorf <barry.kasindorf@amd.com>
	13	+ * @author Jason Yeh <jason.yeh@amd.com>
	14	+ * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
13	15	*/
14	16
15	17	#include <linux/oprofile.h>
16	18	#include <linux/device.h>
17	19	#include <linux/pci.h>
	20	+#include <linux/percpu.h>
18	21
19	22	#include <asm/ptrace.h>
20	23	#include <asm/msr.h>
21	24
...	...	@@ -25,12 +28,23 @@
25	28
26	29	#define NUM_COUNTERS 4
27	30	#define NUM_CONTROLS 4
	31	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	32	+#define NUM_VIRT_COUNTERS 32
	33	+#define NUM_VIRT_CONTROLS 32
	34	+#else
	35	+#define NUM_VIRT_COUNTERS NUM_COUNTERS
	36	+#define NUM_VIRT_CONTROLS NUM_CONTROLS
	37	+#endif
	38	+
28	39	#define OP_EVENT_MASK 0x0FFF
29	40	#define OP_CTR_OVERFLOW (1ULL<<31)
30	41
31	42	#define MSR_AMD_EVENTSEL_RESERVED ((0xFFFFFCF0ULL<<32)\|(1ULL<<21))
32	43
33		-static unsigned long reset_value[NUM_COUNTERS];
	44	+static unsigned long reset_value[NUM_VIRT_COUNTERS];
	45	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	46	+DECLARE_PER_CPU(int, switch_index);
	47	+#endif
34	48
35	49	#ifdef CONFIG_OPROFILE_IBS
36	50
...	...	@@ -82,6 +96,16 @@
82	96	else
83	97	msrs->controls[i].addr = 0;
84	98	}
	99	+
	100	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	101	+ for (i = 0; i < NUM_VIRT_COUNTERS; i++) {
	102	+ int hw_counter = i % NUM_CONTROLS;
	103	+ if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
	104	+ msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter;
	105	+ else
	106	+ msrs->multiplex[i].addr = 0;
	107	+ }
	108	+#endif
85	109	}
86	110
87	111	static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
...	...	@@ -90,6 +114,15 @@
90	114	u64 val;
91	115	int i;
92	116
	117	+ /* setup reset_value */
	118	+ for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
	119	+ if (counter_config[i].enabled) {
	120	+ reset_value[i] = counter_config[i].count;
	121	+ } else {
	122	+ reset_value[i] = 0;
	123	+ }
	124	+ }
	125	+
93	126	/* clear all counters */
94	127	for (i = 0; i < NUM_CONTROLS; ++i) {
95	128	if (unlikely(!msrs->controls[i].addr))
96	129
97	130
98	131
...	...	@@ -108,20 +141,49 @@
108	141
109	142	/* enable active counters */
110	143	for (i = 0; i < NUM_COUNTERS; ++i) {
111		- if (counter_config[i].enabled && msrs->counters[i].addr) {
112		- reset_value[i] = counter_config[i].count;
113		- wrmsrl(msrs->counters[i].addr,
114		- -(u64)counter_config[i].count);
	144	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	145	+ int offset = i + __get_cpu_var(switch_index);
	146	+#else
	147	+ int offset = i;
	148	+#endif
	149	+ if (counter_config[offset].enabled && msrs->counters[i].addr) {
	150	+ /* setup counter registers */
	151	+ wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
	152	+
	153	+ /* setup control registers */
115	154	rdmsrl(msrs->controls[i].addr, val);
116	155	val &= model->reserved;
117		- val \|= op_x86_get_ctrl(model, &counter_config[i]);
	156	+ val \|= op_x86_get_ctrl(model, &counter_config[offset]);
118	157	wrmsrl(msrs->controls[i].addr, val);
119		- } else {
120		- reset_value[i] = 0;
121	158	}
122	159	}
123	160	}
124	161
	162	+
	163	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	164	+
	165	+static void op_amd_switch_ctrl(struct op_x86_model_spec const *model,
	166	+ struct op_msrs const * const msrs)
	167	+{
	168	+ u64 val;
	169	+ int i;
	170	+
	171	+ /* enable active counters */
	172	+ for (i = 0; i < NUM_COUNTERS; ++i) {
	173	+ int offset = i + __get_cpu_var(switch_index);
	174	+ if (counter_config[offset].enabled) {
	175	+ /* setup control registers */
	176	+ rdmsrl(msrs->controls[i].addr, val);
	177	+ val &= model->reserved;
	178	+ val \|= op_x86_get_ctrl(model, &counter_config[offset]);
	179	+ wrmsrl(msrs->controls[i].addr, val);
	180	+ }
	181	+ }
	182	+}
	183	+
	184	+#endif
	185	+
	186	+
125	187	#ifdef CONFIG_OPROFILE_IBS
126	188
127	189	static inline int
128	190
...	...	@@ -230,14 +292,19 @@
230	292	int i;
231	293
232	294	for (i = 0; i < NUM_COUNTERS; ++i) {
233		- if (!reset_value[i])
	295	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	296	+ int offset = i + __get_cpu_var(switch_index);
	297	+#else
	298	+ int offset = i;
	299	+#endif
	300	+ if (!reset_value[offset])
234	301	continue;
235	302	rdmsrl(msrs->counters[i].addr, val);
236	303	/* bit is clear if overflowed: */
237	304	if (val & OP_CTR_OVERFLOW)
238	305	continue;
239		- oprofile_add_sample(regs, i);
240		- wrmsrl(msrs->counters[i].addr, -(u64)reset_value[i]);
	306	+ oprofile_add_sample(regs, offset);
	307	+ wrmsrl(msrs->counters[i].addr, -(u64)reset_value[offset]);
241	308	}
242	309
243	310	op_amd_handle_ibs(regs, msrs);
244	311
...	...	@@ -250,8 +317,14 @@
250	317	{
251	318	u64 val;
252	319	int i;
	320	+
253	321	for (i = 0; i < NUM_COUNTERS; ++i) {
254		- if (reset_value[i]) {
	322	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	323	+ int offset = i + __get_cpu_var(switch_index);
	324	+#else
	325	+ int offset = i;
	326	+#endif
	327	+ if (reset_value[offset]) {
255	328	rdmsrl(msrs->controls[i].addr, val);
256	329	val \|= ARCH_PERFMON_EVENTSEL0_ENABLE;
257	330	wrmsrl(msrs->controls[i].addr, val);
258	331
...	...	@@ -271,7 +344,11 @@
271	344	* pm callback
272	345	*/
273	346	for (i = 0; i < NUM_COUNTERS; ++i) {
	347	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	348	+ if (!reset_value[i + per_cpu(switch_index, smp_processor_id())])
	349	+#else
274	350	if (!reset_value[i])
	351	+#endif
275	352	continue;
276	353	rdmsrl(msrs->controls[i].addr, val);
277	354	val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
...	...	@@ -289,7 +366,7 @@
289	366	if (msrs->counters[i].addr)
290	367	release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
291	368	}
292		- for (i = 0; i < NUM_CONTROLS; ++i) {
	369	+ for (i = 0; i < NUM_COUNTERS; ++i) {
293	370	if (msrs->controls[i].addr)
294	371	release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
295	372	}
...	...	@@ -463,6 +540,8 @@
463	540	struct op_x86_model_spec const op_amd_spec = {
464	541	.num_counters = NUM_COUNTERS,
465	542	.num_controls = NUM_CONTROLS,
	543	+ .num_virt_counters = NUM_VIRT_COUNTERS,
	544	+ .num_virt_controls = NUM_VIRT_CONTROLS,
466	545	.reserved = MSR_AMD_EVENTSEL_RESERVED,
467	546	.event_mask = OP_EVENT_MASK,
468	547	.init = op_amd_init,
...	...	@@ -473,5 +552,8 @@
473	552	.start = &op_amd_start,
474	553	.stop = &op_amd_stop,
475	554	.shutdown = &op_amd_shutdown,
	555	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	556	+ .switch_ctrl = &op_amd_switch_ctrl,
	557	+#endif
476	558	};
...	...	@@ -698,6 +698,8 @@
698	698	struct op_x86_model_spec const op_p4_ht2_spec = {
699	699	.num_counters = NUM_COUNTERS_HT2,
700	700	.num_controls = NUM_CONTROLS_HT2,
	701	+ .num_virt_counters = NUM_COUNTERS_HT2,
	702	+ .num_virt_controls = NUM_CONTROLS_HT2,
701	703	.fill_in_addresses = &p4_fill_in_addresses,
702	704	.setup_ctrs = &p4_setup_ctrs,
703	705	.check_ctrs = &p4_check_ctrs,
...	...	@@ -710,6 +712,8 @@
710	712	struct op_x86_model_spec const op_p4_spec = {
711	713	.num_counters = NUM_COUNTERS_NON_HT,
712	714	.num_controls = NUM_CONTROLS_NON_HT,
	715	+ .num_virt_counters = NUM_COUNTERS_NON_HT,
	716	+ .num_virt_controls = NUM_CONTROLS_NON_HT,
713	717	.fill_in_addresses = &p4_fill_in_addresses,
714	718	.setup_ctrs = &p4_setup_ctrs,
715	719	.check_ctrs = &p4_check_ctrs,
...	...	@@ -206,6 +206,8 @@
206	206	struct op_x86_model_spec const op_ppro_spec = {
207	207	.num_counters = 2,
208	208	.num_controls = 2,
	209	+ .num_virt_counters = 2,
	210	+ .num_virt_controls = 2,
209	211	.reserved = MSR_PPRO_EVENTSEL_RESERVED,
210	212	.fill_in_addresses = &ppro_fill_in_addresses,
211	213	.setup_ctrs = &ppro_setup_ctrs,
...	...	@@ -23,6 +23,7 @@
23	23	struct op_msrs {
24	24	struct op_msr *counters;
25	25	struct op_msr *controls;
	26	+ struct op_msr *multiplex;
26	27	};
27	28
28	29	struct pt_regs;
...	...	@@ -35,6 +36,8 @@
35	36	struct op_x86_model_spec {
36	37	unsigned int num_counters;
37	38	unsigned int num_controls;
	39	+ unsigned int num_virt_counters;
	40	+ unsigned int num_virt_controls;
38	41	u64 reserved;
39	42	u16 event_mask;
40	43	int (init)(struct oprofile_operations ops);
...	...	@@ -47,6 +50,10 @@
47	50	void (start)(struct op_msrs const const msrs);
48	51	void (stop)(struct op_msrs const const msrs);
49	52	void (shutdown)(struct op_msrs const const msrs);
	53	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	54	+ void (switch_ctrl)(struct op_x86_model_spec const model,
	55	+ struct op_msrs const * const msrs);
	56	+#endif
50	57	};
51	58
52	59	struct op_counter_config;
...	...	@@ -12,6 +12,8 @@
12	12	#include <linux/init.h>
13	13	#include <linux/oprofile.h>
14	14	#include <linux/moduleparam.h>
	15	+#include <linux/workqueue.h>
	16	+#include <linux/time.h>
15	17	#include <asm/mutex.h>
16	18
17	19	#include "oprof.h"
...	...	@@ -27,6 +29,15 @@
27	29	static unsigned long is_setup;
28	30	static DEFINE_MUTEX(start_mutex);
29	31
	32	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	33	+
	34	+static void switch_worker(struct work_struct *work);
	35	+static DECLARE_DELAYED_WORK(switch_work, switch_worker);
	36	+unsigned long timeout_jiffies;
	37	+#define MULTIPLEXING_TIMER_DEFAULT 1
	38	+
	39	+#endif
	40	+
30	41	/* timer
31	42	0 - use performance monitoring hardware if available
32	43	1 - use the timer int mechanism regardless
33	44
...	...	@@ -87,7 +98,21 @@
87	98	return err;
88	99	}
89	100
	101	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
90	102
	103	+static void start_switch_worker(void)
	104	+{
	105	+ schedule_delayed_work(&switch_work, timeout_jiffies);
	106	+}
	107	+
	108	+static void switch_worker(struct work_struct *work)
	109	+{
	110	+ if (!oprofile_ops.switch_events())
	111	+ start_switch_worker();
	112	+}
	113	+
	114	+#endif
	115	+
91	116	/* Actually start profiling (echo 1>/dev/oprofile/enable) */
92	117	int oprofile_start(void)
93	118	{
...	...	@@ -108,6 +133,11 @@
108	133	if ((err = oprofile_ops.start()))
109	134	goto out;
110	135
	136	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	137	+ if (oprofile_ops.switch_events)
	138	+ start_switch_worker();
	139	+#endif
	140	+
111	141	oprofile_started = 1;
112	142	out:
113	143	mutex_unlock(&start_mutex);
...	...	@@ -123,6 +153,11 @@
123	153	goto out;
124	154	oprofile_ops.stop();
125	155	oprofile_started = 0;
	156	+
	157	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	158	+ cancel_delayed_work_sync(&switch_work);
	159	+#endif
	160	+
126	161	/* wake up the daemon to read what remains */
127	162	wake_up_buffer_waiter();
128	163	out:
129	164
...	...	@@ -155,7 +190,37 @@
155	190	mutex_unlock(&start_mutex);
156	191	}
157	192
	193	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
158	194
	195	+/* User inputs in ms, converts to jiffies */
	196	+int oprofile_set_timeout(unsigned long val_msec)
	197	+{
	198	+ int err = 0;
	199	+
	200	+ mutex_lock(&start_mutex);
	201	+
	202	+ if (oprofile_started) {
	203	+ err = -EBUSY;
	204	+ goto out;
	205	+ }
	206	+
	207	+ if (!oprofile_ops.switch_events) {
	208	+ err = -EINVAL;
	209	+ goto out;
	210	+ }
	211	+
	212	+ timeout_jiffies = msecs_to_jiffies(val_msec);
	213	+ if (timeout_jiffies == MAX_JIFFY_OFFSET)
	214	+ timeout_jiffies = msecs_to_jiffies(MULTIPLEXING_TIMER_DEFAULT);
	215	+
	216	+out:
	217	+ mutex_unlock(&start_mutex);
	218	+ return err;
	219	+
	220	+}
	221	+
	222	+#endif
	223	+
159	224	int oprofile_set_backtrace(unsigned long val)
160	225	{
161	226	int err = 0;
162	227
...	...	@@ -179,9 +244,22 @@
179	244	return err;
180	245	}
181	246
	247	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	248	+
	249	+static void __init oprofile_multiplexing_init(void)
	250	+{
	251	+ timeout_jiffies = msecs_to_jiffies(MULTIPLEXING_TIMER_DEFAULT);
	252	+}
	253	+
	254	+#endif
	255	+
182	256	static int __init oprofile_init(void)
183	257	{
184	258	int err;
	259	+
	260	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	261	+ oprofile_multiplexing_init();
	262	+#endif
185	263
186	264	err = oprofile_arch_init(&oprofile_ops);
187	265
...	...	@@ -27,6 +27,7 @@
27	27	extern struct oprofile_operations oprofile_ops;
28	28	extern unsigned long oprofile_started;
29	29	extern unsigned long oprofile_backtrace_depth;
	30	+extern unsigned long timeout_jiffies;
30	31
31	32	struct super_block;
32	33	struct dentry;
...	...	@@ -35,6 +36,7 @@
35	36	void oprofile_timer_init(struct oprofile_operations *ops);
36	37
37	38	int oprofile_set_backtrace(unsigned long depth);
	39	+int oprofile_set_timeout(unsigned long time);
38	40
39	41	#endif /* OPROF_H */
...	...	@@ -9,6 +9,7 @@
9	9
10	10	#include <linux/fs.h>
11	11	#include <linux/oprofile.h>
	12	+#include <linux/jiffies.h>
12	13
13	14	#include "event_buffer.h"
14	15	#include "oprofile_stats.h"
...	...	@@ -22,6 +23,45 @@
22	23	unsigned long oprofile_cpu_buffer_size;
23	24	unsigned long oprofile_buffer_watershed;
24	25
	26	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	27	+
	28	+static ssize_t timeout_read(struct file file, char __user buf,
	29	+ size_t count, loff_t *offset)
	30	+{
	31	+ return oprofilefs_ulong_to_user(jiffies_to_msecs(timeout_jiffies),
	32	+ buf, count, offset);
	33	+}
	34	+
	35	+
	36	+static ssize_t timeout_write(struct file file, char const __user buf,
	37	+ size_t count, loff_t *offset)
	38	+{
	39	+ unsigned long val;
	40	+ int retval;
	41	+
	42	+ if (*offset)
	43	+ return -EINVAL;
	44	+
	45	+ retval = oprofilefs_ulong_from_user(&val, buf, count);
	46	+ if (retval)
	47	+ return retval;
	48	+
	49	+ retval = oprofile_set_timeout(val);
	50	+
	51	+ if (retval)
	52	+ return retval;
	53	+ return count;
	54	+}
	55	+
	56	+
	57	+static const struct file_operations timeout_fops = {
	58	+ .read = timeout_read,
	59	+ .write = timeout_write,
	60	+};
	61	+
	62	+#endif
	63	+
	64	+
25	65	static ssize_t depth_read(struct file file, char __user buf, size_t count, loff_t *offset)
26	66	{
27	67	return oprofilefs_ulong_to_user(oprofile_backtrace_depth, buf, count,
...	...	@@ -139,6 +179,9 @@
139	179	oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops);
140	180	oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops);
141	181	oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops);
	182	+#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
	183	+ oprofilefs_create_file(sb, root, "time_slice", &timeout_fops);
	184	+#endif
142	185	oprofile_create_stats_files(sb, root);
143	186	if (oprofile_ops.create_files)
144	187	oprofile_ops.create_files(sb, root);