hw breakpoints: Move to kernel/events/

As part of the events sybsystem unification, relocate hw_breakpoint.c into its new destination. Cc: Frederic Weisbecker <fweisbec@gmail.com> Signed-off-by: Borislav Petkov <borislav.petkov@amd.com>

hw breakpoints: Move to kernel/events/
As part of the events sybsystem unification, relocate hw_breakpoint.c into its new destination. Cc: Frederic Weisbecker <fweisbec@gmail.com> Signed-off-by: Borislav Petkov <borislav.petkov@amd.com>
Borislav Petkov
1 parent fae85b7c8b
Showing 4 changed files with 661 additions and 661 deletions Side-by-side Diff
kernel/Makefile
kernel/events/Makefile
kernel/events/hw_breakpoint.c
kernel/hw_breakpoint.c
@@ -105,7 +105,6 @@
  
 obj-$(CONFIG_PERF_EVENTS) += events/
  
-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
 obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
 obj-$(CONFIG_PADATA) += padata.o
 obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
@@ -2,5 +2,6 @@
 CFLAGS_REMOVE_core.o = -pg
 endif
  
-obj-y += core.o
+obj-y := core.o
+obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2007 Alan Stern
+ * Copyright (C) IBM Corporation, 2009
+ * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
+ *
+ * Thanks to Ingo Molnar for his many suggestions.
+ *
+ * Authors: Alan Stern <stern@rowland.harvard.edu>
+ *          K.Prasad <prasad@linux.vnet.ibm.com>
+ *          Frederic Weisbecker <fweisbec@gmail.com>
+ */
+
+/*
+ * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
+ * using the CPU's debug registers.
+ * This file contains the arch-independent routines.
+ */
+
+#include <linux/irqflags.h>
+#include <linux/kallsyms.h>
+#include <linux/notifier.h>
+#include <linux/kprobes.h>
+#include <linux/kdebug.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/percpu.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/cpu.h>
+#include <linux/smp.h>
+
+#include <linux/hw_breakpoint.h>
+
+
+/*
+ * Constraints data
+ */
+
+/* Number of pinned cpu breakpoints in a cpu */
+static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned[TYPE_MAX]);
+
+/* Number of pinned task breakpoints in a cpu */
+static DEFINE_PER_CPU(unsigned int *, nr_task_bp_pinned[TYPE_MAX]);
+
+/* Number of non-pinned cpu/task breakpoints in a cpu */
+static DEFINE_PER_CPU(unsigned int, nr_bp_flexible[TYPE_MAX]);
+
+static int nr_slots[TYPE_MAX];
+
+/* Keep track of the breakpoints attached to tasks */
+static LIST_HEAD(bp_task_head);
+
+static int constraints_initialized;
+
+/* Gather the number of total pinned and un-pinned bp in a cpuset */
+struct bp_busy_slots {
+	unsigned int pinned;
+	unsigned int flexible;
+};
+
+/* Serialize accesses to the above constraints */
+static DEFINE_MUTEX(nr_bp_mutex);
+
+__weak int hw_breakpoint_weight(struct perf_event *bp)
+{
+	return 1;
+}
+
+static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)
+{
+	if (bp->attr.bp_type & HW_BREAKPOINT_RW)
+		return TYPE_DATA;
+
+	return TYPE_INST;
+}
+
+/*
+ * Report the maximum number of pinned breakpoints a task
+ * have in this cpu
+ */
+static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
+{
+	int i;
+	unsigned int *tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
+
+	for (i = nr_slots[type] - 1; i >= 0; i--) {
+		if (tsk_pinned[i] > 0)
+			return i + 1;
+	}
+
+	return 0;
+}
+
+/*
+ * Count the number of breakpoints of the same type and same task.
+ * The given event must be not on the list.
+ */
+static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
+{
+	struct task_struct *tsk = bp->hw.bp_target;
+	struct perf_event *iter;
+	int count = 0;
+
+	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
+		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
+			count += hw_breakpoint_weight(iter);
+	}
+
+	return count;
+}
+
+/*
+ * Report the number of pinned/un-pinned breakpoints we have in
+ * a given cpu (cpu > -1) or in all of them (cpu = -1).
+ */
+static void
+fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
+		    enum bp_type_idx type)
+{
+	int cpu = bp->cpu;
+	struct task_struct *tsk = bp->hw.bp_target;
+
+	if (cpu >= 0) {
+		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
+		if (!tsk)
+			slots->pinned += max_task_bp_pinned(cpu, type);
+		else
+			slots->pinned += task_bp_pinned(bp, type);
+		slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
+
+		return;
+	}
+
+	for_each_online_cpu(cpu) {
+		unsigned int nr;
+
+		nr = per_cpu(nr_cpu_bp_pinned[type], cpu);
+		if (!tsk)
+			nr += max_task_bp_pinned(cpu, type);
+		else
+			nr += task_bp_pinned(bp, type);
+
+		if (nr > slots->pinned)
+			slots->pinned = nr;
+
+		nr = per_cpu(nr_bp_flexible[type], cpu);
+
+		if (nr > slots->flexible)
+			slots->flexible = nr;
+	}
+}
+
+/*
+ * For now, continue to consider flexible as pinned, until we can
+ * ensure no flexible event can ever be scheduled before a pinned event
+ * in a same cpu.
+ */
+static void
+fetch_this_slot(struct bp_busy_slots *slots, int weight)
+{
+	slots->pinned += weight;
+}
+
+/*
+ * Add a pinned breakpoint for the given task in our constraint table
+ */
+static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
+				enum bp_type_idx type, int weight)
+{
+	unsigned int *tsk_pinned;
+	int old_count = 0;
+	int old_idx = 0;
+	int idx = 0;
+
+	old_count = task_bp_pinned(bp, type);
+	old_idx = old_count - 1;
+	idx = old_idx + weight;
+
+	/* tsk_pinned[n] is the number of tasks having n breakpoints */
+	tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
+	if (enable) {
+		tsk_pinned[idx]++;
+		if (old_count > 0)
+			tsk_pinned[old_idx]--;
+	} else {
+		tsk_pinned[idx]--;
+		if (old_count > 0)
+			tsk_pinned[old_idx]++;
+	}
+}
+
+/*
+ * Add/remove the given breakpoint in our constraint table
+ */
+static void
+toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
+	       int weight)
+{
+	int cpu = bp->cpu;
+	struct task_struct *tsk = bp->hw.bp_target;
+
+	/* Pinned counter cpu profiling */
+	if (!tsk) {
+
+		if (enable)
+			per_cpu(nr_cpu_bp_pinned[type], bp->cpu) += weight;
+		else
+			per_cpu(nr_cpu_bp_pinned[type], bp->cpu) -= weight;
+		return;
+	}
+
+	/* Pinned counter task profiling */
+
+	if (!enable)
+		list_del(&bp->hw.bp_list);
+
+	if (cpu >= 0) {
+		toggle_bp_task_slot(bp, cpu, enable, type, weight);
+	} else {
+		for_each_online_cpu(cpu)
+			toggle_bp_task_slot(bp, cpu, enable, type, weight);
+	}
+
+	if (enable)
+		list_add_tail(&bp->hw.bp_list, &bp_task_head);
+}
+
+/*
+ * Function to perform processor-specific cleanup during unregistration
+ */
+__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
+{
+	/*
+	 * A weak stub function here for those archs that don't define
+	 * it inside arch/.../kernel/hw_breakpoint.c
+	 */
+}
+
+/*
+ * Contraints to check before allowing this new breakpoint counter:
+ *
+ *  == Non-pinned counter == (Considered as pinned for now)
+ *
+ *   - If attached to a single cpu, check:
+ *
+ *       (per_cpu(nr_bp_flexible, cpu) || (per_cpu(nr_cpu_bp_pinned, cpu)
+ *           + max(per_cpu(nr_task_bp_pinned, cpu)))) < HBP_NUM
+ *
+ *       -> If there are already non-pinned counters in this cpu, it means
+ *          there is already a free slot for them.
+ *          Otherwise, we check that the maximum number of per task
+ *          breakpoints (for this cpu) plus the number of per cpu breakpoint
+ *          (for this cpu) doesn't cover every registers.
+ *
+ *   - If attached to every cpus, check:
+ *
+ *       (per_cpu(nr_bp_flexible, *) || (max(per_cpu(nr_cpu_bp_pinned, *))
+ *           + max(per_cpu(nr_task_bp_pinned, *)))) < HBP_NUM
+ *
+ *       -> This is roughly the same, except we check the number of per cpu
+ *          bp for every cpu and we keep the max one. Same for the per tasks
+ *          breakpoints.
+ *
+ *
+ * == Pinned counter ==
+ *
+ *   - If attached to a single cpu, check:
+ *
+ *       ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu)
+ *            + max(per_cpu(nr_task_bp_pinned, cpu))) < HBP_NUM
+ *
+ *       -> Same checks as before. But now the nr_bp_flexible, if any, must keep
+ *          one register at least (or they will never be fed).
+ *
+ *   - If attached to every cpus, check:
+ *
+ *       ((per_cpu(nr_bp_flexible, *) > 1) + max(per_cpu(nr_cpu_bp_pinned, *))
+ *            + max(per_cpu(nr_task_bp_pinned, *))) < HBP_NUM
+ */
+static int __reserve_bp_slot(struct perf_event *bp)
+{
+	struct bp_busy_slots slots = {0};
+	enum bp_type_idx type;
+	int weight;
+
+	/* We couldn't initialize breakpoint constraints on boot */
+	if (!constraints_initialized)
+		return -ENOMEM;
+
+	/* Basic checks */
+	if (bp->attr.bp_type == HW_BREAKPOINT_EMPTY ||
+	    bp->attr.bp_type == HW_BREAKPOINT_INVALID)
+		return -EINVAL;
+
+	type = find_slot_idx(bp);
+	weight = hw_breakpoint_weight(bp);
+
+	fetch_bp_busy_slots(&slots, bp, type);
+	/*
+	 * Simulate the addition of this breakpoint to the constraints
+	 * and see the result.
+	 */
+	fetch_this_slot(&slots, weight);
+
+	/* Flexible counters need to keep at least one slot */
+	if (slots.pinned + (!!slots.flexible) > nr_slots[type])
+		return -ENOSPC;
+
+	toggle_bp_slot(bp, true, type, weight);
+
+	return 0;
+}
+
+int reserve_bp_slot(struct perf_event *bp)
+{
+	int ret;
+
+	mutex_lock(&nr_bp_mutex);
+
+	ret = __reserve_bp_slot(bp);
+
+	mutex_unlock(&nr_bp_mutex);
+
+	return ret;
+}
+
+static void __release_bp_slot(struct perf_event *bp)
+{
+	enum bp_type_idx type;
+	int weight;
+
+	type = find_slot_idx(bp);
+	weight = hw_breakpoint_weight(bp);
+	toggle_bp_slot(bp, false, type, weight);
+}
+
+void release_bp_slot(struct perf_event *bp)
+{
+	mutex_lock(&nr_bp_mutex);
+
+	arch_unregister_hw_breakpoint(bp);
+	__release_bp_slot(bp);
+
+	mutex_unlock(&nr_bp_mutex);
+}
+
+/*
+ * Allow the kernel debugger to reserve breakpoint slots without
+ * taking a lock using the dbg_* variant of for the reserve and
+ * release breakpoint slots.
+ */
+int dbg_reserve_bp_slot(struct perf_event *bp)
+{
+	if (mutex_is_locked(&nr_bp_mutex))
+		return -1;
+
+	return __reserve_bp_slot(bp);
+}
+
+int dbg_release_bp_slot(struct perf_event *bp)
+{
+	if (mutex_is_locked(&nr_bp_mutex))
+		return -1;
+
+	__release_bp_slot(bp);
+
+	return 0;
+}
+
+static int validate_hw_breakpoint(struct perf_event *bp)
+{
+	int ret;
+
+	ret = arch_validate_hwbkpt_settings(bp);
+	if (ret)
+		return ret;
+
+	if (arch_check_bp_in_kernelspace(bp)) {
+		if (bp->attr.exclude_kernel)
+			return -EINVAL;
+		/*
+		 * Don't let unprivileged users set a breakpoint in the trap
+		 * path to avoid trap recursion attacks.
+		 */
+		if (!capable(CAP_SYS_ADMIN))
+			return -EPERM;
+	}
+
+	return 0;
+}
+
+int register_perf_hw_breakpoint(struct perf_event *bp)
+{
+	int ret;
+
+	ret = reserve_bp_slot(bp);
+	if (ret)
+		return ret;
+
+	ret = validate_hw_breakpoint(bp);
+
+	/* if arch_validate_hwbkpt_settings() fails then release bp slot */
+	if (ret)
+		release_bp_slot(bp);
+
+	return ret;
+}
+
+/**
+ * register_user_hw_breakpoint - register a hardware breakpoint for user space
+ * @attr: breakpoint attributes
+ * @triggered: callback to trigger when we hit the breakpoint
+ * @tsk: pointer to 'task_struct' of the process to which the address belongs
+ */
+struct perf_event *
+register_user_hw_breakpoint(struct perf_event_attr *attr,
+			    perf_overflow_handler_t triggered,
+			    struct task_struct *tsk)
+{
+	return perf_event_create_kernel_counter(attr, -1, tsk, triggered);
+}
+EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
+
+/**
+ * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
+ * @bp: the breakpoint structure to modify
+ * @attr: new breakpoint attributes
+ * @triggered: callback to trigger when we hit the breakpoint
+ * @tsk: pointer to 'task_struct' of the process to which the address belongs
+ */
+int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
+{
+	u64 old_addr = bp->attr.bp_addr;
+	u64 old_len = bp->attr.bp_len;
+	int old_type = bp->attr.bp_type;
+	int err = 0;
+
+	perf_event_disable(bp);
+
+	bp->attr.bp_addr = attr->bp_addr;
+	bp->attr.bp_type = attr->bp_type;
+	bp->attr.bp_len = attr->bp_len;
+
+	if (attr->disabled)
+		goto end;
+
+	err = validate_hw_breakpoint(bp);
+	if (!err)
+		perf_event_enable(bp);
+
+	if (err) {
+		bp->attr.bp_addr = old_addr;
+		bp->attr.bp_type = old_type;
+		bp->attr.bp_len = old_len;
+		if (!bp->attr.disabled)
+			perf_event_enable(bp);
+
+		return err;
+	}
+
+end:
+	bp->attr.disabled = attr->disabled;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
+
+/**
+ * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
+ * @bp: the breakpoint structure to unregister
+ */
+void unregister_hw_breakpoint(struct perf_event *bp)
+{
+	if (!bp)
+		return;
+	perf_event_release_kernel(bp);
+}
+EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
+
+/**
+ * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
+ * @attr: breakpoint attributes
+ * @triggered: callback to trigger when we hit the breakpoint
+ *
+ * @return a set of per_cpu pointers to perf events
+ */
+struct perf_event * __percpu *
+register_wide_hw_breakpoint(struct perf_event_attr *attr,
+			    perf_overflow_handler_t triggered)
+{
+	struct perf_event * __percpu *cpu_events, **pevent, *bp;
+	long err;
+	int cpu;
+
+	cpu_events = alloc_percpu(typeof(*cpu_events));
+	if (!cpu_events)
+		return (void __percpu __force *)ERR_PTR(-ENOMEM);
+
+	get_online_cpus();
+	for_each_online_cpu(cpu) {
+		pevent = per_cpu_ptr(cpu_events, cpu);
+		bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered);
+
+		*pevent = bp;
+
+		if (IS_ERR(bp)) {
+			err = PTR_ERR(bp);
+			goto fail;
+		}
+	}
+	put_online_cpus();
+
+	return cpu_events;
+
+fail:
+	for_each_online_cpu(cpu) {
+		pevent = per_cpu_ptr(cpu_events, cpu);
+		if (IS_ERR(*pevent))
+			break;
+		unregister_hw_breakpoint(*pevent);
+	}
+	put_online_cpus();
+
+	free_percpu(cpu_events);
+	return (void __percpu __force *)ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
+
+/**
+ * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
+ * @cpu_events: the per cpu set of events to unregister
+ */
+void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
+{
+	int cpu;
+	struct perf_event **pevent;
+
+	for_each_possible_cpu(cpu) {
+		pevent = per_cpu_ptr(cpu_events, cpu);
+		unregister_hw_breakpoint(*pevent);
+	}
+	free_percpu(cpu_events);
+}
+EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
+
+static struct notifier_block hw_breakpoint_exceptions_nb = {
+	.notifier_call = hw_breakpoint_exceptions_notify,
+	/* we need to be notified first */
+	.priority = 0x7fffffff
+};
+
+static void bp_perf_event_destroy(struct perf_event *event)
+{
+	release_bp_slot(event);
+}
+
+static int hw_breakpoint_event_init(struct perf_event *bp)
+{
+	int err;
+
+	if (bp->attr.type != PERF_TYPE_BREAKPOINT)
+		return -ENOENT;
+
+	err = register_perf_hw_breakpoint(bp);
+	if (err)
+		return err;
+
+	bp->destroy = bp_perf_event_destroy;
+
+	return 0;
+}
+
+static int hw_breakpoint_add(struct perf_event *bp, int flags)
+{
+	if (!(flags & PERF_EF_START))
+		bp->hw.state = PERF_HES_STOPPED;
+
+	return arch_install_hw_breakpoint(bp);
+}
+
+static void hw_breakpoint_del(struct perf_event *bp, int flags)
+{
+	arch_uninstall_hw_breakpoint(bp);
+}
+
+static void hw_breakpoint_start(struct perf_event *bp, int flags)
+{
+	bp->hw.state = 0;
+}
+
+static void hw_breakpoint_stop(struct perf_event *bp, int flags)
+{
+	bp->hw.state = PERF_HES_STOPPED;
+}
+
+static struct pmu perf_breakpoint = {
+	.task_ctx_nr	= perf_sw_context, /* could eventually get its own */
+
+	.event_init	= hw_breakpoint_event_init,
+	.add		= hw_breakpoint_add,
+	.del		= hw_breakpoint_del,
+	.start		= hw_breakpoint_start,
+	.stop		= hw_breakpoint_stop,
+	.read		= hw_breakpoint_pmu_read,
+};
+
+int __init init_hw_breakpoint(void)
+{
+	unsigned int **task_bp_pinned;
+	int cpu, err_cpu;
+	int i;
+
+	for (i = 0; i < TYPE_MAX; i++)
+		nr_slots[i] = hw_breakpoint_slots(i);
+
+	for_each_possible_cpu(cpu) {
+		for (i = 0; i < TYPE_MAX; i++) {
+			task_bp_pinned = &per_cpu(nr_task_bp_pinned[i], cpu);
+			*task_bp_pinned = kzalloc(sizeof(int) * nr_slots[i],
+						  GFP_KERNEL);
+			if (!*task_bp_pinned)
+				goto err_alloc;
+		}
+	}
+
+	constraints_initialized = 1;
+
+	perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
+
+	return register_die_notifier(&hw_breakpoint_exceptions_nb);
+
+ err_alloc:
+	for_each_possible_cpu(err_cpu) {
+		if (err_cpu == cpu)
+			break;
+		for (i = 0; i < TYPE_MAX; i++)
+			kfree(per_cpu(nr_task_bp_pinned[i], cpu));
+	}
+
+	return -ENOMEM;
+}
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright (C) 2007 Alan Stern
- * Copyright (C) IBM Corporation, 2009
- * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
- *
- * Thanks to Ingo Molnar for his many suggestions.
- *
- * Authors: Alan Stern <stern@rowland.harvard.edu>
- *          K.Prasad <prasad@linux.vnet.ibm.com>
- *          Frederic Weisbecker <fweisbec@gmail.com>
- */
-
-/*
- * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
- * using the CPU's debug registers.
- * This file contains the arch-independent routines.
- */
-
-#include <linux/irqflags.h>
-#include <linux/kallsyms.h>
-#include <linux/notifier.h>
-#include <linux/kprobes.h>
-#include <linux/kdebug.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/percpu.h>
-#include <linux/sched.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/list.h>
-#include <linux/cpu.h>
-#include <linux/smp.h>
-
-#include <linux/hw_breakpoint.h>
-
-
-/*
- * Constraints data
- */
-
-/* Number of pinned cpu breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned[TYPE_MAX]);
-
-/* Number of pinned task breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int *, nr_task_bp_pinned[TYPE_MAX]);
-
-/* Number of non-pinned cpu/task breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int, nr_bp_flexible[TYPE_MAX]);
-
-static int nr_slots[TYPE_MAX];
-
-/* Keep track of the breakpoints attached to tasks */
-static LIST_HEAD(bp_task_head);
-
-static int constraints_initialized;
-
-/* Gather the number of total pinned and un-pinned bp in a cpuset */
-struct bp_busy_slots {
-	unsigned int pinned;
-	unsigned int flexible;
-};
-
-/* Serialize accesses to the above constraints */
-static DEFINE_MUTEX(nr_bp_mutex);
-
-__weak int hw_breakpoint_weight(struct perf_event *bp)
-{
-	return 1;
-}
-
-static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)
-{
-	if (bp->attr.bp_type & HW_BREAKPOINT_RW)
-		return TYPE_DATA;
-
-	return TYPE_INST;
-}
-
-/*
- * Report the maximum number of pinned breakpoints a task
- * have in this cpu
- */
-static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
-{
-	int i;
-	unsigned int *tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
-
-	for (i = nr_slots[type] - 1; i >= 0; i--) {
-		if (tsk_pinned[i] > 0)
-			return i + 1;
-	}
-
-	return 0;
-}
-
-/*
- * Count the number of breakpoints of the same type and same task.
- * The given event must be not on the list.
- */
-static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
-{
-	struct task_struct *tsk = bp->hw.bp_target;
-	struct perf_event *iter;
-	int count = 0;
-
-	list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
-		if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
-			count += hw_breakpoint_weight(iter);
-	}
-
-	return count;
-}
-
-/*
- * Report the number of pinned/un-pinned breakpoints we have in
- * a given cpu (cpu > -1) or in all of them (cpu = -1).
- */
-static void
-fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
-		    enum bp_type_idx type)
-{
-	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->hw.bp_target;
-
-	if (cpu >= 0) {
-		slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
-		if (!tsk)
-			slots->pinned += max_task_bp_pinned(cpu, type);
-		else
-			slots->pinned += task_bp_pinned(bp, type);
-		slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
-
-		return;
-	}
-
-	for_each_online_cpu(cpu) {
-		unsigned int nr;
-
-		nr = per_cpu(nr_cpu_bp_pinned[type], cpu);
-		if (!tsk)
-			nr += max_task_bp_pinned(cpu, type);
-		else
-			nr += task_bp_pinned(bp, type);
-
-		if (nr > slots->pinned)
-			slots->pinned = nr;
-
-		nr = per_cpu(nr_bp_flexible[type], cpu);
-
-		if (nr > slots->flexible)
-			slots->flexible = nr;
-	}
-}
-
-/*
- * For now, continue to consider flexible as pinned, until we can
- * ensure no flexible event can ever be scheduled before a pinned event
- * in a same cpu.
- */
-static void
-fetch_this_slot(struct bp_busy_slots *slots, int weight)
-{
-	slots->pinned += weight;
-}
-
-/*
- * Add a pinned breakpoint for the given task in our constraint table
- */
-static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
-				enum bp_type_idx type, int weight)
-{
-	unsigned int *tsk_pinned;
-	int old_count = 0;
-	int old_idx = 0;
-	int idx = 0;
-
-	old_count = task_bp_pinned(bp, type);
-	old_idx = old_count - 1;
-	idx = old_idx + weight;
-
-	/* tsk_pinned[n] is the number of tasks having n breakpoints */
-	tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
-	if (enable) {
-		tsk_pinned[idx]++;
-		if (old_count > 0)
-			tsk_pinned[old_idx]--;
-	} else {
-		tsk_pinned[idx]--;
-		if (old_count > 0)
-			tsk_pinned[old_idx]++;
-	}
-}
-
-/*
- * Add/remove the given breakpoint in our constraint table
- */
-static void
-toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
-	       int weight)
-{
-	int cpu = bp->cpu;
-	struct task_struct *tsk = bp->hw.bp_target;
-
-	/* Pinned counter cpu profiling */
-	if (!tsk) {
-
-		if (enable)
-			per_cpu(nr_cpu_bp_pinned[type], bp->cpu) += weight;
-		else
-			per_cpu(nr_cpu_bp_pinned[type], bp->cpu) -= weight;
-		return;
-	}
-
-	/* Pinned counter task profiling */
-
-	if (!enable)
-		list_del(&bp->hw.bp_list);
-
-	if (cpu >= 0) {
-		toggle_bp_task_slot(bp, cpu, enable, type, weight);
-	} else {
-		for_each_online_cpu(cpu)
-			toggle_bp_task_slot(bp, cpu, enable, type, weight);
-	}
-
-	if (enable)
-		list_add_tail(&bp->hw.bp_list, &bp_task_head);
-}
-
-/*
- * Function to perform processor-specific cleanup during unregistration
- */
-__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
-{
-	/*
-	 * A weak stub function here for those archs that don't define
-	 * it inside arch/.../kernel/hw_breakpoint.c
-	 */
-}
-
-/*
- * Contraints to check before allowing this new breakpoint counter:
- *
- *  == Non-pinned counter == (Considered as pinned for now)
- *
- *   - If attached to a single cpu, check:
- *
- *       (per_cpu(nr_bp_flexible, cpu) || (per_cpu(nr_cpu_bp_pinned, cpu)
- *           + max(per_cpu(nr_task_bp_pinned, cpu)))) < HBP_NUM
- *
- *       -> If there are already non-pinned counters in this cpu, it means
- *          there is already a free slot for them.
- *          Otherwise, we check that the maximum number of per task
- *          breakpoints (for this cpu) plus the number of per cpu breakpoint
- *          (for this cpu) doesn't cover every registers.
- *
- *   - If attached to every cpus, check:
- *
- *       (per_cpu(nr_bp_flexible, *) || (max(per_cpu(nr_cpu_bp_pinned, *))
- *           + max(per_cpu(nr_task_bp_pinned, *)))) < HBP_NUM
- *
- *       -> This is roughly the same, except we check the number of per cpu
- *          bp for every cpu and we keep the max one. Same for the per tasks
- *          breakpoints.
- *
- *
- * == Pinned counter ==
- *
- *   - If attached to a single cpu, check:
- *
- *       ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu)
- *            + max(per_cpu(nr_task_bp_pinned, cpu))) < HBP_NUM
- *
- *       -> Same checks as before. But now the nr_bp_flexible, if any, must keep
- *          one register at least (or they will never be fed).
- *
- *   - If attached to every cpus, check:
- *
- *       ((per_cpu(nr_bp_flexible, *) > 1) + max(per_cpu(nr_cpu_bp_pinned, *))
- *            + max(per_cpu(nr_task_bp_pinned, *))) < HBP_NUM
- */
-static int __reserve_bp_slot(struct perf_event *bp)
-{
-	struct bp_busy_slots slots = {0};
-	enum bp_type_idx type;
-	int weight;
-
-	/* We couldn't initialize breakpoint constraints on boot */
-	if (!constraints_initialized)
-		return -ENOMEM;
-
-	/* Basic checks */
-	if (bp->attr.bp_type == HW_BREAKPOINT_EMPTY ||
-	    bp->attr.bp_type == HW_BREAKPOINT_INVALID)
-		return -EINVAL;
-
-	type = find_slot_idx(bp);
-	weight = hw_breakpoint_weight(bp);
-
-	fetch_bp_busy_slots(&slots, bp, type);
-	/*
-	 * Simulate the addition of this breakpoint to the constraints
-	 * and see the result.
-	 */
-	fetch_this_slot(&slots, weight);
-
-	/* Flexible counters need to keep at least one slot */
-	if (slots.pinned + (!!slots.flexible) > nr_slots[type])
-		return -ENOSPC;
-
-	toggle_bp_slot(bp, true, type, weight);
-
-	return 0;
-}
-
-int reserve_bp_slot(struct perf_event *bp)
-{
-	int ret;
-
-	mutex_lock(&nr_bp_mutex);
-
-	ret = __reserve_bp_slot(bp);
-
-	mutex_unlock(&nr_bp_mutex);
-
-	return ret;
-}
-
-static void __release_bp_slot(struct perf_event *bp)
-{
-	enum bp_type_idx type;
-	int weight;
-
-	type = find_slot_idx(bp);
-	weight = hw_breakpoint_weight(bp);
-	toggle_bp_slot(bp, false, type, weight);
-}
-
-void release_bp_slot(struct perf_event *bp)
-{
-	mutex_lock(&nr_bp_mutex);
-
-	arch_unregister_hw_breakpoint(bp);
-	__release_bp_slot(bp);
-
-	mutex_unlock(&nr_bp_mutex);
-}
-
-/*
- * Allow the kernel debugger to reserve breakpoint slots without
- * taking a lock using the dbg_* variant of for the reserve and
- * release breakpoint slots.
- */
-int dbg_reserve_bp_slot(struct perf_event *bp)
-{
-	if (mutex_is_locked(&nr_bp_mutex))
-		return -1;
-
-	return __reserve_bp_slot(bp);
-}
-
-int dbg_release_bp_slot(struct perf_event *bp)
-{
-	if (mutex_is_locked(&nr_bp_mutex))
-		return -1;
-
-	__release_bp_slot(bp);
-
-	return 0;
-}
-
-static int validate_hw_breakpoint(struct perf_event *bp)
-{
-	int ret;
-
-	ret = arch_validate_hwbkpt_settings(bp);
-	if (ret)
-		return ret;
-
-	if (arch_check_bp_in_kernelspace(bp)) {
-		if (bp->attr.exclude_kernel)
-			return -EINVAL;
-		/*
-		 * Don't let unprivileged users set a breakpoint in the trap
-		 * path to avoid trap recursion attacks.
-		 */
-		if (!capable(CAP_SYS_ADMIN))
-			return -EPERM;
-	}
-
-	return 0;
-}
-
-int register_perf_hw_breakpoint(struct perf_event *bp)
-{
-	int ret;
-
-	ret = reserve_bp_slot(bp);
-	if (ret)
-		return ret;
-
-	ret = validate_hw_breakpoint(bp);
-
-	/* if arch_validate_hwbkpt_settings() fails then release bp slot */
-	if (ret)
-		release_bp_slot(bp);
-
-	return ret;
-}
-
-/**
- * register_user_hw_breakpoint - register a hardware breakpoint for user space
- * @attr: breakpoint attributes
- * @triggered: callback to trigger when we hit the breakpoint
- * @tsk: pointer to 'task_struct' of the process to which the address belongs
- */
-struct perf_event *
-register_user_hw_breakpoint(struct perf_event_attr *attr,
-			    perf_overflow_handler_t triggered,
-			    struct task_struct *tsk)
-{
-	return perf_event_create_kernel_counter(attr, -1, tsk, triggered);
-}
-EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
-
-/**
- * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
- * @bp: the breakpoint structure to modify
- * @attr: new breakpoint attributes
- * @triggered: callback to trigger when we hit the breakpoint
- * @tsk: pointer to 'task_struct' of the process to which the address belongs
- */
-int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr)
-{
-	u64 old_addr = bp->attr.bp_addr;
-	u64 old_len = bp->attr.bp_len;
-	int old_type = bp->attr.bp_type;
-	int err = 0;
-
-	perf_event_disable(bp);
-
-	bp->attr.bp_addr = attr->bp_addr;
-	bp->attr.bp_type = attr->bp_type;
-	bp->attr.bp_len = attr->bp_len;
-
-	if (attr->disabled)
-		goto end;
-
-	err = validate_hw_breakpoint(bp);
-	if (!err)
-		perf_event_enable(bp);
-
-	if (err) {
-		bp->attr.bp_addr = old_addr;
-		bp->attr.bp_type = old_type;
-		bp->attr.bp_len = old_len;
-		if (!bp->attr.disabled)
-			perf_event_enable(bp);
-
-		return err;
-	}
-
-end:
-	bp->attr.disabled = attr->disabled;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
-
-/**
- * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
- * @bp: the breakpoint structure to unregister
- */
-void unregister_hw_breakpoint(struct perf_event *bp)
-{
-	if (!bp)
-		return;
-	perf_event_release_kernel(bp);
-}
-EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
-
-/**
- * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
- * @attr: breakpoint attributes
- * @triggered: callback to trigger when we hit the breakpoint
- *
- * @return a set of per_cpu pointers to perf events
- */
-struct perf_event * __percpu *
-register_wide_hw_breakpoint(struct perf_event_attr *attr,
-			    perf_overflow_handler_t triggered)
-{
-	struct perf_event * __percpu *cpu_events, **pevent, *bp;
-	long err;
-	int cpu;
-
-	cpu_events = alloc_percpu(typeof(*cpu_events));
-	if (!cpu_events)
-		return (void __percpu __force *)ERR_PTR(-ENOMEM);
-
-	get_online_cpus();
-	for_each_online_cpu(cpu) {
-		pevent = per_cpu_ptr(cpu_events, cpu);
-		bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered);
-
-		*pevent = bp;
-
-		if (IS_ERR(bp)) {
-			err = PTR_ERR(bp);
-			goto fail;
-		}
-	}
-	put_online_cpus();
-
-	return cpu_events;
-
-fail:
-	for_each_online_cpu(cpu) {
-		pevent = per_cpu_ptr(cpu_events, cpu);
-		if (IS_ERR(*pevent))
-			break;
-		unregister_hw_breakpoint(*pevent);
-	}
-	put_online_cpus();
-
-	free_percpu(cpu_events);
-	return (void __percpu __force *)ERR_PTR(err);
-}
-EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
-
-/**
- * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
- * @cpu_events: the per cpu set of events to unregister
- */
-void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
-{
-	int cpu;
-	struct perf_event **pevent;
-
-	for_each_possible_cpu(cpu) {
-		pevent = per_cpu_ptr(cpu_events, cpu);
-		unregister_hw_breakpoint(*pevent);
-	}
-	free_percpu(cpu_events);
-}
-EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
-
-static struct notifier_block hw_breakpoint_exceptions_nb = {
-	.notifier_call = hw_breakpoint_exceptions_notify,
-	/* we need to be notified first */
-	.priority = 0x7fffffff
-};
-
-static void bp_perf_event_destroy(struct perf_event *event)
-{
-	release_bp_slot(event);
-}
-
-static int hw_breakpoint_event_init(struct perf_event *bp)
-{
-	int err;
-
-	if (bp->attr.type != PERF_TYPE_BREAKPOINT)
-		return -ENOENT;
-
-	err = register_perf_hw_breakpoint(bp);
-	if (err)
-		return err;
-
-	bp->destroy = bp_perf_event_destroy;
-
-	return 0;
-}
-
-static int hw_breakpoint_add(struct perf_event *bp, int flags)
-{
-	if (!(flags & PERF_EF_START))
-		bp->hw.state = PERF_HES_STOPPED;
-
-	return arch_install_hw_breakpoint(bp);
-}
-
-static void hw_breakpoint_del(struct perf_event *bp, int flags)
-{
-	arch_uninstall_hw_breakpoint(bp);
-}
-
-static void hw_breakpoint_start(struct perf_event *bp, int flags)
-{
-	bp->hw.state = 0;
-}
-
-static void hw_breakpoint_stop(struct perf_event *bp, int flags)
-{
-	bp->hw.state = PERF_HES_STOPPED;
-}
-
-static struct pmu perf_breakpoint = {
-	.task_ctx_nr	= perf_sw_context, /* could eventually get its own */
-
-	.event_init	= hw_breakpoint_event_init,
-	.add		= hw_breakpoint_add,
-	.del		= hw_breakpoint_del,
-	.start		= hw_breakpoint_start,
-	.stop		= hw_breakpoint_stop,
-	.read		= hw_breakpoint_pmu_read,
-};
-
-int __init init_hw_breakpoint(void)
-{
-	unsigned int **task_bp_pinned;
-	int cpu, err_cpu;
-	int i;
-
-	for (i = 0; i < TYPE_MAX; i++)
-		nr_slots[i] = hw_breakpoint_slots(i);
-
-	for_each_possible_cpu(cpu) {
-		for (i = 0; i < TYPE_MAX; i++) {
-			task_bp_pinned = &per_cpu(nr_task_bp_pinned[i], cpu);
-			*task_bp_pinned = kzalloc(sizeof(int) * nr_slots[i],
-						  GFP_KERNEL);
-			if (!*task_bp_pinned)
-				goto err_alloc;
-		}
-	}
-
-	constraints_initialized = 1;
-
-	perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
-
-	return register_die_notifier(&hw_breakpoint_exceptions_nb);
-
- err_alloc:
-	for_each_possible_cpu(err_cpu) {
-		if (err_cpu == cpu)
-			break;
-		for (i = 0; i < TYPE_MAX; i++)
-			kfree(per_cpu(nr_task_bp_pinned[i], cpu));
-	}
-
-	return -ENOMEM;
-}
...	...	@@ -105,7 +105,6 @@
105	105
106	106	obj-$(CONFIG_PERF_EVENTS) += events/
107	107
108		-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
109	108	obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
110	109	obj-$(CONFIG_PADATA) += padata.o
111	110	obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
...	...	@@ -2,5 +2,6 @@
2	2	CFLAGS_REMOVE_core.o = -pg
3	3	endif
4	4
5		-obj-y += core.o
	5	+obj-y := core.o
	6	+obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
	1	+/*
	2	+ * This program is free software; you can redistribute it and/or modify
	3	+ * it under the terms of the GNU General Public License as published by
	4	+ * the Free Software Foundation; either version 2 of the License, or
	5	+ * (at your option) any later version.
	6	+ *
	7	+ * This program is distributed in the hope that it will be useful,
	8	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	+ * GNU General Public License for more details.
	11	+ *
	12	+ * You should have received a copy of the GNU General Public License
	13	+ * along with this program; if not, write to the Free Software
	14	+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	+ *
	16	+ * Copyright (C) 2007 Alan Stern
	17	+ * Copyright (C) IBM Corporation, 2009
	18	+ * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
	19	+ *
	20	+ * Thanks to Ingo Molnar for his many suggestions.
	21	+ *
	22	+ * Authors: Alan Stern <stern@rowland.harvard.edu>
	23	+ * K.Prasad <prasad@linux.vnet.ibm.com>
	24	+ * Frederic Weisbecker <fweisbec@gmail.com>
	25	+ */
	26	+
	27	+/*
	28	+ * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
	29	+ * using the CPU's debug registers.
	30	+ * This file contains the arch-independent routines.
	31	+ */
	32	+
	33	+#include <linux/irqflags.h>
	34	+#include <linux/kallsyms.h>
	35	+#include <linux/notifier.h>
	36	+#include <linux/kprobes.h>
	37	+#include <linux/kdebug.h>
	38	+#include <linux/kernel.h>
	39	+#include <linux/module.h>
	40	+#include <linux/percpu.h>
	41	+#include <linux/sched.h>
	42	+#include <linux/init.h>
	43	+#include <linux/slab.h>
	44	+#include <linux/list.h>
	45	+#include <linux/cpu.h>
	46	+#include <linux/smp.h>
	47	+
	48	+#include <linux/hw_breakpoint.h>
	49	+
	50	+
	51	+/*
	52	+ * Constraints data
	53	+ */
	54	+
	55	+/* Number of pinned cpu breakpoints in a cpu */
	56	+static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned[TYPE_MAX]);
	57	+
	58	+/* Number of pinned task breakpoints in a cpu */
	59	+static DEFINE_PER_CPU(unsigned int *, nr_task_bp_pinned[TYPE_MAX]);
	60	+
	61	+/* Number of non-pinned cpu/task breakpoints in a cpu */
	62	+static DEFINE_PER_CPU(unsigned int, nr_bp_flexible[TYPE_MAX]);
	63	+
	64	+static int nr_slots[TYPE_MAX];
	65	+
	66	+/* Keep track of the breakpoints attached to tasks */
	67	+static LIST_HEAD(bp_task_head);
	68	+
	69	+static int constraints_initialized;
	70	+
	71	+/* Gather the number of total pinned and un-pinned bp in a cpuset */
	72	+struct bp_busy_slots {
	73	+ unsigned int pinned;
	74	+ unsigned int flexible;
	75	+};
	76	+
	77	+/* Serialize accesses to the above constraints */
	78	+static DEFINE_MUTEX(nr_bp_mutex);
	79	+
	80	+__weak int hw_breakpoint_weight(struct perf_event *bp)
	81	+{
	82	+ return 1;
	83	+}
	84	+
	85	+static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)
	86	+{
	87	+ if (bp->attr.bp_type & HW_BREAKPOINT_RW)
	88	+ return TYPE_DATA;
	89	+
	90	+ return TYPE_INST;
	91	+}
	92	+
	93	+/*
	94	+ * Report the maximum number of pinned breakpoints a task
	95	+ * have in this cpu
	96	+ */
	97	+static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
	98	+{
	99	+ int i;
	100	+ unsigned int *tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
	101	+
	102	+ for (i = nr_slots[type] - 1; i >= 0; i--) {
	103	+ if (tsk_pinned[i] > 0)
	104	+ return i + 1;
	105	+ }
	106	+
	107	+ return 0;
	108	+}
	109	+
	110	+/*
	111	+ * Count the number of breakpoints of the same type and same task.
	112	+ * The given event must be not on the list.
	113	+ */
	114	+static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
	115	+{
	116	+ struct task_struct *tsk = bp->hw.bp_target;
	117	+ struct perf_event *iter;
	118	+ int count = 0;
	119	+
	120	+ list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
	121	+ if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
	122	+ count += hw_breakpoint_weight(iter);
	123	+ }
	124	+
	125	+ return count;
	126	+}
	127	+
	128	+/*
	129	+ * Report the number of pinned/un-pinned breakpoints we have in
	130	+ * a given cpu (cpu > -1) or in all of them (cpu = -1).
	131	+ */
	132	+static void
	133	+fetch_bp_busy_slots(struct bp_busy_slots slots, struct perf_event bp,
	134	+ enum bp_type_idx type)
	135	+{
	136	+ int cpu = bp->cpu;
	137	+ struct task_struct *tsk = bp->hw.bp_target;
	138	+
	139	+ if (cpu >= 0) {
	140	+ slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
	141	+ if (!tsk)
	142	+ slots->pinned += max_task_bp_pinned(cpu, type);
	143	+ else
	144	+ slots->pinned += task_bp_pinned(bp, type);
	145	+ slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
	146	+
	147	+ return;
	148	+ }
	149	+
	150	+ for_each_online_cpu(cpu) {
	151	+ unsigned int nr;
	152	+
	153	+ nr = per_cpu(nr_cpu_bp_pinned[type], cpu);
	154	+ if (!tsk)
	155	+ nr += max_task_bp_pinned(cpu, type);
	156	+ else
	157	+ nr += task_bp_pinned(bp, type);
	158	+
	159	+ if (nr > slots->pinned)
	160	+ slots->pinned = nr;
	161	+
	162	+ nr = per_cpu(nr_bp_flexible[type], cpu);
	163	+
	164	+ if (nr > slots->flexible)
	165	+ slots->flexible = nr;
	166	+ }
	167	+}
	168	+
	169	+/*
	170	+ * For now, continue to consider flexible as pinned, until we can
	171	+ * ensure no flexible event can ever be scheduled before a pinned event
	172	+ * in a same cpu.
	173	+ */
	174	+static void
	175	+fetch_this_slot(struct bp_busy_slots *slots, int weight)
	176	+{
	177	+ slots->pinned += weight;
	178	+}
	179	+
	180	+/*
	181	+ * Add a pinned breakpoint for the given task in our constraint table
	182	+ */
	183	+static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
	184	+ enum bp_type_idx type, int weight)
	185	+{
	186	+ unsigned int *tsk_pinned;
	187	+ int old_count = 0;
	188	+ int old_idx = 0;
	189	+ int idx = 0;
	190	+
	191	+ old_count = task_bp_pinned(bp, type);
	192	+ old_idx = old_count - 1;
	193	+ idx = old_idx + weight;
	194	+
	195	+ /* tsk_pinned[n] is the number of tasks having n breakpoints */
	196	+ tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
	197	+ if (enable) {
	198	+ tsk_pinned[idx]++;
	199	+ if (old_count > 0)
	200	+ tsk_pinned[old_idx]--;
	201	+ } else {
	202	+ tsk_pinned[idx]--;
	203	+ if (old_count > 0)
	204	+ tsk_pinned[old_idx]++;
	205	+ }
	206	+}
	207	+
	208	+/*
	209	+ * Add/remove the given breakpoint in our constraint table
	210	+ */
	211	+static void
	212	+toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
	213	+ int weight)
	214	+{
	215	+ int cpu = bp->cpu;
	216	+ struct task_struct *tsk = bp->hw.bp_target;
	217	+
	218	+ /* Pinned counter cpu profiling */
	219	+ if (!tsk) {
	220	+
	221	+ if (enable)
	222	+ per_cpu(nr_cpu_bp_pinned[type], bp->cpu) += weight;
	223	+ else
	224	+ per_cpu(nr_cpu_bp_pinned[type], bp->cpu) -= weight;
	225	+ return;
	226	+ }
	227	+
	228	+ /* Pinned counter task profiling */
	229	+
	230	+ if (!enable)
	231	+ list_del(&bp->hw.bp_list);
	232	+
	233	+ if (cpu >= 0) {
	234	+ toggle_bp_task_slot(bp, cpu, enable, type, weight);
	235	+ } else {
	236	+ for_each_online_cpu(cpu)
	237	+ toggle_bp_task_slot(bp, cpu, enable, type, weight);
	238	+ }
	239	+
	240	+ if (enable)
	241	+ list_add_tail(&bp->hw.bp_list, &bp_task_head);
	242	+}
	243	+
	244	+/*
	245	+ * Function to perform processor-specific cleanup during unregistration
	246	+ */
	247	+__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
	248	+{
	249	+ /*
	250	+ * A weak stub function here for those archs that don't define
	251	+ * it inside arch/.../kernel/hw_breakpoint.c
	252	+ */
	253	+}
	254	+
	255	+/*
	256	+ * Contraints to check before allowing this new breakpoint counter:
	257	+ *
	258	+ * == Non-pinned counter == (Considered as pinned for now)
	259	+ *
	260	+ * - If attached to a single cpu, check:
	261	+ *
	262	+ * (per_cpu(nr_bp_flexible, cpu) \|\| (per_cpu(nr_cpu_bp_pinned, cpu)
	263	+ * + max(per_cpu(nr_task_bp_pinned, cpu)))) < HBP_NUM
	264	+ *
	265	+ * -> If there are already non-pinned counters in this cpu, it means
	266	+ * there is already a free slot for them.
	267	+ * Otherwise, we check that the maximum number of per task
	268	+ * breakpoints (for this cpu) plus the number of per cpu breakpoint
	269	+ * (for this cpu) doesn't cover every registers.
	270	+ *
	271	+ * - If attached to every cpus, check:
	272	+ *
	273	+ * (per_cpu(nr_bp_flexible, ) \|\| (max(per_cpu(nr_cpu_bp_pinned, ))
	274	+ * + max(per_cpu(nr_task_bp_pinned, *)))) < HBP_NUM
	275	+ *
	276	+ * -> This is roughly the same, except we check the number of per cpu
	277	+ * bp for every cpu and we keep the max one. Same for the per tasks
	278	+ * breakpoints.
	279	+ *
	280	+ *
	281	+ * == Pinned counter ==
	282	+ *
	283	+ * - If attached to a single cpu, check:
	284	+ *
	285	+ * ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu)
	286	+ * + max(per_cpu(nr_task_bp_pinned, cpu))) < HBP_NUM
	287	+ *
	288	+ * -> Same checks as before. But now the nr_bp_flexible, if any, must keep
	289	+ * one register at least (or they will never be fed).
	290	+ *
	291	+ * - If attached to every cpus, check:
	292	+ *
	293	+ * ((per_cpu(nr_bp_flexible, ) > 1) + max(per_cpu(nr_cpu_bp_pinned, ))
	294	+ * + max(per_cpu(nr_task_bp_pinned, *))) < HBP_NUM
	295	+ */
	296	+static int __reserve_bp_slot(struct perf_event *bp)
	297	+{
	298	+ struct bp_busy_slots slots = {0};
	299	+ enum bp_type_idx type;
	300	+ int weight;
	301	+
	302	+ /* We couldn't initialize breakpoint constraints on boot */
	303	+ if (!constraints_initialized)
	304	+ return -ENOMEM;
	305	+
	306	+ /* Basic checks */
	307	+ if (bp->attr.bp_type == HW_BREAKPOINT_EMPTY \|\|
	308	+ bp->attr.bp_type == HW_BREAKPOINT_INVALID)
	309	+ return -EINVAL;
	310	+
	311	+ type = find_slot_idx(bp);
	312	+ weight = hw_breakpoint_weight(bp);
	313	+
	314	+ fetch_bp_busy_slots(&slots, bp, type);
	315	+ /*
	316	+ * Simulate the addition of this breakpoint to the constraints
	317	+ * and see the result.
	318	+ */
	319	+ fetch_this_slot(&slots, weight);
	320	+
	321	+ /* Flexible counters need to keep at least one slot */
	322	+ if (slots.pinned + (!!slots.flexible) > nr_slots[type])
	323	+ return -ENOSPC;
	324	+
	325	+ toggle_bp_slot(bp, true, type, weight);
	326	+
	327	+ return 0;
	328	+}
	329	+
	330	+int reserve_bp_slot(struct perf_event *bp)
	331	+{
	332	+ int ret;
	333	+
	334	+ mutex_lock(&nr_bp_mutex);
	335	+
	336	+ ret = __reserve_bp_slot(bp);
	337	+
	338	+ mutex_unlock(&nr_bp_mutex);
	339	+
	340	+ return ret;
	341	+}
	342	+
	343	+static void __release_bp_slot(struct perf_event *bp)
	344	+{
	345	+ enum bp_type_idx type;
	346	+ int weight;
	347	+
	348	+ type = find_slot_idx(bp);
	349	+ weight = hw_breakpoint_weight(bp);
	350	+ toggle_bp_slot(bp, false, type, weight);
	351	+}
	352	+
	353	+void release_bp_slot(struct perf_event *bp)
	354	+{
	355	+ mutex_lock(&nr_bp_mutex);
	356	+
	357	+ arch_unregister_hw_breakpoint(bp);
	358	+ __release_bp_slot(bp);
	359	+
	360	+ mutex_unlock(&nr_bp_mutex);
	361	+}
	362	+
	363	+/*
	364	+ * Allow the kernel debugger to reserve breakpoint slots without
	365	+ * taking a lock using the dbg_* variant of for the reserve and
	366	+ * release breakpoint slots.
	367	+ */
	368	+int dbg_reserve_bp_slot(struct perf_event *bp)
	369	+{
	370	+ if (mutex_is_locked(&nr_bp_mutex))
	371	+ return -1;
	372	+
	373	+ return __reserve_bp_slot(bp);
	374	+}
	375	+
	376	+int dbg_release_bp_slot(struct perf_event *bp)
	377	+{
	378	+ if (mutex_is_locked(&nr_bp_mutex))
	379	+ return -1;
	380	+
	381	+ __release_bp_slot(bp);
	382	+
	383	+ return 0;
	384	+}
	385	+
	386	+static int validate_hw_breakpoint(struct perf_event *bp)
	387	+{
	388	+ int ret;
	389	+
	390	+ ret = arch_validate_hwbkpt_settings(bp);
	391	+ if (ret)
	392	+ return ret;
	393	+
	394	+ if (arch_check_bp_in_kernelspace(bp)) {
	395	+ if (bp->attr.exclude_kernel)
	396	+ return -EINVAL;
	397	+ /*
	398	+ * Don't let unprivileged users set a breakpoint in the trap
	399	+ * path to avoid trap recursion attacks.
	400	+ */
	401	+ if (!capable(CAP_SYS_ADMIN))
	402	+ return -EPERM;
	403	+ }
	404	+
	405	+ return 0;
	406	+}
	407	+
	408	+int register_perf_hw_breakpoint(struct perf_event *bp)
	409	+{
	410	+ int ret;
	411	+
	412	+ ret = reserve_bp_slot(bp);
	413	+ if (ret)
	414	+ return ret;
	415	+
	416	+ ret = validate_hw_breakpoint(bp);
	417	+
	418	+ /* if arch_validate_hwbkpt_settings() fails then release bp slot */
	419	+ if (ret)
	420	+ release_bp_slot(bp);
	421	+
	422	+ return ret;
	423	+}
	424	+
	425	+/**
	426	+ * register_user_hw_breakpoint - register a hardware breakpoint for user space
	427	+ * @attr: breakpoint attributes
	428	+ * @triggered: callback to trigger when we hit the breakpoint
	429	+ * @tsk: pointer to 'task_struct' of the process to which the address belongs
	430	+ */
	431	+struct perf_event *
	432	+register_user_hw_breakpoint(struct perf_event_attr *attr,
	433	+ perf_overflow_handler_t triggered,
	434	+ struct task_struct *tsk)
	435	+{
	436	+ return perf_event_create_kernel_counter(attr, -1, tsk, triggered);
	437	+}
	438	+EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
	439	+
	440	+/**
	441	+ * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
	442	+ * @bp: the breakpoint structure to modify
	443	+ * @attr: new breakpoint attributes
	444	+ * @triggered: callback to trigger when we hit the breakpoint
	445	+ * @tsk: pointer to 'task_struct' of the process to which the address belongs
	446	+ */
	447	+int modify_user_hw_breakpoint(struct perf_event bp, struct perf_event_attr attr)
	448	+{
	449	+ u64 old_addr = bp->attr.bp_addr;
	450	+ u64 old_len = bp->attr.bp_len;
	451	+ int old_type = bp->attr.bp_type;
	452	+ int err = 0;
	453	+
	454	+ perf_event_disable(bp);
	455	+
	456	+ bp->attr.bp_addr = attr->bp_addr;
	457	+ bp->attr.bp_type = attr->bp_type;
	458	+ bp->attr.bp_len = attr->bp_len;
	459	+
	460	+ if (attr->disabled)
	461	+ goto end;
	462	+
	463	+ err = validate_hw_breakpoint(bp);
	464	+ if (!err)
	465	+ perf_event_enable(bp);
	466	+
	467	+ if (err) {
	468	+ bp->attr.bp_addr = old_addr;
	469	+ bp->attr.bp_type = old_type;
	470	+ bp->attr.bp_len = old_len;
	471	+ if (!bp->attr.disabled)
	472	+ perf_event_enable(bp);
	473	+
	474	+ return err;
	475	+ }
	476	+
	477	+end:
	478	+ bp->attr.disabled = attr->disabled;
	479	+
	480	+ return 0;
	481	+}
	482	+EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
	483	+
	484	+/**
	485	+ * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
	486	+ * @bp: the breakpoint structure to unregister
	487	+ */
	488	+void unregister_hw_breakpoint(struct perf_event *bp)
	489	+{
	490	+ if (!bp)
	491	+ return;
	492	+ perf_event_release_kernel(bp);
	493	+}
	494	+EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
	495	+
	496	+/**
	497	+ * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
	498	+ * @attr: breakpoint attributes
	499	+ * @triggered: callback to trigger when we hit the breakpoint
	500	+ *
	501	+ * @return a set of per_cpu pointers to perf events
	502	+ */
	503	+struct perf_event * __percpu *
	504	+register_wide_hw_breakpoint(struct perf_event_attr *attr,
	505	+ perf_overflow_handler_t triggered)
	506	+{
	507	+ struct perf_event * __percpu cpu_events, pevent, bp;
	508	+ long err;
	509	+ int cpu;
	510	+
	511	+ cpu_events = alloc_percpu(typeof(*cpu_events));
	512	+ if (!cpu_events)
	513	+ return (void __percpu __force *)ERR_PTR(-ENOMEM);
	514	+
	515	+ get_online_cpus();
	516	+ for_each_online_cpu(cpu) {
	517	+ pevent = per_cpu_ptr(cpu_events, cpu);
	518	+ bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered);
	519	+
	520	+ *pevent = bp;
	521	+
	522	+ if (IS_ERR(bp)) {
	523	+ err = PTR_ERR(bp);
	524	+ goto fail;
	525	+ }
	526	+ }
	527	+ put_online_cpus();
	528	+
	529	+ return cpu_events;
	530	+
	531	+fail:
	532	+ for_each_online_cpu(cpu) {
	533	+ pevent = per_cpu_ptr(cpu_events, cpu);
	534	+ if (IS_ERR(*pevent))
	535	+ break;
	536	+ unregister_hw_breakpoint(*pevent);
	537	+ }
	538	+ put_online_cpus();
	539	+
	540	+ free_percpu(cpu_events);
	541	+ return (void __percpu __force *)ERR_PTR(err);
	542	+}
	543	+EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
	544	+
	545	+/**
	546	+ * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
	547	+ * @cpu_events: the per cpu set of events to unregister
	548	+ */
	549	+void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
	550	+{
	551	+ int cpu;
	552	+ struct perf_event **pevent;
	553	+
	554	+ for_each_possible_cpu(cpu) {
	555	+ pevent = per_cpu_ptr(cpu_events, cpu);
	556	+ unregister_hw_breakpoint(*pevent);
	557	+ }
	558	+ free_percpu(cpu_events);
	559	+}
	560	+EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
	561	+
	562	+static struct notifier_block hw_breakpoint_exceptions_nb = {
	563	+ .notifier_call = hw_breakpoint_exceptions_notify,
	564	+ /* we need to be notified first */
	565	+ .priority = 0x7fffffff
	566	+};
	567	+
	568	+static void bp_perf_event_destroy(struct perf_event *event)
	569	+{
	570	+ release_bp_slot(event);
	571	+}
	572	+
	573	+static int hw_breakpoint_event_init(struct perf_event *bp)
	574	+{
	575	+ int err;
	576	+
	577	+ if (bp->attr.type != PERF_TYPE_BREAKPOINT)
	578	+ return -ENOENT;
	579	+
	580	+ err = register_perf_hw_breakpoint(bp);
	581	+ if (err)
	582	+ return err;
	583	+
	584	+ bp->destroy = bp_perf_event_destroy;
	585	+
	586	+ return 0;
	587	+}
	588	+
	589	+static int hw_breakpoint_add(struct perf_event *bp, int flags)
	590	+{
	591	+ if (!(flags & PERF_EF_START))
	592	+ bp->hw.state = PERF_HES_STOPPED;
	593	+
	594	+ return arch_install_hw_breakpoint(bp);
	595	+}
	596	+
	597	+static void hw_breakpoint_del(struct perf_event *bp, int flags)
	598	+{
	599	+ arch_uninstall_hw_breakpoint(bp);
	600	+}
	601	+
	602	+static void hw_breakpoint_start(struct perf_event *bp, int flags)
	603	+{
	604	+ bp->hw.state = 0;
	605	+}
	606	+
	607	+static void hw_breakpoint_stop(struct perf_event *bp, int flags)
	608	+{
	609	+ bp->hw.state = PERF_HES_STOPPED;
	610	+}
	611	+
	612	+static struct pmu perf_breakpoint = {
	613	+ .task_ctx_nr = perf_sw_context, /* could eventually get its own */
	614	+
	615	+ .event_init = hw_breakpoint_event_init,
	616	+ .add = hw_breakpoint_add,
	617	+ .del = hw_breakpoint_del,
	618	+ .start = hw_breakpoint_start,
	619	+ .stop = hw_breakpoint_stop,
	620	+ .read = hw_breakpoint_pmu_read,
	621	+};
	622	+
	623	+int __init init_hw_breakpoint(void)
	624	+{
	625	+ unsigned int **task_bp_pinned;
	626	+ int cpu, err_cpu;
	627	+ int i;
	628	+
	629	+ for (i = 0; i < TYPE_MAX; i++)
	630	+ nr_slots[i] = hw_breakpoint_slots(i);
	631	+
	632	+ for_each_possible_cpu(cpu) {
	633	+ for (i = 0; i < TYPE_MAX; i++) {
	634	+ task_bp_pinned = &per_cpu(nr_task_bp_pinned[i], cpu);
	635	+ task_bp_pinned = kzalloc(sizeof(int) nr_slots[i],
	636	+ GFP_KERNEL);
	637	+ if (!*task_bp_pinned)
	638	+ goto err_alloc;
	639	+ }
	640	+ }
	641	+
	642	+ constraints_initialized = 1;
	643	+
	644	+ perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
	645	+
	646	+ return register_die_notifier(&hw_breakpoint_exceptions_nb);
	647	+
	648	+ err_alloc:
	649	+ for_each_possible_cpu(err_cpu) {
	650	+ if (err_cpu == cpu)
	651	+ break;
	652	+ for (i = 0; i < TYPE_MAX; i++)
	653	+ kfree(per_cpu(nr_task_bp_pinned[i], cpu));
	654	+ }
	655	+
	656	+ return -ENOMEM;
	657	+}
1		-/*
2		- * This program is free software; you can redistribute it and/or modify
3		- * it under the terms of the GNU General Public License as published by
4		- * the Free Software Foundation; either version 2 of the License, or
5		- * (at your option) any later version.
6		- *
7		- * This program is distributed in the hope that it will be useful,
8		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
9		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10		- * GNU General Public License for more details.
11		- *
12		- * You should have received a copy of the GNU General Public License
13		- * along with this program; if not, write to the Free Software
14		- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15		- *
16		- * Copyright (C) 2007 Alan Stern
17		- * Copyright (C) IBM Corporation, 2009
18		- * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com>
19		- *
20		- * Thanks to Ingo Molnar for his many suggestions.
21		- *
22		- * Authors: Alan Stern <stern@rowland.harvard.edu>
23		- * K.Prasad <prasad@linux.vnet.ibm.com>
24		- * Frederic Weisbecker <fweisbec@gmail.com>
25		- */
26		-
27		-/*
28		- * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
29		- * using the CPU's debug registers.
30		- * This file contains the arch-independent routines.
31		- */
32		-
33		-#include <linux/irqflags.h>
34		-#include <linux/kallsyms.h>
35		-#include <linux/notifier.h>
36		-#include <linux/kprobes.h>
37		-#include <linux/kdebug.h>
38		-#include <linux/kernel.h>
39		-#include <linux/module.h>
40		-#include <linux/percpu.h>
41		-#include <linux/sched.h>
42		-#include <linux/init.h>
43		-#include <linux/slab.h>
44		-#include <linux/list.h>
45		-#include <linux/cpu.h>
46		-#include <linux/smp.h>
47		-
48		-#include <linux/hw_breakpoint.h>
49		-
50		-
51		-/*
52		- * Constraints data
53		- */
54		-
55		-/* Number of pinned cpu breakpoints in a cpu */
56		-static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned[TYPE_MAX]);
57		-
58		-/* Number of pinned task breakpoints in a cpu */
59		-static DEFINE_PER_CPU(unsigned int *, nr_task_bp_pinned[TYPE_MAX]);
60		-
61		-/* Number of non-pinned cpu/task breakpoints in a cpu */
62		-static DEFINE_PER_CPU(unsigned int, nr_bp_flexible[TYPE_MAX]);
63		-
64		-static int nr_slots[TYPE_MAX];
65		-
66		-/* Keep track of the breakpoints attached to tasks */
67		-static LIST_HEAD(bp_task_head);
68		-
69		-static int constraints_initialized;
70		-
71		-/* Gather the number of total pinned and un-pinned bp in a cpuset */
72		-struct bp_busy_slots {
73		- unsigned int pinned;
74		- unsigned int flexible;
75		-};
76		-
77		-/* Serialize accesses to the above constraints */
78		-static DEFINE_MUTEX(nr_bp_mutex);
79		-
80		-__weak int hw_breakpoint_weight(struct perf_event *bp)
81		-{
82		- return 1;
83		-}
84		-
85		-static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)
86		-{
87		- if (bp->attr.bp_type & HW_BREAKPOINT_RW)
88		- return TYPE_DATA;
89		-
90		- return TYPE_INST;
91		-}
92		-
93		-/*
94		- * Report the maximum number of pinned breakpoints a task
95		- * have in this cpu
96		- */
97		-static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
98		-{
99		- int i;
100		- unsigned int *tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
101		-
102		- for (i = nr_slots[type] - 1; i >= 0; i--) {
103		- if (tsk_pinned[i] > 0)
104		- return i + 1;
105		- }
106		-
107		- return 0;
108		-}
109		-
110		-/*
111		- * Count the number of breakpoints of the same type and same task.
112		- * The given event must be not on the list.
113		- */
114		-static int task_bp_pinned(struct perf_event *bp, enum bp_type_idx type)
115		-{
116		- struct task_struct *tsk = bp->hw.bp_target;
117		- struct perf_event *iter;
118		- int count = 0;
119		-
120		- list_for_each_entry(iter, &bp_task_head, hw.bp_list) {
121		- if (iter->hw.bp_target == tsk && find_slot_idx(iter) == type)
122		- count += hw_breakpoint_weight(iter);
123		- }
124		-
125		- return count;
126		-}
127		-
128		-/*
129		- * Report the number of pinned/un-pinned breakpoints we have in
130		- * a given cpu (cpu > -1) or in all of them (cpu = -1).
131		- */
132		-static void
133		-fetch_bp_busy_slots(struct bp_busy_slots slots, struct perf_event bp,
134		- enum bp_type_idx type)
135		-{
136		- int cpu = bp->cpu;
137		- struct task_struct *tsk = bp->hw.bp_target;
138		-
139		- if (cpu >= 0) {
140		- slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
141		- if (!tsk)
142		- slots->pinned += max_task_bp_pinned(cpu, type);
143		- else
144		- slots->pinned += task_bp_pinned(bp, type);
145		- slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
146		-
147		- return;
148		- }
149		-
150		- for_each_online_cpu(cpu) {
151		- unsigned int nr;
152		-
153		- nr = per_cpu(nr_cpu_bp_pinned[type], cpu);
154		- if (!tsk)
155		- nr += max_task_bp_pinned(cpu, type);
156		- else
157		- nr += task_bp_pinned(bp, type);
158		-
159		- if (nr > slots->pinned)
160		- slots->pinned = nr;
161		-
162		- nr = per_cpu(nr_bp_flexible[type], cpu);
163		-
164		- if (nr > slots->flexible)
165		- slots->flexible = nr;
166		- }
167		-}
168		-
169		-/*
170		- * For now, continue to consider flexible as pinned, until we can
171		- * ensure no flexible event can ever be scheduled before a pinned event
172		- * in a same cpu.
173		- */
174		-static void
175		-fetch_this_slot(struct bp_busy_slots *slots, int weight)
176		-{
177		- slots->pinned += weight;
178		-}
179		-
180		-/*
181		- * Add a pinned breakpoint for the given task in our constraint table
182		- */
183		-static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
184		- enum bp_type_idx type, int weight)
185		-{
186		- unsigned int *tsk_pinned;
187		- int old_count = 0;
188		- int old_idx = 0;
189		- int idx = 0;
190		-
191		- old_count = task_bp_pinned(bp, type);
192		- old_idx = old_count - 1;
193		- idx = old_idx + weight;
194		-
195		- /* tsk_pinned[n] is the number of tasks having n breakpoints */
196		- tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
197		- if (enable) {
198		- tsk_pinned[idx]++;
199		- if (old_count > 0)
200		- tsk_pinned[old_idx]--;
201		- } else {
202		- tsk_pinned[idx]--;
203		- if (old_count > 0)
204		- tsk_pinned[old_idx]++;
205		- }
206		-}
207		-
208		-/*
209		- * Add/remove the given breakpoint in our constraint table
210		- */
211		-static void
212		-toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
213		- int weight)
214		-{
215		- int cpu = bp->cpu;
216		- struct task_struct *tsk = bp->hw.bp_target;
217		-
218		- /* Pinned counter cpu profiling */
219		- if (!tsk) {
220		-
221		- if (enable)
222		- per_cpu(nr_cpu_bp_pinned[type], bp->cpu) += weight;
223		- else
224		- per_cpu(nr_cpu_bp_pinned[type], bp->cpu) -= weight;
225		- return;
226		- }
227		-
228		- /* Pinned counter task profiling */
229		-
230		- if (!enable)
231		- list_del(&bp->hw.bp_list);
232		-
233		- if (cpu >= 0) {
234		- toggle_bp_task_slot(bp, cpu, enable, type, weight);
235		- } else {
236		- for_each_online_cpu(cpu)
237		- toggle_bp_task_slot(bp, cpu, enable, type, weight);
238		- }
239		-
240		- if (enable)
241		- list_add_tail(&bp->hw.bp_list, &bp_task_head);
242		-}
243		-
244		-/*
245		- * Function to perform processor-specific cleanup during unregistration
246		- */
247		-__weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
248		-{
249		- /*
250		- * A weak stub function here for those archs that don't define
251		- * it inside arch/.../kernel/hw_breakpoint.c
252		- */
253		-}
254		-
255		-/*
256		- * Contraints to check before allowing this new breakpoint counter:
257		- *
258		- * == Non-pinned counter == (Considered as pinned for now)
259		- *
260		- * - If attached to a single cpu, check:
261		- *
262		- * (per_cpu(nr_bp_flexible, cpu) \|\| (per_cpu(nr_cpu_bp_pinned, cpu)
263		- * + max(per_cpu(nr_task_bp_pinned, cpu)))) < HBP_NUM
264		- *
265		- * -> If there are already non-pinned counters in this cpu, it means
266		- * there is already a free slot for them.
267		- * Otherwise, we check that the maximum number of per task
268		- * breakpoints (for this cpu) plus the number of per cpu breakpoint
269		- * (for this cpu) doesn't cover every registers.
270		- *
271		- * - If attached to every cpus, check:
272		- *
273		- * (per_cpu(nr_bp_flexible, ) \|\| (max(per_cpu(nr_cpu_bp_pinned, ))
274		- * + max(per_cpu(nr_task_bp_pinned, *)))) < HBP_NUM
275		- *
276		- * -> This is roughly the same, except we check the number of per cpu
277		- * bp for every cpu and we keep the max one. Same for the per tasks
278		- * breakpoints.
279		- *
280		- *
281		- * == Pinned counter ==
282		- *
283		- * - If attached to a single cpu, check:
284		- *
285		- * ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu)
286		- * + max(per_cpu(nr_task_bp_pinned, cpu))) < HBP_NUM
287		- *
288		- * -> Same checks as before. But now the nr_bp_flexible, if any, must keep
289		- * one register at least (or they will never be fed).
290		- *
291		- * - If attached to every cpus, check:
292		- *
293		- * ((per_cpu(nr_bp_flexible, ) > 1) + max(per_cpu(nr_cpu_bp_pinned, ))
294		- * + max(per_cpu(nr_task_bp_pinned, *))) < HBP_NUM
295		- */
296		-static int __reserve_bp_slot(struct perf_event *bp)
297		-{
298		- struct bp_busy_slots slots = {0};
299		- enum bp_type_idx type;
300		- int weight;
301		-
302		- /* We couldn't initialize breakpoint constraints on boot */
303		- if (!constraints_initialized)
304		- return -ENOMEM;
305		-
306		- /* Basic checks */
307		- if (bp->attr.bp_type == HW_BREAKPOINT_EMPTY \|\|
308		- bp->attr.bp_type == HW_BREAKPOINT_INVALID)
309		- return -EINVAL;
310		-
311		- type = find_slot_idx(bp);
312		- weight = hw_breakpoint_weight(bp);
313		-
314		- fetch_bp_busy_slots(&slots, bp, type);
315		- /*
316		- * Simulate the addition of this breakpoint to the constraints
317		- * and see the result.
318		- */
319		- fetch_this_slot(&slots, weight);
320		-
321		- /* Flexible counters need to keep at least one slot */
322		- if (slots.pinned + (!!slots.flexible) > nr_slots[type])
323		- return -ENOSPC;
324		-
325		- toggle_bp_slot(bp, true, type, weight);
326		-
327		- return 0;
328		-}
329		-
330		-int reserve_bp_slot(struct perf_event *bp)
331		-{
332		- int ret;
333		-
334		- mutex_lock(&nr_bp_mutex);
335		-
336		- ret = __reserve_bp_slot(bp);
337		-
338		- mutex_unlock(&nr_bp_mutex);
339		-
340		- return ret;
341		-}
342		-
343		-static void __release_bp_slot(struct perf_event *bp)
344		-{
345		- enum bp_type_idx type;
346		- int weight;
347		-
348		- type = find_slot_idx(bp);
349		- weight = hw_breakpoint_weight(bp);
350		- toggle_bp_slot(bp, false, type, weight);
351		-}
352		-
353		-void release_bp_slot(struct perf_event *bp)
354		-{
355		- mutex_lock(&nr_bp_mutex);
356		-
357		- arch_unregister_hw_breakpoint(bp);
358		- __release_bp_slot(bp);
359		-
360		- mutex_unlock(&nr_bp_mutex);
361		-}
362		-
363		-/*
364		- * Allow the kernel debugger to reserve breakpoint slots without
365		- * taking a lock using the dbg_* variant of for the reserve and
366		- * release breakpoint slots.
367		- */
368		-int dbg_reserve_bp_slot(struct perf_event *bp)
369		-{
370		- if (mutex_is_locked(&nr_bp_mutex))
371		- return -1;
372		-
373		- return __reserve_bp_slot(bp);
374		-}
375		-
376		-int dbg_release_bp_slot(struct perf_event *bp)
377		-{
378		- if (mutex_is_locked(&nr_bp_mutex))
379		- return -1;
380		-
381		- __release_bp_slot(bp);
382		-
383		- return 0;
384		-}
385		-
386		-static int validate_hw_breakpoint(struct perf_event *bp)
387		-{
388		- int ret;
389		-
390		- ret = arch_validate_hwbkpt_settings(bp);
391		- if (ret)
392		- return ret;
393		-
394		- if (arch_check_bp_in_kernelspace(bp)) {
395		- if (bp->attr.exclude_kernel)
396		- return -EINVAL;
397		- /*
398		- * Don't let unprivileged users set a breakpoint in the trap
399		- * path to avoid trap recursion attacks.
400		- */
401		- if (!capable(CAP_SYS_ADMIN))
402		- return -EPERM;
403		- }
404		-
405		- return 0;
406		-}
407		-
408		-int register_perf_hw_breakpoint(struct perf_event *bp)
409		-{
410		- int ret;
411		-
412		- ret = reserve_bp_slot(bp);
413		- if (ret)
414		- return ret;
415		-
416		- ret = validate_hw_breakpoint(bp);
417		-
418		- /* if arch_validate_hwbkpt_settings() fails then release bp slot */
419		- if (ret)
420		- release_bp_slot(bp);
421		-
422		- return ret;
423		-}
424		-
425		-/**
426		- * register_user_hw_breakpoint - register a hardware breakpoint for user space
427		- * @attr: breakpoint attributes
428		- * @triggered: callback to trigger when we hit the breakpoint
429		- * @tsk: pointer to 'task_struct' of the process to which the address belongs
430		- */
431		-struct perf_event *
432		-register_user_hw_breakpoint(struct perf_event_attr *attr,
433		- perf_overflow_handler_t triggered,
434		- struct task_struct *tsk)
435		-{
436		- return perf_event_create_kernel_counter(attr, -1, tsk, triggered);
437		-}
438		-EXPORT_SYMBOL_GPL(register_user_hw_breakpoint);
439		-
440		-/**
441		- * modify_user_hw_breakpoint - modify a user-space hardware breakpoint
442		- * @bp: the breakpoint structure to modify
443		- * @attr: new breakpoint attributes
444		- * @triggered: callback to trigger when we hit the breakpoint
445		- * @tsk: pointer to 'task_struct' of the process to which the address belongs
446		- */
447		-int modify_user_hw_breakpoint(struct perf_event bp, struct perf_event_attr attr)
448		-{
449		- u64 old_addr = bp->attr.bp_addr;
450		- u64 old_len = bp->attr.bp_len;
451		- int old_type = bp->attr.bp_type;
452		- int err = 0;
453		-
454		- perf_event_disable(bp);
455		-
456		- bp->attr.bp_addr = attr->bp_addr;
457		- bp->attr.bp_type = attr->bp_type;
458		- bp->attr.bp_len = attr->bp_len;
459		-
460		- if (attr->disabled)
461		- goto end;
462		-
463		- err = validate_hw_breakpoint(bp);
464		- if (!err)
465		- perf_event_enable(bp);
466		-
467		- if (err) {
468		- bp->attr.bp_addr = old_addr;
469		- bp->attr.bp_type = old_type;
470		- bp->attr.bp_len = old_len;
471		- if (!bp->attr.disabled)
472		- perf_event_enable(bp);
473		-
474		- return err;
475		- }
476		-
477		-end:
478		- bp->attr.disabled = attr->disabled;
479		-
480		- return 0;
481		-}
482		-EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint);
483		-
484		-/**
485		- * unregister_hw_breakpoint - unregister a user-space hardware breakpoint
486		- * @bp: the breakpoint structure to unregister
487		- */
488		-void unregister_hw_breakpoint(struct perf_event *bp)
489		-{
490		- if (!bp)
491		- return;
492		- perf_event_release_kernel(bp);
493		-}
494		-EXPORT_SYMBOL_GPL(unregister_hw_breakpoint);
495		-
496		-/**
497		- * register_wide_hw_breakpoint - register a wide breakpoint in the kernel
498		- * @attr: breakpoint attributes
499		- * @triggered: callback to trigger when we hit the breakpoint
500		- *
501		- * @return a set of per_cpu pointers to perf events
502		- */
503		-struct perf_event * __percpu *
504		-register_wide_hw_breakpoint(struct perf_event_attr *attr,
505		- perf_overflow_handler_t triggered)
506		-{
507		- struct perf_event * __percpu cpu_events, pevent, bp;
508		- long err;
509		- int cpu;
510		-
511		- cpu_events = alloc_percpu(typeof(*cpu_events));
512		- if (!cpu_events)
513		- return (void __percpu __force *)ERR_PTR(-ENOMEM);
514		-
515		- get_online_cpus();
516		- for_each_online_cpu(cpu) {
517		- pevent = per_cpu_ptr(cpu_events, cpu);
518		- bp = perf_event_create_kernel_counter(attr, cpu, NULL, triggered);
519		-
520		- *pevent = bp;
521		-
522		- if (IS_ERR(bp)) {
523		- err = PTR_ERR(bp);
524		- goto fail;
525		- }
526		- }
527		- put_online_cpus();
528		-
529		- return cpu_events;
530		-
531		-fail:
532		- for_each_online_cpu(cpu) {
533		- pevent = per_cpu_ptr(cpu_events, cpu);
534		- if (IS_ERR(*pevent))
535		- break;
536		- unregister_hw_breakpoint(*pevent);
537		- }
538		- put_online_cpus();
539		-
540		- free_percpu(cpu_events);
541		- return (void __percpu __force *)ERR_PTR(err);
542		-}
543		-EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
544		-
545		-/**
546		- * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel
547		- * @cpu_events: the per cpu set of events to unregister
548		- */
549		-void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
550		-{
551		- int cpu;
552		- struct perf_event **pevent;
553		-
554		- for_each_possible_cpu(cpu) {
555		- pevent = per_cpu_ptr(cpu_events, cpu);
556		- unregister_hw_breakpoint(*pevent);
557		- }
558		- free_percpu(cpu_events);
559		-}
560		-EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
561		-
562		-static struct notifier_block hw_breakpoint_exceptions_nb = {
563		- .notifier_call = hw_breakpoint_exceptions_notify,
564		- /* we need to be notified first */
565		- .priority = 0x7fffffff
566		-};
567		-
568		-static void bp_perf_event_destroy(struct perf_event *event)
569		-{
570		- release_bp_slot(event);
571		-}
572		-
573		-static int hw_breakpoint_event_init(struct perf_event *bp)
574		-{
575		- int err;
576		-
577		- if (bp->attr.type != PERF_TYPE_BREAKPOINT)
578		- return -ENOENT;
579		-
580		- err = register_perf_hw_breakpoint(bp);
581		- if (err)
582		- return err;
583		-
584		- bp->destroy = bp_perf_event_destroy;
585		-
586		- return 0;
587		-}
588		-
589		-static int hw_breakpoint_add(struct perf_event *bp, int flags)
590		-{
591		- if (!(flags & PERF_EF_START))
592		- bp->hw.state = PERF_HES_STOPPED;
593		-
594		- return arch_install_hw_breakpoint(bp);
595		-}
596		-
597		-static void hw_breakpoint_del(struct perf_event *bp, int flags)
598		-{
599		- arch_uninstall_hw_breakpoint(bp);
600		-}
601		-
602		-static void hw_breakpoint_start(struct perf_event *bp, int flags)
603		-{
604		- bp->hw.state = 0;
605		-}
606		-
607		-static void hw_breakpoint_stop(struct perf_event *bp, int flags)
608		-{
609		- bp->hw.state = PERF_HES_STOPPED;
610		-}
611		-
612		-static struct pmu perf_breakpoint = {
613		- .task_ctx_nr = perf_sw_context, /* could eventually get its own */
614		-
615		- .event_init = hw_breakpoint_event_init,
616		- .add = hw_breakpoint_add,
617		- .del = hw_breakpoint_del,
618		- .start = hw_breakpoint_start,
619		- .stop = hw_breakpoint_stop,
620		- .read = hw_breakpoint_pmu_read,
621		-};
622		-
623		-int __init init_hw_breakpoint(void)
624		-{
625		- unsigned int **task_bp_pinned;
626		- int cpu, err_cpu;
627		- int i;
628		-
629		- for (i = 0; i < TYPE_MAX; i++)
630		- nr_slots[i] = hw_breakpoint_slots(i);
631		-
632		- for_each_possible_cpu(cpu) {
633		- for (i = 0; i < TYPE_MAX; i++) {
634		- task_bp_pinned = &per_cpu(nr_task_bp_pinned[i], cpu);
635		- task_bp_pinned = kzalloc(sizeof(int) nr_slots[i],
636		- GFP_KERNEL);
637		- if (!*task_bp_pinned)
638		- goto err_alloc;
639		- }
640		- }
641		-
642		- constraints_initialized = 1;
643		-
644		- perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT);
645		-
646		- return register_die_notifier(&hw_breakpoint_exceptions_nb);
647		-
648		- err_alloc:
649		- for_each_possible_cpu(err_cpu) {
650		- if (err_cpu == cpu)
651		- break;
652		- for (i = 0; i < TYPE_MAX; i++)
653		- kfree(per_cpu(nr_task_bp_pinned[i], cpu));
654		- }
655		-
656		- return -ENOMEM;
657		-}