perf: Simplify and fix __perf_install_in_context()

Currently __perf_install_in_context() will try and schedule in the event irrespective of our event scheduling rules, that is, we try to schedule CPU-pinned, TASK-pinned, CPU-flexible, TASK-flexible, but when creating a new event we simply try and schedule it on top of whatever is already on the PMU, this can lead to errors for pinned events. Therefore, simplify things and simply schedule everything out, add the event to the corresponding context and schedule everything back in. This also nicely handles the case where with __ARCH_WANT_INTERRUPTS_ON_CTXSW the IPI can come right in the middle of schedule, before we managed to call perf_event_task_sched_in(). Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20110409192141.870894224@chello.nl Signed-off-by: Ingo Molnar <mingo@elte.hu>

perf: Simplify and fix __perf_install_in_context()
Currently __perf_install_in_context() will try and schedule in the event irrespective of our event scheduling rules, that is, we try to schedule CPU-pinned, TASK-pinned, CPU-flexible, TASK-flexible, but when creating a new event we simply try and schedule it on top of whatever is already on the PMU, this can lead to errors for pinned events. Therefore, simplify things and simply schedule everything out, add the event to the corresponding context and schedule everything back in. This also nicely handles the case where with __ARCH_WANT_INTERRUPTS_ON_CTXSW the IPI can come right in the middle of schedule, before we managed to call perf_event_task_sched_in(). Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Link: http://lkml.kernel.org/r/20110409192141.870894224@chello.nl Signed-off-by: Ingo Molnar <mingo@elte.hu>
Peter Zijlstra · Ingo Molnar
1 parent 04dc2dbbfe
Showing 1 changed file with 35 additions and 45 deletions Side-by-side Diff
kernel/events/core.c
@@ -1469,8 +1469,12 @@
 	event->tstamp_stopped = tstamp;
 }
  
-static void perf_event_context_sched_in(struct perf_event_context *ctx,
-					struct task_struct *tsk);
+static void task_ctx_sched_out(struct perf_event_context *ctx);
+static void
+ctx_sched_in(struct perf_event_context *ctx,
+	     struct perf_cpu_context *cpuctx,
+	     enum event_type_t event_type,
+	     struct task_struct *task);
  
 /*
  * Cross CPU call to install and enable a performance event
  
  
  
  
  
@@ -1481,20 +1485,31 @@
 {
 	struct perf_event *event = info;
 	struct perf_event_context *ctx = event->ctx;
-	struct perf_event *leader = event->group_leader;
 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
-	int err;
+	struct perf_event_context *task_ctx = cpuctx->task_ctx;
+	struct task_struct *task = current;
  
+	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
+	perf_pmu_disable(cpuctx->ctx.pmu);
+
 	/*
-	 * In case we're installing a new context to an already running task,
-	 * could also happen before perf_event_task_sched_in() on architectures
-	 * which do context switches with IRQs enabled.
+	 * If there was an active task_ctx schedule it out.
 	 */
-	if (ctx->task && !cpuctx->task_ctx)
-		perf_event_context_sched_in(ctx, ctx->task);
+	if (task_ctx) {
+		task_ctx_sched_out(task_ctx);
+		/*
+		 * If the context we're installing events in is not the
+		 * active task_ctx, flip them.
+		 */
+		if (ctx->task && task_ctx != ctx) {
+			raw_spin_unlock(&cpuctx->ctx.lock);
+			raw_spin_lock(&ctx->lock);
+			cpuctx->task_ctx = task_ctx = ctx;
+		}
+		task = task_ctx->task;
+	}
+	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
  
-	raw_spin_lock(&ctx->lock);
-	ctx->is_active = 1;
 	update_context_time(ctx);
 	/*
 	 * update cgrp time only if current cgrp
  
  
  
@@ -1505,43 +1520,18 @@
  
 	add_event_to_ctx(event, ctx);
  
-	if (!event_filter_match(event))
-		goto unlock;
-
 	/*
-	 * Don't put the event on if it is disabled or if
-	 * it is in a group and the group isn't on.
+	 * Schedule everything back in
 	 */
-	if (event->state != PERF_EVENT_STATE_INACTIVE ||
-	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
-		goto unlock;
+	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
+	if (task_ctx)
+		ctx_sched_in(task_ctx, cpuctx, EVENT_PINNED, task);
+	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
+	if (task_ctx)
+		ctx_sched_in(task_ctx, cpuctx, EVENT_FLEXIBLE, task);
  
-	/*
-	 * An exclusive event can't go on if there are already active
-	 * hardware events, and no hardware event can go on if there
-	 * is already an exclusive event on.
-	 */
-	if (!group_can_go_on(event, cpuctx, 1))
-		err = -EEXIST;
-	else
-		err = event_sched_in(event, cpuctx, ctx);
-
-	if (err) {
-		/*
-		 * This event couldn't go on.  If it is in a group
-		 * then we have to pull the whole group off.
-		 * If the event group is pinned then put it in error state.
-		 */
-		if (leader != event)
-			group_sched_out(leader, cpuctx, ctx);
-		if (leader->attr.pinned) {
-			update_group_times(leader);
-			leader->state = PERF_EVENT_STATE_ERROR;
-		}
-	}
-
-unlock:
-	raw_spin_unlock(&ctx->lock);
+	perf_pmu_enable(cpuctx->ctx.pmu);
+	perf_ctx_unlock(cpuctx, task_ctx);
  
 	return 0;
 }
...	...	@@ -1469,8 +1469,12 @@
1469	1469	event->tstamp_stopped = tstamp;
1470	1470	}
1471	1471
1472		-static void perf_event_context_sched_in(struct perf_event_context *ctx,
1473		- struct task_struct *tsk);
	1472	+static void task_ctx_sched_out(struct perf_event_context *ctx);
	1473	+static void
	1474	+ctx_sched_in(struct perf_event_context *ctx,
	1475	+ struct perf_cpu_context *cpuctx,
	1476	+ enum event_type_t event_type,
	1477	+ struct task_struct *task);
1474	1478
1475	1479	/*
1476	1480	* Cross CPU call to install and enable a performance event
1477	1481
1478	1482
1479	1483
1480	1484
1481	1485
...	...	@@ -1481,20 +1485,31 @@
1481	1485	{
1482	1486	struct perf_event *event = info;
1483	1487	struct perf_event_context *ctx = event->ctx;
1484		- struct perf_event *leader = event->group_leader;
1485	1488	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1486		- int err;
	1489	+ struct perf_event_context *task_ctx = cpuctx->task_ctx;
	1490	+ struct task_struct *task = current;
1487	1491
	1492	+ perf_ctx_lock(cpuctx, cpuctx->task_ctx);
	1493	+ perf_pmu_disable(cpuctx->ctx.pmu);
	1494	+
1488	1495	/*
1489		- * In case we're installing a new context to an already running task,
1490		- * could also happen before perf_event_task_sched_in() on architectures
1491		- * which do context switches with IRQs enabled.
	1496	+ * If there was an active task_ctx schedule it out.
1492	1497	*/
1493		- if (ctx->task && !cpuctx->task_ctx)
1494		- perf_event_context_sched_in(ctx, ctx->task);
	1498	+ if (task_ctx) {
	1499	+ task_ctx_sched_out(task_ctx);
	1500	+ /*
	1501	+ * If the context we're installing events in is not the
	1502	+ * active task_ctx, flip them.
	1503	+ */
	1504	+ if (ctx->task && task_ctx != ctx) {
	1505	+ raw_spin_unlock(&cpuctx->ctx.lock);
	1506	+ raw_spin_lock(&ctx->lock);
	1507	+ cpuctx->task_ctx = task_ctx = ctx;
	1508	+ }
	1509	+ task = task_ctx->task;
	1510	+ }
	1511	+ cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1495	1512
1496		- raw_spin_lock(&ctx->lock);
1497		- ctx->is_active = 1;
1498	1513	update_context_time(ctx);
1499	1514	/*
1500	1515	* update cgrp time only if current cgrp
1501	1516
1502	1517
1503	1518
...	...	@@ -1505,43 +1520,18 @@
1505	1520
1506	1521	add_event_to_ctx(event, ctx);
1507	1522
1508		- if (!event_filter_match(event))
1509		- goto unlock;
1510		-
1511	1523	/*
1512		- * Don't put the event on if it is disabled or if
1513		- * it is in a group and the group isn't on.
	1524	+ * Schedule everything back in
1514	1525	*/
1515		- if (event->state != PERF_EVENT_STATE_INACTIVE \|\|
1516		- (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))
1517		- goto unlock;
	1526	+ cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
	1527	+ if (task_ctx)
	1528	+ ctx_sched_in(task_ctx, cpuctx, EVENT_PINNED, task);
	1529	+ cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
	1530	+ if (task_ctx)
	1531	+ ctx_sched_in(task_ctx, cpuctx, EVENT_FLEXIBLE, task);
1518	1532
1519		- /*
1520		- * An exclusive event can't go on if there are already active
1521		- * hardware events, and no hardware event can go on if there
1522		- * is already an exclusive event on.
1523		- */
1524		- if (!group_can_go_on(event, cpuctx, 1))
1525		- err = -EEXIST;
1526		- else
1527		- err = event_sched_in(event, cpuctx, ctx);
1528		-
1529		- if (err) {
1530		- /*
1531		- * This event couldn't go on. If it is in a group
1532		- * then we have to pull the whole group off.
1533		- * If the event group is pinned then put it in error state.
1534		- */
1535		- if (leader != event)
1536		- group_sched_out(leader, cpuctx, ctx);
1537		- if (leader->attr.pinned) {
1538		- update_group_times(leader);
1539		- leader->state = PERF_EVENT_STATE_ERROR;
1540		- }
1541		- }
1542		-
1543		-unlock:
1544		- raw_spin_unlock(&ctx->lock);
	1533	+ perf_pmu_enable(cpuctx->ctx.pmu);
	1534	+ perf_ctx_unlock(cpuctx, task_ctx);
1545	1535
1546	1536	return 0;
1547	1537	}