/*

   * Performance events core code:

   *

   *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>

   *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>

   *

   * For licensing details see kernel-base/COPYING

   */
  
  #include <linux/fs.h>

  #include <linux/mm.h>

  #include <linux/cpu.h>
  #include <linux/smp.h>

  #include <linux/idr.h>

  #include <linux/file.h>

  #include <linux/poll.h>

  #include <linux/slab.h>

  #include <linux/hash.h>

  #include <linux/sysfs.h>

  #include <linux/dcache.h>

  #include <linux/percpu.h>

  #include <linux/ptrace.h>

  #include <linux/reboot.h>

  #include <linux/vmstat.h>

  #include <linux/device.h>

  #include <linux/vmalloc.h>

  #include <linux/hardirq.h>
  #include <linux/rculist.h>

  #include <linux/uaccess.h>
  #include <linux/syscalls.h>
  #include <linux/anon_inodes.h>

  #include <linux/kernel_stat.h>

  #include <linux/perf_event.h>

  #include <linux/ftrace_event.h>

  #include <linux/hw_breakpoint.h>

  #include <asm/irq_regs.h>

  enum event_type_t {
  	EVENT_FLEXIBLE = 0x1,
  	EVENT_PINNED = 0x2,
  	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
  };

  atomic_t perf_task_events __read_mostly;

  static atomic_t nr_mmap_events __read_mostly;
  static atomic_t nr_comm_events __read_mostly;
  static atomic_t nr_task_events __read_mostly;

  static LIST_HEAD(pmus);
  static DEFINE_MUTEX(pmus_lock);
  static struct srcu_struct pmus_srcu;

  /*

   * perf event paranoia level:

   *  -1 - not paranoid at all
   *   0 - disallow raw tracepoint access for unpriv

   *   1 - disallow cpu events for unpriv

   *   2 - disallow kernel profiling for unpriv

   */

  int sysctl_perf_event_paranoid __read_mostly = 1;

  int sysctl_perf_event_mlock __read_mostly = 512; /* 'free' kb per user */

  
  /*

   * max perf event sample rate

   */

  int sysctl_perf_event_sample_rate __read_mostly = 100000;

  static atomic64_t perf_event_id;

  static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
  			      enum event_type_t event_type);
  
  static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
  			     enum event_type_t event_type);

  void __weak perf_event_print_debug(void)	{ }

  extern __weak const char *perf_pmu_name(void)

  {

  	return "pmu";

  }

  static inline u64 perf_clock(void)
  {
  	return local_clock();
  }

  void perf_pmu_disable(struct pmu *pmu)

  {

  	int *count = this_cpu_ptr(pmu->pmu_disable_count);
  	if (!(*count)++)
  		pmu->pmu_disable(pmu);

  }

  void perf_pmu_enable(struct pmu *pmu)

  {

  	int *count = this_cpu_ptr(pmu->pmu_disable_count);
  	if (!--(*count))
  		pmu->pmu_enable(pmu);

  }

  static DEFINE_PER_CPU(struct list_head, rotation_list);
  
  /*
   * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
   * because they're strictly cpu affine and rotate_start is called with IRQs
   * disabled, while rotate_context is called from IRQ context.
   */

  static void perf_pmu_rotate_start(struct pmu *pmu)

  {

  	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

  	struct list_head *head = &__get_cpu_var(rotation_list);

  	WARN_ON(!irqs_disabled());

  	if (list_empty(&cpuctx->rotation_list))
  		list_add(&cpuctx->rotation_list, head);

  }

  static void get_ctx(struct perf_event_context *ctx)

  {

  	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));

  }

  static void free_ctx(struct rcu_head *head)
  {

  	struct perf_event_context *ctx;

  	ctx = container_of(head, struct perf_event_context, rcu_head);

  	kfree(ctx);
  }

  static void put_ctx(struct perf_event_context *ctx)

  {

  	if (atomic_dec_and_test(&ctx->refcount)) {
  		if (ctx->parent_ctx)
  			put_ctx(ctx->parent_ctx);

  		if (ctx->task)
  			put_task_struct(ctx->task);
  		call_rcu(&ctx->rcu_head, free_ctx);

  	}

  }

  static void unclone_ctx(struct perf_event_context *ctx)

  {
  	if (ctx->parent_ctx) {
  		put_ctx(ctx->parent_ctx);
  		ctx->parent_ctx = NULL;
  	}
  }

  static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
  {
  	/*
  	 * only top level events have the pid namespace they were created in
  	 */
  	if (event->parent)
  		event = event->parent;
  
  	return task_tgid_nr_ns(p, event->ns);
  }
  
  static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
  {
  	/*
  	 * only top level events have the pid namespace they were created in
  	 */
  	if (event->parent)
  		event = event->parent;
  
  	return task_pid_nr_ns(p, event->ns);
  }

  /*

   * If we inherit events we want to return the parent event id

   * to userspace.
   */

  static u64 primary_event_id(struct perf_event *event)

  {

  	u64 id = event->id;

  	if (event->parent)
  		id = event->parent->id;

  
  	return id;
  }

  /*

   * Get the perf_event_context for a task and lock it.

   * This has to cope with with the fact that until it is locked,
   * the context could get moved to another task.
   */

  static struct perf_event_context *

  perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)

  {

  	struct perf_event_context *ctx;

  
  	rcu_read_lock();

  retry:

  	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);

  	if (ctx) {
  		/*
  		 * If this context is a clone of another, it might
  		 * get swapped for another underneath us by

  		 * perf_event_task_sched_out, though the

  		 * rcu_read_lock() protects us from any context
  		 * getting freed.  Lock the context and check if it
  		 * got swapped before we could get the lock, and retry
  		 * if so.  If we locked the right context, then it
  		 * can't get swapped on us any more.
  		 */

  		raw_spin_lock_irqsave(&ctx->lock, *flags);

  		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {

  			raw_spin_unlock_irqrestore(&ctx->lock, *flags);

  			goto retry;
  		}

  
  		if (!atomic_inc_not_zero(&ctx->refcount)) {

  			raw_spin_unlock_irqrestore(&ctx->lock, *flags);

  			ctx = NULL;
  		}

  	}
  	rcu_read_unlock();
  	return ctx;
  }
  
  /*
   * Get the context for a task and increment its pin_count so it
   * can't get swapped to another task.  This also increments its
   * reference count so that the context can't get freed.
   */

  static struct perf_event_context *
  perf_pin_task_context(struct task_struct *task, int ctxn)

  {

  	struct perf_event_context *ctx;

  	unsigned long flags;

  	ctx = perf_lock_task_context(task, ctxn, &flags);

  	if (ctx) {
  		++ctx->pin_count;

  		raw_spin_unlock_irqrestore(&ctx->lock, flags);

  	}
  	return ctx;
  }

  static void perf_unpin_context(struct perf_event_context *ctx)

  {
  	unsigned long flags;

  	raw_spin_lock_irqsave(&ctx->lock, flags);

  	--ctx->pin_count;

  	raw_spin_unlock_irqrestore(&ctx->lock, flags);

  	put_ctx(ctx);
  }

  /*
   * Update the record of the current time in a context.
   */
  static void update_context_time(struct perf_event_context *ctx)
  {
  	u64 now = perf_clock();
  
  	ctx->time += now - ctx->timestamp;
  	ctx->timestamp = now;
  }

  static u64 perf_event_time(struct perf_event *event)
  {
  	struct perf_event_context *ctx = event->ctx;
  	return ctx ? ctx->time : 0;
  }

  /*
   * Update the total_time_enabled and total_time_running fields for a event.
   */
  static void update_event_times(struct perf_event *event)
  {
  	struct perf_event_context *ctx = event->ctx;
  	u64 run_end;
  
  	if (event->state < PERF_EVENT_STATE_INACTIVE ||
  	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
  		return;

  	if (ctx->is_active)

  		run_end = perf_event_time(event);

  	else
  		run_end = event->tstamp_stopped;
  
  	event->total_time_enabled = run_end - event->tstamp_enabled;

  
  	if (event->state == PERF_EVENT_STATE_INACTIVE)
  		run_end = event->tstamp_stopped;
  	else

  		run_end = perf_event_time(event);

  
  	event->total_time_running = run_end - event->tstamp_running;
  }

  /*
   * Update total_time_enabled and total_time_running for all events in a group.
   */
  static void update_group_times(struct perf_event *leader)
  {
  	struct perf_event *event;
  
  	update_event_times(leader);
  	list_for_each_entry(event, &leader->sibling_list, group_entry)
  		update_event_times(event);
  }

  static struct list_head *
  ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
  {
  	if (event->attr.pinned)
  		return &ctx->pinned_groups;
  	else
  		return &ctx->flexible_groups;
  }

  /*

   * Add a event from the lists for its context.

   * Must be called with ctx->mutex and ctx->lock held.
   */

  static void

  list_add_event(struct perf_event *event, struct perf_event_context *ctx)

  {

  	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
  	event->attach_state |= PERF_ATTACH_CONTEXT;

  
  	/*

  	 * If we're a stand alone event or group leader, we go to the context
  	 * list, group events are kept attached to the group so that
  	 * perf_group_detach can, at all times, locate all siblings.

  	 */

  	if (event->group_leader == event) {

  		struct list_head *list;

  		if (is_software_event(event))
  			event->group_flags |= PERF_GROUP_SOFTWARE;

  		list = ctx_group_list(event, ctx);
  		list_add_tail(&event->group_entry, list);

  	}

  	list_add_rcu(&event->event_entry, &ctx->event_list);

  	if (!ctx->nr_events)

  		perf_pmu_rotate_start(ctx->pmu);

  	ctx->nr_events++;
  	if (event->attr.inherit_stat)

  		ctx->nr_stat++;

  }

  /*
   * Called at perf_event creation and when events are attached/detached from a
   * group.
   */
  static void perf_event__read_size(struct perf_event *event)
  {
  	int entry = sizeof(u64); /* value */
  	int size = 0;
  	int nr = 1;
  
  	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
  		size += sizeof(u64);
  
  	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
  		size += sizeof(u64);
  
  	if (event->attr.read_format & PERF_FORMAT_ID)
  		entry += sizeof(u64);
  
  	if (event->attr.read_format & PERF_FORMAT_GROUP) {
  		nr += event->group_leader->nr_siblings;
  		size += sizeof(u64);
  	}
  
  	size += entry * nr;
  	event->read_size = size;
  }
  
  static void perf_event__header_size(struct perf_event *event)
  {
  	struct perf_sample_data *data;
  	u64 sample_type = event->attr.sample_type;
  	u16 size = 0;
  
  	perf_event__read_size(event);
  
  	if (sample_type & PERF_SAMPLE_IP)
  		size += sizeof(data->ip);

  	if (sample_type & PERF_SAMPLE_ADDR)
  		size += sizeof(data->addr);
  
  	if (sample_type & PERF_SAMPLE_PERIOD)
  		size += sizeof(data->period);
  
  	if (sample_type & PERF_SAMPLE_READ)
  		size += event->read_size;
  
  	event->header_size = size;
  }
  
  static void perf_event__id_header_size(struct perf_event *event)
  {
  	struct perf_sample_data *data;
  	u64 sample_type = event->attr.sample_type;
  	u16 size = 0;

  	if (sample_type & PERF_SAMPLE_TID)
  		size += sizeof(data->tid_entry);
  
  	if (sample_type & PERF_SAMPLE_TIME)
  		size += sizeof(data->time);

  	if (sample_type & PERF_SAMPLE_ID)
  		size += sizeof(data->id);
  
  	if (sample_type & PERF_SAMPLE_STREAM_ID)
  		size += sizeof(data->stream_id);
  
  	if (sample_type & PERF_SAMPLE_CPU)
  		size += sizeof(data->cpu_entry);

  	event->id_header_size = size;

  }

  static void perf_group_attach(struct perf_event *event)
  {

  	struct perf_event *group_leader = event->group_leader, *pos;

  	/*
  	 * We can have double attach due to group movement in perf_event_open.
  	 */
  	if (event->attach_state & PERF_ATTACH_GROUP)
  		return;

  	event->attach_state |= PERF_ATTACH_GROUP;
  
  	if (group_leader == event)
  		return;
  
  	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
  			!is_software_event(event))
  		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
  
  	list_add_tail(&event->group_entry, &group_leader->sibling_list);
  	group_leader->nr_siblings++;

  
  	perf_event__header_size(group_leader);
  
  	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
  		perf_event__header_size(pos);

  }

  /*

   * Remove a event from the lists for its context.

   * Must be called with ctx->mutex and ctx->lock held.

   */

  static void

  list_del_event(struct perf_event *event, struct perf_event_context *ctx)

  {

  	/*
  	 * We can have double detach due to exit/hot-unplug + close.
  	 */
  	if (!(event->attach_state & PERF_ATTACH_CONTEXT))

  		return;

  
  	event->attach_state &= ~PERF_ATTACH_CONTEXT;

  	ctx->nr_events--;
  	if (event->attr.inherit_stat)

  		ctx->nr_stat--;

  	list_del_rcu(&event->event_entry);

  	if (event->group_leader == event)
  		list_del_init(&event->group_entry);

  	update_group_times(event);

  
  	/*
  	 * If event was in error state, then keep it
  	 * that way, otherwise bogus counts will be
  	 * returned on read(). The only way to get out
  	 * of error state is by explicit re-enabling
  	 * of the event
  	 */
  	if (event->state > PERF_EVENT_STATE_OFF)
  		event->state = PERF_EVENT_STATE_OFF;

  }

  static void perf_group_detach(struct perf_event *event)

  {
  	struct perf_event *sibling, *tmp;

  	struct list_head *list = NULL;
  
  	/*
  	 * We can have double detach due to exit/hot-unplug + close.
  	 */
  	if (!(event->attach_state & PERF_ATTACH_GROUP))
  		return;
  
  	event->attach_state &= ~PERF_ATTACH_GROUP;
  
  	/*
  	 * If this is a sibling, remove it from its group.
  	 */
  	if (event->group_leader != event) {
  		list_del_init(&event->group_entry);
  		event->group_leader->nr_siblings--;

  		goto out;

  	}
  
  	if (!list_empty(&event->group_entry))
  		list = &event->group_entry;

  	/*

  	 * If this was a group event with sibling events then
  	 * upgrade the siblings to singleton events by adding them

  	 * to whatever list we are on.

  	 */

  	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {

  		if (list)
  			list_move_tail(&sibling->group_entry, list);

  		sibling->group_leader = sibling;

  
  		/* Inherit group flags from the previous leader */
  		sibling->group_flags = event->group_flags;

  	}

  
  out:
  	perf_event__header_size(event->group_leader);
  
  	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
  		perf_event__header_size(tmp);

  }

  static inline int
  event_filter_match(struct perf_event *event)
  {
  	return event->cpu == -1 || event->cpu == smp_processor_id();
  }

  static void
  event_sched_out(struct perf_event *event,

  		  struct perf_cpu_context *cpuctx,

  		  struct perf_event_context *ctx)

  {

  	u64 tstamp = perf_event_time(event);

  	u64 delta;
  	/*
  	 * An event which could not be activated because of
  	 * filter mismatch still needs to have its timings
  	 * maintained, otherwise bogus information is return
  	 * via read() for time_enabled, time_running:
  	 */
  	if (event->state == PERF_EVENT_STATE_INACTIVE
  	    && !event_filter_match(event)) {
  		delta = ctx->time - event->tstamp_stopped;
  		event->tstamp_running += delta;

  		event->tstamp_stopped = tstamp;

  	}

  	if (event->state != PERF_EVENT_STATE_ACTIVE)

  		return;

  	event->state = PERF_EVENT_STATE_INACTIVE;
  	if (event->pending_disable) {
  		event->pending_disable = 0;
  		event->state = PERF_EVENT_STATE_OFF;

  	}

  	event->tstamp_stopped = tstamp;

  	event->pmu->del(event, 0);

  	event->oncpu = -1;

  	if (!is_software_event(event))

  		cpuctx->active_oncpu--;
  	ctx->nr_active--;

  	if (event->attr.exclusive || !cpuctx->active_oncpu)

  		cpuctx->exclusive = 0;
  }

  static void

  group_sched_out(struct perf_event *group_event,

  		struct perf_cpu_context *cpuctx,

  		struct perf_event_context *ctx)

  {

  	struct perf_event *event;

  	int state = group_event->state;

  	event_sched_out(group_event, cpuctx, ctx);

  
  	/*
  	 * Schedule out siblings (if any):
  	 */

  	list_for_each_entry(event, &group_event->sibling_list, group_entry)
  		event_sched_out(event, cpuctx, ctx);

  	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)

  		cpuctx->exclusive = 0;
  }

  static inline struct perf_cpu_context *
  __get_cpu_context(struct perf_event_context *ctx)
  {
  	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
  }

  /*

   * Cross CPU call to remove a performance event

   *

   * We disable the event on the hardware level first. After that we

   * remove it from the context list.
   */

  static void __perf_event_remove_from_context(void *info)

  {

  	struct perf_event *event = info;
  	struct perf_event_context *ctx = event->ctx;

  	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

  
  	/*
  	 * If this is a task context, we need to check whether it is
  	 * the current task context of this cpu. If not it has been
  	 * scheduled out before the smp call arrived.
  	 */

  	if (ctx->task && cpuctx->task_ctx != ctx)

  		return;

  	raw_spin_lock(&ctx->lock);

  	event_sched_out(event, cpuctx, ctx);

  	list_del_event(event, ctx);

  	raw_spin_unlock(&ctx->lock);

  }
  
  
  /*

   * Remove the event from a task's (or a CPU's) list of events.

   *

   * Must be called with ctx->mutex held.

   *

   * CPU events are removed with a smp call. For task events we only

   * call when the task is on a CPU.

   *

   * If event->ctx is a cloned context, callers must make sure that
   * every task struct that event->ctx->task could possibly point to

   * remains valid.  This is OK when called from perf_release since
   * that only calls us on the top-level context, which can't be a clone.

   * When called from perf_event_exit_task, it's OK because the

   * context has been detached from its task.

   */

  static void perf_event_remove_from_context(struct perf_event *event)

  {

  	struct perf_event_context *ctx = event->ctx;

  	struct task_struct *task = ctx->task;
  
  	if (!task) {
  		/*

  		 * Per cpu events are removed via an smp call and

  		 * the removal is always successful.

  		 */

  		smp_call_function_single(event->cpu,
  					 __perf_event_remove_from_context,
  					 event, 1);

  		return;
  	}
  
  retry:

  	task_oncpu_function_call(task, __perf_event_remove_from_context,
  				 event);

  	raw_spin_lock_irq(&ctx->lock);

  	/*
  	 * If the context is active we need to retry the smp call.
  	 */

  	if (ctx->nr_active && !list_empty(&event->group_entry)) {

  		raw_spin_unlock_irq(&ctx->lock);

  		goto retry;
  	}
  
  	/*
  	 * The lock prevents that this context is scheduled in so we

  	 * can remove the event safely, if the call above did not

  	 * succeed.
  	 */

  	if (!list_empty(&event->group_entry))

  		list_del_event(event, ctx);

  	raw_spin_unlock_irq(&ctx->lock);

  }

  /*

   * Cross CPU call to disable a performance event

   */

  static void __perf_event_disable(void *info)

  {

  	struct perf_event *event = info;

  	struct perf_event_context *ctx = event->ctx;

  	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

  
  	/*

  	 * If this is a per-task event, need to check whether this
  	 * event's task is the current task on this cpu.

  	 */

  	if (ctx->task && cpuctx->task_ctx != ctx)

  		return;

  	raw_spin_lock(&ctx->lock);

  
  	/*

  	 * If the event is on, turn it off.

  	 * If it is in error state, leave it in error state.
  	 */

  	if (event->state >= PERF_EVENT_STATE_INACTIVE) {

  		update_context_time(ctx);

  		update_group_times(event);
  		if (event == event->group_leader)
  			group_sched_out(event, cpuctx, ctx);

  		else

  			event_sched_out(event, cpuctx, ctx);
  		event->state = PERF_EVENT_STATE_OFF;

  	}

  	raw_spin_unlock(&ctx->lock);

  }
  
  /*

   * Disable a event.

   *

   * If event->ctx is a cloned context, callers must make sure that
   * every task struct that event->ctx->task could possibly point to

   * remains valid.  This condition is satisifed when called through

   * perf_event_for_each_child or perf_event_for_each because they
   * hold the top-level event's child_mutex, so any descendant that
   * goes to exit will block in sync_child_event.
   * When called from perf_pending_event it's OK because event->ctx

   * is the current context on this CPU and preemption is disabled,

   * hence we can't get into perf_event_task_sched_out for this context.

   */

  void perf_event_disable(struct perf_event *event)

  {

  	struct perf_event_context *ctx = event->ctx;

  	struct task_struct *task = ctx->task;
  
  	if (!task) {
  		/*

  		 * Disable the event on the cpu that it's on

  		 */

  		smp_call_function_single(event->cpu, __perf_event_disable,
  					 event, 1);

  		return;
  	}

  retry:

  	task_oncpu_function_call(task, __perf_event_disable, event);

  	raw_spin_lock_irq(&ctx->lock);

  	/*

  	 * If the event is still active, we need to retry the cross-call.

  	 */

  	if (event->state == PERF_EVENT_STATE_ACTIVE) {

  		raw_spin_unlock_irq(&ctx->lock);

  		goto retry;
  	}
  
  	/*
  	 * Since we have the lock this context can't be scheduled
  	 * in, so we can change the state safely.
  	 */

  	if (event->state == PERF_EVENT_STATE_INACTIVE) {
  		update_group_times(event);
  		event->state = PERF_EVENT_STATE_OFF;

  	}

  	raw_spin_unlock_irq(&ctx->lock);

  }

  static int

  event_sched_in(struct perf_event *event,

  		 struct perf_cpu_context *cpuctx,

  		 struct perf_event_context *ctx)

  {

  	u64 tstamp = perf_event_time(event);

  	if (event->state <= PERF_EVENT_STATE_OFF)

  		return 0;

  	event->state = PERF_EVENT_STATE_ACTIVE;

  	event->oncpu = smp_processor_id();

  	/*
  	 * The new state must be visible before we turn it on in the hardware:
  	 */
  	smp_wmb();

  	if (event->pmu->add(event, PERF_EF_START)) {

  		event->state = PERF_EVENT_STATE_INACTIVE;
  		event->oncpu = -1;

  		return -EAGAIN;
  	}

  	event->tstamp_running += tstamp - event->tstamp_stopped;

  	event->shadow_ctx_time = tstamp - ctx->timestamp;

  	if (!is_software_event(event))

  		cpuctx->active_oncpu++;

  	ctx->nr_active++;

  	if (event->attr.exclusive)

  		cpuctx->exclusive = 1;

  	return 0;
  }

  static int

  group_sched_in(struct perf_event *group_event,

  	       struct perf_cpu_context *cpuctx,

  	       struct perf_event_context *ctx)

  {

  	struct perf_event *event, *partial_group = NULL;

  	struct pmu *pmu = group_event->pmu;

  	u64 now = ctx->time;
  	bool simulate = false;

  	if (group_event->state == PERF_EVENT_STATE_OFF)

  		return 0;

  	pmu->start_txn(pmu);

  	if (event_sched_in(group_event, cpuctx, ctx)) {

  		pmu->cancel_txn(pmu);

  		return -EAGAIN;

  	}

  
  	/*
  	 * Schedule in siblings as one group (if any):
  	 */

  	list_for_each_entry(event, &group_event->sibling_list, group_entry) {

  		if (event_sched_in(event, cpuctx, ctx)) {

  			partial_group = event;

  			goto group_error;
  		}
  	}

  	if (!pmu->commit_txn(pmu))

  		return 0;

  group_error:
  	/*
  	 * Groups can be scheduled in as one unit only, so undo any
  	 * partial group before returning:

  	 * The events up to the failed event are scheduled out normally,
  	 * tstamp_stopped will be updated.
  	 *
  	 * The failed events and the remaining siblings need to have
  	 * their timings updated as if they had gone thru event_sched_in()
  	 * and event_sched_out(). This is required to get consistent timings
  	 * across the group. This also takes care of the case where the group
  	 * could never be scheduled by ensuring tstamp_stopped is set to mark
  	 * the time the event was actually stopped, such that time delta
  	 * calculation in update_event_times() is correct.

  	 */

  	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
  		if (event == partial_group)

  			simulate = true;
  
  		if (simulate) {
  			event->tstamp_running += now - event->tstamp_stopped;
  			event->tstamp_stopped = now;
  		} else {
  			event_sched_out(event, cpuctx, ctx);
  		}

  	}

  	event_sched_out(group_event, cpuctx, ctx);

  	pmu->cancel_txn(pmu);

  	return -EAGAIN;
  }

  /*

   * Work out whether we can put this event group on the CPU now.

   */

  static int group_can_go_on(struct perf_event *event,

  			   struct perf_cpu_context *cpuctx,
  			   int can_add_hw)
  {
  	/*

  	 * Groups consisting entirely of software events can always go on.

  	 */

  	if (event->group_flags & PERF_GROUP_SOFTWARE)

  		return 1;
  	/*
  	 * If an exclusive group is already on, no other hardware

  	 * events can go on.

  	 */
  	if (cpuctx->exclusive)
  		return 0;
  	/*
  	 * If this group is exclusive and there are already

  	 * events on the CPU, it can't go on.

  	 */

  	if (event->attr.exclusive && cpuctx->active_oncpu)

  		return 0;
  	/*
  	 * Otherwise, try to add it if all previous groups were able
  	 * to go on.
  	 */
  	return can_add_hw;
  }

  static void add_event_to_ctx(struct perf_event *event,
  			       struct perf_event_context *ctx)

  {

  	u64 tstamp = perf_event_time(event);

  	list_add_event(event, ctx);

  	perf_group_attach(event);

  	event->tstamp_enabled = tstamp;
  	event->tstamp_running = tstamp;
  	event->tstamp_stopped = tstamp;

  }

  /*

   * Cross CPU call to install and enable a performance event

   *
   * Must be called with ctx->mutex held

   */
  static void __perf_install_in_context(void *info)
  {

  	struct perf_event *event = info;
  	struct perf_event_context *ctx = event->ctx;
  	struct perf_event *leader = event->group_leader;

  	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

  	int err;

  
  	/*
  	 * If this is a task context, we need to check whether it is
  	 * the current task context of this cpu. If not it has been
  	 * scheduled out before the smp call arrived.

  	 * Or possibly this is the right context but it isn't

  	 * on this cpu because it had no events.

  	 */

  	if (ctx->task && cpuctx->task_ctx != ctx) {

  		if (cpuctx->task_ctx || ctx->task != current)

  			return;
  		cpuctx->task_ctx = ctx;
  	}

  	raw_spin_lock(&ctx->lock);

  	ctx->is_active = 1;

  	update_context_time(ctx);

  	add_event_to_ctx(event, ctx);

  	if (!event_filter_match(event))

  		goto unlock;

  	/*

  	 * Don't put the event on if it is disabled or if

  	 * it is in a group and the group isn't on.
  	 */

  	if (event->state != PERF_EVENT_STATE_INACTIVE ||
  	    (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE))

  		goto unlock;
  
  	/*

  	 * An exclusive event can't go on if there are already active
  	 * hardware events, and no hardware event can go on if there
  	 * is already an exclusive event on.

  	 */

  	if (!group_can_go_on(event, cpuctx, 1))

  		err = -EEXIST;
  	else

  		err = event_sched_in(event, cpuctx, ctx);

  	if (err) {
  		/*

  		 * This event couldn't go on.  If it is in a group

  		 * then we have to pull the whole group off.

  		 * If the event group is pinned then put it in error state.

  		 */

  		if (leader != event)

  			group_sched_out(leader, cpuctx, ctx);

  		if (leader->attr.pinned) {

  			update_group_times(leader);

  			leader->state = PERF_EVENT_STATE_ERROR;

  		}

  	}

  unlock:

  	raw_spin_unlock(&ctx->lock);

  }
  
  /*

   * Attach a performance event to a context

   *

   * First we add the event to the list with the hardware enable bit
   * in event->hw_config cleared.

   *

   * If the event is attached to a task which is on a CPU we use a smp

   * call to enable it in the task context. The task might have been
   * scheduled away, but we check this in the smp call again.

   *
   * Must be called with ctx->mutex held.

   */
  static void

  perf_install_in_context(struct perf_event_context *ctx,
  			struct perf_event *event,

  			int cpu)
  {
  	struct task_struct *task = ctx->task;

  	event->ctx = ctx;

  	if (!task) {
  		/*

  		 * Per cpu events are installed via an smp call and

  		 * the install is always successful.

  		 */
  		smp_call_function_single(cpu, __perf_install_in_context,

  					 event, 1);

  		return;
  	}

  retry:
  	task_oncpu_function_call(task, __perf_install_in_context,

  				 event);

  	raw_spin_lock_irq(&ctx->lock);

  	/*

  	 * we need to retry the smp call.
  	 */

  	if (ctx->is_active && list_empty(&event->group_entry)) {

  		raw_spin_unlock_irq(&ctx->lock);

  		goto retry;
  	}
  
  	/*
  	 * The lock prevents that this context is scheduled in so we

  	 * can add the event safely, if it the call above did not

  	 * succeed.
  	 */

  	if (list_empty(&event->group_entry))
  		add_event_to_ctx(event, ctx);

  	raw_spin_unlock_irq(&ctx->lock);

  }

  /*

   * Put a event into inactive state and update time fields.

   * Enabling the leader of a group effectively enables all
   * the group members that aren't explicitly disabled, so we
   * have to update their ->tstamp_enabled also.
   * Note: this works for group members as well as group leaders
   * since the non-leader members' sibling_lists will be empty.
   */

  static void __perf_event_mark_enabled(struct perf_event *event,
  					struct perf_event_context *ctx)

  {

  	struct perf_event *sub;

  	u64 tstamp = perf_event_time(event);

  	event->state = PERF_EVENT_STATE_INACTIVE;

  	event->tstamp_enabled = tstamp - event->total_time_enabled;

  	list_for_each_entry(sub, &event->sibling_list, group_entry) {

  		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
  			sub->tstamp_enabled = tstamp - sub->total_time_enabled;

  	}

  }
  
  /*

   * Cross CPU call to enable a performance event

   */

  static void __perf_event_enable(void *info)

  {

  	struct perf_event *event = info;

  	struct perf_event_context *ctx = event->ctx;
  	struct perf_event *leader = event->group_leader;

  	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

  	int err;

  	/*

  	 * If this is a per-task event, need to check whether this
  	 * event's task is the current task on this cpu.

  	 */

  	if (ctx->task && cpuctx->task_ctx != ctx) {

  		if (cpuctx->task_ctx || ctx->task != current)

  			return;
  		cpuctx->task_ctx = ctx;
  	}

  	raw_spin_lock(&ctx->lock);

  	ctx->is_active = 1;

  	update_context_time(ctx);

  	if (event->state >= PERF_EVENT_STATE_INACTIVE)

  		goto unlock;

  	__perf_event_mark_enabled(event, ctx);

  	if (!event_filter_match(event))

  		goto unlock;

  	/*

  	 * If the event is in a group and isn't the group leader,

  	 * then don't put it on unless the group is on.

  	 */

  	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)

  		goto unlock;

  	if (!group_can_go_on(event, cpuctx, 1)) {

  		err = -EEXIST;

  	} else {

  		if (event == leader)

  			err = group_sched_in(event, cpuctx, ctx);

  		else

  			err = event_sched_in(event, cpuctx, ctx);

  	}

  
  	if (err) {
  		/*

  		 * If this event can't go on and it's part of a

  		 * group, then the whole group has to come off.
  		 */

  		if (leader != event)

  			group_sched_out(leader, cpuctx, ctx);

  		if (leader->attr.pinned) {

  			update_group_times(leader);

  			leader->state = PERF_EVENT_STATE_ERROR;

  		}

  	}

  unlock:

  	raw_spin_unlock(&ctx->lock);

  }
  
  /*

   * Enable a event.

   *

   * If event->ctx is a cloned context, callers must make sure that
   * every task struct that event->ctx->task could possibly point to

   * remains valid.  This condition is satisfied when called through

   * perf_event_for_each_child or perf_event_for_each as described
   * for perf_event_disable.

   */

  void perf_event_enable(struct perf_event *event)

  {

  	struct perf_event_context *ctx = event->ctx;

  	struct task_struct *task = ctx->task;
  
  	if (!task) {
  		/*

  		 * Enable the event on the cpu that it's on

  		 */

  		smp_call_function_single(event->cpu, __perf_event_enable,
  					 event, 1);

  		return;
  	}

  	raw_spin_lock_irq(&ctx->lock);

  	if (event->state >= PERF_EVENT_STATE_INACTIVE)

  		goto out;
  
  	/*

  	 * If the event is in error state, clear that first.
  	 * That way, if we see the event in error state below, we

  	 * know that it has gone back into error state, as distinct
  	 * from the task having been scheduled away before the
  	 * cross-call arrived.
  	 */

  	if (event->state == PERF_EVENT_STATE_ERROR)
  		event->state = PERF_EVENT_STATE_OFF;

  retry:

  	raw_spin_unlock_irq(&ctx->lock);

  	task_oncpu_function_call(task, __perf_event_enable, event);

  	raw_spin_lock_irq(&ctx->lock);

  
  	/*

  	 * If the context is active and the event is still off,

  	 * we need to retry the cross-call.
  	 */

  	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF)

  		goto retry;
  
  	/*
  	 * Since we have the lock this context can't be scheduled
  	 * in, so we can change the state safely.
  	 */

  	if (event->state == PERF_EVENT_STATE_OFF)
  		__perf_event_mark_enabled(event, ctx);

  out:

  	raw_spin_unlock_irq(&ctx->lock);

  }

  static int perf_event_refresh(struct perf_event *event, int refresh)

  {

  	/*

  	 * not supported on inherited events

  	 */

  	if (event->attr.inherit || !is_sampling_event(event))

  		return -EINVAL;

  	atomic_add(refresh, &event->event_limit);
  	perf_event_enable(event);

  
  	return 0;

  }

  static void ctx_sched_out(struct perf_event_context *ctx,
  			  struct perf_cpu_context *cpuctx,
  			  enum event_type_t event_type)

  {

  	struct perf_event *event;

  	raw_spin_lock(&ctx->lock);

  	perf_pmu_disable(ctx->pmu);

  	ctx->is_active = 0;

  	if (likely(!ctx->nr_events))

  		goto out;

  	update_context_time(ctx);

  	if (!ctx->nr_active)

  		goto out;