/*
   *  Copyright (C) 1991, 1992  Linus Torvalds
   *  Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs

   *  Copyright (C) 2011	Don Zickus Red Hat, Inc.

   *
   *  Pentium III FXSR, SSE support
   *	Gareth Hughes <gareth@valinux.com>, May 2000
   */
  
  /*
   * Handle hardware traps and faults.
   */
  #include <linux/spinlock.h>
  #include <linux/kprobes.h>
  #include <linux/kdebug.h>
  #include <linux/nmi.h>

  #include <linux/delay.h>
  #include <linux/hardirq.h>
  #include <linux/slab.h>

  #include <linux/export.h>

  #include <linux/mca.h>

  #if defined(CONFIG_EDAC)
  #include <linux/edac.h>
  #endif
  
  #include <linux/atomic.h>
  #include <asm/traps.h>
  #include <asm/mach_traps.h>

  #include <asm/nmi.h>

  #include <asm/x86_init.h>

  
  #define NMI_MAX_NAMELEN	16
  struct nmiaction {
  	struct list_head list;
  	nmi_handler_t handler;
  	unsigned int flags;
  	char *name;
  };
  
  struct nmi_desc {
  	spinlock_t lock;
  	struct list_head head;
  };
  
  static struct nmi_desc nmi_desc[NMI_MAX] = 
  {
  	{
  		.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
  		.head = LIST_HEAD_INIT(nmi_desc[0].head),
  	},
  	{
  		.lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
  		.head = LIST_HEAD_INIT(nmi_desc[1].head),
  	},
  
  };

  struct nmi_stats {
  	unsigned int normal;
  	unsigned int unknown;
  	unsigned int external;
  	unsigned int swallow;
  };
  
  static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);

  static int ignore_nmis;
  
  int unknown_nmi_panic;
  /*
   * Prevent NMI reason port (0x61) being accessed simultaneously, can
   * only be used in NMI handler.
   */
  static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
  
  static int __init setup_unknown_nmi_panic(char *str)
  {
  	unknown_nmi_panic = 1;
  	return 1;
  }
  __setup("unknown_nmi_panic", setup_unknown_nmi_panic);

  #define nmi_to_desc(type) (&nmi_desc[type])

  static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)

  {
  	struct nmi_desc *desc = nmi_to_desc(type);
  	struct nmiaction *a;
  	int handled=0;
  
  	rcu_read_lock();
  
  	/*
  	 * NMIs are edge-triggered, which means if you have enough
  	 * of them concurrently, you can lose some because only one
  	 * can be latched at any given time.  Walk the whole list
  	 * to handle those situations.
  	 */

  	list_for_each_entry_rcu(a, &desc->head, list)

  		handled += a->handler(type, regs);

  	rcu_read_unlock();
  
  	/* return total number of NMI events handled */
  	return handled;
  }
  
  static int __setup_nmi(unsigned int type, struct nmiaction *action)
  {
  	struct nmi_desc *desc = nmi_to_desc(type);
  	unsigned long flags;
  
  	spin_lock_irqsave(&desc->lock, flags);
  
  	/*

  	 * most handlers of type NMI_UNKNOWN never return because
  	 * they just assume the NMI is theirs.  Just a sanity check
  	 * to manage expectations
  	 */
  	WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
  
  	/*

  	 * some handlers need to be executed first otherwise a fake
  	 * event confuses some handlers (kdump uses this flag)
  	 */
  	if (action->flags & NMI_FLAG_FIRST)
  		list_add_rcu(&action->list, &desc->head);
  	else
  		list_add_tail_rcu(&action->list, &desc->head);
  	
  	spin_unlock_irqrestore(&desc->lock, flags);
  	return 0;
  }
  
  static struct nmiaction *__free_nmi(unsigned int type, const char *name)
  {
  	struct nmi_desc *desc = nmi_to_desc(type);
  	struct nmiaction *n;
  	unsigned long flags;
  
  	spin_lock_irqsave(&desc->lock, flags);
  
  	list_for_each_entry_rcu(n, &desc->head, list) {
  		/*
  		 * the name passed in to describe the nmi handler
  		 * is used as the lookup key
  		 */
  		if (!strcmp(n->name, name)) {
  			WARN(in_nmi(),
  				"Trying to free NMI (%s) from NMI context!
  ", n->name);
  			list_del_rcu(&n->list);
  			break;
  		}
  	}
  
  	spin_unlock_irqrestore(&desc->lock, flags);
  	synchronize_rcu();
  	return (n);
  }
  
  int register_nmi_handler(unsigned int type, nmi_handler_t handler,
  			unsigned long nmiflags, const char *devname)
  {
  	struct nmiaction *action;
  	int retval = -ENOMEM;
  
  	if (!handler)
  		return -EINVAL;
  
  	action = kzalloc(sizeof(struct nmiaction), GFP_KERNEL);
  	if (!action)
  		goto fail_action;
  
  	action->handler = handler;
  	action->flags = nmiflags;
  	action->name = kstrndup(devname, NMI_MAX_NAMELEN, GFP_KERNEL);
  	if (!action->name)
  		goto fail_action_name;
  
  	retval = __setup_nmi(type, action);
  
  	if (retval)
  		goto fail_setup_nmi;
  
  	return retval;
  
  fail_setup_nmi:
  	kfree(action->name);
  fail_action_name:
  	kfree(action);
  fail_action:	
  
  	return retval;
  }
  EXPORT_SYMBOL_GPL(register_nmi_handler);
  
  void unregister_nmi_handler(unsigned int type, const char *name)
  {
  	struct nmiaction *a;
  
  	a = __free_nmi(type, name);
  	if (a) {
  		kfree(a->name);
  		kfree(a);
  	}
  }
  
  EXPORT_SYMBOL_GPL(unregister_nmi_handler);

  static notrace __kprobes void
  pci_serr_error(unsigned char reason, struct pt_regs *regs)
  {
  	pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.
  ",
  		 reason, smp_processor_id());
  
  	/*
  	 * On some machines, PCI SERR line is used to report memory
  	 * errors. EDAC makes use of it.
  	 */
  #if defined(CONFIG_EDAC)
  	if (edac_handler_set()) {
  		edac_atomic_assert_error();
  		return;
  	}
  #endif
  
  	if (panic_on_unrecovered_nmi)
  		panic("NMI: Not continuing");
  
  	pr_emerg("Dazed and confused, but trying to continue
  ");
  
  	/* Clear and disable the PCI SERR error line. */
  	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
  	outb(reason, NMI_REASON_PORT);
  }
  
  static notrace __kprobes void
  io_check_error(unsigned char reason, struct pt_regs *regs)
  {
  	unsigned long i;
  
  	pr_emerg(
  	"NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.
  ",
  		 reason, smp_processor_id());
  	show_registers(regs);
  
  	if (panic_on_io_nmi)
  		panic("NMI IOCK error: Not continuing");
  
  	/* Re-enable the IOCK line, wait for a few seconds */
  	reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
  	outb(reason, NMI_REASON_PORT);
  
  	i = 20000;
  	while (--i) {
  		touch_nmi_watchdog();
  		udelay(100);
  	}
  
  	reason &= ~NMI_REASON_CLEAR_IOCHK;
  	outb(reason, NMI_REASON_PORT);
  }
  
  static notrace __kprobes void
  unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
  {

  	int handled;

  	/*
  	 * Use 'false' as back-to-back NMIs are dealt with one level up.
  	 * Of course this makes having multiple 'unknown' handlers useless
  	 * as only the first one is ever run (unless it can actually determine
  	 * if it caused the NMI)
  	 */
  	handled = nmi_handle(NMI_UNKNOWN, regs, false);

  	if (handled) {
  		__this_cpu_add(nmi_stats.unknown, handled);

  		return;

  	}
  
  	__this_cpu_add(nmi_stats.unknown, 1);

  #ifdef CONFIG_MCA
  	/*
  	 * Might actually be able to figure out what the guilty party
  	 * is:
  	 */
  	if (MCA_bus) {
  		mca_handle_nmi();
  		return;
  	}
  #endif
  	pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.
  ",
  		 reason, smp_processor_id());
  
  	pr_emerg("Do you have a strange power saving mode enabled?
  ");
  	if (unknown_nmi_panic || panic_on_unrecovered_nmi)
  		panic("NMI: Not continuing");
  
  	pr_emerg("Dazed and confused, but trying to continue
  ");
  }

  static DEFINE_PER_CPU(bool, swallow_nmi);
  static DEFINE_PER_CPU(unsigned long, last_nmi_rip);

  static notrace __kprobes void default_do_nmi(struct pt_regs *regs)
  {
  	unsigned char reason = 0;

  	int handled;

  	bool b2b = false;

  
  	/*
  	 * CPU-specific NMI must be processed before non-CPU-specific
  	 * NMI, otherwise we may lose it, because the CPU-specific
  	 * NMI can not be detected/processed on other CPUs.
  	 */

  
  	/*
  	 * Back-to-back NMIs are interesting because they can either
  	 * be two NMI or more than two NMIs (any thing over two is dropped
  	 * due to NMI being edge-triggered).  If this is the second half
  	 * of the back-to-back NMI, assume we dropped things and process
  	 * more handlers.  Otherwise reset the 'swallow' NMI behaviour
  	 */
  	if (regs->ip == __this_cpu_read(last_nmi_rip))
  		b2b = true;
  	else
  		__this_cpu_write(swallow_nmi, false);
  
  	__this_cpu_write(last_nmi_rip, regs->ip);
  
  	handled = nmi_handle(NMI_LOCAL, regs, b2b);

  	__this_cpu_add(nmi_stats.normal, handled);

  	if (handled) {
  		/*
  		 * There are cases when a NMI handler handles multiple
  		 * events in the current NMI.  One of these events may
  		 * be queued for in the next NMI.  Because the event is
  		 * already handled, the next NMI will result in an unknown
  		 * NMI.  Instead lets flag this for a potential NMI to
  		 * swallow.
  		 */
  		if (handled > 1)
  			__this_cpu_write(swallow_nmi, true);

  		return;

  	}

  
  	/* Non-CPU-specific NMI: NMI sources can be processed on any CPU */
  	raw_spin_lock(&nmi_reason_lock);

  	reason = x86_platform.get_nmi_reason();

  
  	if (reason & NMI_REASON_MASK) {
  		if (reason & NMI_REASON_SERR)
  			pci_serr_error(reason, regs);
  		else if (reason & NMI_REASON_IOCHK)
  			io_check_error(reason, regs);
  #ifdef CONFIG_X86_32
  		/*
  		 * Reassert NMI in case it became active
  		 * meanwhile as it's edge-triggered:
  		 */
  		reassert_nmi();
  #endif

  		__this_cpu_add(nmi_stats.external, 1);

  		raw_spin_unlock(&nmi_reason_lock);
  		return;
  	}
  	raw_spin_unlock(&nmi_reason_lock);

  	/*
  	 * Only one NMI can be latched at a time.  To handle
  	 * this we may process multiple nmi handlers at once to
  	 * cover the case where an NMI is dropped.  The downside
  	 * to this approach is we may process an NMI prematurely,
  	 * while its real NMI is sitting latched.  This will cause
  	 * an unknown NMI on the next run of the NMI processing.
  	 *
  	 * We tried to flag that condition above, by setting the
  	 * swallow_nmi flag when we process more than one event.
  	 * This condition is also only present on the second half
  	 * of a back-to-back NMI, so we flag that condition too.
  	 *
  	 * If both are true, we assume we already processed this
  	 * NMI previously and we swallow it.  Otherwise we reset
  	 * the logic.
  	 *
  	 * There are scenarios where we may accidentally swallow
  	 * a 'real' unknown NMI.  For example, while processing
  	 * a perf NMI another perf NMI comes in along with a
  	 * 'real' unknown NMI.  These two NMIs get combined into
  	 * one (as descibed above).  When the next NMI gets
  	 * processed, it will be flagged by perf as handled, but
  	 * noone will know that there was a 'real' unknown NMI sent
  	 * also.  As a result it gets swallowed.  Or if the first
  	 * perf NMI returns two events handled then the second
  	 * NMI will get eaten by the logic below, again losing a
  	 * 'real' unknown NMI.  But this is the best we can do
  	 * for now.
  	 */
  	if (b2b && __this_cpu_read(swallow_nmi))

  		__this_cpu_add(nmi_stats.swallow, 1);

  	else
  		unknown_nmi_error(reason, regs);

  }

  /*
   * NMIs can hit breakpoints which will cause it to lose its
   * NMI context with the CPU when the breakpoint does an iret.
   */
  #ifdef CONFIG_X86_32
  /*
   * For i386, NMIs use the same stack as the kernel, and we can
   * add a workaround to the iret problem in C. Simply have 3 states
   * the NMI can be in.
   *
   *  1) not running
   *  2) executing
   *  3) latched
   *
   * When no NMI is in progress, it is in the "not running" state.
   * When an NMI comes in, it goes into the "executing" state.
   * Normally, if another NMI is triggered, it does not interrupt
   * the running NMI and the HW will simply latch it so that when
   * the first NMI finishes, it will restart the second NMI.
   * (Note, the latch is binary, thus multiple NMIs triggering,
   *  when one is running, are ignored. Only one NMI is restarted.)
   *
   * If an NMI hits a breakpoint that executes an iret, another
   * NMI can preempt it. We do not want to allow this new NMI
   * to run, but we want to execute it when the first one finishes.
   * We set the state to "latched", and the first NMI will perform
   * an cmpxchg on the state, and if it doesn't successfully
   * reset the state to "not running" it will restart the next
   * NMI.
   */
  enum nmi_states {
  	NMI_NOT_RUNNING,
  	NMI_EXECUTING,
  	NMI_LATCHED,
  };
  static DEFINE_PER_CPU(enum nmi_states, nmi_state);
  
  #define nmi_nesting_preprocess(regs)					\
  	do {								\
  		if (__get_cpu_var(nmi_state) != NMI_NOT_RUNNING) {	\
  			__get_cpu_var(nmi_state) = NMI_LATCHED;		\
  			return;						\
  		}							\
  	nmi_restart:							\
  		__get_cpu_var(nmi_state) = NMI_EXECUTING;		\
  	} while (0)
  
  #define nmi_nesting_postprocess()					\
  	do {								\
  		if (cmpxchg(&__get_cpu_var(nmi_state),			\
  		    NMI_EXECUTING, NMI_NOT_RUNNING) != NMI_EXECUTING)	\
  			goto nmi_restart;				\
  	} while (0)
  #else /* x86_64 */
  /*
   * In x86_64 things are a bit more difficult. This has the same problem
   * where an NMI hitting a breakpoint that calls iret will remove the
   * NMI context, allowing a nested NMI to enter. What makes this more
   * difficult is that both NMIs and breakpoints have their own stack.
   * When a new NMI or breakpoint is executed, the stack is set to a fixed
   * point. If an NMI is nested, it will have its stack set at that same
   * fixed address that the first NMI had, and will start corrupting the
   * stack. This is handled in entry_64.S, but the same problem exists with
   * the breakpoint stack.
   *
   * If a breakpoint is being processed, and the debug stack is being used,
   * if an NMI comes in and also hits a breakpoint, the stack pointer
   * will be set to the same fixed address as the breakpoint that was
   * interrupted, causing that stack to be corrupted. To handle this case,
   * check if the stack that was interrupted is the debug stack, and if
   * so, change the IDT so that new breakpoints will use the current stack
   * and not switch to the fixed address. On return of the NMI, switch back
   * to the original IDT.
   */
  static DEFINE_PER_CPU(int, update_debug_stack);

  static inline void nmi_nesting_preprocess(struct pt_regs *regs)
  {

  	/*
  	 * If we interrupted a breakpoint, it is possible that
  	 * the nmi handler will have breakpoints too. We need to
  	 * change the IDT such that breakpoints that happen here
  	 * continue to use the NMI stack.
  	 */
  	if (unlikely(is_debug_stack(regs->sp))) {
  		debug_stack_set_zero();

  		__get_cpu_var(update_debug_stack) = 1;

  	}

  }
  
  static inline void nmi_nesting_postprocess(void)
  {
  	if (unlikely(__get_cpu_var(update_debug_stack)))
  		debug_stack_reset();
  }
  #endif
  
  dotraplinkage notrace __kprobes void
  do_nmi(struct pt_regs *regs, long error_code)
  {
  	nmi_nesting_preprocess(regs);

  	nmi_enter();
  
  	inc_irq_stat(__nmi_count);
  
  	if (!ignore_nmis)
  		default_do_nmi(regs);
  
  	nmi_exit();

  	/* On i386, may loop back to preprocess */
  	nmi_nesting_postprocess();

  }
  
  void stop_nmi(void)
  {
  	ignore_nmis++;
  }
  
  void restart_nmi(void)
  {
  	ignore_nmis--;
  }

  
  /* reset the back-to-back NMI logic */
  void local_touch_nmi(void)
  {
  	__this_cpu_write(last_nmi_rip, 0);
  }