Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit d04c56f73c30a5e593202ecfcf25ed43d42363a2

Authored by Paul Mackerras 2006-10-04 14:47:49 +0800

Exists in master and in 7 other branches

[POWERPC] Lazy interrupt disabling for 64-bit machines

This implements a lazy strategy for disabling interrupts. This means
that local_irq_disable() et al. just clear the 'interrupts are
enabled' flag in the paca. If an interrupt comes along, the interrupt
entry code notices that interrupts are supposed to be disabled, and
clears the EE bit in SRR1, clears the 'interrupts are hard-enabled'
flag in the paca, and returns. This means that interrupts only
actually get disabled in the processor when an interrupt comes along.

When interrupts are enabled by local_irq_enable() et al., the code
sets the interrupts-enabled flag in the paca, and then checks whether
interrupts got hard-disabled. If so, it also sets the EE bit in the
MSR to hard-enable the interrupts.

This has the potential to improve performance, and also makes it
easier to make a kernel that can boot on iSeries and on other 64-bit
machines, since this lazy-disable strategy is very similar to the
soft-disable strategy that iSeries already uses.

This version renames paca->proc_enabled to paca->soft_enabled, and
changes a couple of soft-disables in the kexec code to hard-disables,
which should fix the crash that Michael Ellerman saw. This doesn't
yet use a reserved CR field for the soft_enabled and hard_enabled
flags. This applies on top of Stephen Rothwell's patches to make it
possible to build a combined iSeries/other kernel.

Signed-off-by: Paul Mackerras <paulus@samba.org>

Showing 12 changed files with 160 additions and 111 deletions Inline Diff

arch/powerpc/kernel/asm-offsets.c
arch/powerpc/kernel/crash.c
arch/powerpc/kernel/entry_64.S
arch/powerpc/kernel/head_64.S
arch/powerpc/kernel/idle_power4.S
arch/powerpc/kernel/irq.c
arch/powerpc/kernel/ppc_ksyms.c
arch/powerpc/kernel/setup_64.c
arch/powerpc/platforms/iseries/ksyms.c
arch/powerpc/platforms/iseries/misc.S
include/asm-powerpc/hw_irq.h
include/asm-powerpc/paca.h

arch/powerpc/kernel/asm-offsets.c

Diff comments View file @ d04c56f

 /*
  * This program is used to generate definitions needed by
  * assembly language modules.
  *
  * We use the technique used in the OSF Mach kernel code:
  * generate asm statements containing #defines,
  * compile this file to assembler, and then extract the
  * #defines from the assembly-language output.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/mman.h>
 #include <linux/mm.h>
 #ifdef CONFIG_PPC64
 #include <linux/time.h>
 #include <linux/hardirq.h>
 #else
 #include <linux/ptrace.h>
 #include <linux/suspend.h>
 #endif
 #include <asm/io.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/cputable.h>
 #include <asm/thread_info.h>
 #include <asm/rtas.h>
 #include <asm/vdso_datapage.h>
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 #include <asm/lppaca.h>
 #include <asm/cache.h>
 #include <asm/compat.h>
 #include <asm/mmu.h>
 #include <asm/hvcall.h>
 #endif
 #define DEFINE(sym, val) \
 	asm volatile("\n->" #sym " %0 " #val : : "i" (val))
 #define BLANK() asm volatile("\n->" : : )
 int main(void)
 {
 	DEFINE(THREAD, offsetof(struct task_struct, thread));
 	DEFINE(MM, offsetof(struct task_struct, mm));
 #ifdef CONFIG_PPC64
 	DEFINE(AUDITCONTEXT, offsetof(struct task_struct, audit_context));
 #else
 	DEFINE(THREAD_INFO, offsetof(struct task_struct, thread_info));
 	DEFINE(PTRACE, offsetof(struct task_struct, ptrace));
 #endif /* CONFIG_PPC64 */
 	DEFINE(KSP, offsetof(struct thread_struct, ksp));
 	DEFINE(PT_REGS, offsetof(struct thread_struct, regs));
 	DEFINE(THREAD_FPEXC_MODE, offsetof(struct thread_struct, fpexc_mode));
 	DEFINE(THREAD_FPR0, offsetof(struct thread_struct, fpr[0]));
 	DEFINE(THREAD_FPSCR, offsetof(struct thread_struct, fpscr));
 #ifdef CONFIG_ALTIVEC
 	DEFINE(THREAD_VR0, offsetof(struct thread_struct, vr[0]));
 	DEFINE(THREAD_VRSAVE, offsetof(struct thread_struct, vrsave));
 	DEFINE(THREAD_VSCR, offsetof(struct thread_struct, vscr));
 	DEFINE(THREAD_USED_VR, offsetof(struct thread_struct, used_vr));
 #endif /* CONFIG_ALTIVEC */
 #ifdef CONFIG_PPC64
 	DEFINE(KSP_VSID, offsetof(struct thread_struct, ksp_vsid));
 #else /* CONFIG_PPC64 */
 	DEFINE(PGDIR, offsetof(struct thread_struct, pgdir));
 	DEFINE(LAST_SYSCALL, offsetof(struct thread_struct, last_syscall));
 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 	DEFINE(THREAD_DBCR0, offsetof(struct thread_struct, dbcr0));
 	DEFINE(PT_PTRACED, PT_PTRACED);
 #endif
 #ifdef CONFIG_SPE
 	DEFINE(THREAD_EVR0, offsetof(struct thread_struct, evr[0]));
 	DEFINE(THREAD_ACC, offsetof(struct thread_struct, acc));
 	DEFINE(THREAD_SPEFSCR, offsetof(struct thread_struct, spefscr));
 	DEFINE(THREAD_USED_SPE, offsetof(struct thread_struct, used_spe));
 #endif /* CONFIG_SPE */
 #endif /* CONFIG_PPC64 */
 	DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
 	DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
 	DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
 	DEFINE(TI_TASK, offsetof(struct thread_info, task));
 #ifdef CONFIG_PPC32
 	DEFINE(TI_EXECDOMAIN, offsetof(struct thread_info, exec_domain));
 	DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
 #endif /* CONFIG_PPC32 */
 #ifdef CONFIG_PPC64
 	DEFINE(DCACHEL1LINESIZE, offsetof(struct ppc64_caches, dline_size));
 	DEFINE(DCACHEL1LOGLINESIZE, offsetof(struct ppc64_caches, log_dline_size));
 	DEFINE(DCACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, dlines_per_page));
 	DEFINE(ICACHEL1LINESIZE, offsetof(struct ppc64_caches, iline_size));
 	DEFINE(ICACHEL1LOGLINESIZE, offsetof(struct ppc64_caches, log_iline_size));
 	DEFINE(ICACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, ilines_per_page));
 	/* paca */
 	DEFINE(PACA_SIZE, sizeof(struct paca_struct));
 	DEFINE(PACAPACAINDEX, offsetof(struct paca_struct, paca_index));
 	DEFINE(PACAPROCSTART, offsetof(struct paca_struct, cpu_start));
 	DEFINE(PACAKSAVE, offsetof(struct paca_struct, kstack));
 	DEFINE(PACACURRENT, offsetof(struct paca_struct, __current));
 	DEFINE(PACASAVEDMSR, offsetof(struct paca_struct, saved_msr));
 	DEFINE(PACASTABREAL, offsetof(struct paca_struct, stab_real));
 	DEFINE(PACASTABVIRT, offsetof(struct paca_struct, stab_addr));
 	DEFINE(PACASTABRR, offsetof(struct paca_struct, stab_rr));
 	DEFINE(PACAR1, offsetof(struct paca_struct, saved_r1));
 	DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
-	DEFINE(PACAPROCENABLED, offsetof(struct paca_struct, proc_enabled));
+	DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
+	DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
 	DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));
 	DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
 	DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
 	DEFINE(PACACONTEXTSLLP, offsetof(struct paca_struct, context.sllp));
 	DEFINE(PACAVMALLOCSLLP, offsetof(struct paca_struct, vmalloc_sllp));
 #ifdef CONFIG_HUGETLB_PAGE
 	DEFINE(PACALOWHTLBAREAS, offsetof(struct paca_struct, context.low_htlb_areas));
 	DEFINE(PACAHIGHHTLBAREAS, offsetof(struct paca_struct, context.high_htlb_areas));
 #endif /* CONFIG_HUGETLB_PAGE */
 	DEFINE(PACA_EXGEN, offsetof(struct paca_struct, exgen));
 	DEFINE(PACA_EXMC, offsetof(struct paca_struct, exmc));
 	DEFINE(PACA_EXSLB, offsetof(struct paca_struct, exslb));
 	DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp));
 	DEFINE(PACALPPACAPTR, offsetof(struct paca_struct, lppaca_ptr));
 	DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id));
 	DEFINE(PACA_STARTPURR, offsetof(struct paca_struct, startpurr));
 	DEFINE(PACA_USER_TIME, offsetof(struct paca_struct, user_time));
 	DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time));
 	DEFINE(PACA_SLBSHADOWPTR, offsetof(struct paca_struct, slb_shadow_ptr));
 	DEFINE(PACA_DATA_OFFSET, offsetof(struct paca_struct, data_offset));
 	DEFINE(SLBSHADOW_STACKVSID,
 	       offsetof(struct slb_shadow, save_area[SLB_NUM_BOLTED - 1].vsid));
 	DEFINE(SLBSHADOW_STACKESID,
 	       offsetof(struct slb_shadow, save_area[SLB_NUM_BOLTED - 1].esid));
 	DEFINE(LPPACASRR0, offsetof(struct lppaca, saved_srr0));
 	DEFINE(LPPACASRR1, offsetof(struct lppaca, saved_srr1));
 	DEFINE(LPPACAANYINT, offsetof(struct lppaca, int_dword.any_int));
 	DEFINE(LPPACADECRINT, offsetof(struct lppaca, int_dword.fields.decr_int));
 	DEFINE(SLBSHADOW_SAVEAREA, offsetof(struct slb_shadow, save_area));
 #endif /* CONFIG_PPC64 */
 	/* RTAS */
 	DEFINE(RTASBASE, offsetof(struct rtas_t, base));
 	DEFINE(RTASENTRY, offsetof(struct rtas_t, entry));
 	/* Interrupt register frame */
 	DEFINE(STACK_FRAME_OVERHEAD, STACK_FRAME_OVERHEAD);
 #ifndef CONFIG_PPC64
 	DEFINE(INT_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs));
 #else /* CONFIG_PPC64 */
 	DEFINE(SWITCH_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs));
 	/* 288 = # of volatile regs, int & fp, for leaf routines */
 	/* which do not stack a frame.  See the PPC64 ABI.       */
 	DEFINE(INT_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 288);
 	/* Create extra stack space for SRR0 and SRR1 when calling prom/rtas. */
 	DEFINE(PROM_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
 	DEFINE(RTAS_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
 	/* hcall statistics */
 	DEFINE(HCALL_STAT_SIZE, sizeof(struct hcall_stats));
 	DEFINE(HCALL_STAT_CALLS, offsetof(struct hcall_stats, num_calls));
 	DEFINE(HCALL_STAT_TB, offsetof(struct hcall_stats, tb_total));
 	DEFINE(HCALL_STAT_PURR, offsetof(struct hcall_stats, purr_total));
 #endif /* CONFIG_PPC64 */
 	DEFINE(GPR0, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[0]));
 	DEFINE(GPR1, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[1]));
 	DEFINE(GPR2, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[2]));
 	DEFINE(GPR3, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[3]));
 	DEFINE(GPR4, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[4]));
 	DEFINE(GPR5, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[5]));
 	DEFINE(GPR6, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[6]));
 	DEFINE(GPR7, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[7]));
 	DEFINE(GPR8, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[8]));
 	DEFINE(GPR9, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[9]));
 	DEFINE(GPR10, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[10]));
 	DEFINE(GPR11, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[11]));
 	DEFINE(GPR12, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[12]));
 	DEFINE(GPR13, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[13]));
 #ifndef CONFIG_PPC64
 	DEFINE(GPR14, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[14]));
 	DEFINE(GPR15, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[15]));
 	DEFINE(GPR16, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[16]));
 	DEFINE(GPR17, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[17]));
 	DEFINE(GPR18, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[18]));
 	DEFINE(GPR19, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[19]));
 	DEFINE(GPR20, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[20]));
 	DEFINE(GPR21, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[21]));
 	DEFINE(GPR22, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[22]));
 	DEFINE(GPR23, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[23]));
 	DEFINE(GPR24, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[24]));
 	DEFINE(GPR25, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[25]));
 	DEFINE(GPR26, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[26]));
 	DEFINE(GPR27, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[27]));
 	DEFINE(GPR28, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[28]));
 	DEFINE(GPR29, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[29]));
 	DEFINE(GPR30, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[30]));
 	DEFINE(GPR31, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[31]));
 #endif /* CONFIG_PPC64 */
 	/*
 	 * Note: these symbols include _ because they overlap with special
 	 * register names
 	 */
 	DEFINE(_NIP, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, nip));
 	DEFINE(_MSR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, msr));
 	DEFINE(_CTR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, ctr));
 	DEFINE(_LINK, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, link));
 	DEFINE(_CCR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, ccr));
 	DEFINE(_XER, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, xer));
 	DEFINE(_DAR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dar));
 	DEFINE(_DSISR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dsisr));
 	DEFINE(ORIG_GPR3, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, orig_gpr3));
 	DEFINE(RESULT, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, result));
 	DEFINE(_TRAP, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, trap));
 #ifndef CONFIG_PPC64
 	DEFINE(_MQ, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, mq));
 	/*
 	 * The PowerPC 400-class & Book-E processors have neither the DAR
 	 * nor the DSISR SPRs. Hence, we overload them to hold the similar
 	 * DEAR and ESR SPRs for such processors.  For critical interrupts
 	 * we use them to hold SRR0 and SRR1.
 	 */
 	DEFINE(_DEAR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dar));
 	DEFINE(_ESR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dsisr));
 #else /* CONFIG_PPC64 */
 	DEFINE(SOFTE, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, softe));
 	/* These _only_ to be used with {PROM,RTAS}_FRAME_SIZE!!! */
 	DEFINE(_SRR0, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs));
 	DEFINE(_SRR1, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs)+8);
 #endif /* CONFIG_PPC64 */
 	DEFINE(CLONE_VM, CLONE_VM);
 	DEFINE(CLONE_UNTRACED, CLONE_UNTRACED);
 #ifndef CONFIG_PPC64
 	DEFINE(MM_PGD, offsetof(struct mm_struct, pgd));
 #endif /* ! CONFIG_PPC64 */
 	/* About the CPU features table */
 	DEFINE(CPU_SPEC_ENTRY_SIZE, sizeof(struct cpu_spec));
 	DEFINE(CPU_SPEC_PVR_MASK, offsetof(struct cpu_spec, pvr_mask));
 	DEFINE(CPU_SPEC_PVR_VALUE, offsetof(struct cpu_spec, pvr_value));
 	DEFINE(CPU_SPEC_FEATURES, offsetof(struct cpu_spec, cpu_features));
 	DEFINE(CPU_SPEC_SETUP, offsetof(struct cpu_spec, cpu_setup));
 	DEFINE(CPU_SPEC_RESTORE, offsetof(struct cpu_spec, cpu_restore));
 #ifndef CONFIG_PPC64
 	DEFINE(pbe_address, offsetof(struct pbe, address));
 	DEFINE(pbe_orig_address, offsetof(struct pbe, orig_address));
 	DEFINE(pbe_next, offsetof(struct pbe, next));
 	DEFINE(TASK_SIZE, TASK_SIZE);
 	DEFINE(NUM_USER_SEGMENTS, TASK_SIZE>>28);
 #endif /* ! CONFIG_PPC64 */
 	/* datapage offsets for use by vdso */
 	DEFINE(CFG_TB_ORIG_STAMP, offsetof(struct vdso_data, tb_orig_stamp));
 	DEFINE(CFG_TB_TICKS_PER_SEC, offsetof(struct vdso_data, tb_ticks_per_sec));
 	DEFINE(CFG_TB_TO_XS, offsetof(struct vdso_data, tb_to_xs));
 	DEFINE(CFG_STAMP_XSEC, offsetof(struct vdso_data, stamp_xsec));
 	DEFINE(CFG_TB_UPDATE_COUNT, offsetof(struct vdso_data, tb_update_count));
 	DEFINE(CFG_TZ_MINUTEWEST, offsetof(struct vdso_data, tz_minuteswest));
 	DEFINE(CFG_TZ_DSTTIME, offsetof(struct vdso_data, tz_dsttime));
 	DEFINE(CFG_SYSCALL_MAP32, offsetof(struct vdso_data, syscall_map_32));
 	DEFINE(WTOM_CLOCK_SEC, offsetof(struct vdso_data, wtom_clock_sec));
 	DEFINE(WTOM_CLOCK_NSEC, offsetof(struct vdso_data, wtom_clock_nsec));
 #ifdef CONFIG_PPC64
 	DEFINE(CFG_SYSCALL_MAP64, offsetof(struct vdso_data, syscall_map_64));
 	DEFINE(TVAL64_TV_SEC, offsetof(struct timeval, tv_sec));
 	DEFINE(TVAL64_TV_USEC, offsetof(struct timeval, tv_usec));
 	DEFINE(TVAL32_TV_SEC, offsetof(struct compat_timeval, tv_sec));
 	DEFINE(TVAL32_TV_USEC, offsetof(struct compat_timeval, tv_usec));
 	DEFINE(TSPC64_TV_SEC, offsetof(struct timespec, tv_sec));
 	DEFINE(TSPC64_TV_NSEC, offsetof(struct timespec, tv_nsec));
 	DEFINE(TSPC32_TV_SEC, offsetof(struct compat_timespec, tv_sec));
 	DEFINE(TSPC32_TV_NSEC, offsetof(struct compat_timespec, tv_nsec));
 #else
 	DEFINE(TVAL32_TV_SEC, offsetof(struct timeval, tv_sec));
 	DEFINE(TVAL32_TV_USEC, offsetof(struct timeval, tv_usec));
 	DEFINE(TSPC32_TV_SEC, offsetof(struct timespec, tv_sec));
 	DEFINE(TSPC32_TV_NSEC, offsetof(struct timespec, tv_nsec));
 #endif
 	/* timeval/timezone offsets for use by vdso */
 	DEFINE(TZONE_TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
 	DEFINE(TZONE_TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
 	/* Other bits used by the vdso */
 	DEFINE(CLOCK_REALTIME, CLOCK_REALTIME);
 	DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC);
 	DEFINE(NSEC_PER_SEC, NSEC_PER_SEC);
 	DEFINE(CLOCK_REALTIME_RES, TICK_NSEC);
 	return 0;
 }

arch/powerpc/kernel/crash.c

Diff comments View file @ d04c56f

1	/*	1	/*
2	* Architecture specific (PPC64) functions for kexec based crash dumps.	2	* Architecture specific (PPC64) functions for kexec based crash dumps.
3	*	3	*
4	* Copyright (C) 2005, IBM Corp.	4	* Copyright (C) 2005, IBM Corp.
5	*	5	*
6	* Created by: Haren Myneni	6	* Created by: Haren Myneni
7	*	7	*
8	* This source code is licensed under the GNU General Public License,	8	* This source code is licensed under the GNU General Public License,
9	* Version 2. See the file COPYING for more details.	9	* Version 2. See the file COPYING for more details.
10	*	10	*
11	*/	11	*/
12		12
13	#undef DEBUG	13	#undef DEBUG
14		14
15	#include <linux/kernel.h>	15	#include <linux/kernel.h>
16	#include <linux/smp.h>	16	#include <linux/smp.h>
17	#include <linux/reboot.h>	17	#include <linux/reboot.h>
18	#include <linux/kexec.h>	18	#include <linux/kexec.h>
19	#include <linux/bootmem.h>	19	#include <linux/bootmem.h>
20	#include <linux/crash_dump.h>	20	#include <linux/crash_dump.h>
21	#include <linux/delay.h>	21	#include <linux/delay.h>
22	#include <linux/elf.h>	22	#include <linux/elf.h>
23	#include <linux/elfcore.h>	23	#include <linux/elfcore.h>
24	#include <linux/init.h>	24	#include <linux/init.h>
25	#include <linux/irq.h>	25	#include <linux/irq.h>
26	#include <linux/types.h>	26	#include <linux/types.h>
27	#include <linux/irq.h>	27	#include <linux/irq.h>
28		28
29	#include <asm/processor.h>	29	#include <asm/processor.h>
30	#include <asm/machdep.h>	30	#include <asm/machdep.h>
31	#include <asm/kexec.h>	31	#include <asm/kexec.h>
32	#include <asm/kdump.h>	32	#include <asm/kdump.h>
33	#include <asm/lmb.h>	33	#include <asm/lmb.h>
34	#include <asm/firmware.h>	34	#include <asm/firmware.h>
35	#include <asm/smp.h>	35	#include <asm/smp.h>
36		36
37	#ifdef DEBUG	37	#ifdef DEBUG
38	#include <asm/udbg.h>	38	#include <asm/udbg.h>
39	#define DBG(fmt...) udbg_printf(fmt)	39	#define DBG(fmt...) udbg_printf(fmt)
40	#else	40	#else
41	#define DBG(fmt...)	41	#define DBG(fmt...)
42	#endif	42	#endif
43		43
44	/* This keeps a track of which one is crashing cpu. */	44	/* This keeps a track of which one is crashing cpu. */
45	int crashing_cpu = -1;	45	int crashing_cpu = -1;
46	static cpumask_t cpus_in_crash = CPU_MASK_NONE;	46	static cpumask_t cpus_in_crash = CPU_MASK_NONE;
47	cpumask_t cpus_in_sr = CPU_MASK_NONE;	47	cpumask_t cpus_in_sr = CPU_MASK_NONE;
48		48
49	static u32 append_elf_note(u32 buf, char name, unsigned type, void data,	49	static u32 append_elf_note(u32 buf, char name, unsigned type, void data,
50	size_t data_len)	50	size_t data_len)
51	{	51	{
52	struct elf_note note;	52	struct elf_note note;
53		53
54	note.n_namesz = strlen(name) + 1;	54	note.n_namesz = strlen(name) + 1;
55	note.n_descsz = data_len;	55	note.n_descsz = data_len;
56	note.n_type = type;	56	note.n_type = type;
57	memcpy(buf, &note, sizeof(note));	57	memcpy(buf, &note, sizeof(note));
58	buf += (sizeof(note) +3)/4;	58	buf += (sizeof(note) +3)/4;
59	memcpy(buf, name, note.n_namesz);	59	memcpy(buf, name, note.n_namesz);
60	buf += (note.n_namesz + 3)/4;	60	buf += (note.n_namesz + 3)/4;
61	memcpy(buf, data, note.n_descsz);	61	memcpy(buf, data, note.n_descsz);
62	buf += (note.n_descsz + 3)/4;	62	buf += (note.n_descsz + 3)/4;
63		63
64	return buf;	64	return buf;
65	}	65	}
66		66
67	static void final_note(u32 *buf)	67	static void final_note(u32 *buf)
68	{	68	{
69	struct elf_note note;	69	struct elf_note note;
70		70
71	note.n_namesz = 0;	71	note.n_namesz = 0;
72	note.n_descsz = 0;	72	note.n_descsz = 0;
73	note.n_type = 0;	73	note.n_type = 0;
74	memcpy(buf, &note, sizeof(note));	74	memcpy(buf, &note, sizeof(note));
75	}	75	}
76		76
77	static void crash_save_this_cpu(struct pt_regs *regs, int cpu)	77	static void crash_save_this_cpu(struct pt_regs *regs, int cpu)
78	{	78	{
79	struct elf_prstatus prstatus;	79	struct elf_prstatus prstatus;
80	u32 *buf;	80	u32 *buf;
81		81
82	if ((cpu < 0) \|\| (cpu >= NR_CPUS))	82	if ((cpu < 0) \|\| (cpu >= NR_CPUS))
83	return;	83	return;
84		84
85	/* Using ELF notes here is opportunistic.	85	/* Using ELF notes here is opportunistic.
86	* I need a well defined structure format	86	* I need a well defined structure format
87	* for the data I pass, and I need tags	87	* for the data I pass, and I need tags
88	* on the data to indicate what information I have	88	* on the data to indicate what information I have
89	* squirrelled away. ELF notes happen to provide	89	* squirrelled away. ELF notes happen to provide
90	* all of that that no need to invent something new.	90	* all of that that no need to invent something new.
91	*/	91	*/
92	buf = (u32*)per_cpu_ptr(crash_notes, cpu);	92	buf = (u32*)per_cpu_ptr(crash_notes, cpu);
93	if (!buf)	93	if (!buf)
94	return;	94	return;
95		95
96	memset(&prstatus, 0, sizeof(prstatus));	96	memset(&prstatus, 0, sizeof(prstatus));
97	prstatus.pr_pid = current->pid;	97	prstatus.pr_pid = current->pid;
98	elf_core_copy_regs(&prstatus.pr_reg, regs);	98	elf_core_copy_regs(&prstatus.pr_reg, regs);
99	buf = append_elf_note(buf, "CORE", NT_PRSTATUS, &prstatus,	99	buf = append_elf_note(buf, "CORE", NT_PRSTATUS, &prstatus,
100	sizeof(prstatus));	100	sizeof(prstatus));
101	final_note(buf);	101	final_note(buf);
102	}	102	}
103		103
104	#ifdef CONFIG_SMP	104	#ifdef CONFIG_SMP
105	static atomic_t enter_on_soft_reset = ATOMIC_INIT(0);	105	static atomic_t enter_on_soft_reset = ATOMIC_INIT(0);
106		106
107	void crash_ipi_callback(struct pt_regs *regs)	107	void crash_ipi_callback(struct pt_regs *regs)
108	{	108	{
109	int cpu = smp_processor_id();	109	int cpu = smp_processor_id();
110		110
111	if (!cpu_online(cpu))	111	if (!cpu_online(cpu))
112	return;	112	return;
113		113
114	local_irq_disable();	114	hard_irq_disable();
115	if (!cpu_isset(cpu, cpus_in_crash))	115	if (!cpu_isset(cpu, cpus_in_crash))
116	crash_save_this_cpu(regs, cpu);	116	crash_save_this_cpu(regs, cpu);
117	cpu_set(cpu, cpus_in_crash);	117	cpu_set(cpu, cpus_in_crash);
118		118
119	/*	119	/*
120	* Entered via soft-reset - could be the kdump	120	* Entered via soft-reset - could be the kdump
121	* process is invoked using soft-reset or user activated	121	* process is invoked using soft-reset or user activated
122	* it if some CPU did not respond to an IPI.	122	* it if some CPU did not respond to an IPI.
123	* For soft-reset, the secondary CPU can enter this func	123	* For soft-reset, the secondary CPU can enter this func
124	* twice. 1 - using IPI, and 2. soft-reset.	124	* twice. 1 - using IPI, and 2. soft-reset.
125	* Tell the kexec CPU that entered via soft-reset and ready	125	* Tell the kexec CPU that entered via soft-reset and ready
126	* to go down.	126	* to go down.
127	*/	127	*/
128	if (cpu_isset(cpu, cpus_in_sr)) {	128	if (cpu_isset(cpu, cpus_in_sr)) {
129	cpu_clear(cpu, cpus_in_sr);	129	cpu_clear(cpu, cpus_in_sr);
130	atomic_inc(&enter_on_soft_reset);	130	atomic_inc(&enter_on_soft_reset);
131	}	131	}
132		132
133	/*	133	/*
134	* Starting the kdump boot.	134	* Starting the kdump boot.
135	* This barrier is needed to make sure that all CPUs are stopped.	135	* This barrier is needed to make sure that all CPUs are stopped.
136	* If not, soft-reset will be invoked to bring other CPUs.	136	* If not, soft-reset will be invoked to bring other CPUs.
137	*/	137	*/
138	while (!cpu_isset(crashing_cpu, cpus_in_crash))	138	while (!cpu_isset(crashing_cpu, cpus_in_crash))
139	cpu_relax();	139	cpu_relax();
140		140
141	if (ppc_md.kexec_cpu_down)	141	if (ppc_md.kexec_cpu_down)
142	ppc_md.kexec_cpu_down(1, 1);	142	ppc_md.kexec_cpu_down(1, 1);
143		143
144	#ifdef CONFIG_PPC64	144	#ifdef CONFIG_PPC64
145	kexec_smp_wait();	145	kexec_smp_wait();
146	#else	146	#else
147	for (;;); /* FIXME */	147	for (;;); /* FIXME */
148	#endif	148	#endif
149		149
150	/* NOTREACHED */	150	/* NOTREACHED */
151	}	151	}
152		152
153	/*	153	/*
154	* Wait until all CPUs are entered via soft-reset.	154	* Wait until all CPUs are entered via soft-reset.
155	*/	155	*/
156	static void crash_soft_reset_check(int cpu)	156	static void crash_soft_reset_check(int cpu)
157	{	157	{
158	unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */	158	unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */
159		159
160	cpu_clear(cpu, cpus_in_sr);	160	cpu_clear(cpu, cpus_in_sr);
161	while (atomic_read(&enter_on_soft_reset) != ncpus)	161	while (atomic_read(&enter_on_soft_reset) != ncpus)
162	cpu_relax();	162	cpu_relax();
163	}	163	}
164		164
165		165
166	static void crash_kexec_prepare_cpus(int cpu)	166	static void crash_kexec_prepare_cpus(int cpu)
167	{	167	{
168	unsigned int msecs;	168	unsigned int msecs;
169		169
170	unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */	170	unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */
171		171
172	crash_send_ipi(crash_ipi_callback);	172	crash_send_ipi(crash_ipi_callback);
173	smp_wmb();	173	smp_wmb();
174		174
175	/*	175	/*
176	* FIXME: Until we will have the way to stop other CPUSs reliabally,	176	* FIXME: Until we will have the way to stop other CPUSs reliabally,
177	* the crash CPU will send an IPI and wait for other CPUs to	177	* the crash CPU will send an IPI and wait for other CPUs to
178	* respond.	178	* respond.
179	* Delay of at least 10 seconds.	179	* Delay of at least 10 seconds.
180	*/	180	*/
181	printk(KERN_EMERG "Sending IPI to other cpus...\n");	181	printk(KERN_EMERG "Sending IPI to other cpus...\n");
182	msecs = 10000;	182	msecs = 10000;
183	while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) {	183	while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) {
184	cpu_relax();	184	cpu_relax();
185	mdelay(1);	185	mdelay(1);
186	}	186	}
187		187
188	/* Would it be better to replace the trap vector here? */	188	/* Would it be better to replace the trap vector here? */
189		189
190	/*	190	/*
191	* FIXME: In case if we do not get all CPUs, one possibility: ask the	191	* FIXME: In case if we do not get all CPUs, one possibility: ask the
192	* user to do soft reset such that we get all.	192	* user to do soft reset such that we get all.
193	* Soft-reset will be used until better mechanism is implemented.	193	* Soft-reset will be used until better mechanism is implemented.
194	*/	194	*/
195	if (cpus_weight(cpus_in_crash) < ncpus) {	195	if (cpus_weight(cpus_in_crash) < ncpus) {
196	printk(KERN_EMERG "done waiting: %d cpu(s) not responding\n",	196	printk(KERN_EMERG "done waiting: %d cpu(s) not responding\n",
197	ncpus - cpus_weight(cpus_in_crash));	197	ncpus - cpus_weight(cpus_in_crash));
198	printk(KERN_EMERG "Activate soft-reset to stop other cpu(s)\n");	198	printk(KERN_EMERG "Activate soft-reset to stop other cpu(s)\n");
199	cpus_in_sr = CPU_MASK_NONE;	199	cpus_in_sr = CPU_MASK_NONE;
200	atomic_set(&enter_on_soft_reset, 0);	200	atomic_set(&enter_on_soft_reset, 0);
201	while (cpus_weight(cpus_in_crash) < ncpus)	201	while (cpus_weight(cpus_in_crash) < ncpus)
202	cpu_relax();	202	cpu_relax();
203	}	203	}
204	/*	204	/*
205	* Make sure all CPUs are entered via soft-reset if the kdump is	205	* Make sure all CPUs are entered via soft-reset if the kdump is
206	* invoked using soft-reset.	206	* invoked using soft-reset.
207	*/	207	*/
208	if (cpu_isset(cpu, cpus_in_sr))	208	if (cpu_isset(cpu, cpus_in_sr))
209	crash_soft_reset_check(cpu);	209	crash_soft_reset_check(cpu);
210	/* Leave the IPI callback set */	210	/* Leave the IPI callback set */
211	}	211	}
212		212
213	/*	213	/*
214	* This function will be called by secondary cpus or by kexec cpu	214	* This function will be called by secondary cpus or by kexec cpu
215	* if soft-reset is activated to stop some CPUs.	215	* if soft-reset is activated to stop some CPUs.
216	*/	216	*/
217	void crash_kexec_secondary(struct pt_regs *regs)	217	void crash_kexec_secondary(struct pt_regs *regs)
218	{	218	{
219	int cpu = smp_processor_id();	219	int cpu = smp_processor_id();
220	unsigned long flags;	220	unsigned long flags;
221	int msecs = 5;	221	int msecs = 5;
222		222
223	local_irq_save(flags);	223	local_irq_save(flags);
224	/* Wait 5ms if the kexec CPU is not entered yet. */	224	/* Wait 5ms if the kexec CPU is not entered yet. */
225	while (crashing_cpu < 0) {	225	while (crashing_cpu < 0) {
226	if (--msecs < 0) {	226	if (--msecs < 0) {
227	/*	227	/*
228	* Either kdump image is not loaded or	228	* Either kdump image is not loaded or
229	* kdump process is not started - Probably xmon	229	* kdump process is not started - Probably xmon
230	* exited using 'x'(exit and recover) or	230	* exited using 'x'(exit and recover) or
231	* kexec_should_crash() failed for all running tasks.	231	* kexec_should_crash() failed for all running tasks.
232	*/	232	*/
233	cpu_clear(cpu, cpus_in_sr);	233	cpu_clear(cpu, cpus_in_sr);
234	local_irq_restore(flags);	234	local_irq_restore(flags);
235	return;	235	return;
236	}	236	}
237	mdelay(1);	237	mdelay(1);
238	cpu_relax();	238	cpu_relax();
239	}	239	}
240	if (cpu == crashing_cpu) {	240	if (cpu == crashing_cpu) {
241	/*	241	/*
242	* Panic CPU will enter this func only via soft-reset.	242	* Panic CPU will enter this func only via soft-reset.
243	* Wait until all secondary CPUs entered and	243	* Wait until all secondary CPUs entered and
244	* then start kexec boot.	244	* then start kexec boot.
245	*/	245	*/
246	crash_soft_reset_check(cpu);	246	crash_soft_reset_check(cpu);
247	cpu_set(crashing_cpu, cpus_in_crash);	247	cpu_set(crashing_cpu, cpus_in_crash);
248	if (ppc_md.kexec_cpu_down)	248	if (ppc_md.kexec_cpu_down)
249	ppc_md.kexec_cpu_down(1, 0);	249	ppc_md.kexec_cpu_down(1, 0);
250	machine_kexec(kexec_crash_image);	250	machine_kexec(kexec_crash_image);
251	/* NOTREACHED */	251	/* NOTREACHED */
252	}	252	}
253	crash_ipi_callback(regs);	253	crash_ipi_callback(regs);
254	}	254	}
255		255
256	#else	256	#else
257	static void crash_kexec_prepare_cpus(int cpu)	257	static void crash_kexec_prepare_cpus(int cpu)
258	{	258	{
259	/*	259	/*
260	* move the secondarys to us so that we can copy	260	* move the secondarys to us so that we can copy
261	* the new kernel 0-0x100 safely	261	* the new kernel 0-0x100 safely
262	*	262	*
263	* do this if kexec in setup.c ?	263	* do this if kexec in setup.c ?
264	*/	264	*/
265	#ifdef CONFIG_PPC64	265	#ifdef CONFIG_PPC64
266	smp_release_cpus();	266	smp_release_cpus();
267	#else	267	#else
268	/* FIXME */	268	/* FIXME */
269	#endif	269	#endif
270	}	270	}
271		271
272	void crash_kexec_secondary(struct pt_regs *regs)	272	void crash_kexec_secondary(struct pt_regs *regs)
273	{	273	{
274	cpus_in_sr = CPU_MASK_NONE;	274	cpus_in_sr = CPU_MASK_NONE;
275	}	275	}
276	#endif	276	#endif
277		277
278	void default_machine_crash_shutdown(struct pt_regs *regs)	278	void default_machine_crash_shutdown(struct pt_regs *regs)
279	{	279	{
280	unsigned int irq;	280	unsigned int irq;
281		281
282	/*	282	/*
283	* This function is only called after the system	283	* This function is only called after the system
284	* has panicked or is otherwise in a critical state.	284	* has panicked or is otherwise in a critical state.
285	* The minimum amount of code to allow a kexec'd kernel	285	* The minimum amount of code to allow a kexec'd kernel
286	* to run successfully needs to happen here.	286	* to run successfully needs to happen here.
287	*	287	*
288	* In practice this means stopping other cpus in	288	* In practice this means stopping other cpus in
289	* an SMP system.	289	* an SMP system.
290	* The kernel is broken so disable interrupts.	290	* The kernel is broken so disable interrupts.
291	*/	291	*/
292	local_irq_disable();	292	hard_irq_disable();
293		293
294	for_each_irq(irq) {	294	for_each_irq(irq) {
295	struct irq_desc *desc = irq_desc + irq;	295	struct irq_desc *desc = irq_desc + irq;
296		296
297	if (desc->status & IRQ_INPROGRESS)	297	if (desc->status & IRQ_INPROGRESS)
298	desc->chip->eoi(irq);	298	desc->chip->eoi(irq);
299		299
300	if (!(desc->status & IRQ_DISABLED))	300	if (!(desc->status & IRQ_DISABLED))
301	desc->chip->disable(irq);	301	desc->chip->disable(irq);
302	}	302	}
303		303
304	/*	304	/*
305	* Make a note of crashing cpu. Will be used in machine_kexec	305	* Make a note of crashing cpu. Will be used in machine_kexec
306	* such that another IPI will not be sent.	306	* such that another IPI will not be sent.
307	*/	307	*/
308	crashing_cpu = smp_processor_id();	308	crashing_cpu = smp_processor_id();
309	crash_save_this_cpu(regs, crashing_cpu);	309	crash_save_this_cpu(regs, crashing_cpu);
310	crash_kexec_prepare_cpus(crashing_cpu);	310	crash_kexec_prepare_cpus(crashing_cpu);
311	cpu_set(crashing_cpu, cpus_in_crash);	311	cpu_set(crashing_cpu, cpus_in_crash);
312	if (ppc_md.kexec_cpu_down)	312	if (ppc_md.kexec_cpu_down)
313	ppc_md.kexec_cpu_down(1, 0);	313	ppc_md.kexec_cpu_down(1, 0);
314	}	314	}
315		315

arch/powerpc/kernel/entry_64.S

Diff comments View file @ d04c56f

1	/*	1	/*
2	* PowerPC version	2	* PowerPC version
3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)	3	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4	* Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP	4	* Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5	* Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>	5	* Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6	* Adapted for Power Macintosh by Paul Mackerras.	6	* Adapted for Power Macintosh by Paul Mackerras.
7	* Low-level exception handlers and MMU support	7	* Low-level exception handlers and MMU support
8	* rewritten by Paul Mackerras.	8	* rewritten by Paul Mackerras.
9	* Copyright (C) 1996 Paul Mackerras.	9	* Copyright (C) 1996 Paul Mackerras.
10	* MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).	10	* MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11	*	11	*
12	* This file contains the system call entry code, context switch	12	* This file contains the system call entry code, context switch
13	* code, and exception/interrupt return code for PowerPC.	13	* code, and exception/interrupt return code for PowerPC.
14	*	14	*
15	* This program is free software; you can redistribute it and/or	15	* This program is free software; you can redistribute it and/or
16	* modify it under the terms of the GNU General Public License	16	* modify it under the terms of the GNU General Public License
17	* as published by the Free Software Foundation; either version	17	* as published by the Free Software Foundation; either version
18	* 2 of the License, or (at your option) any later version.	18	* 2 of the License, or (at your option) any later version.
19	*/	19	*/
20		20
21	#include <linux/errno.h>	21	#include <linux/errno.h>
22	#include <asm/unistd.h>	22	#include <asm/unistd.h>
23	#include <asm/processor.h>	23	#include <asm/processor.h>
24	#include <asm/page.h>	24	#include <asm/page.h>
25	#include <asm/mmu.h>	25	#include <asm/mmu.h>
26	#include <asm/thread_info.h>	26	#include <asm/thread_info.h>
27	#include <asm/ppc_asm.h>	27	#include <asm/ppc_asm.h>
28	#include <asm/asm-offsets.h>	28	#include <asm/asm-offsets.h>
29	#include <asm/cputable.h>	29	#include <asm/cputable.h>
30	#include <asm/firmware.h>	30	#include <asm/firmware.h>
31		31
32	/*	32	/*
33	* System calls.	33	* System calls.
34	*/	34	*/
35	.section ".toc","aw"	35	.section ".toc","aw"
36	.SYS_CALL_TABLE:	36	.SYS_CALL_TABLE:
37	.tc .sys_call_table[TC],.sys_call_table	37	.tc .sys_call_table[TC],.sys_call_table
38		38
39	/* This value is used to mark exception frames on the stack. */	39	/* This value is used to mark exception frames on the stack. */
40	exception_marker:	40	exception_marker:
41	.tc ID_72656773_68657265[TC],0x7265677368657265	41	.tc ID_72656773_68657265[TC],0x7265677368657265
42		42
43	.section ".text"	43	.section ".text"
44	.align 7	44	.align 7
45		45
46	#undef SHOW_SYSCALLS	46	#undef SHOW_SYSCALLS
47		47
48	.globl system_call_common	48	.globl system_call_common
49	system_call_common:	49	system_call_common:
50	andi. r10,r12,MSR_PR	50	andi. r10,r12,MSR_PR
51	mr r10,r1	51	mr r10,r1
52	addi r1,r1,-INT_FRAME_SIZE	52	addi r1,r1,-INT_FRAME_SIZE
53	beq- 1f	53	beq- 1f
54	ld r1,PACAKSAVE(r13)	54	ld r1,PACAKSAVE(r13)
55	1: std r10,0(r1)	55	1: std r10,0(r1)
56	crclr so	56	crclr so
57	std r11,_NIP(r1)	57	std r11,_NIP(r1)
58	std r12,_MSR(r1)	58	std r12,_MSR(r1)
59	std r0,GPR0(r1)	59	std r0,GPR0(r1)
60	std r10,GPR1(r1)	60	std r10,GPR1(r1)
61	ACCOUNT_CPU_USER_ENTRY(r10, r11)	61	ACCOUNT_CPU_USER_ENTRY(r10, r11)
62	std r2,GPR2(r1)	62	std r2,GPR2(r1)
63	std r3,GPR3(r1)	63	std r3,GPR3(r1)
64	std r4,GPR4(r1)	64	std r4,GPR4(r1)
65	std r5,GPR5(r1)	65	std r5,GPR5(r1)
66	std r6,GPR6(r1)	66	std r6,GPR6(r1)
67	std r7,GPR7(r1)	67	std r7,GPR7(r1)
68	std r8,GPR8(r1)	68	std r8,GPR8(r1)
69	li r11,0	69	li r11,0
70	std r11,GPR9(r1)	70	std r11,GPR9(r1)
71	std r11,GPR10(r1)	71	std r11,GPR10(r1)
72	std r11,GPR11(r1)	72	std r11,GPR11(r1)
73	std r11,GPR12(r1)	73	std r11,GPR12(r1)
74	std r9,GPR13(r1)	74	std r9,GPR13(r1)
75	mfcr r9	75	mfcr r9
76	mflr r10	76	mflr r10
77	li r11,0xc01	77	li r11,0xc01
78	std r9,_CCR(r1)	78	std r9,_CCR(r1)
79	std r10,_LINK(r1)	79	std r10,_LINK(r1)
80	std r11,_TRAP(r1)	80	std r11,_TRAP(r1)
81	mfxer r9	81	mfxer r9
82	mfctr r10	82	mfctr r10
83	std r9,_XER(r1)	83	std r9,_XER(r1)
84	std r10,_CTR(r1)	84	std r10,_CTR(r1)
85	std r3,ORIG_GPR3(r1)	85	std r3,ORIG_GPR3(r1)
86	ld r2,PACATOC(r13)	86	ld r2,PACATOC(r13)
87	addi r9,r1,STACK_FRAME_OVERHEAD	87	addi r9,r1,STACK_FRAME_OVERHEAD
88	ld r11,exception_marker@toc(r2)	88	ld r11,exception_marker@toc(r2)
89	std r11,-16(r9) /* "regshere" marker */	89	std r11,-16(r9) /* "regshere" marker */
		90	li r10,1
		91	stb r10,PACASOFTIRQEN(r13)
		92	stb r10,PACAHARDIRQEN(r13)
		93	std r10,SOFTE(r1)
90	#ifdef CONFIG_PPC_ISERIES	94	#ifdef CONFIG_PPC_ISERIES
91	BEGIN_FW_FTR_SECTION	95	BEGIN_FW_FTR_SECTION
92	/* Hack for handling interrupts when soft-enabling on iSeries */	96	/* Hack for handling interrupts when soft-enabling on iSeries */
93	cmpdi cr1,r0,0x5555 /* syscall 0x5555 */	97	cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
94	andi. r10,r12,MSR_PR /* from kernel */	98	andi. r10,r12,MSR_PR /* from kernel */
95	crand 4cr0+eq,4cr1+eq,4*cr0+eq	99	crand 4cr0+eq,4cr1+eq,4*cr0+eq
96	beq hardware_interrupt_entry	100	beq hardware_interrupt_entry
97	lbz r10,PACAPROCENABLED(r13)
98	std r10,SOFTE(r1)
99	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)	101	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
100	#endif	102	#endif
101	mfmsr r11	103	mfmsr r11
102	ori r11,r11,MSR_EE	104	ori r11,r11,MSR_EE
103	mtmsrd r11,1	105	mtmsrd r11,1
104		106
105	#ifdef SHOW_SYSCALLS	107	#ifdef SHOW_SYSCALLS
106	bl .do_show_syscall	108	bl .do_show_syscall
107	REST_GPR(0,r1)	109	REST_GPR(0,r1)
108	REST_4GPRS(3,r1)	110	REST_4GPRS(3,r1)
109	REST_2GPRS(7,r1)	111	REST_2GPRS(7,r1)
110	addi r9,r1,STACK_FRAME_OVERHEAD	112	addi r9,r1,STACK_FRAME_OVERHEAD
111	#endif	113	#endif
112	clrrdi r11,r1,THREAD_SHIFT	114	clrrdi r11,r1,THREAD_SHIFT
113	ld r10,TI_FLAGS(r11)	115	ld r10,TI_FLAGS(r11)
114	andi. r11,r10,_TIF_SYSCALL_T_OR_A	116	andi. r11,r10,_TIF_SYSCALL_T_OR_A
115	bne- syscall_dotrace	117	bne- syscall_dotrace
116	syscall_dotrace_cont:	118	syscall_dotrace_cont:
117	cmpldi 0,r0,NR_syscalls	119	cmpldi 0,r0,NR_syscalls
118	bge- syscall_enosys	120	bge- syscall_enosys
119		121
120	system_call: /* label this so stack traces look sane */	122	system_call: /* label this so stack traces look sane */
121	/*	123	/*
122	* Need to vector to 32 Bit or default sys_call_table here,	124	* Need to vector to 32 Bit or default sys_call_table here,
123	* based on caller's run-mode / personality.	125	* based on caller's run-mode / personality.
124	*/	126	*/
125	ld r11,.SYS_CALL_TABLE@toc(2)	127	ld r11,.SYS_CALL_TABLE@toc(2)
126	andi. r10,r10,_TIF_32BIT	128	andi. r10,r10,_TIF_32BIT
127	beq 15f	129	beq 15f
128	addi r11,r11,8 /* use 32-bit syscall entries */	130	addi r11,r11,8 /* use 32-bit syscall entries */
129	clrldi r3,r3,32	131	clrldi r3,r3,32
130	clrldi r4,r4,32	132	clrldi r4,r4,32
131	clrldi r5,r5,32	133	clrldi r5,r5,32
132	clrldi r6,r6,32	134	clrldi r6,r6,32
133	clrldi r7,r7,32	135	clrldi r7,r7,32
134	clrldi r8,r8,32	136	clrldi r8,r8,32
135	15:	137	15:
136	slwi r0,r0,4	138	slwi r0,r0,4
137	ldx r10,r11,r0 /* Fetch system call handler [ptr] */	139	ldx r10,r11,r0 /* Fetch system call handler [ptr] */
138	mtctr r10	140	mtctr r10
139	bctrl /* Call handler */	141	bctrl /* Call handler */
140		142
141	syscall_exit:	143	syscall_exit:
142	std r3,RESULT(r1)	144	std r3,RESULT(r1)
143	#ifdef SHOW_SYSCALLS	145	#ifdef SHOW_SYSCALLS
144	bl .do_show_syscall_exit	146	bl .do_show_syscall_exit
145	ld r3,RESULT(r1)	147	ld r3,RESULT(r1)
146	#endif	148	#endif
147	clrrdi r12,r1,THREAD_SHIFT	149	clrrdi r12,r1,THREAD_SHIFT
148		150
149	/* disable interrupts so current_thread_info()->flags can't change,	151	/* disable interrupts so current_thread_info()->flags can't change,
150	and so that we don't get interrupted after loading SRR0/1. */	152	and so that we don't get interrupted after loading SRR0/1. */
151	ld r8,_MSR(r1)	153	ld r8,_MSR(r1)
152	andi. r10,r8,MSR_RI	154	andi. r10,r8,MSR_RI
153	beq- unrecov_restore	155	beq- unrecov_restore
154	mfmsr r10	156	mfmsr r10
155	rldicl r10,r10,48,1	157	rldicl r10,r10,48,1
156	rotldi r10,r10,16	158	rotldi r10,r10,16
157	mtmsrd r10,1	159	mtmsrd r10,1
158	ld r9,TI_FLAGS(r12)	160	ld r9,TI_FLAGS(r12)
159	li r11,-_LAST_ERRNO	161	li r11,-_LAST_ERRNO
160	andi. r0,r9,(_TIF_SYSCALL_T_OR_A\|_TIF_SINGLESTEP\|_TIF_USER_WORK_MASK\|_TIF_PERSYSCALL_MASK)	162	andi. r0,r9,(_TIF_SYSCALL_T_OR_A\|_TIF_SINGLESTEP\|_TIF_USER_WORK_MASK\|_TIF_PERSYSCALL_MASK)
161	bne- syscall_exit_work	163	bne- syscall_exit_work
162	cmpld r3,r11	164	cmpld r3,r11
163	ld r5,_CCR(r1)	165	ld r5,_CCR(r1)
164	bge- syscall_error	166	bge- syscall_error
165	syscall_error_cont:	167	syscall_error_cont:
166	ld r7,_NIP(r1)	168	ld r7,_NIP(r1)
167	stdcx. r0,0,r1 /* to clear the reservation */	169	stdcx. r0,0,r1 /* to clear the reservation */
168	andi. r6,r8,MSR_PR	170	andi. r6,r8,MSR_PR
169	ld r4,_LINK(r1)	171	ld r4,_LINK(r1)
170	beq- 1f	172	beq- 1f
171	ACCOUNT_CPU_USER_EXIT(r11, r12)	173	ACCOUNT_CPU_USER_EXIT(r11, r12)
172	ld r13,GPR13(r1) /* only restore r13 if returning to usermode */	174	ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
173	1: ld r2,GPR2(r1)	175	1: ld r2,GPR2(r1)
174	li r12,MSR_RI	176	li r12,MSR_RI
175	andc r11,r10,r12	177	andc r11,r10,r12
176	mtmsrd r11,1 /* clear MSR.RI */	178	mtmsrd r11,1 /* clear MSR.RI */
177	ld r1,GPR1(r1)	179	ld r1,GPR1(r1)
178	mtlr r4	180	mtlr r4
179	mtcr r5	181	mtcr r5
180	mtspr SPRN_SRR0,r7	182	mtspr SPRN_SRR0,r7
181	mtspr SPRN_SRR1,r8	183	mtspr SPRN_SRR1,r8
182	rfid	184	rfid
183	b . /* prevent speculative execution */	185	b . /* prevent speculative execution */
184		186
185	syscall_error:	187	syscall_error:
186	oris r5,r5,0x1000 /* Set SO bit in CR */	188	oris r5,r5,0x1000 /* Set SO bit in CR */
187	neg r3,r3	189	neg r3,r3
188	std r5,_CCR(r1)	190	std r5,_CCR(r1)
189	b syscall_error_cont	191	b syscall_error_cont
190		192
191	/* Traced system call support */	193	/* Traced system call support */
192	syscall_dotrace:	194	syscall_dotrace:
193	bl .save_nvgprs	195	bl .save_nvgprs
194	addi r3,r1,STACK_FRAME_OVERHEAD	196	addi r3,r1,STACK_FRAME_OVERHEAD
195	bl .do_syscall_trace_enter	197	bl .do_syscall_trace_enter
196	ld r0,GPR0(r1) /* Restore original registers */	198	ld r0,GPR0(r1) /* Restore original registers */
197	ld r3,GPR3(r1)	199	ld r3,GPR3(r1)
198	ld r4,GPR4(r1)	200	ld r4,GPR4(r1)
199	ld r5,GPR5(r1)	201	ld r5,GPR5(r1)
200	ld r6,GPR6(r1)	202	ld r6,GPR6(r1)
201	ld r7,GPR7(r1)	203	ld r7,GPR7(r1)
202	ld r8,GPR8(r1)	204	ld r8,GPR8(r1)
203	addi r9,r1,STACK_FRAME_OVERHEAD	205	addi r9,r1,STACK_FRAME_OVERHEAD
204	clrrdi r10,r1,THREAD_SHIFT	206	clrrdi r10,r1,THREAD_SHIFT
205	ld r10,TI_FLAGS(r10)	207	ld r10,TI_FLAGS(r10)
206	b syscall_dotrace_cont	208	b syscall_dotrace_cont
207		209
208	syscall_enosys:	210	syscall_enosys:
209	li r3,-ENOSYS	211	li r3,-ENOSYS
210	b syscall_exit	212	b syscall_exit
211		213
212	syscall_exit_work:	214	syscall_exit_work:
213	/* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.	215	/* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
214	If TIF_NOERROR is set, just save r3 as it is. */	216	If TIF_NOERROR is set, just save r3 as it is. */
215		217
216	andi. r0,r9,_TIF_RESTOREALL	218	andi. r0,r9,_TIF_RESTOREALL
217	beq+ 0f	219	beq+ 0f
218	REST_NVGPRS(r1)	220	REST_NVGPRS(r1)
219	b 2f	221	b 2f
220	0: cmpld r3,r11 /* r10 is -LAST_ERRNO */	222	0: cmpld r3,r11 /* r10 is -LAST_ERRNO */
221	blt+ 1f	223	blt+ 1f
222	andi. r0,r9,_TIF_NOERROR	224	andi. r0,r9,_TIF_NOERROR
223	bne- 1f	225	bne- 1f
224	ld r5,_CCR(r1)	226	ld r5,_CCR(r1)
225	neg r3,r3	227	neg r3,r3
226	oris r5,r5,0x1000 /* Set SO bit in CR */	228	oris r5,r5,0x1000 /* Set SO bit in CR */
227	std r5,_CCR(r1)	229	std r5,_CCR(r1)
228	1: std r3,GPR3(r1)	230	1: std r3,GPR3(r1)
229	2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)	231	2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
230	beq 4f	232	beq 4f
231		233
232	/* Clear per-syscall TIF flags if any are set. */	234	/* Clear per-syscall TIF flags if any are set. */
233		235
234	li r11,_TIF_PERSYSCALL_MASK	236	li r11,_TIF_PERSYSCALL_MASK
235	addi r12,r12,TI_FLAGS	237	addi r12,r12,TI_FLAGS
236	3: ldarx r10,0,r12	238	3: ldarx r10,0,r12
237	andc r10,r10,r11	239	andc r10,r10,r11
238	stdcx. r10,0,r12	240	stdcx. r10,0,r12
239	bne- 3b	241	bne- 3b
240	subi r12,r12,TI_FLAGS	242	subi r12,r12,TI_FLAGS
241		243
242	4: /* Anything else left to do? */	244	4: /* Anything else left to do? */
243	andi. r0,r9,(_TIF_SYSCALL_T_OR_A\|_TIF_SINGLESTEP)	245	andi. r0,r9,(_TIF_SYSCALL_T_OR_A\|_TIF_SINGLESTEP)
244	beq .ret_from_except_lite	246	beq .ret_from_except_lite
245		247
246	/* Re-enable interrupts */	248	/* Re-enable interrupts */
247	mfmsr r10	249	mfmsr r10
248	ori r10,r10,MSR_EE	250	ori r10,r10,MSR_EE
249	mtmsrd r10,1	251	mtmsrd r10,1
250		252
251	bl .save_nvgprs	253	bl .save_nvgprs
252	addi r3,r1,STACK_FRAME_OVERHEAD	254	addi r3,r1,STACK_FRAME_OVERHEAD
253	bl .do_syscall_trace_leave	255	bl .do_syscall_trace_leave
254	b .ret_from_except	256	b .ret_from_except
255		257
256	/* Save non-volatile GPRs, if not already saved. */	258	/* Save non-volatile GPRs, if not already saved. */
257	_GLOBAL(save_nvgprs)	259	_GLOBAL(save_nvgprs)
258	ld r11,_TRAP(r1)	260	ld r11,_TRAP(r1)
259	andi. r0,r11,1	261	andi. r0,r11,1
260	beqlr-	262	beqlr-
261	SAVE_NVGPRS(r1)	263	SAVE_NVGPRS(r1)
262	clrrdi r0,r11,1	264	clrrdi r0,r11,1
263	std r0,_TRAP(r1)	265	std r0,_TRAP(r1)
264	blr	266	blr
265		267
266		268
267	/*	269	/*
268	* The sigsuspend and rt_sigsuspend system calls can call do_signal	270	* The sigsuspend and rt_sigsuspend system calls can call do_signal
269	* and thus put the process into the stopped state where we might	271	* and thus put the process into the stopped state where we might
270	* want to examine its user state with ptrace. Therefore we need	272	* want to examine its user state with ptrace. Therefore we need
271	* to save all the nonvolatile registers (r14 - r31) before calling	273	* to save all the nonvolatile registers (r14 - r31) before calling
272	* the C code. Similarly, fork, vfork and clone need the full	274	* the C code. Similarly, fork, vfork and clone need the full
273	* register state on the stack so that it can be copied to the child.	275	* register state on the stack so that it can be copied to the child.
274	*/	276	*/
275		277
276	_GLOBAL(ppc_fork)	278	_GLOBAL(ppc_fork)
277	bl .save_nvgprs	279	bl .save_nvgprs
278	bl .sys_fork	280	bl .sys_fork
279	b syscall_exit	281	b syscall_exit
280		282
281	_GLOBAL(ppc_vfork)	283	_GLOBAL(ppc_vfork)
282	bl .save_nvgprs	284	bl .save_nvgprs
283	bl .sys_vfork	285	bl .sys_vfork
284	b syscall_exit	286	b syscall_exit
285		287
286	_GLOBAL(ppc_clone)	288	_GLOBAL(ppc_clone)
287	bl .save_nvgprs	289	bl .save_nvgprs
288	bl .sys_clone	290	bl .sys_clone
289	b syscall_exit	291	b syscall_exit
290		292
291	_GLOBAL(ppc32_swapcontext)	293	_GLOBAL(ppc32_swapcontext)
292	bl .save_nvgprs	294	bl .save_nvgprs
293	bl .compat_sys_swapcontext	295	bl .compat_sys_swapcontext
294	b syscall_exit	296	b syscall_exit
295		297
296	_GLOBAL(ppc64_swapcontext)	298	_GLOBAL(ppc64_swapcontext)
297	bl .save_nvgprs	299	bl .save_nvgprs
298	bl .sys_swapcontext	300	bl .sys_swapcontext
299	b syscall_exit	301	b syscall_exit
300		302
301	_GLOBAL(ret_from_fork)	303	_GLOBAL(ret_from_fork)
302	bl .schedule_tail	304	bl .schedule_tail
303	REST_NVGPRS(r1)	305	REST_NVGPRS(r1)
304	li r3,0	306	li r3,0
305	b syscall_exit	307	b syscall_exit
306		308
307	/*	309	/*
308	* This routine switches between two different tasks. The process	310	* This routine switches between two different tasks. The process
309	* state of one is saved on its kernel stack. Then the state	311	* state of one is saved on its kernel stack. Then the state
310	* of the other is restored from its kernel stack. The memory	312	* of the other is restored from its kernel stack. The memory
311	* management hardware is updated to the second process's state.	313	* management hardware is updated to the second process's state.
312	* Finally, we can return to the second process, via ret_from_except.	314	* Finally, we can return to the second process, via ret_from_except.
313	* On entry, r3 points to the THREAD for the current task, r4	315	* On entry, r3 points to the THREAD for the current task, r4
314	* points to the THREAD for the new task.	316	* points to the THREAD for the new task.
315	*	317	*
316	* Note: there are two ways to get to the "going out" portion	318	* Note: there are two ways to get to the "going out" portion
317	* of this code; either by coming in via the entry (_switch)	319	* of this code; either by coming in via the entry (_switch)
318	* or via "fork" which must set up an environment equivalent	320	* or via "fork" which must set up an environment equivalent
319	* to the "_switch" path. If you change this you'll have to change	321	* to the "_switch" path. If you change this you'll have to change
320	* the fork code also.	322	* the fork code also.
321	*	323	*
322	* The code which creates the new task context is in 'copy_thread'	324	* The code which creates the new task context is in 'copy_thread'
323	* in arch/powerpc/kernel/process.c	325	* in arch/powerpc/kernel/process.c
324	*/	326	*/
325	.align 7	327	.align 7
326	_GLOBAL(_switch)	328	_GLOBAL(_switch)
327	mflr r0	329	mflr r0
328	std r0,16(r1)	330	std r0,16(r1)
329	stdu r1,-SWITCH_FRAME_SIZE(r1)	331	stdu r1,-SWITCH_FRAME_SIZE(r1)
330	/* r3-r13 are caller saved -- Cort */	332	/* r3-r13 are caller saved -- Cort */
331	SAVE_8GPRS(14, r1)	333	SAVE_8GPRS(14, r1)
332	SAVE_10GPRS(22, r1)	334	SAVE_10GPRS(22, r1)
333	mflr r20 /* Return to switch caller */	335	mflr r20 /* Return to switch caller */
334	mfmsr r22	336	mfmsr r22
335	li r0, MSR_FP	337	li r0, MSR_FP
336	#ifdef CONFIG_ALTIVEC	338	#ifdef CONFIG_ALTIVEC
337	BEGIN_FTR_SECTION	339	BEGIN_FTR_SECTION
338	oris r0,r0,MSR_VEC@h /* Disable altivec */	340	oris r0,r0,MSR_VEC@h /* Disable altivec */
339	mfspr r24,SPRN_VRSAVE /* save vrsave register value */	341	mfspr r24,SPRN_VRSAVE /* save vrsave register value */
340	std r24,THREAD_VRSAVE(r3)	342	std r24,THREAD_VRSAVE(r3)
341	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)	343	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
342	#endif /* CONFIG_ALTIVEC */	344	#endif /* CONFIG_ALTIVEC */
343	and. r0,r0,r22	345	and. r0,r0,r22
344	beq+ 1f	346	beq+ 1f
345	andc r22,r22,r0	347	andc r22,r22,r0
346	mtmsrd r22	348	mtmsrd r22
347	isync	349	isync
348	1: std r20,_NIP(r1)	350	1: std r20,_NIP(r1)
349	mfcr r23	351	mfcr r23
350	std r23,_CCR(r1)	352	std r23,_CCR(r1)
351	std r1,KSP(r3) /* Set old stack pointer */	353	std r1,KSP(r3) /* Set old stack pointer */
352		354
353	#ifdef CONFIG_SMP	355	#ifdef CONFIG_SMP
354	/* We need a sync somewhere here to make sure that if the	356	/* We need a sync somewhere here to make sure that if the
355	* previous task gets rescheduled on another CPU, it sees all	357	* previous task gets rescheduled on another CPU, it sees all
356	* stores it has performed on this one.	358	* stores it has performed on this one.
357	*/	359	*/
358	sync	360	sync
359	#endif /* CONFIG_SMP */	361	#endif /* CONFIG_SMP */
360		362
361	addi r6,r4,-THREAD /* Convert THREAD to 'current' */	363	addi r6,r4,-THREAD /* Convert THREAD to 'current' */
362	std r6,PACACURRENT(r13) /* Set new 'current' */	364	std r6,PACACURRENT(r13) /* Set new 'current' */
363		365
364	ld r8,KSP(r4) /* new stack pointer */	366	ld r8,KSP(r4) /* new stack pointer */
365	BEGIN_FTR_SECTION	367	BEGIN_FTR_SECTION
366	clrrdi r6,r8,28 /* get its ESID */	368	clrrdi r6,r8,28 /* get its ESID */
367	clrrdi r9,r1,28 /* get current sp ESID */	369	clrrdi r9,r1,28 /* get current sp ESID */
368	clrldi. r0,r6,2 /* is new ESID c00000000? */	370	clrldi. r0,r6,2 /* is new ESID c00000000? */
369	cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */	371	cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
370	cror eq,4*cr1+eq,eq	372	cror eq,4*cr1+eq,eq
371	beq 2f /* if yes, don't slbie it */	373	beq 2f /* if yes, don't slbie it */
372		374
373	/* Bolt in the new stack SLB entry */	375	/* Bolt in the new stack SLB entry */
374	ld r7,KSP_VSID(r4) /* Get new stack's VSID */	376	ld r7,KSP_VSID(r4) /* Get new stack's VSID */
375	oris r0,r6,(SLB_ESID_V)@h	377	oris r0,r6,(SLB_ESID_V)@h
376	ori r0,r0,(SLB_NUM_BOLTED-1)@l	378	ori r0,r0,(SLB_NUM_BOLTED-1)@l
377		379
378	/* Update the last bolted SLB */	380	/* Update the last bolted SLB */
379	ld r9,PACA_SLBSHADOWPTR(r13)	381	ld r9,PACA_SLBSHADOWPTR(r13)
380	li r12,0	382	li r12,0
381	std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */	383	std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
382	std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */	384	std r7,SLBSHADOW_STACKVSID(r9) /* Save VSID */
383	std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */	385	std r0,SLBSHADOW_STACKESID(r9) /* Save ESID */
384		386
385	slbie r6	387	slbie r6
386	slbie r6 /* Workaround POWER5 < DD2.1 issue */	388	slbie r6 /* Workaround POWER5 < DD2.1 issue */
387	slbmte r7,r0	389	slbmte r7,r0
388	isync	390	isync
389		391
390	2:	392	2:
391	END_FTR_SECTION_IFSET(CPU_FTR_SLB)	393	END_FTR_SECTION_IFSET(CPU_FTR_SLB)
392	clrrdi r7,r8,THREAD_SHIFT /* base of new stack */	394	clrrdi r7,r8,THREAD_SHIFT /* base of new stack */
393	/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE	395	/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
394	because we don't need to leave the 288-byte ABI gap at the	396	because we don't need to leave the 288-byte ABI gap at the
395	top of the kernel stack. */	397	top of the kernel stack. */
396	addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE	398	addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
397		399
398	mr r1,r8 /* start using new stack pointer */	400	mr r1,r8 /* start using new stack pointer */
399	std r7,PACAKSAVE(r13)	401	std r7,PACAKSAVE(r13)
400		402
401	ld r6,_CCR(r1)	403	ld r6,_CCR(r1)
402	mtcrf 0xFF,r6	404	mtcrf 0xFF,r6
403		405
404	#ifdef CONFIG_ALTIVEC	406	#ifdef CONFIG_ALTIVEC
405	BEGIN_FTR_SECTION	407	BEGIN_FTR_SECTION
406	ld r0,THREAD_VRSAVE(r4)	408	ld r0,THREAD_VRSAVE(r4)
407	mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */	409	mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
408	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)	410	END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
409	#endif /* CONFIG_ALTIVEC */	411	#endif /* CONFIG_ALTIVEC */
410		412
411	/* r3-r13 are destroyed -- Cort */	413	/* r3-r13 are destroyed -- Cort */
412	REST_8GPRS(14, r1)	414	REST_8GPRS(14, r1)
413	REST_10GPRS(22, r1)	415	REST_10GPRS(22, r1)
414		416
415	/* convert old thread to its task_struct for return value */	417	/* convert old thread to its task_struct for return value */
416	addi r3,r3,-THREAD	418	addi r3,r3,-THREAD
417	ld r7,_NIP(r1) /* Return to _switch caller in new task */	419	ld r7,_NIP(r1) /* Return to _switch caller in new task */
418	mtlr r7	420	mtlr r7
419	addi r1,r1,SWITCH_FRAME_SIZE	421	addi r1,r1,SWITCH_FRAME_SIZE
420	blr	422	blr
421		423
422	.align 7	424	.align 7
423	_GLOBAL(ret_from_except)	425	_GLOBAL(ret_from_except)
424	ld r11,_TRAP(r1)	426	ld r11,_TRAP(r1)
425	andi. r0,r11,1	427	andi. r0,r11,1
426	bne .ret_from_except_lite	428	bne .ret_from_except_lite
427	REST_NVGPRS(r1)	429	REST_NVGPRS(r1)
428		430
429	_GLOBAL(ret_from_except_lite)	431	_GLOBAL(ret_from_except_lite)
430	/*	432	/*
431	* Disable interrupts so that current_thread_info()->flags	433	* Disable interrupts so that current_thread_info()->flags
432	* can't change between when we test it and when we return	434	* can't change between when we test it and when we return
433	* from the interrupt.	435	* from the interrupt.
434	*/	436	*/
435	mfmsr r10 /* Get current interrupt state */	437	mfmsr r10 /* Get current interrupt state */
436	rldicl r9,r10,48,1 /* clear MSR_EE */	438	rldicl r9,r10,48,1 /* clear MSR_EE */
437	rotldi r9,r9,16	439	rotldi r9,r9,16
438	mtmsrd r9,1 /* Update machine state */	440	mtmsrd r9,1 /* Update machine state */
439		441
440	#ifdef CONFIG_PREEMPT	442	#ifdef CONFIG_PREEMPT
441	clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */	443	clrrdi r9,r1,THREAD_SHIFT /* current_thread_info() */
442	li r0,_TIF_NEED_RESCHED /* bits to check */	444	li r0,_TIF_NEED_RESCHED /* bits to check */
443	ld r3,_MSR(r1)	445	ld r3,_MSR(r1)
444	ld r4,TI_FLAGS(r9)	446	ld r4,TI_FLAGS(r9)
445	/* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */	447	/* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
446	rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING	448	rlwimi r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
447	and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */	449	and. r0,r4,r0 /* check NEED_RESCHED and maybe SIGPENDING */
448	bne do_work	450	bne do_work
449		451
450	#else /* !CONFIG_PREEMPT */	452	#else /* !CONFIG_PREEMPT */
451	ld r3,_MSR(r1) /* Returning to user mode? */	453	ld r3,_MSR(r1) /* Returning to user mode? */
452	andi. r3,r3,MSR_PR	454	andi. r3,r3,MSR_PR
453	beq restore /* if not, just restore regs and return */	455	beq restore /* if not, just restore regs and return */
454		456
455	/* Check current_thread_info()->flags */	457	/* Check current_thread_info()->flags */
456	clrrdi r9,r1,THREAD_SHIFT	458	clrrdi r9,r1,THREAD_SHIFT
457	ld r4,TI_FLAGS(r9)	459	ld r4,TI_FLAGS(r9)
458	andi. r0,r4,_TIF_USER_WORK_MASK	460	andi. r0,r4,_TIF_USER_WORK_MASK
459	bne do_work	461	bne do_work
460	#endif	462	#endif
461		463
462	restore:	464	restore:
		465	ld r5,SOFTE(r1)
463	#ifdef CONFIG_PPC_ISERIES	466	#ifdef CONFIG_PPC_ISERIES
464	BEGIN_FW_FTR_SECTION	467	BEGIN_FW_FTR_SECTION
465	ld r5,SOFTE(r1)
466	cmpdi 0,r5,0	468	cmpdi 0,r5,0
467	beq 4f	469	beq 4f
468	/* Check for pending interrupts (iSeries) */	470	/* Check for pending interrupts (iSeries) */
469	ld r3,PACALPPACAPTR(r13)	471	ld r3,PACALPPACAPTR(r13)
470	ld r3,LPPACAANYINT(r3)	472	ld r3,LPPACAANYINT(r3)
471	cmpdi r3,0	473	cmpdi r3,0
472	beq+ 4f /* skip do_IRQ if no interrupts */	474	beq+ 4f /* skip do_IRQ if no interrupts */
473		475
474	li r3,0	476	li r3,0
475	stb r3,PACAPROCENABLED(r13) /* ensure we are soft-disabled */	477	stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
476	ori r10,r10,MSR_EE	478	ori r10,r10,MSR_EE
477	mtmsrd r10 /* hard-enable again */	479	mtmsrd r10 /* hard-enable again */
478	addi r3,r1,STACK_FRAME_OVERHEAD	480	addi r3,r1,STACK_FRAME_OVERHEAD
479	bl .do_IRQ	481	bl .do_IRQ
480	b .ret_from_except_lite /* loop back and handle more */	482	b .ret_from_except_lite /* loop back and handle more */
481		483	4:
482	4: stb r5,PACAPROCENABLED(r13)
483	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)	484	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
484	#endif	485	#endif
		486	stb r5,PACASOFTIRQEN(r13)
485		487
486	ld r3,_MSR(r1)	488	ld r3,_MSR(r1)
487	andi. r0,r3,MSR_RI	489	andi. r0,r3,MSR_RI
488	beq- unrecov_restore	490	beq- unrecov_restore
489		491
490	andi. r0,r3,MSR_PR	492	andi. r0,r3,MSR_PR
491		493
492	/*	494	/*
493	* r13 is our per cpu area, only restore it if we are returning to	495	* r13 is our per cpu area, only restore it if we are returning to
494	* userspace	496	* userspace
495	*/	497	*/
496	beq 1f	498	beq 1f
497	ACCOUNT_CPU_USER_EXIT(r3, r4)	499	ACCOUNT_CPU_USER_EXIT(r3, r4)
498	REST_GPR(13, r1)	500	REST_GPR(13, r1)
499	1:	501	1:
500	ld r3,_CTR(r1)	502	ld r3,_CTR(r1)
501	ld r0,_LINK(r1)	503	ld r0,_LINK(r1)
502	mtctr r3	504	mtctr r3
503	mtlr r0	505	mtlr r0
504	ld r3,_XER(r1)	506	ld r3,_XER(r1)
505	mtspr SPRN_XER,r3	507	mtspr SPRN_XER,r3
506		508
507	REST_8GPRS(5, r1)	509	REST_8GPRS(5, r1)
508		510
509	stdcx. r0,0,r1 /* to clear the reservation */	511	stdcx. r0,0,r1 /* to clear the reservation */
510		512
511	mfmsr r0	513	mfmsr r0
512	li r2, MSR_RI	514	li r2, MSR_RI
513	andc r0,r0,r2	515	andc r0,r0,r2
514	mtmsrd r0,1	516	mtmsrd r0,1
515		517
516	ld r0,_MSR(r1)	518	ld r0,_MSR(r1)
517	mtspr SPRN_SRR1,r0	519	mtspr SPRN_SRR1,r0
518		520
519	ld r2,_CCR(r1)	521	ld r2,_CCR(r1)
520	mtcrf 0xFF,r2	522	mtcrf 0xFF,r2
521	ld r2,_NIP(r1)	523	ld r2,_NIP(r1)
522	mtspr SPRN_SRR0,r2	524	mtspr SPRN_SRR0,r2
523		525
524	ld r0,GPR0(r1)	526	ld r0,GPR0(r1)
525	ld r2,GPR2(r1)	527	ld r2,GPR2(r1)
526	ld r3,GPR3(r1)	528	ld r3,GPR3(r1)
527	ld r4,GPR4(r1)	529	ld r4,GPR4(r1)
528	ld r1,GPR1(r1)	530	ld r1,GPR1(r1)
529		531
530	rfid	532	rfid
531	b . /* prevent speculative execution */	533	b . /* prevent speculative execution */
532		534
533	/* Note: this must change if we start using the TIF_NOTIFY_RESUME bit */	535	/* Note: this must change if we start using the TIF_NOTIFY_RESUME bit */
534	do_work:	536	do_work:
535	#ifdef CONFIG_PREEMPT	537	#ifdef CONFIG_PREEMPT
536	andi. r0,r3,MSR_PR /* Returning to user mode? */	538	andi. r0,r3,MSR_PR /* Returning to user mode? */
537	bne user_work	539	bne user_work
538	/* Check that preempt_count() == 0 and interrupts are enabled */	540	/* Check that preempt_count() == 0 and interrupts are enabled */
539	lwz r8,TI_PREEMPT(r9)	541	lwz r8,TI_PREEMPT(r9)
540	cmpwi cr1,r8,0	542	cmpwi cr1,r8,0
541	#ifdef CONFIG_PPC_ISERIES
542	BEGIN_FW_FTR_SECTION
543	ld r0,SOFTE(r1)	543	ld r0,SOFTE(r1)
544	cmpdi r0,0	544	cmpdi r0,0
545	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
546	#endif
547	BEGIN_FW_FTR_SECTION
548	andi. r0,r3,MSR_EE
549	END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
550	crandc eq,cr1*4+eq,eq	545	crandc eq,cr1*4+eq,eq
551	bne restore	546	bne restore
552	/* here we are preempting the current task */	547	/* here we are preempting the current task */
553	1:	548	1:
554	#ifdef CONFIG_PPC_ISERIES
555	BEGIN_FW_FTR_SECTION
556	li r0,1	549	li r0,1
557	stb r0,PACAPROCENABLED(r13)	550	stb r0,PACASOFTIRQEN(r13)
558	END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)	551	stb r0,PACAHARDIRQEN(r13)
559	#endif
560	ori r10,r10,MSR_EE	552	ori r10,r10,MSR_EE
561	mtmsrd r10,1 /* reenable interrupts */	553	mtmsrd r10,1 /* reenable interrupts */
562	bl .preempt_schedule	554	bl .preempt_schedule
563	mfmsr r10	555	mfmsr r10
564	clrrdi r9,r1,THREAD_SHIFT	556	clrrdi r9,r1,THREAD_SHIFT
565	rldicl r10,r10,48,1 /* disable interrupts again */	557	rldicl r10,r10,48,1 /* disable interrupts again */
566	rotldi r10,r10,16	558	rotldi r10,r10,16
567	mtmsrd r10,1	559	mtmsrd r10,1
568	ld r4,TI_FLAGS(r9)	560	ld r4,TI_FLAGS(r9)
569	andi. r0,r4,_TIF_NEED_RESCHED	561	andi. r0,r4,_TIF_NEED_RESCHED
570	bne 1b	562	bne 1b
571	b restore	563	b restore
572		564
573	user_work:	565	user_work:
574	#endif	566	#endif
575	/* Enable interrupts */	567	/* Enable interrupts */
576	ori r10,r10,MSR_EE	568	ori r10,r10,MSR_EE
577	mtmsrd r10,1	569	mtmsrd r10,1
578		570
579	andi. r0,r4,_TIF_NEED_RESCHED	571	andi. r0,r4,_TIF_NEED_RESCHED
580	beq 1f	572	beq 1f
581	bl .schedule	573	bl .schedule
582	b .ret_from_except_lite	574	b .ret_from_except_lite
583		575
584	1: bl .save_nvgprs	576	1: bl .save_nvgprs
585	li r3,0	577	li r3,0
586	addi r4,r1,STACK_FRAME_OVERHEAD	578	addi r4,r1,STACK_FRAME_OVERHEAD
587	bl .do_signal	579	bl .do_signal
588	b .ret_from_except	580	b .ret_from_except
589		581
590	unrecov_restore:	582	unrecov_restore:
591	addi r3,r1,STACK_FRAME_OVERHEAD	583	addi r3,r1,STACK_FRAME_OVERHEAD
592	bl .unrecoverable_exception	584	bl .unrecoverable_exception
593	b unrecov_restore	585	b unrecov_restore
594		586
595	#ifdef CONFIG_PPC_RTAS	587	#ifdef CONFIG_PPC_RTAS
596	/*	588	/*
597	* On CHRP, the Run-Time Abstraction Services (RTAS) have to be	589	* On CHRP, the Run-Time Abstraction Services (RTAS) have to be
598	* called with the MMU off.	590	* called with the MMU off.
599	*	591	*
600	* In addition, we need to be in 32b mode, at least for now.	592	* In addition, we need to be in 32b mode, at least for now.
601	*	593	*
602	* Note: r3 is an input parameter to rtas, so don't trash it...	594	* Note: r3 is an input parameter to rtas, so don't trash it...
603	*/	595	*/
604	_GLOBAL(enter_rtas)	596	_GLOBAL(enter_rtas)
605	mflr r0	597	mflr r0
606	std r0,16(r1)	598	std r0,16(r1)
607	stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */	599	stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
608		600
609	/* Because RTAS is running in 32b mode, it clobbers the high order half	601	/* Because RTAS is running in 32b mode, it clobbers the high order half
610	* of all registers that it saves. We therefore save those registers	602	* of all registers that it saves. We therefore save those registers
611	* RTAS might touch to the stack. (r0, r3-r13 are caller saved)	603	* RTAS might touch to the stack. (r0, r3-r13 are caller saved)
612	*/	604	*/
613	SAVE_GPR(2, r1) /* Save the TOC */	605	SAVE_GPR(2, r1) /* Save the TOC */
614	SAVE_GPR(13, r1) /* Save paca */	606	SAVE_GPR(13, r1) /* Save paca */
615	SAVE_8GPRS(14, r1) /* Save the non-volatiles */	607	SAVE_8GPRS(14, r1) /* Save the non-volatiles */
616	SAVE_10GPRS(22, r1) /* ditto */	608	SAVE_10GPRS(22, r1) /* ditto */
617		609
618	mfcr r4	610	mfcr r4
619	std r4,_CCR(r1)	611	std r4,_CCR(r1)
620	mfctr r5	612	mfctr r5
621	std r5,_CTR(r1)	613	std r5,_CTR(r1)
622	mfspr r6,SPRN_XER	614	mfspr r6,SPRN_XER
623	std r6,_XER(r1)	615	std r6,_XER(r1)
624	mfdar r7	616	mfdar r7
625	std r7,_DAR(r1)	617	std r7,_DAR(r1)
626	mfdsisr r8	618	mfdsisr r8
627	std r8,_DSISR(r1)	619	std r8,_DSISR(r1)
628	mfsrr0 r9	620	mfsrr0 r9
629	std r9,_SRR0(r1)	621	std r9,_SRR0(r1)
630	mfsrr1 r10	622	mfsrr1 r10
631	std r10,_SRR1(r1)	623	std r10,_SRR1(r1)
632		624
633	/* Temporary workaround to clear CR until RTAS can be modified to	625	/* Temporary workaround to clear CR until RTAS can be modified to
634	* ignore all bits.	626	* ignore all bits.
635	*/	627	*/
636	li r0,0	628	li r0,0
637	mtcr r0	629	mtcr r0
638		630
639	/* There is no way it is acceptable to get here with interrupts enabled,	631	/* There is no way it is acceptable to get here with interrupts enabled,
640	* check it with the asm equivalent of WARN_ON	632	* check it with the asm equivalent of WARN_ON
641	*/	633	*/
642	mfmsr r6	634	lbz r0,PACASOFTIRQEN(r13)
643	andi. r0,r6,MSR_EE
644	1: tdnei r0,0	635	1: tdnei r0,0
645	.section __bug_table,"a"	636	.section __bug_table,"a"
646	.llong 1b,__LINE__ + 0x1000000, 1f, 2f	637	.llong 1b,__LINE__ + 0x1000000, 1f, 2f
647	.previous	638	.previous
648	.section .rodata,"a"	639	.section .rodata,"a"
649	1: .asciz __FILE__	640	1: .asciz __FILE__
650	2: .asciz "enter_rtas"	641	2: .asciz "enter_rtas"
651	.previous	642	.previous
652		643
		644	/* Hard-disable interrupts */
		645	mfmsr r6
		646	rldicl r7,r6,48,1
		647	rotldi r7,r7,16
		648	mtmsrd r7,1
		649
653	/* Unfortunately, the stack pointer and the MSR are also clobbered,	650	/* Unfortunately, the stack pointer and the MSR are also clobbered,
654	* so they are saved in the PACA which allows us to restore	651	* so they are saved in the PACA which allows us to restore
655	* our original state after RTAS returns.	652	* our original state after RTAS returns.
656	*/	653	*/
657	std r1,PACAR1(r13)	654	std r1,PACAR1(r13)
658	std r6,PACASAVEDMSR(r13)	655	std r6,PACASAVEDMSR(r13)
659		656
660	/* Setup our real return addr */	657	/* Setup our real return addr */
661	LOAD_REG_ADDR(r4,.rtas_return_loc)	658	LOAD_REG_ADDR(r4,.rtas_return_loc)
662	clrldi r4,r4,2 /* convert to realmode address */	659	clrldi r4,r4,2 /* convert to realmode address */
663	mtlr r4	660	mtlr r4
664		661
665	li r0,0	662	li r0,0
666	ori r0,r0,MSR_EE\|MSR_SE\|MSR_BE\|MSR_RI	663	ori r0,r0,MSR_EE\|MSR_SE\|MSR_BE\|MSR_RI
667	andc r0,r6,r0	664	andc r0,r6,r0
668		665
669	li r9,1	666	li r9,1
670	rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)	667	rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
671	ori r9,r9,MSR_IR\|MSR_DR\|MSR_FE0\|MSR_FE1\|MSR_FP	668	ori r9,r9,MSR_IR\|MSR_DR\|MSR_FE0\|MSR_FE1\|MSR_FP
672	andc r6,r0,r9	669	andc r6,r0,r9
673	ori r6,r6,MSR_RI	670	ori r6,r6,MSR_RI
674	sync /* disable interrupts so SRR0/1 */	671	sync /* disable interrupts so SRR0/1 */
675	mtmsrd r0 /* don't get trashed */	672	mtmsrd r0 /* don't get trashed */
676		673
677	LOAD_REG_ADDR(r4, rtas)	674	LOAD_REG_ADDR(r4, rtas)
678	ld r5,RTASENTRY(r4) /* get the rtas->entry value */	675	ld r5,RTASENTRY(r4) /* get the rtas->entry value */
679	ld r4,RTASBASE(r4) /* get the rtas->base value */	676	ld r4,RTASBASE(r4) /* get the rtas->base value */
680		677
681	mtspr SPRN_SRR0,r5	678	mtspr SPRN_SRR0,r5
682	mtspr SPRN_SRR1,r6	679	mtspr SPRN_SRR1,r6
683	rfid	680	rfid
684	b . /* prevent speculative execution */	681	b . /* prevent speculative execution */
685		682
686	_STATIC(rtas_return_loc)	683	_STATIC(rtas_return_loc)
687	/* relocation is off at this point */	684	/* relocation is off at this point */
688	mfspr r4,SPRN_SPRG3 /* Get PACA */	685	mfspr r4,SPRN_SPRG3 /* Get PACA */
689	clrldi r4,r4,2 /* convert to realmode address */	686	clrldi r4,r4,2 /* convert to realmode address */
690		687
691	mfmsr r6	688	mfmsr r6
692	li r0,MSR_RI	689	li r0,MSR_RI
693	andc r6,r6,r0	690	andc r6,r6,r0
694	sync	691	sync
695	mtmsrd r6	692	mtmsrd r6
696		693
697	ld r1,PACAR1(r4) /* Restore our SP */	694	ld r1,PACAR1(r4) /* Restore our SP */
698	LOAD_REG_IMMEDIATE(r3,.rtas_restore_regs)	695	LOAD_REG_IMMEDIATE(r3,.rtas_restore_regs)
699	ld r4,PACASAVEDMSR(r4) /* Restore our MSR */	696	ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
700		697
701	mtspr SPRN_SRR0,r3	698	mtspr SPRN_SRR0,r3
702	mtspr SPRN_SRR1,r4	699	mtspr SPRN_SRR1,r4
703	rfid	700	rfid
704	b . /* prevent speculative execution */	701	b . /* prevent speculative execution */
705		702
706	_STATIC(rtas_restore_regs)	703	_STATIC(rtas_restore_regs)
707	/* relocation is on at this point */	704	/* relocation is on at this point */
708	REST_GPR(2, r1) /* Restore the TOC */	705	REST_GPR(2, r1) /* Restore the TOC */
709	REST_GPR(13, r1) /* Restore paca */	706	REST_GPR(13, r1) /* Restore paca */
710	REST_8GPRS(14, r1) /* Restore the non-volatiles */	707	REST_8GPRS(14, r1) /* Restore the non-volatiles */
711	REST_10GPRS(22, r1) /* ditto */	708	REST_10GPRS(22, r1) /* ditto */
712		709
713	mfspr r13,SPRN_SPRG3	710	mfspr r13,SPRN_SPRG3
714		711
715	ld r4,_CCR(r1)	712	ld r4,_CCR(r1)
716	mtcr r4	713	mtcr r4
717	ld r5,_CTR(r1)	714	ld r5,_CTR(r1)
718	mtctr r5	715	mtctr r5
719	ld r6,_XER(r1)	716	ld r6,_XER(r1)
720	mtspr SPRN_XER,r6	717	mtspr SPRN_XER,r6
721	ld r7,_DAR(r1)	718	ld r7,_DAR(r1)
722	mtdar r7	719	mtdar r7
723	ld r8,_DSISR(r1)	720	ld r8,_DSISR(r1)
724	mtdsisr r8	721	mtdsisr r8
725	ld r9,_SRR0(r1)	722	ld r9,_SRR0(r1)
726	mtsrr0 r9	723	mtsrr0 r9
727	ld r10,_SRR1(r1)	724	ld r10,_SRR1(r1)
728	mtsrr1 r10	725	mtsrr1 r10
729		726
730	addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */	727	addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
731	ld r0,16(r1) /* get return address */	728	ld r0,16(r1) /* get return address */
732		729
733	mtlr r0	730	mtlr r0
734	blr /* return to caller */	731	blr /* return to caller */
735		732
736	#endif /* CONFIG_PPC_RTAS */	733	#endif /* CONFIG_PPC_RTAS */
737		734
738	#ifdef CONFIG_PPC_MULTIPLATFORM	735	#ifdef CONFIG_PPC_MULTIPLATFORM
739		736
740	_GLOBAL(enter_prom)	737	_GLOBAL(enter_prom)
741	mflr r0	738	mflr r0
742	std r0,16(r1)	739	std r0,16(r1)
743	stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */	740	stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
744		741
745	/* Because PROM is running in 32b mode, it clobbers the high order half	742	/* Because PROM is running in 32b mode, it clobbers the high order half
746	* of all registers that it saves. We therefore save those registers	743	* of all registers that it saves. We therefore save those registers
747	* PROM might touch to the stack. (r0, r3-r13 are caller saved)	744	* PROM might touch to the stack. (r0, r3-r13 are caller saved)
748	*/	745	*/
749	SAVE_8GPRS(2, r1)	746	SAVE_8GPRS(2, r1)
750	SAVE_GPR(13, r1)	747	SAVE_GPR(13, r1)
751	SAVE_8GPRS(14, r1)	748	SAVE_8GPRS(14, r1)
752	SAVE_10GPRS(22, r1)	749	SAVE_10GPRS(22, r1)
753	mfcr r4	750	mfcr r4
754	std r4,_CCR(r1)	751	std r4,_CCR(r1)
755	mfctr r5	752	mfctr r5
756	std r5,_CTR(r1)	753	std r5,_CTR(r1)
757	mfspr r6,SPRN_XER	754	mfspr r6,SPRN_XER
758	std r6,_XER(r1)	755	std r6,_XER(r1)
759	mfdar r7	756	mfdar r7
760	std r7,_DAR(r1)	757	std r7,_DAR(r1)
761	mfdsisr r8	758	mfdsisr r8
762	std r8,_DSISR(r1)	759	std r8,_DSISR(r1)
763	mfsrr0 r9	760	mfsrr0 r9
764	std r9,_SRR0(r1)	761	std r9,_SRR0(r1)
765	mfsrr1 r10	762	mfsrr1 r10
766	std r10,_SRR1(r1)	763	std r10,_SRR1(r1)
767	mfmsr r11	764	mfmsr r11
768	std r11,_MSR(r1)	765	std r11,_MSR(r1)
769		766
770	/* Get the PROM entrypoint */	767	/* Get the PROM entrypoint */
771	ld r0,GPR4(r1)	768	ld r0,GPR4(r1)
772	mtlr r0	769	mtlr r0
773		770
774	/* Switch MSR to 32 bits mode	771	/* Switch MSR to 32 bits mode
775	*/	772	*/
776	mfmsr r11	773	mfmsr r11
777	li r12,1	774	li r12,1
778	rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)	775	rldicr r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
779	andc r11,r11,r12	776	andc r11,r11,r12
780	li r12,1	777	li r12,1
781	rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)	778	rldicr r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
782	andc r11,r11,r12	779	andc r11,r11,r12
783	mtmsrd r11	780	mtmsrd r11
784	isync	781	isync
785		782
786	/* Restore arguments & enter PROM here... */	783	/* Restore arguments & enter PROM here... */
787	ld r3,GPR3(r1)	784	ld r3,GPR3(r1)
788	blrl	785	blrl
789		786
790	/* Just make sure that r1 top 32 bits didn't get	787	/* Just make sure that r1 top 32 bits didn't get
791	* corrupt by OF	788	* corrupt by OF
792	*/	789	*/
793	rldicl r1,r1,0,32	790	rldicl r1,r1,0,32
794		791
795	/* Restore the MSR (back to 64 bits) */	792	/* Restore the MSR (back to 64 bits) */
796	ld r0,_MSR(r1)	793	ld r0,_MSR(r1)
797	mtmsrd r0	794	mtmsrd r0
798	isync	795	isync
799		796
800	/* Restore other registers */	797	/* Restore other registers */
801	REST_GPR(2, r1)	798	REST_GPR(2, r1)
802	REST_GPR(13, r1)	799	REST_GPR(13, r1)
803	REST_8GPRS(14, r1)	800	REST_8GPRS(14, r1)
804	REST_10GPRS(22, r1)	801	REST_10GPRS(22, r1)
805	ld r4,_CCR(r1)	802	ld r4,_CCR(r1)
806	mtcr r4	803	mtcr r4
807	ld r5,_CTR(r1)	804	ld r5,_CTR(r1)
808	mtctr r5	805	mtctr r5
809	ld r6,_XER(r1)	806	ld r6,_XER(r1)
810	mtspr SPRN_XER,r6	807	mtspr SPRN_XER,r6
811	ld r7,_DAR(r1)	808	ld r7,_DAR(r1)
812	mtdar r7	809	mtdar r7
813	ld r8,_DSISR(r1)	810	ld r8,_DSISR(r1)
814	mtdsisr r8	811	mtdsisr r8

arch/powerpc/kernel/head_64.S

Diff comments View file @ d04c56f

 /*
  *  PowerPC version
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *
  *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
  *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
  *  Adapted for Power Macintosh by Paul Mackerras.
  *  Low-level exception handlers and MMU support
  *  rewritten by Paul Mackerras.
  *    Copyright (C) 1996 Paul Mackerras.
  *
  *  Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
  *    Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
  *
  *  This file contains the low-level support and setup for the
  *  PowerPC-64 platform, including trap and interrupt dispatch.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  */
 #include <linux/threads.h>
 #include <asm/reg.h>
 #include <asm/page.h>
 #include <asm/mmu.h>
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/bug.h>
 #include <asm/cputable.h>
 #include <asm/setup.h>
 #include <asm/hvcall.h>
 #include <asm/iseries/lpar_map.h>
 #include <asm/thread_info.h>
 #include <asm/firmware.h>
-#ifdef CONFIG_PPC_ISERIES
 #define DO_SOFT_DISABLE
-#endif
 /*
  * We layout physical memory as follows:
  * 0x0000 - 0x00ff : Secondary processor spin code
  * 0x0100 - 0x2fff : pSeries Interrupt prologs
  * 0x3000 - 0x5fff : interrupt support, iSeries and common interrupt prologs
  * 0x6000 - 0x6fff : Initial (CPU0) segment table
  * 0x7000 - 0x7fff : FWNMI data area
  * 0x8000 -        : Early init and support code
  */
 /*
  *   SPRG Usage
  *
  *   Register	Definition
  *
  *   SPRG0	reserved for hypervisor
  *   SPRG1	temp - used to save gpr
  *   SPRG2	temp - used to save gpr
  *   SPRG3	virt addr of paca
  */
 /*
  * Entering into this code we make the following assumptions:
  *  For pSeries:
  *   1. The MMU is off & open firmware is running in real mode.
  *   2. The kernel is entered at __start
  *
  *  For iSeries:
  *   1. The MMU is on (as it always is for iSeries)
  *   2. The kernel is entered at system_reset_iSeries
  */
 	.text
 	.globl  _stext
 _stext:
 #ifdef CONFIG_PPC_MULTIPLATFORM
 _GLOBAL(__start)
 	/* NOP this out unconditionally */
 BEGIN_FTR_SECTION
 	b	.__start_initialization_multiplatform
 END_FTR_SECTION(0, 1)
 #endif /* CONFIG_PPC_MULTIPLATFORM */
 	/* Catch branch to 0 in real mode */
 	trap
 	/* Secondary processors spin on this value until it goes to 1. */
 	.globl  __secondary_hold_spinloop
 __secondary_hold_spinloop:
 	.llong	0x0
 	/* Secondary processors write this value with their cpu # */
 	/* after they enter the spin loop immediately below.	  */
 	.globl	__secondary_hold_acknowledge
 __secondary_hold_acknowledge:
 	.llong	0x0
 #ifdef CONFIG_PPC_ISERIES
 	/*
 	 * At offset 0x20, there is a pointer to iSeries LPAR data.
 	 * This is required by the hypervisor
 	 */
 	. = 0x20
 	.llong hvReleaseData-KERNELBASE
 #endif /* CONFIG_PPC_ISERIES */
 	. = 0x60
 /*
  * The following code is used on pSeries to hold secondary processors
  * in a spin loop after they have been freed from OpenFirmware, but
  * before the bulk of the kernel has been relocated.  This code
  * is relocated to physical address 0x60 before prom_init is run.
  * All of it must fit below the first exception vector at 0x100.
  */
 _GLOBAL(__secondary_hold)
 	mfmsr	r24
 	ori	r24,r24,MSR_RI
 	mtmsrd	r24			/* RI on */
 	/* Grab our physical cpu number */
 	mr	r24,r3
 	/* Tell the master cpu we're here */
 	/* Relocation is off & we are located at an address less */
 	/* than 0x100, so only need to grab low order offset.    */
 	std	r24,__secondary_hold_acknowledge@l(0)
 	sync
 	/* All secondary cpus wait here until told to start. */
 100:	ld	r4,__secondary_hold_spinloop@l(0)
 	cmpdi	0,r4,1
 	bne	100b
 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
 	LOAD_REG_IMMEDIATE(r4, .generic_secondary_smp_init)
 	mtctr	r4
 	mr	r3,r24
 	bctr
 #else
 	BUG_OPCODE
 #endif
 /* This value is used to mark exception frames on the stack. */
 	.section ".toc","aw"
 exception_marker:
 	.tc	ID_72656773_68657265[TC],0x7265677368657265
 	.text
 /*
  * The following macros define the code that appears as
  * the prologue to each of the exception handlers.  They
  * are split into two parts to allow a single kernel binary
  * to be used for pSeries and iSeries.
  * LOL.  One day... - paulus
  */
 /*
  * We make as much of the exception code common between native
  * exception handlers (including pSeries LPAR) and iSeries LPAR
  * implementations as possible.
  */
 /*
  * This is the start of the interrupt handlers for pSeries
  * This code runs with relocation off.
  */
 #define EX_R9		0
 #define EX_R10		8
 #define EX_R11		16
 #define EX_R12		24
 #define EX_R13		32
 #define EX_SRR0		40
 #define EX_DAR		48
 #define EX_DSISR	56
 #define EX_CCR		60
 #define EX_R3		64
 #define EX_LR		72
 /*
  * We're short on space and time in the exception prolog, so we can't
  * use the normal SET_REG_IMMEDIATE macro. Normally we just need the
  * low halfword of the address, but for Kdump we need the whole low
  * word.
  */
 #ifdef CONFIG_CRASH_DUMP
 #define LOAD_HANDLER(reg, label)					\
 	oris	reg,reg,(label)@h;	/* virt addr of handler ... */	\
 	ori	reg,reg,(label)@l;	/* .. and the rest */
 #else
 #define LOAD_HANDLER(reg, label)					\
 	ori	reg,reg,(label)@l;	/* virt addr of handler ... */
 #endif
 /*
  * Equal to EXCEPTION_PROLOG_PSERIES, except that it forces 64bit mode.
  * The firmware calls the registered system_reset_fwnmi and
  * machine_check_fwnmi handlers in 32bit mode if the cpu happens to run
  * a 32bit application at the time of the event.
  * This firmware bug is present on POWER4 and JS20.
  */
 #define EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(area, label)		\
 	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
 	std	r9,area+EX_R9(r13);	/* save r9 - r12 */		\
 	std	r10,area+EX_R10(r13);					\
 	std	r11,area+EX_R11(r13);					\
 	std	r12,area+EX_R12(r13);					\
 	mfspr	r9,SPRN_SPRG1;						\
 	std	r9,area+EX_R13(r13);					\
 	mfcr	r9;							\
 	clrrdi	r12,r13,32;		/* get high part of &label */	\
 	mfmsr	r10;							\
 	/* force 64bit mode */						\
 	li	r11,5;			/* MSR_SF_LG|MSR_ISF_LG */	\
 	rldimi	r10,r11,61,0;		/* insert into top 3 bits */	\
 	/* done 64bit mode */						\
 	mfspr	r11,SPRN_SRR0;		/* save SRR0 */			\
 	LOAD_HANDLER(r12,label)						\
 	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI;				\
 	mtspr	SPRN_SRR0,r12;						\
 	mfspr	r12,SPRN_SRR1;		/* and SRR1 */			\
 	mtspr	SPRN_SRR1,r10;						\
 	rfid;								\
 	b	.	/* prevent speculative execution */
 #define EXCEPTION_PROLOG_PSERIES(area, label)				\
 	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
 	std	r9,area+EX_R9(r13);	/* save r9 - r12 */		\
 	std	r10,area+EX_R10(r13);					\
 	std	r11,area+EX_R11(r13);					\
 	std	r12,area+EX_R12(r13);					\
 	mfspr	r9,SPRN_SPRG1;						\
 	std	r9,area+EX_R13(r13);					\
 	mfcr	r9;							\
 	clrrdi	r12,r13,32;		/* get high part of &label */	\
 	mfmsr	r10;							\
 	mfspr	r11,SPRN_SRR0;		/* save SRR0 */			\
 	LOAD_HANDLER(r12,label)						\
 	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI;				\
 	mtspr	SPRN_SRR0,r12;						\
 	mfspr	r12,SPRN_SRR1;		/* and SRR1 */			\
 	mtspr	SPRN_SRR1,r10;						\
 	rfid;								\
 	b	.	/* prevent speculative execution */
 /*
  * This is the start of the interrupt handlers for iSeries
  * This code runs with relocation on.
  */
 #define EXCEPTION_PROLOG_ISERIES_1(area)				\
 	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
 	std	r9,area+EX_R9(r13);	/* save r9 - r12 */		\
 	std	r10,area+EX_R10(r13);					\
 	std	r11,area+EX_R11(r13);					\
 	std	r12,area+EX_R12(r13);					\
 	mfspr	r9,SPRN_SPRG1;						\
 	std	r9,area+EX_R13(r13);					\
 	mfcr	r9
 #define EXCEPTION_PROLOG_ISERIES_2					\
 	mfmsr	r10;							\
 	ld	r12,PACALPPACAPTR(r13);					\
 	ld	r11,LPPACASRR0(r12);					\
 	ld	r12,LPPACASRR1(r12);					\
 	ori	r10,r10,MSR_RI;						\
 	mtmsrd	r10,1
 /*
  * The common exception prolog is used for all except a few exceptions
  * such as a segment miss on a kernel address.  We have to be prepared
  * to take another exception from the point where we first touch the
  * kernel stack onwards.
  *
  * On entry r13 points to the paca, r9-r13 are saved in the paca,
  * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and
  * SRR1, and relocation is on.
  */
 #define EXCEPTION_PROLOG_COMMON(n, area)				   \
 	andi.	r10,r12,MSR_PR;		/* See if coming from user	*/ \
 	mr	r10,r1;			/* Save r1			*/ \
 	subi	r1,r1,INT_FRAME_SIZE;	/* alloc frame on kernel stack	*/ \
 	beq-	1f;							   \
 	ld	r1,PACAKSAVE(r13);	/* kernel stack to use		*/ \
 1:	cmpdi	cr1,r1,0;		/* check if r1 is in userspace	*/ \
 	bge-	cr1,bad_stack;		/* abort if it is		*/ \
 	std	r9,_CCR(r1);		/* save CR in stackframe	*/ \
 	std	r11,_NIP(r1);		/* save SRR0 in stackframe	*/ \
 	std	r12,_MSR(r1);		/* save SRR1 in stackframe	*/ \
 	std	r10,0(r1);		/* make stack chain pointer	*/ \
 	std	r0,GPR0(r1);		/* save r0 in stackframe	*/ \
 	std	r10,GPR1(r1);		/* save r1 in stackframe	*/ \
 	ACCOUNT_CPU_USER_ENTRY(r9, r10);				   \
 	std	r2,GPR2(r1);		/* save r2 in stackframe	*/ \
 	SAVE_4GPRS(3, r1);		/* save r3 - r6 in stackframe	*/ \
 	SAVE_2GPRS(7, r1);		/* save r7, r8 in stackframe	*/ \
 	ld	r9,area+EX_R9(r13);	/* move r9, r10 to stackframe	*/ \
 	ld	r10,area+EX_R10(r13);					   \
 	std	r9,GPR9(r1);						   \
 	std	r10,GPR10(r1);						   \
 	ld	r9,area+EX_R11(r13);	/* move r11 - r13 to stackframe	*/ \
 	ld	r10,area+EX_R12(r13);					   \
 	ld	r11,area+EX_R13(r13);					   \
 	std	r9,GPR11(r1);						   \
 	std	r10,GPR12(r1);						   \
 	std	r11,GPR13(r1);						   \
 	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
 	mflr	r9;			/* save LR in stackframe	*/ \
 	std	r9,_LINK(r1);						   \
 	mfctr	r10;			/* save CTR in stackframe	*/ \
 	std	r10,_CTR(r1);						   \
+	lbz	r10,PACASOFTIRQEN(r13);				   \
 	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
+	std	r10,SOFTE(r1);						   \
 	std	r11,_XER(r1);						   \
 	li	r9,(n)+1;						   \
 	std	r9,_TRAP(r1);		/* set trap number		*/ \
 	li	r10,0;							   \
 	ld	r11,exception_marker@toc(r2);				   \
 	std	r10,RESULT(r1);		/* clear regs->result		*/ \
 	std	r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame	*/
 /*
  * Exception vectors.
  */
 #define STD_EXCEPTION_PSERIES(n, label)			\
 	. = n;						\
 	.globl label##_pSeries;				\
 label##_pSeries:					\
 	HMT_MEDIUM;					\
 	mtspr	SPRN_SPRG1,r13;		/* save r13 */	\
 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
 #define HSTD_EXCEPTION_PSERIES(n, label)		\
 	. = n;						\
 	.globl label##_pSeries;				\
 label##_pSeries:					\
 	HMT_MEDIUM;					\
 	mtspr	SPRN_SPRG1,r20;		/* save r20 */	\
 	mfspr	r20,SPRN_HSRR0;		/* copy HSRR0 to SRR0 */ \
 	mtspr	SPRN_SRR0,r20;				\
 	mfspr	r20,SPRN_HSRR1;		/* copy HSRR0 to SRR0 */ \
 	mtspr	SPRN_SRR1,r20;				\
 	mfspr	r20,SPRN_SPRG1;		/* restore r20 */ \
 	mtspr	SPRN_SPRG1,r13;		/* save r13 */	\
 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
+#define MASKABLE_EXCEPTION_PSERIES(n, label)				\
+	. = n;								\
+	.globl label##_pSeries;						\
+label##_pSeries:							\
+	HMT_MEDIUM;							\
+	mtspr	SPRN_SPRG1,r13;		/* save r13 */			\
+	mfspr	r13,SPRN_SPRG3;		/* get paca address into r13 */	\
+	std	r9,PACA_EXGEN+EX_R9(r13);	/* save r9, r10 */	\
+	std	r10,PACA_EXGEN+EX_R10(r13);				\
+	lbz	r10,PACASOFTIRQEN(r13);					\
+	mfcr	r9;							\
+	cmpwi	r10,0;							\
+	beq	masked_interrupt;					\
+	mfspr	r10,SPRN_SPRG1;						\
+	std	r10,PACA_EXGEN+EX_R13(r13);				\
+	std	r11,PACA_EXGEN+EX_R11(r13);				\
+	std	r12,PACA_EXGEN+EX_R12(r13);				\
+	clrrdi	r12,r13,32;		/* get high part of &label */	\
+	mfmsr	r10;							\
+	mfspr	r11,SPRN_SRR0;		/* save SRR0 */			\
+	LOAD_HANDLER(r12,label##_common)				\
+	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI;				\
+	mtspr	SPRN_SRR0,r12;						\
+	mfspr	r12,SPRN_SRR1;		/* and SRR1 */			\
+	mtspr	SPRN_SRR1,r10;						\
+	rfid;								\
+	b	.	/* prevent speculative execution */
 #define STD_EXCEPTION_ISERIES(n, label, area)		\
 	.globl label##_iSeries;				\
 label##_iSeries:					\
 	HMT_MEDIUM;					\
 	mtspr	SPRN_SPRG1,r13;		/* save r13 */	\
 	EXCEPTION_PROLOG_ISERIES_1(area);		\
 	EXCEPTION_PROLOG_ISERIES_2;			\
 	b	label##_common
 #define MASKABLE_EXCEPTION_ISERIES(n, label)				\
 	.globl label##_iSeries;						\
 label##_iSeries:							\
 	HMT_MEDIUM;							\
 	mtspr	SPRN_SPRG1,r13;		/* save r13 */			\
 	EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);				\
-	lbz	r10,PACAPROCENABLED(r13);				\
+	lbz	r10,PACASOFTIRQEN(r13);					\
 	cmpwi	0,r10,0;						\
 	beq-	label##_iSeries_masked;					\
 	EXCEPTION_PROLOG_ISERIES_2;					\
 	b	label##_common;						\
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 #define DISABLE_INTS				\
-BEGIN_FW_FTR_SECTION;				\
-	lbz	r10,PACAPROCENABLED(r13);	\
 	li	r11,0;				\
-	std	r10,SOFTE(r1);			\
+	stb	r11,PACASOFTIRQEN(r13);		\
+BEGIN_FW_FTR_SECTION;				\
+	stb	r11,PACAHARDIRQEN(r13);		\
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\
+BEGIN_FW_FTR_SECTION;				\
 	mfmsr	r10;				\
-	stb	r11,PACAPROCENABLED(r13);	\
 	ori	r10,r10,MSR_EE;			\
 	mtmsrd	r10,1;				\
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
-#define ENABLE_INTS				\
+#else
-BEGIN_FW_FTR_SECTION;				\
+#define DISABLE_INTS				\
-	lbz	r10,PACAPROCENABLED(r13);	\
+	li	r11,0;				\
-	mfmsr	r11;				\
+	stb	r11,PACASOFTIRQEN(r13);		\
-	std	r10,SOFTE(r1);			\
+	stb	r11,PACAHARDIRQEN(r13)
-	ori	r11,r11,MSR_EE;			\
-END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES);	\
-BEGIN_FW_FTR_SECTION;				\
-	ld	r12,_MSR(r1);			\
-	mfmsr	r11;				\
-	rlwimi	r11,r12,0,MSR_EE;		\
-END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\
-	mtmsrd	r11,1
-#else	/* hard enable/disable interrupts */
+#endif /* CONFIG_PPC_ISERIES */
-#define DISABLE_INTS
 #define ENABLE_INTS				\
 	ld	r12,_MSR(r1);			\
 	mfmsr	r11;				\
 	rlwimi	r11,r12,0,MSR_EE;		\
 	mtmsrd	r11,1
-#endif
 #define STD_EXCEPTION_COMMON(trap, label, hdlr)		\
 	.align	7;					\
 	.globl label##_common;				\
 label##_common:						\
 	EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);	\
 	DISABLE_INTS;					\
 	bl	.save_nvgprs;				\
 	addi	r3,r1,STACK_FRAME_OVERHEAD;		\
 	bl	hdlr;					\
 	b	.ret_from_except
 /*
  * Like STD_EXCEPTION_COMMON, but for exceptions that can occur
  * in the idle task and therefore need the special idle handling.
  */
 #define STD_EXCEPTION_COMMON_IDLE(trap, label, hdlr)	\
 	.align	7;					\
 	.globl label##_common;				\
 label##_common:						\
 	EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);	\
 	FINISH_NAP;					\
 	DISABLE_INTS;					\
 	bl	.save_nvgprs;				\
 	addi	r3,r1,STACK_FRAME_OVERHEAD;		\
 	bl	hdlr;					\
 	b	.ret_from_except
 #define STD_EXCEPTION_COMMON_LITE(trap, label, hdlr)	\
 	.align	7;					\
 	.globl label##_common;				\
 label##_common:						\
 	EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);	\
 	FINISH_NAP;					\
 	DISABLE_INTS;					\
 	bl	.ppc64_runlatch_on;			\
 	addi	r3,r1,STACK_FRAME_OVERHEAD;		\
 	bl	hdlr;					\
 	b	.ret_from_except_lite
 /*
  * When the idle code in power4_idle puts the CPU into NAP mode,
  * it has to do so in a loop, and relies on the external interrupt
  * and decrementer interrupt entry code to get it out of the loop.
  * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
  * to signal that it is in the loop and needs help to get out.
  */
 #ifdef CONFIG_PPC_970_NAP
 #define FINISH_NAP				\
 BEGIN_FTR_SECTION				\
 	clrrdi	r11,r1,THREAD_SHIFT;		\
 	ld	r9,TI_LOCAL_FLAGS(r11);		\
 	andi.	r10,r9,_TLF_NAPPING;		\
 	bnel	power4_fixup_nap;		\
 END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
 #else
 #define FINISH_NAP
 #endif
 /*
  * Start of pSeries system interrupt routines
  */
 	. = 0x100
 	.globl __start_interrupts
 __start_interrupts:
 	STD_EXCEPTION_PSERIES(0x100, system_reset)
 	. = 0x200
 _machine_check_pSeries:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13		/* save r13 */
 	EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
 	. = 0x300
 	.globl data_access_pSeries
 data_access_pSeries:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13
 BEGIN_FTR_SECTION
 	mtspr	SPRN_SPRG2,r12
 	mfspr	r13,SPRN_DAR
 	mfspr	r12,SPRN_DSISR
 	srdi	r13,r13,60
 	rlwimi	r13,r12,16,0x20
 	mfcr	r12
 	cmpwi	r13,0x2c
 	beq	.do_stab_bolted_pSeries
 	mtcrf	0x80,r12
 	mfspr	r12,SPRN_SPRG2
 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common)
 	. = 0x380
 	.globl data_access_slb_pSeries
 data_access_slb_pSeries:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13
 	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
 	std	r3,PACA_EXSLB+EX_R3(r13)
 	mfspr	r3,SPRN_DAR
 	std	r9,PACA_EXSLB+EX_R9(r13)	/* save r9 - r12 */
 	mfcr	r9
 #ifdef __DISABLED__
 	/* Keep that around for when we re-implement dynamic VSIDs */
 	cmpdi	r3,0
 	bge	slb_miss_user_pseries
 #endif /* __DISABLED__ */
 	std	r10,PACA_EXSLB+EX_R10(r13)
 	std	r11,PACA_EXSLB+EX_R11(r13)
 	std	r12,PACA_EXSLB+EX_R12(r13)
 	mfspr	r10,SPRN_SPRG1
 	std	r10,PACA_EXSLB+EX_R13(r13)
 	mfspr	r12,SPRN_SRR1		/* and SRR1 */
 	b	.slb_miss_realmode	/* Rel. branch works in real mode */
 	STD_EXCEPTION_PSERIES(0x400, instruction_access)
 	. = 0x480
 	.globl instruction_access_slb_pSeries
 instruction_access_slb_pSeries:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13
 	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
 	std	r3,PACA_EXSLB+EX_R3(r13)
 	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
 	std	r9,PACA_EXSLB+EX_R9(r13)	/* save r9 - r12 */
 	mfcr	r9
 #ifdef __DISABLED__
 	/* Keep that around for when we re-implement dynamic VSIDs */
 	cmpdi	r3,0
 	bge	slb_miss_user_pseries
 #endif /* __DISABLED__ */
 	std	r10,PACA_EXSLB+EX_R10(r13)
 	std	r11,PACA_EXSLB+EX_R11(r13)
 	std	r12,PACA_EXSLB+EX_R12(r13)
 	mfspr	r10,SPRN_SPRG1
 	std	r10,PACA_EXSLB+EX_R13(r13)
 	mfspr	r12,SPRN_SRR1		/* and SRR1 */
 	b	.slb_miss_realmode	/* Rel. branch works in real mode */
-	STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
+	MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
 	STD_EXCEPTION_PSERIES(0x600, alignment)
 	STD_EXCEPTION_PSERIES(0x700, program_check)
 	STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
-	STD_EXCEPTION_PSERIES(0x900, decrementer)
+	MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
 	STD_EXCEPTION_PSERIES(0xa00, trap_0a)
 	STD_EXCEPTION_PSERIES(0xb00, trap_0b)
 	. = 0xc00
 	.globl	system_call_pSeries
 system_call_pSeries:
 	HMT_MEDIUM
 	mr	r9,r13
 	mfmsr	r10
 	mfspr	r13,SPRN_SPRG3
 	mfspr	r11,SPRN_SRR0
 	clrrdi	r12,r13,32
 	oris	r12,r12,system_call_common@h
 	ori	r12,r12,system_call_common@l
 	mtspr	SPRN_SRR0,r12
 	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI
 	mfspr	r12,SPRN_SRR1
 	mtspr	SPRN_SRR1,r10
 	rfid
 	b	.	/* prevent speculative execution */
 	STD_EXCEPTION_PSERIES(0xd00, single_step)
 	STD_EXCEPTION_PSERIES(0xe00, trap_0e)
 	/* We need to deal with the Altivec unavailable exception
 	 * here which is at 0xf20, thus in the middle of the
 	 * prolog code of the PerformanceMonitor one. A little
 	 * trickery is thus necessary
 	 */
 	. = 0xf00
 	b	performance_monitor_pSeries
 	STD_EXCEPTION_PSERIES(0xf20, altivec_unavailable)
 #ifdef CONFIG_CBE_RAS
 	HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
 #endif /* CONFIG_CBE_RAS */
 	STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
 #ifdef CONFIG_CBE_RAS
 	HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
 #endif /* CONFIG_CBE_RAS */
 	STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
 #ifdef CONFIG_CBE_RAS
 	HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
 #endif /* CONFIG_CBE_RAS */
 	. = 0x3000
 /*** pSeries interrupt support ***/
 	/* moved from 0xf00 */
-	STD_EXCEPTION_PSERIES(., performance_monitor)
+	MASKABLE_EXCEPTION_PSERIES(., performance_monitor)
+/*
+ * An interrupt came in while soft-disabled; clear EE in SRR1,
+ * clear paca->hard_enabled and return.
+ */
+masked_interrupt:
+	stb	r10,PACAHARDIRQEN(r13)
+	mtcrf	0x80,r9
+	ld	r9,PACA_EXGEN+EX_R9(r13)
+	mfspr	r10,SPRN_SRR1
+	rldicl	r10,r10,48,1		/* clear MSR_EE */
+	rotldi	r10,r10,16
+	mtspr	SPRN_SRR1,r10
+	ld	r10,PACA_EXGEN+EX_R10(r13)
+	mfspr	r13,SPRN_SPRG1
+	rfid
+	b	.
 	.align	7
 _GLOBAL(do_stab_bolted_pSeries)
 	mtcrf	0x80,r12
 	mfspr	r12,SPRN_SPRG2
 	EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
 /*
  * We have some room here  we use that to put
  * the peries slb miss user trampoline code so it's reasonably
  * away from slb_miss_user_common to avoid problems with rfid
  *
  * This is used for when the SLB miss handler has to go virtual,
  * which doesn't happen for now anymore but will once we re-implement
  * dynamic VSIDs for shared page tables
  */
 #ifdef __DISABLED__
 slb_miss_user_pseries:
 	std	r10,PACA_EXGEN+EX_R10(r13)
 	std	r11,PACA_EXGEN+EX_R11(r13)
 	std	r12,PACA_EXGEN+EX_R12(r13)
 	mfspr	r10,SPRG1
 	ld	r11,PACA_EXSLB+EX_R9(r13)
 	ld	r12,PACA_EXSLB+EX_R3(r13)
 	std	r10,PACA_EXGEN+EX_R13(r13)
 	std	r11,PACA_EXGEN+EX_R9(r13)
 	std	r12,PACA_EXGEN+EX_R3(r13)
 	clrrdi	r12,r13,32
 	mfmsr	r10
 	mfspr	r11,SRR0			/* save SRR0 */
 	ori	r12,r12,slb_miss_user_common@l	/* virt addr of handler */
 	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI
 	mtspr	SRR0,r12
 	mfspr	r12,SRR1			/* and SRR1 */
 	mtspr	SRR1,r10
 	rfid
 	b	.				/* prevent spec. execution */
 #endif /* __DISABLED__ */
 /*
  * Vectors for the FWNMI option.  Share common code.
  */
 	.globl system_reset_fwnmi
       .align 7
 system_reset_fwnmi:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13		/* save r13 */
 	EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXGEN, system_reset_common)
 	.globl machine_check_fwnmi
       .align 7
 machine_check_fwnmi:
 	HMT_MEDIUM
 	mtspr	SPRN_SPRG1,r13		/* save r13 */
 	EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXMC, machine_check_common)
 #ifdef CONFIG_PPC_ISERIES
 /***  ISeries-LPAR interrupt handlers ***/
 	STD_EXCEPTION_ISERIES(0x200, machine_check, PACA_EXMC)
 	.globl data_access_iSeries
 data_access_iSeries:
 	mtspr	SPRN_SPRG1,r13
 BEGIN_FTR_SECTION
 	mtspr	SPRN_SPRG2,r12
 	mfspr	r13,SPRN_DAR
 	mfspr	r12,SPRN_DSISR
 	srdi	r13,r13,60
 	rlwimi	r13,r12,16,0x20
 	mfcr	r12
 	cmpwi	r13,0x2c
 	beq	.do_stab_bolted_iSeries
 	mtcrf	0x80,r12
 	mfspr	r12,SPRN_SPRG2
 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 	EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN)
 	EXCEPTION_PROLOG_ISERIES_2
 	b	data_access_common
 .do_stab_bolted_iSeries:
 	mtcrf	0x80,r12
 	mfspr	r12,SPRN_SPRG2
 	EXCEPTION_PROLOG_ISERIES_1(PACA_EXSLB)
 	EXCEPTION_PROLOG_ISERIES_2
 	b	.do_stab_bolted
 	.globl	data_access_slb_iSeries
 data_access_slb_iSeries:
 	mtspr	SPRN_SPRG1,r13		/* save r13 */
 	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
 	std	r3,PACA_EXSLB+EX_R3(r13)
 	mfspr	r3,SPRN_DAR
 	std	r9,PACA_EXSLB+EX_R9(r13)
 	mfcr	r9
 #ifdef __DISABLED__
 	cmpdi	r3,0
 	bge	slb_miss_user_iseries
 #endif
 	std	r10,PACA_EXSLB+EX_R10(r13)
 	std	r11,PACA_EXSLB+EX_R11(r13)
 	std	r12,PACA_EXSLB+EX_R12(r13)
 	mfspr	r10,SPRN_SPRG1
 	std	r10,PACA_EXSLB+EX_R13(r13)
 	ld	r12,PACALPPACAPTR(r13)
 	ld	r12,LPPACASRR1(r12)
 	b	.slb_miss_realmode
 	STD_EXCEPTION_ISERIES(0x400, instruction_access, PACA_EXGEN)
 	.globl	instruction_access_slb_iSeries
 instruction_access_slb_iSeries:
 	mtspr	SPRN_SPRG1,r13		/* save r13 */
 	mfspr	r13,SPRN_SPRG3		/* get paca address into r13 */
 	std	r3,PACA_EXSLB+EX_R3(r13)
 	ld	r3,PACALPPACAPTR(r13)
 	ld	r3,LPPACASRR0(r3)	/* get SRR0 value */
 	std	r9,PACA_EXSLB+EX_R9(r13)
 	mfcr	r9
 #ifdef __DISABLED__
 	cmpdi	r3,0
 	bge	.slb_miss_user_iseries
 #endif
 	std	r10,PACA_EXSLB+EX_R10(r13)
 	std	r11,PACA_EXSLB+EX_R11(r13)
 	std	r12,PACA_EXSLB+EX_R12(r13)
 	mfspr	r10,SPRN_SPRG1
 	std	r10,PACA_EXSLB+EX_R13(r13)
 	ld	r12,PACALPPACAPTR(r13)
 	ld	r12,LPPACASRR1(r12)
 	b	.slb_miss_realmode
 #ifdef __DISABLED__
 slb_miss_user_iseries:
 	std	r10,PACA_EXGEN+EX_R10(r13)
 	std	r11,PACA_EXGEN+EX_R11(r13)
 	std	r12,PACA_EXGEN+EX_R12(r13)
 	mfspr	r10,SPRG1
 	ld	r11,PACA_EXSLB+EX_R9(r13)
 	ld	r12,PACA_EXSLB+EX_R3(r13)
 	std	r10,PACA_EXGEN+EX_R13(r13)
 	std	r11,PACA_EXGEN+EX_R9(r13)
 	std	r12,PACA_EXGEN+EX_R3(r13)
 	EXCEPTION_PROLOG_ISERIES_2
 	b	slb_miss_user_common
 #endif
 	MASKABLE_EXCEPTION_ISERIES(0x500, hardware_interrupt)
 	STD_EXCEPTION_ISERIES(0x600, alignment, PACA_EXGEN)
 	STD_EXCEPTION_ISERIES(0x700, program_check, PACA_EXGEN)
 	STD_EXCEPTION_ISERIES(0x800, fp_unavailable, PACA_EXGEN)
 	MASKABLE_EXCEPTION_ISERIES(0x900, decrementer)
 	STD_EXCEPTION_ISERIES(0xa00, trap_0a, PACA_EXGEN)
 	STD_EXCEPTION_ISERIES(0xb00, trap_0b, PACA_EXGEN)
 	.globl	system_call_iSeries
 system_call_iSeries:
 	mr	r9,r13
 	mfspr	r13,SPRN_SPRG3
 	EXCEPTION_PROLOG_ISERIES_2
 	b	system_call_common
 	STD_EXCEPTION_ISERIES( 0xd00, single_step, PACA_EXGEN)
 	STD_EXCEPTION_ISERIES( 0xe00, trap_0e, PACA_EXGEN)
 	STD_EXCEPTION_ISERIES( 0xf00, performance_monitor, PACA_EXGEN)
 	.globl system_reset_iSeries
 system_reset_iSeries:
 	mfspr	r13,SPRN_SPRG3		/* Get paca address */
 	mfmsr	r24
 	ori	r24,r24,MSR_RI
 	mtmsrd	r24			/* RI on */
 	lhz	r24,PACAPACAINDEX(r13)	/* Get processor # */
 	cmpwi	0,r24,0			/* Are we processor 0? */
 	beq	.__start_initialization_iSeries	/* Start up the first processor */
 	mfspr	r4,SPRN_CTRLF
 	li	r5,CTRL_RUNLATCH	/* Turn off the run light */
 	andc	r4,r4,r5
 	mtspr	SPRN_CTRLT,r4
 1:
 	HMT_LOW
 #ifdef CONFIG_SMP
 	lbz	r23,PACAPROCSTART(r13)	/* Test if this processor
 					 * should start */
 	sync
 	LOAD_REG_IMMEDIATE(r3,current_set)
 	sldi	r28,r24,3		/* get current_set[cpu#] */
 	ldx	r3,r3,r28
 	addi	r1,r3,THREAD_SIZE
 	subi	r1,r1,STACK_FRAME_OVERHEAD
 	cmpwi	0,r23,0
 	beq	iSeries_secondary_smp_loop	/* Loop until told to go */
 	bne	.__secondary_start		/* Loop until told to go */
 iSeries_secondary_smp_loop:
 	/* Let the Hypervisor know we are alive */
 	/* 8002 is a call to HvCallCfg::getLps, a harmless Hypervisor function */
 	lis	r3,0x8002
 	rldicr	r3,r3,32,15		/* r0 = (r3 << 32) & 0xffff000000000000 */
 #else /* CONFIG_SMP */
 	/* Yield the processor.  This is required for non-SMP kernels
 		which are running on multi-threaded machines. */
 	lis	r3,0x8000
 	rldicr	r3,r3,32,15		/* r3 = (r3 << 32) & 0xffff000000000000 */
 	addi	r3,r3,18		/* r3 = 0x8000000000000012 which is "yield" */
 	li	r4,0			/* "yield timed" */
 	li	r5,-1			/* "yield forever" */
 #endif /* CONFIG_SMP */
 	li	r0,-1			/* r0=-1 indicates a Hypervisor call */
 	sc				/* Invoke the hypervisor via a system call */
 	mfspr	r13,SPRN_SPRG3		/* Put r13 back ???? */
 	b	1b			/* If SMP not configured, secondaries
 					 * loop forever */
 	.globl decrementer_iSeries_masked
 decrementer_iSeries_masked:
 	/* We may not have a valid TOC pointer in here. */
 	li	r11,1
 	ld	r12,PACALPPACAPTR(r13)
 	stb	r11,LPPACADECRINT(r12)
 	LOAD_REG_IMMEDIATE(r12, tb_ticks_per_jiffy)
 	lwz	r12,0(r12)
 	mtspr	SPRN_DEC,r12
 	/* fall through */
 	.globl hardware_interrupt_iSeries_masked
 hardware_interrupt_iSeries_masked:
 	mtcrf	0x80,r9		/* Restore regs */
 	ld	r12,PACALPPACAPTR(r13)
 	ld	r11,LPPACASRR0(r12)
 	ld	r12,LPPACASRR1(r12)
 	mtspr	SPRN_SRR0,r11
 	mtspr	SPRN_SRR1,r12
 	ld	r9,PACA_EXGEN+EX_R9(r13)
 	ld	r10,PACA_EXGEN+EX_R10(r13)
 	ld	r11,PACA_EXGEN+EX_R11(r13)
 	ld	r12,PACA_EXGEN+EX_R12(r13)
 	ld	r13,PACA_EXGEN+EX_R13(r13)
 	rfid
 	b	.	/* prevent speculative execution */
 #endif /* CONFIG_PPC_ISERIES */
 /*** Common interrupt handlers ***/
 	STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
 	/*
 	 * Machine check is different because we use a different
 	 * save area: PACA_EXMC instead of PACA_EXGEN.
 	 */
 	.align	7
 	.globl machine_check_common
 machine_check_common:
 	EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
 	FINISH_NAP
 	DISABLE_INTS
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.machine_check_exception
 	b	.ret_from_except
 	STD_EXCEPTION_COMMON_LITE(0x900, decrementer, .timer_interrupt)
 	STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
 	STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
 	STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
 	STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
 	STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
 	STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
 #ifdef CONFIG_ALTIVEC
 	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
 #else
 	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
 #endif
 #ifdef CONFIG_CBE_RAS
 	STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
 	STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
 	STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
 #endif /* CONFIG_CBE_RAS */
 /*
  * Here we have detected that the kernel stack pointer is bad.
  * R9 contains the saved CR, r13 points to the paca,
  * r10 contains the (bad) kernel stack pointer,
  * r11 and r12 contain the saved SRR0 and SRR1.
  * We switch to using an emergency stack, save the registers there,
  * and call kernel_bad_stack(), which panics.
  */
 bad_stack:
 	ld	r1,PACAEMERGSP(r13)
 	subi	r1,r1,64+INT_FRAME_SIZE
 	std	r9,_CCR(r1)
 	std	r10,GPR1(r1)
 	std	r11,_NIP(r1)
 	std	r12,_MSR(r1)
 	mfspr	r11,SPRN_DAR
 	mfspr	r12,SPRN_DSISR
 	std	r11,_DAR(r1)
 	std	r12,_DSISR(r1)
 	mflr	r10
 	mfctr	r11
 	mfxer	r12
 	std	r10,_LINK(r1)
 	std	r11,_CTR(r1)
 	std	r12,_XER(r1)
 	SAVE_GPR(0,r1)
 	SAVE_GPR(2,r1)
 	SAVE_4GPRS(3,r1)
 	SAVE_2GPRS(7,r1)
 	SAVE_10GPRS(12,r1)
 	SAVE_10GPRS(22,r1)
 	addi	r11,r1,INT_FRAME_SIZE
 	std	r11,0(r1)
 	li	r12,0
 	std	r12,0(r11)
 	ld	r2,PACATOC(r13)
 1:	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.kernel_bad_stack
 	b	1b
 /*
  * Return from an exception with minimal checks.
  * The caller is assumed to have done EXCEPTION_PROLOG_COMMON.
  * If interrupts have been enabled, or anything has been
  * done that might have changed the scheduling status of
  * any task or sent any task a signal, you should use
  * ret_from_except or ret_from_except_lite instead of this.
  */
 	.globl	fast_exception_return
 fast_exception_return:
 	ld	r12,_MSR(r1)
 	ld	r11,_NIP(r1)
 	andi.	r3,r12,MSR_RI		/* check if RI is set */
 	beq-	unrecov_fer
 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
 	andi.	r3,r12,MSR_PR
 	beq	2f
 	ACCOUNT_CPU_USER_EXIT(r3, r4)
 2:
 #endif
 	ld	r3,_CCR(r1)
 	ld	r4,_LINK(r1)
 	ld	r5,_CTR(r1)
 	ld	r6,_XER(r1)
 	mtcr	r3
 	mtlr	r4
 	mtctr	r5
 	mtxer	r6
 	REST_GPR(0, r1)
 	REST_8GPRS(2, r1)
 	mfmsr	r10
-	clrrdi	r10,r10,2		/* clear RI (LE is 0 already) */
+	rldicl	r10,r10,48,1		/* clear EE */
+	rldicr	r10,r10,16,61		/* clear RI (LE is 0 already) */
 	mtmsrd	r10,1
 	mtspr	SPRN_SRR1,r12
 	mtspr	SPRN_SRR0,r11
 	REST_4GPRS(10, r1)
 	ld	r1,GPR1(r1)
 	rfid
 	b	.	/* prevent speculative execution */
 unrecov_fer:
 	bl	.save_nvgprs
 1:	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.unrecoverable_exception
 	b	1b
 /*
  * Here r13 points to the paca, r9 contains the saved CR,
  * SRR0 and SRR1 are saved in r11 and r12,
  * r9 - r13 are saved in paca->exgen.
  */
 	.align	7
 	.globl data_access_common
 data_access_common:
 	mfspr	r10,SPRN_DAR
 	std	r10,PACA_EXGEN+EX_DAR(r13)
 	mfspr	r10,SPRN_DSISR
 	stw	r10,PACA_EXGEN+EX_DSISR(r13)
 	EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
 	ld	r3,PACA_EXGEN+EX_DAR(r13)
 	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
 	li	r5,0x300
 	b	.do_hash_page	 	/* Try to handle as hpte fault */
 	.align	7
 	.globl instruction_access_common
 instruction_access_common:
 	EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
 	ld	r3,_NIP(r1)
 	andis.	r4,r12,0x5820
 	li	r5,0x400
 	b	.do_hash_page		/* Try to handle as hpte fault */
 /*
  * Here is the common SLB miss user that is used when going to virtual
  * mode for SLB misses, that is currently not used
  */
 #ifdef __DISABLED__
 	.align	7
 	.globl	slb_miss_user_common
 slb_miss_user_common:
 	mflr	r10
 	std	r3,PACA_EXGEN+EX_DAR(r13)
 	stw	r9,PACA_EXGEN+EX_CCR(r13)
 	std	r10,PACA_EXGEN+EX_LR(r13)
 	std	r11,PACA_EXGEN+EX_SRR0(r13)
 	bl	.slb_allocate_user
 	ld	r10,PACA_EXGEN+EX_LR(r13)
 	ld	r3,PACA_EXGEN+EX_R3(r13)
 	lwz	r9,PACA_EXGEN+EX_CCR(r13)
 	ld	r11,PACA_EXGEN+EX_SRR0(r13)
 	mtlr	r10
 	beq-	slb_miss_fault
 	andi.	r10,r12,MSR_RI		/* check for unrecoverable exception */
 	beq-	unrecov_user_slb
 	mfmsr	r10
 .machine push
 .machine "power4"
 	mtcrf	0x80,r9
 .machine pop
 	clrrdi	r10,r10,2		/* clear RI before setting SRR0/1 */
 	mtmsrd	r10,1
 	mtspr	SRR0,r11
 	mtspr	SRR1,r12
 	ld	r9,PACA_EXGEN+EX_R9(r13)
 	ld	r10,PACA_EXGEN+EX_R10(r13)
 	ld	r11,PACA_EXGEN+EX_R11(r13)
 	ld	r12,PACA_EXGEN+EX_R12(r13)
 	ld	r13,PACA_EXGEN+EX_R13(r13)
 	rfid
 	b	.
 slb_miss_fault:
 	EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
 	ld	r4,PACA_EXGEN+EX_DAR(r13)
 	li	r5,0
 	std	r4,_DAR(r1)
 	std	r5,_DSISR(r1)
 	b	.handle_page_fault
 unrecov_user_slb:
 	EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
 	DISABLE_INTS
 	bl	.save_nvgprs
 1:	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.unrecoverable_exception
 	b	1b
 #endif /* __DISABLED__ */
 /*
  * r13 points to the PACA, r9 contains the saved CR,
  * r12 contain the saved SRR1, SRR0 is still ready for return
  * r3 has the faulting address
  * r9 - r13 are saved in paca->exslb.
  * r3 is saved in paca->slb_r3
  * We assume we aren't going to take any exceptions during this procedure.
  */
 _GLOBAL(slb_miss_realmode)
 	mflr	r10
 	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
 	std	r10,PACA_EXSLB+EX_LR(r13)	/* save LR */
 	bl	.slb_allocate_realmode
 	/* All done -- return from exception. */
 	ld	r10,PACA_EXSLB+EX_LR(r13)
 	ld	r3,PACA_EXSLB+EX_R3(r13)
 	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
 #ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
 	ld	r11,PACALPPACAPTR(r13)
 	ld	r11,LPPACASRR0(r11)		/* get SRR0 value */
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif /* CONFIG_PPC_ISERIES */
 	mtlr	r10
 	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
 	beq-	unrecov_slb
 .machine	push
 .machine	"power4"
 	mtcrf	0x80,r9
 	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
 .machine	pop
 #ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
 	mtspr	SPRN_SRR0,r11
 	mtspr	SPRN_SRR1,r12
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif /* CONFIG_PPC_ISERIES */
 	ld	r9,PACA_EXSLB+EX_R9(r13)
 	ld	r10,PACA_EXSLB+EX_R10(r13)
 	ld	r11,PACA_EXSLB+EX_R11(r13)
 	ld	r12,PACA_EXSLB+EX_R12(r13)
 	ld	r13,PACA_EXSLB+EX_R13(r13)
 	rfid
 	b	.	/* prevent speculative execution */
 unrecov_slb:
 	EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
 	DISABLE_INTS
 	bl	.save_nvgprs
 1:	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.unrecoverable_exception
 	b	1b
 	.align	7
 	.globl hardware_interrupt_common
 	.globl hardware_interrupt_entry
 hardware_interrupt_common:
 	EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
 	FINISH_NAP
 hardware_interrupt_entry:
 	DISABLE_INTS
 	bl	.ppc64_runlatch_on
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_IRQ
 	b	.ret_from_except_lite
 #ifdef CONFIG_PPC_970_NAP
 power4_fixup_nap:
 	andc	r9,r9,r10
 	std	r9,TI_LOCAL_FLAGS(r11)
 	ld	r10,_LINK(r1)		/* make idle task do the */
 	std	r10,_NIP(r1)		/* equivalent of a blr */
 	blr
 #endif
 	.align	7
 	.globl alignment_common
 alignment_common:
 	mfspr	r10,SPRN_DAR
 	std	r10,PACA_EXGEN+EX_DAR(r13)
 	mfspr	r10,SPRN_DSISR
 	stw	r10,PACA_EXGEN+EX_DSISR(r13)
 	EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
 	ld	r3,PACA_EXGEN+EX_DAR(r13)
 	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
 	std	r3,_DAR(r1)
 	std	r4,_DSISR(r1)
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	ENABLE_INTS
 	bl	.alignment_exception
 	b	.ret_from_except
 	.align	7
 	.globl program_check_common
 program_check_common:
 	EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	ENABLE_INTS
 	bl	.program_check_exception
 	b	.ret_from_except
 	.align	7
 	.globl fp_unavailable_common
 fp_unavailable_common:
 	EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
 	bne	.load_up_fpu		/* if from user, just load it up */
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	ENABLE_INTS
 	bl	.kernel_fp_unavailable_exception
 	BUG_OPCODE
 	.align	7
 	.globl altivec_unavailable_common
 altivec_unavailable_common:
 	EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
 #ifdef CONFIG_ALTIVEC
 BEGIN_FTR_SECTION
 	bne	.load_up_altivec	/* if from user, just load it up */
 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 #endif
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	ENABLE_INTS
 	bl	.altivec_unavailable_exception
 	b	.ret_from_except
 #ifdef CONFIG_ALTIVEC
 /*
  * load_up_altivec(unused, unused, tsk)
  * Disable VMX for the task which had it previously,
  * and save its vector registers in its thread_struct.
  * Enables the VMX for use in the kernel on return.
  * On SMP we know the VMX is free, since we give it up every
  * switch (ie, no lazy save of the vector registers).
  * On entry: r13 == 'current' && last_task_used_altivec != 'current'
  */
 _STATIC(load_up_altivec)
 	mfmsr	r5			/* grab the current MSR */
 	oris	r5,r5,MSR_VEC@h
 	mtmsrd	r5			/* enable use of VMX now */
 	isync
 /*
  * For SMP, we don't do lazy VMX switching because it just gets too
  * horrendously complex, especially when a task switches from one CPU
  * to another.  Instead we call giveup_altvec in switch_to.
  * VRSAVE isn't dealt with here, that is done in the normal context
  * switch code. Note that we could rely on vrsave value to eventually
  * avoid saving all of the VREGs here...
  */
 #ifndef CONFIG_SMP
 	ld	r3,last_task_used_altivec@got(r2)
 	ld	r4,0(r3)
 	cmpdi	0,r4,0
 	beq	1f
 	/* Save VMX state to last_task_used_altivec's THREAD struct */
 	addi	r4,r4,THREAD
 	SAVE_32VRS(0,r5,r4)
 	mfvscr	vr0
 	li	r10,THREAD_VSCR
 	stvx	vr0,r10,r4
 	/* Disable VMX for last_task_used_altivec */
 	ld	r5,PT_REGS(r4)
 	ld	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
 	lis	r6,MSR_VEC@h
 	andc	r4,r4,r6
 	std	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
 1:
 #endif /* CONFIG_SMP */
 	/* Hack: if we get an altivec unavailable trap with VRSAVE
 	 * set to all zeros, we assume this is a broken application
 	 * that fails to set it properly, and thus we switch it to
 	 * all 1's
 	 */
 	mfspr	r4,SPRN_VRSAVE
 	cmpdi	0,r4,0
 	bne+	1f
 	li	r4,-1
 	mtspr	SPRN_VRSAVE,r4
 1:
 	/* enable use of VMX after return */
 	ld	r4,PACACURRENT(r13)
 	addi	r5,r4,THREAD		/* Get THREAD */
 	oris	r12,r12,MSR_VEC@h
 	std	r12,_MSR(r1)
 	li	r4,1
 	li	r10,THREAD_VSCR
 	stw	r4,THREAD_USED_VR(r5)
 	lvx	vr0,r10,r5
 	mtvscr	vr0
 	REST_32VRS(0,r4,r5)
 #ifndef CONFIG_SMP
 	/* Update last_task_used_math to 'current' */
 	subi	r4,r5,THREAD		/* Back to 'current' */
 	std	r4,0(r3)
 #endif /* CONFIG_SMP */
 	/* restore registers and return */
 	b	fast_exception_return
 #endif /* CONFIG_ALTIVEC */
 /*
  * Hash table stuff
  */
 	.align	7
 _GLOBAL(do_hash_page)
 	std	r3,_DAR(r1)
 	std	r4,_DSISR(r1)
 	andis.	r0,r4,0xa450		/* weird error? */
 	bne-	.handle_page_fault	/* if not, try to insert a HPTE */
 BEGIN_FTR_SECTION
 	andis.	r0,r4,0x0020		/* Is it a segment table fault? */
 	bne-	.do_ste_alloc		/* If so handle it */
 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 	/*
 	 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
 	 * accessing a userspace segment (even from the kernel). We assume
 	 * kernel addresses always have the high bit set.
 	 */
 	rlwinm	r4,r4,32-25+9,31-9,31-9	/* DSISR_STORE -> _PAGE_RW */
 	rotldi	r0,r3,15		/* Move high bit into MSR_PR posn */
 	orc	r0,r12,r0		/* MSR_PR | ~high_bit */
 	rlwimi	r4,r0,32-13,30,30	/* becomes _PAGE_USER access bit */
 	ori	r4,r4,1			/* add _PAGE_PRESENT */
 	rlwimi	r4,r5,22+2,31-2,31-2	/* Set _PAGE_EXEC if trap is 0x400 */
 	/*
 	 * On iSeries, we soft-disable interrupts here, then
 	 * hard-enable interrupts so that the hash_page code can spin on
 	 * the hash_table_lock without problems on a shared processor.
 	 */
 	DISABLE_INTS
 	/*
 	 * r3 contains the faulting address
 	 * r4 contains the required access permissions
 	 * r5 contains the trap number
 	 *
 	 * at return r3 = 0 for success
 	 */
 	bl	.hash_page		/* build HPTE if possible */
 	cmpdi	r3,0			/* see if hash_page succeeded */
 #ifdef DO_SOFT_DISABLE
 BEGIN_FW_FTR_SECTION
 	/*
 	 * If we had interrupts soft-enabled at the point where the
 	 * DSI/ISI occurred, and an interrupt came in during hash_page,
 	 * handle it now.
 	 * We jump to ret_from_except_lite rather than fast_exception_return
 	 * because ret_from_except_lite will check for and handle pending
 	 * interrupts if necessary.
 	 */
 	beq	.ret_from_except_lite
 	/* For a hash failure, we don't bother re-enabling interrupts */
 	ble-	12f
 	/*
 	 * hash_page couldn't handle it, set soft interrupt enable back
 	 * to what it was before the trap.  Note that .local_irq_restore
 	 * handles any interrupts pending at this point.
 	 */
 	ld	r3,SOFTE(r1)
 	bl	.local_irq_restore
 	b	11f
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
 BEGIN_FW_FTR_SECTION
 	beq	fast_exception_return   /* Return from exception on success */
 	ble-	12f			/* Failure return from hash_page */
 	/* fall through */
 END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 /* Here we have a page fault that hash_page can't handle. */
 _GLOBAL(handle_page_fault)
 	ENABLE_INTS
 11:	ld	r4,_DAR(r1)
 	ld	r5,_DSISR(r1)
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_page_fault
 	cmpdi	r3,0
 	beq+	.ret_from_except_lite
 	bl	.save_nvgprs
 	mr	r5,r3
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	lwz	r4,_DAR(r1)
 	bl	.bad_page_fault
 	b	.ret_from_except
 /* We have a page fault that hash_page could handle but HV refused
  * the PTE insertion
  */
 12:	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	lwz	r4,_DAR(r1)
 	bl	.low_hash_fault
 	b	.ret_from_except
 	/* here we have a segment miss */
 _GLOBAL(do_ste_alloc)
 	bl	.ste_allocate		/* try to insert stab entry */
 	cmpdi	r3,0
 	beq+	fast_exception_return
 	b	.handle_page_fault
 /*
  * r13 points to the PACA, r9 contains the saved CR,
  * r11 and r12 contain the saved SRR0 and SRR1.
  * r9 - r13 are saved in paca->exslb.
  * We assume we aren't going to take any exceptions during this procedure.
  * We assume (DAR >> 60) == 0xc.
  */
 	.align	7
 _GLOBAL(do_stab_bolted)
 	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
 	std	r11,PACA_EXSLB+EX_SRR0(r13)	/* save SRR0 in exc. frame */
 	/* Hash to the primary group */
 	ld	r10,PACASTABVIRT(r13)
 	mfspr	r11,SPRN_DAR
 	srdi	r11,r11,28
 	rldimi	r10,r11,7,52	/* r10 = first ste of the group */
 	/* Calculate VSID */
 	/* This is a kernel address, so protovsid = ESID */
 	ASM_VSID_SCRAMBLE(r11, r9)
 	rldic	r9,r11,12,16	/* r9 = vsid << 12 */
 	/* Search the primary group for a free entry */
 1:	ld	r11,0(r10)	/* Test valid bit of the current ste	*/
 	andi.	r11,r11,0x80
 	beq	2f
 	addi	r10,r10,16
 	andi.	r11,r10,0x70
 	bne	1b
 	/* Stick for only searching the primary group for now.		*/
 	/* At least for now, we use a very simple random castout scheme */
 	/* Use the TB as a random number ;  OR in 1 to avoid entry 0	*/
 	mftb	r11
 	rldic	r11,r11,4,57	/* r11 = (r11 << 4) & 0x70 */
 	ori	r11,r11,0x10
 	/* r10 currently points to an ste one past the group of interest */
 	/* make it point to the randomly selected entry			*/
 	subi	r10,r10,128
 	or 	r10,r10,r11	/* r10 is the entry to invalidate	*/
 	isync			/* mark the entry invalid		*/
 	ld	r11,0(r10)
 	rldicl	r11,r11,56,1	/* clear the valid bit */
 	rotldi	r11,r11,8
 	std	r11,0(r10)
 	sync
 	clrrdi	r11,r11,28	/* Get the esid part of the ste		*/
 	slbie	r11
 2:	std	r9,8(r10)	/* Store the vsid part of the ste	*/
 	eieio
 	mfspr	r11,SPRN_DAR		/* Get the new esid			*/
 	clrrdi	r11,r11,28	/* Permits a full 32b of ESID		*/
 	ori	r11,r11,0x90	/* Turn on valid and kp			*/
 	std	r11,0(r10)	/* Put new entry back into the stab	*/
 	sync
 	/* All done -- return from exception. */
 	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
 	ld	r11,PACA_EXSLB+EX_SRR0(r13)	/* get saved SRR0 */
 	andi.	r10,r12,MSR_RI
 	beq-	unrecov_slb
 	mtcrf	0x80,r9			/* restore CR */
 	mfmsr	r10
 	clrrdi	r10,r10,2
 	mtmsrd	r10,1
 	mtspr	SPRN_SRR0,r11
 	mtspr	SPRN_SRR1,r12
 	ld	r9,PACA_EXSLB+EX_R9(r13)
 	ld	r10,PACA_EXSLB+EX_R10(r13)
 	ld	r11,PACA_EXSLB+EX_R11(r13)
 	ld	r12,PACA_EXSLB+EX_R12(r13)
 	ld	r13,PACA_EXSLB+EX_R13(r13)
 	rfid
 	b	.	/* prevent speculative execution */
 /*
  * Space for CPU0's segment table.
  *
  * On iSeries, the hypervisor must fill in at least one entry before
  * we get control (with relocate on).  The address is give to the hv
  * as a page number (see xLparMap in lpardata.c), so this must be at a
  * fixed address (the linker can't compute (u64)&initial_stab >>
  * PAGE_SHIFT).
  */
 	. = STAB0_OFFSET	/* 0x6000 */
 	.globl initial_stab
 initial_stab:
 	.space	4096
 /*
  * Data area reserved for FWNMI option.
  * This address (0x7000) is fixed by the RPA.
  */
 	.= 0x7000
 	.globl fwnmi_data_area
 fwnmi_data_area:
 	/* iSeries does not use the FWNMI stuff, so it is safe to put
 	 * this here, even if we later allow kernels that will boot on
 	 * both pSeries and iSeries */
 #ifdef CONFIG_PPC_ISERIES
         . = LPARMAP_PHYS
 #include "lparmap.s"
 /*
  * This ".text" is here for old compilers that generate a trailing
  * .note section when compiling .c files to .s
  */
 	.text
 #endif /* CONFIG_PPC_ISERIES */
         . = 0x8000
 /*
  * On pSeries and most other platforms, secondary processors spin
  * in the following code.
  * At entry, r3 = this processor's number (physical cpu id)
  */
 _GLOBAL(generic_secondary_smp_init)
 	mr	r24,r3
 	/* turn on 64-bit mode */
 	bl	.enable_64b_mode
 	isync
 	/* Set up a paca value for this processor. Since we have the
 	 * physical cpu id in r24, we need to search the pacas to find
 	 * which logical id maps to our physical one.
 	 */
 	LOAD_REG_IMMEDIATE(r13, paca)	/* Get base vaddr of paca array	 */
 	li	r5,0			/* logical cpu id                */
 1:	lhz	r6,PACAHWCPUID(r13)	/* Load HW procid from paca      */
 	cmpw	r6,r24			/* Compare to our id             */
 	beq	2f
 	addi	r13,r13,PACA_SIZE	/* Loop to next PACA on miss     */
 	addi	r5,r5,1
 	cmpwi	r5,NR_CPUS
 	blt	1b
 	mr	r3,r24			/* not found, copy phys to r3	 */
 	b	.kexec_wait		/* next kernel might do better	 */
 2:	mtspr	SPRN_SPRG3,r13		/* Save vaddr of paca in SPRG3	 */
 	/* From now on, r24 is expected to be logical cpuid */
 	mr	r24,r5
 3:	HMT_LOW
 	lbz	r23,PACAPROCSTART(r13)	/* Test if this processor should */
 					/* start.			 */
 	sync
 #ifndef CONFIG_SMP
 	b	3b			/* Never go on non-SMP		 */
 #else
 	cmpwi	0,r23,0
 	beq	3b			/* Loop until told to go	 */
 	/* See if we need to call a cpu state restore handler */
 	LOAD_REG_IMMEDIATE(r23, cur_cpu_spec)
 	ld	r23,0(r23)
 	ld	r23,CPU_SPEC_RESTORE(r23)
 	cmpdi	0,r23,0
 	beq	4f
 	ld	r23,0(r23)
 	mtctr	r23
 	bctrl
 4:	/* Create a temp kernel stack for use before relocation is on.	*/
 	ld	r1,PACAEMERGSP(r13)
 	subi	r1,r1,STACK_FRAME_OVERHEAD
 	b	.__secondary_start
 #endif
 #ifdef CONFIG_PPC_ISERIES
 _STATIC(__start_initialization_iSeries)
 	/* Clear out the BSS */
 	LOAD_REG_IMMEDIATE(r11,__bss_stop)
 	LOAD_REG_IMMEDIATE(r8,__bss_start)
 	sub	r11,r11,r8		/* bss size			*/
 	addi	r11,r11,7		/* round up to an even double word */
 	rldicl. r11,r11,61,3		/* shift right by 3		*/
 	beq	4f
 	addi	r8,r8,-8
 	li	r0,0
 	mtctr	r11			/* zero this many doublewords	*/
 3:	stdu	r0,8(r8)
 	bdnz	3b
 4:
 	LOAD_REG_IMMEDIATE(r1,init_thread_union)
 	addi	r1,r1,THREAD_SIZE
 	li	r0,0
 	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
 	LOAD_REG_IMMEDIATE(r3,cpu_specs)
 	LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
 	li	r5,0
 	bl	.identify_cpu
 	LOAD_REG_IMMEDIATE(r2,__toc_start)
 	addi	r2,r2,0x4000
 	addi	r2,r2,0x4000
 	bl	.iSeries_early_setup
 	bl	.early_setup
 	/* relocation is on at this point */
 	b	.start_here_common
 #endif /* CONFIG_PPC_ISERIES */
 #ifdef CONFIG_PPC_MULTIPLATFORM
 _STATIC(__mmu_off)
 	mfmsr	r3
 	andi.	r0,r3,MSR_IR|MSR_DR
 	beqlr
 	andc	r3,r3,r0
 	mtspr	SPRN_SRR0,r4
 	mtspr	SPRN_SRR1,r3
 	sync
 	rfid
 	b	.	/* prevent speculative execution */
 /*
  * Here is our main kernel entry point. We support currently 2 kind of entries
  * depending on the value of r5.
  *
  *   r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
  *                 in r3...r7
  *
  *   r5 == NULL -> kexec style entry. r3 is a physical pointer to the
  *                 DT block, r4 is a physical pointer to the kernel itself
  *
  */
 _GLOBAL(__start_initialization_multiplatform)
 #ifdef CONFIG_PPC_MULTIPLATFORM
 	/*
 	 * Are we booted from a PROM Of-type client-interface ?
 	 */
 	cmpldi	cr0,r5,0
 	bne	.__boot_from_prom		/* yes -> prom */
 #endif
 	/* Save parameters */
 	mr	r31,r3
 	mr	r30,r4
 	/* Make sure we are running in 64 bits mode */
 	bl	.enable_64b_mode
 	/* Setup some critical 970 SPRs before switching MMU off */
 	mfspr	r0,SPRN_PVR
 	srwi	r0,r0,16
 	cmpwi	r0,0x39		/* 970 */
 	beq	1f
 	cmpwi	r0,0x3c		/* 970FX */
 	beq	1f
 	cmpwi	r0,0x44		/* 970MP */
 	bne	2f
 1:	bl	.__cpu_preinit_ppc970
 2:
 	/* Switch off MMU if not already */
 	LOAD_REG_IMMEDIATE(r4, .__after_prom_start - KERNELBASE)
 	add	r4,r4,r30
 	bl	.__mmu_off
 	b	.__after_prom_start
 #ifdef CONFIG_PPC_MULTIPLATFORM
 _STATIC(__boot_from_prom)
 	/* Save parameters */
 	mr	r31,r3
 	mr	r30,r4
 	mr	r29,r5
 	mr	r28,r6
 	mr	r27,r7
 	/*
 	 * Align the stack to 16-byte boundary
 	 * Depending on the size and layout of the ELF sections in the initial
 	 * boot binary, the stack pointer will be unalignet on PowerMac
 	 */
 	rldicr	r1,r1,0,59
 	/* Make sure we are running in 64 bits mode */
 	bl	.enable_64b_mode
 	/* put a relocation offset into r3 */
 	bl	.reloc_offset
 	LOAD_REG_IMMEDIATE(r2,__toc_start)
 	addi	r2,r2,0x4000
 	addi	r2,r2,0x4000
 	/* Relocate the TOC from a virt addr to a real addr */
 	add	r2,r2,r3
 	/* Restore parameters */
 	mr	r3,r31
 	mr	r4,r30
 	mr	r5,r29
 	mr	r6,r28
 	mr	r7,r27
 	/* Do all of the interaction with OF client interface */
 	bl	.prom_init
 	/* We never return */
 	trap
 #endif
 /*
  * At this point, r3 contains the physical address we are running at,
  * returned by prom_init()
  */
 _STATIC(__after_prom_start)
 /*
  * We need to run with __start at physical address PHYSICAL_START.
  * This will leave some code in the first 256B of
  * real memory, which are reserved for software use.
  * The remainder of the first page is loaded with the fixed
  * interrupt vectors.  The next two pages are filled with
  * unknown exception placeholders.
  *
  * Note: This process overwrites the OF exception vectors.
  *	r26 == relocation offset
  *	r27 == KERNELBASE
  */
 	bl	.reloc_offset
 	mr	r26,r3
 	LOAD_REG_IMMEDIATE(r27, KERNELBASE)
 	LOAD_REG_IMMEDIATE(r3, PHYSICAL_START)	/* target addr */
 	// XXX FIXME: Use phys returned by OF (r30)
 	add	r4,r27,r26 		/* source addr			 */
 					/* current address of _start	 */
 					/*   i.e. where we are running	 */
 					/*	the source addr		 */
 	cmpdi	r4,0			/* In some cases the loader may  */
 	beq	.start_here_multiplatform /* have already put us at zero */
 					/* so we can skip the copy.      */
 	LOAD_REG_IMMEDIATE(r5,copy_to_here) /* # bytes of memory to copy */
 	sub	r5,r5,r27
 	li	r6,0x100		/* Start offset, the first 0x100 */
 					/* bytes were copied earlier.	 */
 	bl	.copy_and_flush		/* copy the first n bytes	 */
 					/* this includes the code being	 */
 					/* executed here.		 */
 	LOAD_REG_IMMEDIATE(r0, 4f)	/* Jump to the copy of this code */
 	mtctr	r0			/* that we just made/relocated	 */
 	bctr
 4:	LOAD_REG_IMMEDIATE(r5,klimit)
 	add	r5,r5,r26
 	ld	r5,0(r5)		/* get the value of klimit */
 	sub	r5,r5,r27
 	bl	.copy_and_flush		/* copy the rest */
 	b	.start_here_multiplatform
 #endif /* CONFIG_PPC_MULTIPLATFORM */
 /*
  * Copy routine used to copy the kernel to start at physical address 0
  * and flush and invalidate the caches as needed.
  * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
  * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
  *
  * Note: this routine *only* clobbers r0, r6 and lr
  */
 _GLOBAL(copy_and_flush)
 	addi	r5,r5,-8
 	addi	r6,r6,-8
 4:	li	r0,8			/* Use the smallest common	*/
 					/* denominator cache line	*/
 					/* size.  This results in	*/
 					/* extra cache line flushes	*/
 					/* but operation is correct.	*/
 					/* Can't get cache line size	*/
 					/* from NACA as it is being	*/
 					/* moved too.			*/
 	mtctr	r0			/* put # words/line in ctr	*/
 3:	addi	r6,r6,8			/* copy a cache line		*/
 	ldx	r0,r6,r4
 	stdx	r0,r6,r3
 	bdnz	3b
 	dcbst	r6,r3			/* write it to memory		*/
 	sync
 	icbi	r6,r3			/* flush the icache line	*/
 	cmpld	0,r6,r5
 	blt	4b
 	sync
 	addi	r5,r5,8
 	addi	r6,r6,8
 	blr
 .align 8
 copy_to_here:
 #ifdef CONFIG_SMP
 #ifdef CONFIG_PPC_PMAC
 /*
  * On PowerMac, secondary processors starts from the reset vector, which
  * is temporarily turned into a call to one of the functions below.
  */
 	.section ".text";
 	.align 2 ;
 	.globl	__secondary_start_pmac_0
 __secondary_start_pmac_0:
 	/* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
 	li	r24,0
 	b	1f
 	li	r24,1
 	b	1f
 	li	r24,2
 	b	1f
 	li	r24,3
 1:
 _GLOBAL(pmac_secondary_start)
 	/* turn on 64-bit mode */
 	bl	.enable_64b_mode
 	isync
 	/* Copy some CPU settings from CPU 0 */
 	bl	.__restore_cpu_ppc970
 	/* pSeries do that early though I don't think we really need it */
 	mfmsr	r3
 	ori	r3,r3,MSR_RI
 	mtmsrd	r3			/* RI on */
 	/* Set up a paca value for this processor. */
 	LOAD_REG_IMMEDIATE(r4, paca)	/* Get base vaddr of paca array	*/
 	mulli	r13,r24,PACA_SIZE	 /* Calculate vaddr of right paca */
 	add	r13,r13,r4		/* for this processor.		*/
 	mtspr	SPRN_SPRG3,r13		 /* Save vaddr of paca in SPRG3	*/
 	/* Create a temp kernel stack for use before relocation is on.	*/
 	ld	r1,PACAEMERGSP(r13)
 	subi	r1,r1,STACK_FRAME_OVERHEAD
 	b	.__secondary_start
 #endif /* CONFIG_PPC_PMAC */
 /*
  * This function is called after the master CPU has released the
  * secondary processors.  The execution environment is relocation off.
  * The paca for this processor has the following fields initialized at
  * this point:
  *   1. Processor number
  *   2. Segment table pointer (virtual address)
  * On entry the following are set:
  *   r1	= stack pointer.  vaddr for iSeries, raddr (temp stack) for pSeries
  *   r24   = cpu# (in Linux terms)
  *   r13   = paca virtual address
  *   SPRG3 = paca virtual address
  */
 _GLOBAL(__secondary_start)
 	/* Set thread priority to MEDIUM */
 	HMT_MEDIUM
 	/* Load TOC */
 	ld	r2,PACATOC(r13)
 	/* Do early setup for that CPU (stab, slb, hash table pointer) */
 	bl	.early_setup_secondary
 	/* Initialize the kernel stack.  Just a repeat for iSeries.	 */
 	LOAD_REG_ADDR(r3, current_set)
 	sldi	r28,r24,3		/* get current_set[cpu#]	 */
 	ldx	r1,r3,r28
 	addi	r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
 	std	r1,PACAKSAVE(r13)
 	/* Clear backchain so we get nice backtraces */
 	li	r7,0
 	mtlr	r7
 	/* enable MMU and jump to start_secondary */
 	LOAD_REG_ADDR(r3, .start_secondary_prolog)
 	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
 	ori	r4,r4,MSR_EE
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+	stb	r7,PACASOFTIRQEN(r13)
+	stb	r7,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 	mtspr	SPRN_SRR0,r3
 	mtspr	SPRN_SRR1,r4
 	rfid
 	b	.	/* prevent speculative execution */
 /*
  * Running with relocation on at this point.  All we want to do is
  * zero the stack back-chain pointer before going into C code.
  */
 _GLOBAL(start_secondary_prolog)
 	li	r3,0
 	std	r3,0(r1)		/* Zero the stack frame pointer	*/
 	bl	.start_secondary
 	b	.
 #endif
 /*
  * This subroutine clobbers r11 and r12
  */
 _GLOBAL(enable_64b_mode)
 	mfmsr	r11			/* grab the current MSR */
 	li	r12,1
 	rldicr	r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
 	or	r11,r11,r12
 	li	r12,1
 	rldicr	r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
 	or	r11,r11,r12
 	mtmsrd	r11
 	isync
 	blr
 #ifdef CONFIG_PPC_MULTIPLATFORM
 /*
  * This is where the main kernel code starts.
  */
 _STATIC(start_here_multiplatform)
 	/* get a new offset, now that the kernel has moved. */
 	bl	.reloc_offset
 	mr	r26,r3
 	/* Clear out the BSS. It may have been done in prom_init,
 	 * already but that's irrelevant since prom_init will soon
 	 * be detached from the kernel completely. Besides, we need
 	 * to clear it now for kexec-style entry.
 	 */
 	LOAD_REG_IMMEDIATE(r11,__bss_stop)
 	LOAD_REG_IMMEDIATE(r8,__bss_start)
 	sub	r11,r11,r8		/* bss size			*/
 	addi	r11,r11,7		/* round up to an even double word */
 	rldicl. r11,r11,61,3		/* shift right by 3		*/
 	beq	4f
 	addi	r8,r8,-8
 	li	r0,0
 	mtctr	r11			/* zero this many doublewords	*/
 3:	stdu	r0,8(r8)
 	bdnz	3b
 4:
 	mfmsr	r6
 	ori	r6,r6,MSR_RI
 	mtmsrd	r6			/* RI on */
 	/* The following gets the stack and TOC set up with the regs */
 	/* pointing to the real addr of the kernel stack.  This is   */
 	/* all done to support the C function call below which sets  */
 	/* up the htab.  This is done because we have relocated the  */
 	/* kernel but are still running in real mode. */
 	LOAD_REG_IMMEDIATE(r3,init_thread_union)
 	add	r3,r3,r26
 	/* set up a stack pointer (physical address) */
 	addi	r1,r3,THREAD_SIZE
 	li	r0,0
 	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
 	/* set up the TOC (physical address) */
 	LOAD_REG_IMMEDIATE(r2,__toc_start)
 	addi	r2,r2,0x4000
 	addi	r2,r2,0x4000
 	add	r2,r2,r26
 	LOAD_REG_IMMEDIATE(r3, cpu_specs)
 	add	r3,r3,r26
 	LOAD_REG_IMMEDIATE(r4,cur_cpu_spec)
 	add	r4,r4,r26
 	mr	r5,r26
 	bl	.identify_cpu
 	/* Do very early kernel initializations, including initial hash table,
 	 * stab and slb setup before we turn on relocation.	*/
 	/* Restore parameters passed from prom_init/kexec */
 	mr	r3,r31
  	bl	.early_setup
 	LOAD_REG_IMMEDIATE(r3, .start_here_common)
 	LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
 	mtspr	SPRN_SRR0,r3
 	mtspr	SPRN_SRR1,r4
 	rfid
 	b	.	/* prevent speculative execution */
 #endif /* CONFIG_PPC_MULTIPLATFORM */
 	/* This is where all platforms converge execution */
 _STATIC(start_here_common)
 	/* relocation is on at this point */
 	/* The following code sets up the SP and TOC now that we are */
 	/* running with translation enabled. */
 	LOAD_REG_IMMEDIATE(r3,init_thread_union)
 	/* set up the stack */
 	addi	r1,r3,THREAD_SIZE
 	li	r0,0
 	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
 	/* Apply the CPUs-specific fixups (nop out sections not relevant
 	 * to this CPU
 	 */
 	li	r3,0
 	bl	.do_cpu_ftr_fixups
 	bl	.do_fw_ftr_fixups
 	/* ptr to current */
 	LOAD_REG_IMMEDIATE(r4, init_task)
 	std	r4,PACACURRENT(r13)
 	/* Load the TOC */
 	ld	r2,PACATOC(r13)
 	std	r1,PACAKSAVE(r13)
 	bl	.setup_system
 	/* Load up the kernel context */
 5:
-#ifdef DO_SOFT_DISABLE
-BEGIN_FW_FTR_SECTION
 	li	r5,0
-	stb	r5,PACAPROCENABLED(r13)	/* Soft Disabled */
+	stb	r5,PACASOFTIRQEN(r13)	/* Soft Disabled */
+#ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
 	mfmsr	r5
 	ori	r5,r5,MSR_EE		/* Hard Enabled */
 	mtmsrd	r5
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+	stb	r5,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 	bl .start_kernel
 	/* Not reached */
 	BUG_OPCODE
 /*
  * We put a few things here that have to be page-aligned.
  * This stuff goes at the beginning of the bss, which is page-aligned.
  */
 	.section ".bss"

arch/powerpc/kernel/idle_power4.S

Diff comments View file @ d04c56f

 /*
  *  This file contains the power_save function for 970-family CPUs.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  */
 #include <linux/threads.h>
 #include <asm/processor.h>
 #include <asm/page.h>
 #include <asm/cputable.h>
 #include <asm/thread_info.h>
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #undef DEBUG
 	.text
 _GLOBAL(power4_idle)
 BEGIN_FTR_SECTION
 	blr
 END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
 	/* Now check if user or arch enabled NAP mode */
 	LOAD_REG_ADDRBASE(r3,powersave_nap)
 	lwz	r4,ADDROFF(powersave_nap)(r3)
 	cmpwi	0,r4,0
 	beqlr
 	/* Go to NAP now */
+	mfmsr	r7
+	rldicl	r0,r7,48,1
+	rotldi	r0,r0,16
+	mtmsrd	r0,1			/* hard-disable interrupts */
+	li	r0,1
+	stb	r0,PACASOFTIRQEN(r13)	/* we'll hard-enable shortly */
+	stb	r0,PACAHARDIRQEN(r13)
 BEGIN_FTR_SECTION
 	DSSALL
 	sync
 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 	clrrdi	r9,r1,THREAD_SHIFT	/* current thread_info */
 	ld	r8,TI_LOCAL_FLAGS(r9)	/* set napping bit */
 	ori	r8,r8,_TLF_NAPPING	/* so when we take an exception */
 	std	r8,TI_LOCAL_FLAGS(r9)	/* it will return to our caller */
-	mfmsr	r7
 	ori	r7,r7,MSR_EE
 	oris	r7,r7,MSR_POW@h
 1:	sync
 	isync
 	mtmsrd	r7
 	isync
 	b	1b

arch/powerpc/kernel/irq.c

Diff comments View file @ d04c56f

 /*
  *  Derived from arch/i386/kernel/irq.c
  *    Copyright (C) 1992 Linus Torvalds
  *  Adapted from arch/i386 by Gary Thomas
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *  Updated and modified by Cort Dougan <cort@fsmlabs.com>
  *    Copyright (C) 1996-2001 Cort Dougan
  *  Adapted for Power Macintosh by Paul Mackerras
  *    Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
  *  Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk).
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  *
  * This file contains the code used by various IRQ handling routines:
  * asking for different IRQ's should be done through these routines
  * instead of just grabbing them. Thus setups with different IRQ numbers
  * shouldn't result in any weird surprises, and installing new handlers
  * should be easier.
  *
  * The MPC8xx has an interrupt mask in the SIU.  If a bit is set, the
  * interrupt is _enabled_.  As expected, IRQ0 is bit 0 in the 32-bit
  * mask register (of which only 16 are defined), hence the weird shifting
  * and complement of the cached_irq_mask.  I want to be able to stuff
  * this right into the SIU SMASK register.
  * Many of the prep/chrp functions are conditional compiled on CONFIG_8xx
  * to reduce code space and undefined function references.
  */
 #undef DEBUG
 #include <linux/module.h>
 #include <linux/threads.h>
 #include <linux/kernel_stat.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/ptrace.h>
 #include <linux/ioport.h>
 #include <linux/interrupt.h>
 #include <linux/timex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/irq.h>
 #include <linux/seq_file.h>
 #include <linux/cpumask.h>
 #include <linux/profile.h>
 #include <linux/bitops.h>
 #include <linux/list.h>
 #include <linux/radix-tree.h>
 #include <linux/mutex.h>
 #include <linux/bootmem.h>
 #include <linux/pci.h>
 #include <asm/uaccess.h>
 #include <asm/system.h>
 #include <asm/io.h>
 #include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <asm/cache.h>
 #include <asm/prom.h>
 #include <asm/ptrace.h>
 #include <asm/machdep.h>
 #include <asm/udbg.h>
-#ifdef CONFIG_PPC_ISERIES
+#ifdef CONFIG_PPC64
 #include <asm/paca.h>
+#include <asm/firmware.h>
 #endif
 int __irq_offset_value;
 static int ppc_spurious_interrupts;
 #ifdef CONFIG_PPC32
 EXPORT_SYMBOL(__irq_offset_value);
 atomic_t ppc_n_lost_interrupts;
 #ifndef CONFIG_PPC_MERGE
 #define NR_MASK_WORDS	((NR_IRQS + 31) / 32)
 unsigned long ppc_cached_irq_mask[NR_MASK_WORDS];
 #endif
 #ifdef CONFIG_TAU_INT
 extern int tau_initialized;
 extern int tau_interrupts(int);
 #endif
 #endif /* CONFIG_PPC32 */
 #if defined(CONFIG_SMP) && !defined(CONFIG_PPC_MERGE)
 extern atomic_t ipi_recv;
 extern atomic_t ipi_sent;
 #endif
 #ifdef CONFIG_PPC64
 EXPORT_SYMBOL(irq_desc);
 int distribute_irqs = 1;
+void local_irq_restore(unsigned long en)
+{
+	get_paca()->soft_enabled = en;
+	if (!en)
+		return;
+	if (firmware_has_feature(FW_FEATURE_ISERIES)) {
+		if (get_paca()->lppaca_ptr->int_dword.any_int)
+			iseries_handle_interrupts();
+		return;
+	}
+	if (get_paca()->hard_enabled)
+		return;
+	/* need to hard-enable interrupts here */
+	get_paca()->hard_enabled = en;
+	if ((int)mfspr(SPRN_DEC) < 0)
+		mtspr(SPRN_DEC, 1);
+	hard_irq_enable();
+}
 #endif /* CONFIG_PPC64 */
 int show_interrupts(struct seq_file *p, void *v)
 {
 	int i = *(loff_t *)v, j;
 	struct irqaction *action;
 	irq_desc_t *desc;
 	unsigned long flags;
 	if (i == 0) {
 		seq_puts(p, "           ");
 		for_each_online_cpu(j)
 			seq_printf(p, "CPU%d       ", j);
 		seq_putc(p, '\n');
 	}
 	if (i < NR_IRQS) {
 		desc = get_irq_desc(i);
 		spin_lock_irqsave(&desc->lock, flags);
 		action = desc->action;
 		if (!action || !action->handler)
 			goto skip;
 		seq_printf(p, "%3d: ", i);
 #ifdef CONFIG_SMP
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
 #else
 		seq_printf(p, "%10u ", kstat_irqs(i));
 #endif /* CONFIG_SMP */
 		if (desc->chip)
 			seq_printf(p, " %s ", desc->chip->typename);
 		else
 			seq_puts(p, "  None      ");
 		seq_printf(p, "%s", (desc->status & IRQ_LEVEL) ? "Level " : "Edge  ");
 		seq_printf(p, "    %s", action->name);
 		for (action = action->next; action; action = action->next)
 			seq_printf(p, ", %s", action->name);
 		seq_putc(p, '\n');
 skip:
 		spin_unlock_irqrestore(&desc->lock, flags);
 	} else if (i == NR_IRQS) {
 #ifdef CONFIG_PPC32
 #ifdef CONFIG_TAU_INT
 		if (tau_initialized){
 			seq_puts(p, "TAU: ");
 			for_each_online_cpu(j)
 				seq_printf(p, "%10u ", tau_interrupts(j));
 			seq_puts(p, "  PowerPC             Thermal Assist (cpu temp)\n");
 		}
 #endif
 #if defined(CONFIG_SMP) && !defined(CONFIG_PPC_MERGE)
 		/* should this be per processor send/receive? */
 		seq_printf(p, "IPI (recv/sent): %10u/%u\n",
 				atomic_read(&ipi_recv), atomic_read(&ipi_sent));
 #endif
 #endif /* CONFIG_PPC32 */
 		seq_printf(p, "BAD: %10u\n", ppc_spurious_interrupts);
 	}
 	return 0;
 }
 #ifdef CONFIG_HOTPLUG_CPU
 void fixup_irqs(cpumask_t map)
 {
 	unsigned int irq;
 	static int warned;
 	for_each_irq(irq) {
 		cpumask_t mask;
 		if (irq_desc[irq].status & IRQ_PER_CPU)
 			continue;
 		cpus_and(mask, irq_desc[irq].affinity, map);
 		if (any_online_cpu(mask) == NR_CPUS) {
 			printk("Breaking affinity for irq %i\n", irq);
 			mask = map;
 		}
 		if (irq_desc[irq].chip->set_affinity)
 			irq_desc[irq].chip->set_affinity(irq, mask);
 		else if (irq_desc[irq].action && !(warned++))
 			printk("Cannot set affinity for irq %i\n", irq);
 	}
 	local_irq_enable();
 	mdelay(1);
 	local_irq_disable();
 }
 #endif
 void do_IRQ(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 	unsigned int irq;
 #ifdef CONFIG_IRQSTACKS
 	struct thread_info *curtp, *irqtp;
 #endif
         irq_enter();
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 	/* Debugging check for stack overflow: is there less than 2KB free? */
 	{
 		long sp;
 		sp = __get_SP() & (THREAD_SIZE-1);
 		if (unlikely(sp < (sizeof(struct thread_info) + 2048))) {
 			printk("do_IRQ: stack overflow: %ld\n",
 				sp - sizeof(struct thread_info));
 			dump_stack();
 		}
 	}
 #endif
 	/*
 	 * Every platform is required to implement ppc_md.get_irq.
 	 * This function will either return an irq number or -1 to
 	 * indicate there are no more pending.
 	 * The value -2 is for buggy hardware and means that this IRQ
 	 * has already been handled. -- Tom
 	 */
 	irq = ppc_md.get_irq();
 	if (irq != NO_IRQ && irq != NO_IRQ_IGNORE) {
 #ifdef CONFIG_IRQSTACKS
 		/* Switch to the irq stack to handle this */
 		curtp = current_thread_info();
 		irqtp = hardirq_ctx[smp_processor_id()];
 		if (curtp != irqtp) {
 			struct irq_desc *desc = irq_desc + irq;
 			void *handler = desc->handle_irq;
 			if (handler == NULL)
 				handler = &__do_IRQ;
 			irqtp->task = curtp->task;
 			irqtp->flags = 0;
 			call_handle_irq(irq, desc, irqtp, handler);
 			irqtp->task = NULL;
 			if (irqtp->flags)
 				set_bits(irqtp->flags, &curtp->flags);
 		} else
 #endif
 			generic_handle_irq(irq);
 	} else if (irq != NO_IRQ_IGNORE)
 		/* That's not SMP safe ... but who cares ? */
 		ppc_spurious_interrupts++;
         irq_exit();
 	set_irq_regs(old_regs);
 #ifdef CONFIG_PPC_ISERIES
 	if (get_lppaca()->int_dword.fields.decr_int) {
 		get_lppaca()->int_dword.fields.decr_int = 0;
 		/* Signal a fake decrementer interrupt */
 		timer_interrupt(regs);
 	}
 #endif
 }
 void __init init_IRQ(void)
 {
 	ppc_md.init_IRQ();
 #ifdef CONFIG_PPC64
 	irq_ctx_init();
 #endif
 }
 #ifdef CONFIG_IRQSTACKS
 struct thread_info *softirq_ctx[NR_CPUS] __read_mostly;
 struct thread_info *hardirq_ctx[NR_CPUS] __read_mostly;
 void irq_ctx_init(void)
 {
 	struct thread_info *tp;
 	int i;
 	for_each_possible_cpu(i) {
 		memset((void *)softirq_ctx[i], 0, THREAD_SIZE);
 		tp = softirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = SOFTIRQ_OFFSET;
 		memset((void *)hardirq_ctx[i], 0, THREAD_SIZE);
 		tp = hardirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = HARDIRQ_OFFSET;
 	}
 }
 static inline void do_softirq_onstack(void)
 {
 	struct thread_info *curtp, *irqtp;
 	curtp = current_thread_info();
 	irqtp = softirq_ctx[smp_processor_id()];
 	irqtp->task = curtp->task;
 	call_do_softirq(irqtp);
 	irqtp->task = NULL;
 }
 #else
 #define do_softirq_onstack()	__do_softirq()
 #endif /* CONFIG_IRQSTACKS */
 void do_softirq(void)
 {
 	unsigned long flags;
 	if (in_interrupt())
 		return;
 	local_irq_save(flags);
 	if (local_softirq_pending())
 		do_softirq_onstack();
 	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(do_softirq);
 /*
  * IRQ controller and virtual interrupts
  */
 #ifdef CONFIG_PPC_MERGE
 static LIST_HEAD(irq_hosts);
 static spinlock_t irq_big_lock = SPIN_LOCK_UNLOCKED;
 static DEFINE_PER_CPU(unsigned int, irq_radix_reader);
 static unsigned int irq_radix_writer;
 struct irq_map_entry irq_map[NR_IRQS];
 static unsigned int irq_virq_count = NR_IRQS;
 static struct irq_host *irq_default_host;
 struct irq_host *irq_alloc_host(unsigned int revmap_type,
 				unsigned int revmap_arg,
 				struct irq_host_ops *ops,
 				irq_hw_number_t inval_irq)
 {
 	struct irq_host *host;
 	unsigned int size = sizeof(struct irq_host);
 	unsigned int i;
 	unsigned int *rmap;
 	unsigned long flags;
 	/* Allocate structure and revmap table if using linear mapping */
 	if (revmap_type == IRQ_HOST_MAP_LINEAR)
 		size += revmap_arg * sizeof(unsigned int);
 	if (mem_init_done)
 		host = kzalloc(size, GFP_KERNEL);
 	else {
 		host = alloc_bootmem(size);
 		if (host)
 			memset(host, 0, size);
 	}
 	if (host == NULL)
 		return NULL;
 	/* Fill structure */
 	host->revmap_type = revmap_type;
 	host->inval_irq = inval_irq;
 	host->ops = ops;
 	spin_lock_irqsave(&irq_big_lock, flags);
 	/* If it's a legacy controller, check for duplicates and
 	 * mark it as allocated (we use irq 0 host pointer for that
 	 */
 	if (revmap_type == IRQ_HOST_MAP_LEGACY) {
 		if (irq_map[0].host != NULL) {
 			spin_unlock_irqrestore(&irq_big_lock, flags);
 			/* If we are early boot, we can't free the structure,
 			 * too bad...
 			 * this will be fixed once slab is made available early
 			 * instead of the current cruft
 			 */
 			if (mem_init_done)
 				kfree(host);
 			return NULL;
 		}
 		irq_map[0].host = host;
 	}
 	list_add(&host->link, &irq_hosts);
 	spin_unlock_irqrestore(&irq_big_lock, flags);
 	/* Additional setups per revmap type */
 	switch(revmap_type) {
 	case IRQ_HOST_MAP_LEGACY:
 		/* 0 is always the invalid number for legacy */
 		host->inval_irq = 0;
 		/* setup us as the host for all legacy interrupts */
 		for (i = 1; i < NUM_ISA_INTERRUPTS; i++) {
 			irq_map[i].hwirq = 0;
 			smp_wmb();
 			irq_map[i].host = host;
 			smp_wmb();
 			/* Clear norequest flags */
 			get_irq_desc(i)->status &= ~IRQ_NOREQUEST;
 			/* Legacy flags are left to default at this point,
 			 * one can then use irq_create_mapping() to
 			 * explicitely change them
 			 */
 			ops->map(host, i, i);
 		}
 		break;
 	case IRQ_HOST_MAP_LINEAR:
 		rmap = (unsigned int *)(host + 1);
 		for (i = 0; i < revmap_arg; i++)
 			rmap[i] = IRQ_NONE;
 		host->revmap_data.linear.size = revmap_arg;
 		smp_wmb();
 		host->revmap_data.linear.revmap = rmap;
 		break;
 	default:
 		break;
 	}
 	pr_debug("irq: Allocated host of type %d @0x%p\n", revmap_type, host);
 	return host;
 }
 struct irq_host *irq_find_host(struct device_node *node)
 {
 	struct irq_host *h, *found = NULL;
 	unsigned long flags;
 	/* We might want to match the legacy controller last since
 	 * it might potentially be set to match all interrupts in
 	 * the absence of a device node. This isn't a problem so far
 	 * yet though...
 	 */
 	spin_lock_irqsave(&irq_big_lock, flags);
 	list_for_each_entry(h, &irq_hosts, link)
 		if (h->ops->match == NULL || h->ops->match(h, node)) {
 			found = h;
 			break;
 		}
 	spin_unlock_irqrestore(&irq_big_lock, flags);
 	return found;
 }
 EXPORT_SYMBOL_GPL(irq_find_host);
 void irq_set_default_host(struct irq_host *host)
 {
 	pr_debug("irq: Default host set to @0x%p\n", host);
 	irq_default_host = host;
 }
 void irq_set_virq_count(unsigned int count)
 {
 	pr_debug("irq: Trying to set virq count to %d\n", count);
 	BUG_ON(count < NUM_ISA_INTERRUPTS);
 	if (count < NR_IRQS)
 		irq_virq_count = count;
 }
 /* radix tree not lockless safe ! we use a brlock-type mecanism
  * for now, until we can use a lockless radix tree
  */
 static void irq_radix_wrlock(unsigned long *flags)
 {
 	unsigned int cpu, ok;
 	spin_lock_irqsave(&irq_big_lock, *flags);
 	irq_radix_writer = 1;
 	smp_mb();
 	do {
 		barrier();
 		ok = 1;
 		for_each_possible_cpu(cpu) {
 			if (per_cpu(irq_radix_reader, cpu)) {
 				ok = 0;
 				break;
 			}
 		}
 		if (!ok)
 			cpu_relax();
 	} while(!ok);
 }
 static void irq_radix_wrunlock(unsigned long flags)
 {
 	smp_wmb();
 	irq_radix_writer = 0;
 	spin_unlock_irqrestore(&irq_big_lock, flags);
 }
 static void irq_radix_rdlock(unsigned long *flags)
 {
 	local_irq_save(*flags);
 	__get_cpu_var(irq_radix_reader) = 1;
 	smp_mb();
 	if (likely(irq_radix_writer == 0))
 		return;
 	__get_cpu_var(irq_radix_reader) = 0;
 	smp_wmb();
 	spin_lock(&irq_big_lock);
 	__get_cpu_var(irq_radix_reader) = 1;
 	spin_unlock(&irq_big_lock);
 }
 static void irq_radix_rdunlock(unsigned long flags)
 {
 	__get_cpu_var(irq_radix_reader) = 0;
 	local_irq_restore(flags);
 }
 unsigned int irq_create_mapping(struct irq_host *host,
 				irq_hw_number_t hwirq)
 {
 	unsigned int virq, hint;
 	pr_debug("irq: irq_create_mapping(0x%p, 0x%lx)\n", host, hwirq);
 	/* Look for default host if nececssary */
 	if (host == NULL)
 		host = irq_default_host;
 	if (host == NULL) {
 		printk(KERN_WARNING "irq_create_mapping called for"
 		       " NULL host, hwirq=%lx\n", hwirq);
 		WARN_ON(1);
 		return NO_IRQ;
 	}
 	pr_debug("irq: -> using host @%p\n", host);
 	/* Check if mapping already exist, if it does, call
 	 * host->ops->map() to update the flags
 	 */
 	virq = irq_find_mapping(host, hwirq);
 	if (virq != IRQ_NONE) {
 		pr_debug("irq: -> existing mapping on virq %d\n", virq);
 		return virq;
 	}
 	/* Get a virtual interrupt number */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY) {
 		/* Handle legacy */
 		virq = (unsigned int)hwirq;
 		if (virq == 0 || virq >= NUM_ISA_INTERRUPTS)
 			return NO_IRQ;
 		return virq;
 	} else {
 		/* Allocate a virtual interrupt number */
 		hint = hwirq % irq_virq_count;
 		virq = irq_alloc_virt(host, 1, hint);
 		if (virq == NO_IRQ) {
 			pr_debug("irq: -> virq allocation failed\n");
 			return NO_IRQ;
 		}
 	}
 	pr_debug("irq: -> obtained virq %d\n", virq);
 	/* Clear IRQ_NOREQUEST flag */
 	get_irq_desc(virq)->status &= ~IRQ_NOREQUEST;
 	/* map it */
 	smp_wmb();
 	irq_map[virq].hwirq = hwirq;
 	smp_mb();
 	if (host->ops->map(host, virq, hwirq)) {
 		pr_debug("irq: -> mapping failed, freeing\n");
 		irq_free_virt(virq, 1);
 		return NO_IRQ;
 	}
 	return virq;
 }
 EXPORT_SYMBOL_GPL(irq_create_mapping);
 unsigned int irq_create_of_mapping(struct device_node *controller,
 				   u32 *intspec, unsigned int intsize)
 {
 	struct irq_host *host;
 	irq_hw_number_t hwirq;
 	unsigned int type = IRQ_TYPE_NONE;
 	unsigned int virq;
 	if (controller == NULL)
 		host = irq_default_host;
 	else
 		host = irq_find_host(controller);
 	if (host == NULL) {
 		printk(KERN_WARNING "irq: no irq host found for %s !\n",
 		       controller->full_name);
 		return NO_IRQ;
 	}
 	/* If host has no translation, then we assume interrupt line */
 	if (host->ops->xlate == NULL)
 		hwirq = intspec[0];
 	else {
 		if (host->ops->xlate(host, controller, intspec, intsize,
 				     &hwirq, &type))
 			return NO_IRQ;
 	}
 	/* Create mapping */
 	virq = irq_create_mapping(host, hwirq);
 	if (virq == NO_IRQ)
 		return virq;
 	/* Set type if specified and different than the current one */
 	if (type != IRQ_TYPE_NONE &&
 	    type != (get_irq_desc(virq)->status & IRQF_TRIGGER_MASK))
 		set_irq_type(virq, type);
 	return virq;
 }
 EXPORT_SYMBOL_GPL(irq_create_of_mapping);
 unsigned int irq_of_parse_and_map(struct device_node *dev, int index)
 {
 	struct of_irq oirq;
 	if (of_irq_map_one(dev, index, &oirq))
 		return NO_IRQ;
 	return irq_create_of_mapping(oirq.controller, oirq.specifier,
 				     oirq.size);
 }
 EXPORT_SYMBOL_GPL(irq_of_parse_and_map);
 void irq_dispose_mapping(unsigned int virq)
 {
 	struct irq_host *host = irq_map[virq].host;
 	irq_hw_number_t hwirq;
 	unsigned long flags;
 	WARN_ON (host == NULL);
 	if (host == NULL)
 		return;
 	/* Never unmap legacy interrupts */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY)
 		return;
 	/* remove chip and handler */
 	set_irq_chip_and_handler(virq, NULL, NULL);
 	/* Make sure it's completed */
 	synchronize_irq(virq);
 	/* Tell the PIC about it */
 	if (host->ops->unmap)
 		host->ops->unmap(host, virq);
 	smp_mb();
 	/* Clear reverse map */
 	hwirq = irq_map[virq].hwirq;
 	switch(host->revmap_type) {
 	case IRQ_HOST_MAP_LINEAR:
 		if (hwirq < host->revmap_data.linear.size)
 			host->revmap_data.linear.revmap[hwirq] = IRQ_NONE;
 		break;
 	case IRQ_HOST_MAP_TREE:
 		/* Check if radix tree allocated yet */
 		if (host->revmap_data.tree.gfp_mask == 0)
 			break;
 		irq_radix_wrlock(&flags);
 		radix_tree_delete(&host->revmap_data.tree, hwirq);
 		irq_radix_wrunlock(flags);
 		break;
 	}
 	/* Destroy map */
 	smp_mb();
 	irq_map[virq].hwirq = host->inval_irq;
 	/* Set some flags */
 	get_irq_desc(virq)->status |= IRQ_NOREQUEST;
 	/* Free it */
 	irq_free_virt(virq, 1);
 }
 EXPORT_SYMBOL_GPL(irq_dispose_mapping);
 unsigned int irq_find_mapping(struct irq_host *host,
 			      irq_hw_number_t hwirq)
 {
 	unsigned int i;
 	unsigned int hint = hwirq % irq_virq_count;
 	/* Look for default host if nececssary */
 	if (host == NULL)
 		host = irq_default_host;
 	if (host == NULL)
 		return NO_IRQ;
 	/* legacy -> bail early */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY)
 		return hwirq;
 	/* Slow path does a linear search of the map */
 	if (hint < NUM_ISA_INTERRUPTS)
 		hint = NUM_ISA_INTERRUPTS;
 	i = hint;
 	do  {
 		if (irq_map[i].host == host &&
 		    irq_map[i].hwirq == hwirq)
 			return i;
 		i++;
 		if (i >= irq_virq_count)
 			i = NUM_ISA_INTERRUPTS;
 	} while(i != hint);
 	return NO_IRQ;
 }
 EXPORT_SYMBOL_GPL(irq_find_mapping);
 unsigned int irq_radix_revmap(struct irq_host *host,
 			      irq_hw_number_t hwirq)
 {
 	struct radix_tree_root *tree;
 	struct irq_map_entry *ptr;
 	unsigned int virq;
 	unsigned long flags;
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE);
 	/* Check if the radix tree exist yet. We test the value of
 	 * the gfp_mask for that. Sneaky but saves another int in the
 	 * structure. If not, we fallback to slow mode
 	 */
 	tree = &host->revmap_data.tree;
 	if (tree->gfp_mask == 0)
 		return irq_find_mapping(host, hwirq);
 	/* Now try to resolve */
 	irq_radix_rdlock(&flags);
 	ptr = radix_tree_lookup(tree, hwirq);
 	irq_radix_rdunlock(flags);
 	/* Found it, return */
 	if (ptr) {
 		virq = ptr - irq_map;
 		return virq;
 	}
 	/* If not there, try to insert it */
 	virq = irq_find_mapping(host, hwirq);
 	if (virq != NO_IRQ) {
 		irq_radix_wrlock(&flags);
 		radix_tree_insert(tree, hwirq, &irq_map[virq]);
 		irq_radix_wrunlock(flags);
 	}
 	return virq;
 }
 unsigned int irq_linear_revmap(struct irq_host *host,
 			       irq_hw_number_t hwirq)
 {
 	unsigned int *revmap;
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_LINEAR);
 	/* Check revmap bounds */
 	if (unlikely(hwirq >= host->revmap_data.linear.size))
 		return irq_find_mapping(host, hwirq);
 	/* Check if revmap was allocated */
 	revmap = host->revmap_data.linear.revmap;
 	if (unlikely(revmap == NULL))
 		return irq_find_mapping(host, hwirq);
 	/* Fill up revmap with slow path if no mapping found */
 	if (unlikely(revmap[hwirq] == NO_IRQ))
 		revmap[hwirq] = irq_find_mapping(host, hwirq);
 	return revmap[hwirq];
 }
 unsigned int irq_alloc_virt(struct irq_host *host,
 			    unsigned int count,
 			    unsigned int hint)
 {
 	unsigned long flags;
 	unsigned int i, j, found = NO_IRQ;
 	if (count == 0 || count > (irq_virq_count - NUM_ISA_INTERRUPTS))
 		return NO_IRQ;
 	spin_lock_irqsave(&irq_big_lock, flags);
 	/* Use hint for 1 interrupt if any */
 	if (count == 1 && hint >= NUM_ISA_INTERRUPTS &&
 	    hint < irq_virq_count && irq_map[hint].host == NULL) {
 		found = hint;
 		goto hint_found;
 	}
 	/* Look for count consecutive numbers in the allocatable
 	 * (non-legacy) space
 	 */
 	for (i = NUM_ISA_INTERRUPTS, j = 0; i < irq_virq_count; i++) {
 		if (irq_map[i].host != NULL)
 			j = 0;
 		else
 			j++;
 		if (j == count) {
 			found = i - count + 1;
 			break;
 		}
 	}
 	if (found == NO_IRQ) {
 		spin_unlock_irqrestore(&irq_big_lock, flags);
 		return NO_IRQ;
 	}
  hint_found:
 	for (i = found; i < (found + count); i++) {
 		irq_map[i].hwirq = host->inval_irq;
 		smp_wmb();
 		irq_map[i].host = host;
 	}
 	spin_unlock_irqrestore(&irq_big_lock, flags);
 	return found;
 }
 void irq_free_virt(unsigned int virq, unsigned int count)
 {
 	unsigned long flags;
 	unsigned int i;
 	WARN_ON (virq < NUM_ISA_INTERRUPTS);
 	WARN_ON (count == 0 || (virq + count) > irq_virq_count);
 	spin_lock_irqsave(&irq_big_lock, flags);
 	for (i = virq; i < (virq + count); i++) {
 		struct irq_host *host;
 		if (i < NUM_ISA_INTERRUPTS ||
 		    (virq + count) > irq_virq_count)
 			continue;
 		host = irq_map[i].host;
 		irq_map[i].hwirq = host->inval_irq;
 		smp_wmb();
 		irq_map[i].host = NULL;
 	}
 	spin_unlock_irqrestore(&irq_big_lock, flags);
 }
 void irq_early_init(void)
 {
 	unsigned int i;
 	for (i = 0; i < NR_IRQS; i++)
 		get_irq_desc(i)->status |= IRQ_NOREQUEST;
 }
 /* We need to create the radix trees late */
 static int irq_late_init(void)
 {
 	struct irq_host *h;
 	unsigned long flags;
 	irq_radix_wrlock(&flags);
 	list_for_each_entry(h, &irq_hosts, link) {
 		if (h->revmap_type == IRQ_HOST_MAP_TREE)
 			INIT_RADIX_TREE(&h->revmap_data.tree, GFP_ATOMIC);
 	}
 	irq_radix_wrunlock(flags);
 	return 0;
 }
 arch_initcall(irq_late_init);
 #endif /* CONFIG_PPC_MERGE */
 #ifdef CONFIG_PCI_MSI
 int pci_enable_msi(struct pci_dev * pdev)
 {
 	if (ppc_md.enable_msi)
 		return ppc_md.enable_msi(pdev);
 	else
 		return -1;
 }
 EXPORT_SYMBOL(pci_enable_msi);
 void pci_disable_msi(struct pci_dev * pdev)
 {
 	if (ppc_md.disable_msi)
 		ppc_md.disable_msi(pdev);
 }
 EXPORT_SYMBOL(pci_disable_msi);
 void pci_scan_msi_device(struct pci_dev *dev) {}
 int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) {return -1;}
 void pci_disable_msix(struct pci_dev *dev) {}
 void msi_remove_pci_irq_vectors(struct pci_dev *dev) {}
 void disable_msi_mode(struct pci_dev *dev, int pos, int type) {}
 void pci_no_msi(void) {}
 EXPORT_SYMBOL(pci_enable_msix);
 EXPORT_SYMBOL(pci_disable_msix);
 #endif
 #ifdef CONFIG_PPC64
 static int __init setup_noirqdistrib(char *str)
 {
 	distribute_irqs = 0;
 	return 1;
 }
 __setup("noirqdistrib", setup_noirqdistrib);
 #endif /* CONFIG_PPC64 */

arch/powerpc/kernel/ppc_ksyms.c

Diff comments View file @ d04c56f

 #include <linux/module.h>
 #include <linux/threads.h>
 #include <linux/smp.h>
 #include <linux/sched.h>
 #include <linux/elfcore.h>
 #include <linux/string.h>
 #include <linux/interrupt.h>
 #include <linux/screen_info.h>
 #include <linux/vt_kern.h>
 #include <linux/nvram.h>
 #include <linux/console.h>
 #include <linux/irq.h>
 #include <linux/pci.h>
 #include <linux/delay.h>
 #include <linux/ide.h>
 #include <linux/bitops.h>
 #include <asm/page.h>
 #include <asm/semaphore.h>
 #include <asm/processor.h>
 #include <asm/uaccess.h>
 #include <asm/io.h>
 #include <asm/ide.h>
 #include <asm/atomic.h>
 #include <asm/checksum.h>
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
 #include <linux/adb.h>
 #include <linux/cuda.h>
 #include <linux/pmu.h>
 #include <asm/prom.h>
 #include <asm/system.h>
 #include <asm/pci-bridge.h>
 #include <asm/irq.h>
 #include <asm/pmac_feature.h>
 #include <asm/dma.h>
 #include <asm/machdep.h>
 #include <asm/hw_irq.h>
 #include <asm/nvram.h>
 #include <asm/mmu_context.h>
 #include <asm/backlight.h>
 #include <asm/time.h>
 #include <asm/cputable.h>
 #include <asm/btext.h>
 #include <asm/div64.h>
 #include <asm/signal.h>
 #ifdef  CONFIG_8xx
 #include <asm/commproc.h>
 #endif
+#ifdef CONFIG_PPC64
+EXPORT_SYMBOL(local_irq_restore);
+#endif
 #ifdef CONFIG_PPC32
 extern void transfer_to_handler(void);
 extern void do_IRQ(struct pt_regs *regs);
 extern void machine_check_exception(struct pt_regs *regs);
 extern void alignment_exception(struct pt_regs *regs);
 extern void program_check_exception(struct pt_regs *regs);
 extern void single_step_exception(struct pt_regs *regs);
 extern int sys_sigreturn(struct pt_regs *regs);
 EXPORT_SYMBOL(clear_pages);
 EXPORT_SYMBOL(ISA_DMA_THRESHOLD);
 EXPORT_SYMBOL(DMA_MODE_READ);
 EXPORT_SYMBOL(DMA_MODE_WRITE);
 EXPORT_SYMBOL(__div64_32);
 EXPORT_SYMBOL(do_signal);
 EXPORT_SYMBOL(transfer_to_handler);
 EXPORT_SYMBOL(do_IRQ);
 EXPORT_SYMBOL(machine_check_exception);
 EXPORT_SYMBOL(alignment_exception);
 EXPORT_SYMBOL(program_check_exception);
 EXPORT_SYMBOL(single_step_exception);
 EXPORT_SYMBOL(sys_sigreturn);
 #endif
 EXPORT_SYMBOL(strcpy);
 EXPORT_SYMBOL(strncpy);
 EXPORT_SYMBOL(strcat);
 EXPORT_SYMBOL(strlen);
 EXPORT_SYMBOL(strcmp);
 EXPORT_SYMBOL(strcasecmp);
 EXPORT_SYMBOL(strncasecmp);
 EXPORT_SYMBOL(csum_partial);
 EXPORT_SYMBOL(csum_partial_copy_generic);
 EXPORT_SYMBOL(ip_fast_csum);
 EXPORT_SYMBOL(csum_tcpudp_magic);
 EXPORT_SYMBOL(__copy_tofrom_user);
 EXPORT_SYMBOL(__clear_user);
 EXPORT_SYMBOL(__strncpy_from_user);
 EXPORT_SYMBOL(__strnlen_user);
 #ifdef CONFIG_PPC64
 EXPORT_SYMBOL(copy_4K_page);
 #endif
 #if defined(CONFIG_PPC32) && (defined(CONFIG_BLK_DEV_IDE) || defined(CONFIG_BLK_DEV_IDE_MODULE))
 EXPORT_SYMBOL(ppc_ide_md);
 #endif
 #if defined(CONFIG_PCI) && defined(CONFIG_PPC32)
 EXPORT_SYMBOL(isa_io_base);
 EXPORT_SYMBOL(isa_mem_base);
 EXPORT_SYMBOL(pci_dram_offset);
 EXPORT_SYMBOL(pci_alloc_consistent);
 EXPORT_SYMBOL(pci_free_consistent);
 EXPORT_SYMBOL(pci_bus_io_base);
 EXPORT_SYMBOL(pci_bus_io_base_phys);
 EXPORT_SYMBOL(pci_bus_mem_base_phys);
 EXPORT_SYMBOL(pci_bus_to_hose);
 #endif /* CONFIG_PCI */
 EXPORT_SYMBOL(start_thread);
 EXPORT_SYMBOL(kernel_thread);
 EXPORT_SYMBOL(giveup_fpu);
 #ifdef CONFIG_ALTIVEC
 EXPORT_SYMBOL(giveup_altivec);
 #endif /* CONFIG_ALTIVEC */
 #ifdef CONFIG_SPE
 EXPORT_SYMBOL(giveup_spe);
 #endif /* CONFIG_SPE */
 #ifndef CONFIG_PPC64
 EXPORT_SYMBOL(flush_instruction_cache);
 EXPORT_SYMBOL(flush_tlb_kernel_range);
 EXPORT_SYMBOL(flush_tlb_page);
 EXPORT_SYMBOL(_tlbie);
 #endif
 EXPORT_SYMBOL(__flush_icache_range);
 EXPORT_SYMBOL(flush_dcache_range);
 #ifdef CONFIG_SMP
 #ifdef CONFIG_PPC32
 EXPORT_SYMBOL(smp_hw_index);
 #endif
 #endif
 #ifdef CONFIG_ADB
 EXPORT_SYMBOL(adb_request);
 EXPORT_SYMBOL(adb_register);
 EXPORT_SYMBOL(adb_unregister);
 EXPORT_SYMBOL(adb_poll);
 EXPORT_SYMBOL(adb_try_handler_change);
 #endif /* CONFIG_ADB */
 #ifdef CONFIG_ADB_CUDA
 EXPORT_SYMBOL(cuda_request);
 EXPORT_SYMBOL(cuda_poll);
 #endif /* CONFIG_ADB_CUDA */
 #ifdef CONFIG_VT
 EXPORT_SYMBOL(kd_mksound);
 #endif
 EXPORT_SYMBOL(to_tm);
 #ifdef CONFIG_PPC32
 long long __ashrdi3(long long, int);
 long long __ashldi3(long long, int);
 long long __lshrdi3(long long, int);
 EXPORT_SYMBOL(__ashrdi3);
 EXPORT_SYMBOL(__ashldi3);
 EXPORT_SYMBOL(__lshrdi3);
 #endif
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memset);
 EXPORT_SYMBOL(memmove);
 EXPORT_SYMBOL(memcmp);
 EXPORT_SYMBOL(memchr);
 #if defined(CONFIG_FB_VGA16_MODULE)
 EXPORT_SYMBOL(screen_info);
 #endif
 #ifdef CONFIG_PPC32
 EXPORT_SYMBOL(timer_interrupt);
 EXPORT_SYMBOL(irq_desc);
 EXPORT_SYMBOL(tb_ticks_per_jiffy);
 EXPORT_SYMBOL(console_drivers);
 EXPORT_SYMBOL(cacheable_memcpy);
 #endif
 #ifdef  CONFIG_8xx
 EXPORT_SYMBOL(cpm_install_handler);
 EXPORT_SYMBOL(cpm_free_handler);
 #endif /* CONFIG_8xx */
 #if defined(CONFIG_8xx) || defined(CONFIG_40x)
 EXPORT_SYMBOL(__res);
 #endif
 #ifdef CONFIG_PPC32
 EXPORT_SYMBOL(next_mmu_context);
 EXPORT_SYMBOL(set_context);
 #endif
 #ifdef CONFIG_PPC_STD_MMU_32
 extern long mol_trampoline;
 EXPORT_SYMBOL(mol_trampoline); /* For MOL */
 EXPORT_SYMBOL(flush_hash_pages); /* For MOL */
 #ifdef CONFIG_SMP
 extern int mmu_hash_lock;
 EXPORT_SYMBOL(mmu_hash_lock); /* For MOL */
 #endif /* CONFIG_SMP */
 extern long *intercept_table;
 EXPORT_SYMBOL(intercept_table);
 #endif /* CONFIG_PPC_STD_MMU_32 */
 #if defined(CONFIG_40x) || defined(CONFIG_BOOKE)
 EXPORT_SYMBOL(__mtdcr);
 EXPORT_SYMBOL(__mfdcr);
 #endif

arch/powerpc/kernel/setup_64.c

Diff comments View file @ d04c56f

1	/*	1	/*
2	*	2	*
3	* Common boot and setup code.	3	* Common boot and setup code.
4	*	4	*
5	* Copyright (C) 2001 PPC64 Team, IBM Corp	5	* Copyright (C) 2001 PPC64 Team, IBM Corp
6	*	6	*
7	* This program is free software; you can redistribute it and/or	7	* This program is free software; you can redistribute it and/or
8	* modify it under the terms of the GNU General Public License	8	* modify it under the terms of the GNU General Public License
9	* as published by the Free Software Foundation; either version	9	* as published by the Free Software Foundation; either version
10	* 2 of the License, or (at your option) any later version.	10	* 2 of the License, or (at your option) any later version.
11	*/	11	*/
12		12
13	#undef DEBUG	13	#undef DEBUG
14		14
15	#include <linux/module.h>	15	#include <linux/module.h>
16	#include <linux/string.h>	16	#include <linux/string.h>
17	#include <linux/sched.h>	17	#include <linux/sched.h>
18	#include <linux/init.h>	18	#include <linux/init.h>
19	#include <linux/kernel.h>	19	#include <linux/kernel.h>
20	#include <linux/reboot.h>	20	#include <linux/reboot.h>
21	#include <linux/delay.h>	21	#include <linux/delay.h>
22	#include <linux/initrd.h>	22	#include <linux/initrd.h>
23	#include <linux/ide.h>	23	#include <linux/ide.h>
24	#include <linux/seq_file.h>	24	#include <linux/seq_file.h>
25	#include <linux/ioport.h>	25	#include <linux/ioport.h>
26	#include <linux/console.h>	26	#include <linux/console.h>
27	#include <linux/utsname.h>	27	#include <linux/utsname.h>
28	#include <linux/tty.h>	28	#include <linux/tty.h>
29	#include <linux/root_dev.h>	29	#include <linux/root_dev.h>
30	#include <linux/notifier.h>	30	#include <linux/notifier.h>
31	#include <linux/cpu.h>	31	#include <linux/cpu.h>
32	#include <linux/unistd.h>	32	#include <linux/unistd.h>
33	#include <linux/serial.h>	33	#include <linux/serial.h>
34	#include <linux/serial_8250.h>	34	#include <linux/serial_8250.h>
35	#include <linux/bootmem.h>	35	#include <linux/bootmem.h>
36	#include <asm/io.h>	36	#include <asm/io.h>
37	#include <asm/kdump.h>	37	#include <asm/kdump.h>
38	#include <asm/prom.h>	38	#include <asm/prom.h>
39	#include <asm/processor.h>	39	#include <asm/processor.h>
40	#include <asm/pgtable.h>	40	#include <asm/pgtable.h>
41	#include <asm/smp.h>	41	#include <asm/smp.h>
42	#include <asm/elf.h>	42	#include <asm/elf.h>
43	#include <asm/machdep.h>	43	#include <asm/machdep.h>
44	#include <asm/paca.h>	44	#include <asm/paca.h>
45	#include <asm/time.h>	45	#include <asm/time.h>
46	#include <asm/cputable.h>	46	#include <asm/cputable.h>
47	#include <asm/sections.h>	47	#include <asm/sections.h>
48	#include <asm/btext.h>	48	#include <asm/btext.h>
49	#include <asm/nvram.h>	49	#include <asm/nvram.h>
50	#include <asm/setup.h>	50	#include <asm/setup.h>
51	#include <asm/system.h>	51	#include <asm/system.h>
52	#include <asm/rtas.h>	52	#include <asm/rtas.h>
53	#include <asm/iommu.h>	53	#include <asm/iommu.h>
54	#include <asm/serial.h>	54	#include <asm/serial.h>
55	#include <asm/cache.h>	55	#include <asm/cache.h>
56	#include <asm/page.h>	56	#include <asm/page.h>
57	#include <asm/mmu.h>	57	#include <asm/mmu.h>
58	#include <asm/lmb.h>	58	#include <asm/lmb.h>
59	#include <asm/firmware.h>	59	#include <asm/firmware.h>
60	#include <asm/xmon.h>	60	#include <asm/xmon.h>
61	#include <asm/udbg.h>	61	#include <asm/udbg.h>
62	#include <asm/kexec.h>	62	#include <asm/kexec.h>
63		63
64	#include "setup.h"	64	#include "setup.h"
65		65
66	#ifdef DEBUG	66	#ifdef DEBUG
67	#define DBG(fmt...) udbg_printf(fmt)	67	#define DBG(fmt...) udbg_printf(fmt)
68	#else	68	#else
69	#define DBG(fmt...)	69	#define DBG(fmt...)
70	#endif	70	#endif
71		71
72	int have_of = 1;	72	int have_of = 1;
73	int boot_cpuid = 0;	73	int boot_cpuid = 0;
74	dev_t boot_dev;	74	dev_t boot_dev;
75	u64 ppc64_pft_size;	75	u64 ppc64_pft_size;
76		76
77	/* Pick defaults since we might want to patch instructions	77	/* Pick defaults since we might want to patch instructions
78	* before we've read this from the device tree.	78	* before we've read this from the device tree.
79	*/	79	*/
80	struct ppc64_caches ppc64_caches = {	80	struct ppc64_caches ppc64_caches = {
81	.dline_size = 0x40,	81	.dline_size = 0x40,
82	.log_dline_size = 6,	82	.log_dline_size = 6,
83	.iline_size = 0x40,	83	.iline_size = 0x40,
84	.log_iline_size = 6	84	.log_iline_size = 6
85	};	85	};
86	EXPORT_SYMBOL_GPL(ppc64_caches);	86	EXPORT_SYMBOL_GPL(ppc64_caches);
87		87
88	/*	88	/*
89	* These are used in binfmt_elf.c to put aux entries on the stack	89	* These are used in binfmt_elf.c to put aux entries on the stack
90	* for each elf executable being started.	90	* for each elf executable being started.
91	*/	91	*/
92	int dcache_bsize;	92	int dcache_bsize;
93	int icache_bsize;	93	int icache_bsize;
94	int ucache_bsize;	94	int ucache_bsize;
95		95
96	#ifdef CONFIG_SMP	96	#ifdef CONFIG_SMP
97		97
98	static int smt_enabled_cmdline;	98	static int smt_enabled_cmdline;
99		99
100	/* Look for ibm,smt-enabled OF option */	100	/* Look for ibm,smt-enabled OF option */
101	static void check_smt_enabled(void)	101	static void check_smt_enabled(void)
102	{	102	{
103	struct device_node *dn;	103	struct device_node *dn;
104	const char *smt_option;	104	const char *smt_option;
105		105
106	/* Allow the command line to overrule the OF option */	106	/* Allow the command line to overrule the OF option */
107	if (smt_enabled_cmdline)	107	if (smt_enabled_cmdline)
108	return;	108	return;
109		109
110	dn = of_find_node_by_path("/options");	110	dn = of_find_node_by_path("/options");
111		111
112	if (dn) {	112	if (dn) {
113	smt_option = get_property(dn, "ibm,smt-enabled", NULL);	113	smt_option = get_property(dn, "ibm,smt-enabled", NULL);
114		114
115	if (smt_option) {	115	if (smt_option) {
116	if (!strcmp(smt_option, "on"))	116	if (!strcmp(smt_option, "on"))
117	smt_enabled_at_boot = 1;	117	smt_enabled_at_boot = 1;
118	else if (!strcmp(smt_option, "off"))	118	else if (!strcmp(smt_option, "off"))
119	smt_enabled_at_boot = 0;	119	smt_enabled_at_boot = 0;
120	}	120	}
121	}	121	}
122	}	122	}
123		123
124	/* Look for smt-enabled= cmdline option */	124	/* Look for smt-enabled= cmdline option */
125	static int __init early_smt_enabled(char *p)	125	static int __init early_smt_enabled(char *p)
126	{	126	{
127	smt_enabled_cmdline = 1;	127	smt_enabled_cmdline = 1;
128		128
129	if (!p)	129	if (!p)
130	return 0;	130	return 0;
131		131
132	if (!strcmp(p, "on") \|\| !strcmp(p, "1"))	132	if (!strcmp(p, "on") \|\| !strcmp(p, "1"))
133	smt_enabled_at_boot = 1;	133	smt_enabled_at_boot = 1;
134	else if (!strcmp(p, "off") \|\| !strcmp(p, "0"))	134	else if (!strcmp(p, "off") \|\| !strcmp(p, "0"))
135	smt_enabled_at_boot = 0;	135	smt_enabled_at_boot = 0;
136		136
137	return 0;	137	return 0;
138	}	138	}
139	early_param("smt-enabled", early_smt_enabled);	139	early_param("smt-enabled", early_smt_enabled);
140		140
141	#else	141	#else
142	#define check_smt_enabled()	142	#define check_smt_enabled()
143	#endif /* CONFIG_SMP */	143	#endif /* CONFIG_SMP */
144		144
145	/* Put the paca pointer into r13 and SPRG3 */	145	/* Put the paca pointer into r13 and SPRG3 */
146	void __init setup_paca(int cpu)	146	void __init setup_paca(int cpu)
147	{	147	{
148	local_paca = &paca[cpu];	148	local_paca = &paca[cpu];
149	mtspr(SPRN_SPRG3, local_paca);	149	mtspr(SPRN_SPRG3, local_paca);
150	}	150	}
151		151
152	/*	152	/*
153	* Early initialization entry point. This is called by head.S	153	* Early initialization entry point. This is called by head.S
154	* with MMU translation disabled. We rely on the "feature" of	154	* with MMU translation disabled. We rely on the "feature" of
155	* the CPU that ignores the top 2 bits of the address in real	155	* the CPU that ignores the top 2 bits of the address in real
156	* mode so we can access kernel globals normally provided we	156	* mode so we can access kernel globals normally provided we
157	* only toy with things in the RMO region. From here, we do	157	* only toy with things in the RMO region. From here, we do
158	* some early parsing of the device-tree to setup out LMB	158	* some early parsing of the device-tree to setup out LMB
159	* data structures, and allocate & initialize the hash table	159	* data structures, and allocate & initialize the hash table
160	* and segment tables so we can start running with translation	160	* and segment tables so we can start running with translation
161	* enabled.	161	* enabled.
162	*	162	*
163	* It is this function which will call the probe() callback of	163	* It is this function which will call the probe() callback of
164	* the various platform types and copy the matching one to the	164	* the various platform types and copy the matching one to the
165	* global ppc_md structure. Your platform can eventually do	165	* global ppc_md structure. Your platform can eventually do
166	* some very early initializations from the probe() routine, but	166	* some very early initializations from the probe() routine, but
167	* this is not recommended, be very careful as, for example, the	167	* this is not recommended, be very careful as, for example, the
168	* device-tree is not accessible via normal means at this point.	168	* device-tree is not accessible via normal means at this point.
169	*/	169	*/
170		170
171	void __init early_setup(unsigned long dt_ptr)	171	void __init early_setup(unsigned long dt_ptr)
172	{	172	{
173	/* Assume we're on cpu 0 for now. Don't write to the paca yet! */	173	/* Assume we're on cpu 0 for now. Don't write to the paca yet! */
174	setup_paca(0);	174	setup_paca(0);
175		175
176	/* Enable early debugging if any specified (see udbg.h) */	176	/* Enable early debugging if any specified (see udbg.h) */
177	udbg_early_init();	177	udbg_early_init();
178		178
179	DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr);	179	DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr);
180		180
181	/*	181	/*
182	* Do early initializations using the flattened device	182	* Do early initializations using the flattened device
183	* tree, like retreiving the physical memory map or	183	* tree, like retreiving the physical memory map or
184	* calculating/retreiving the hash table size	184	* calculating/retreiving the hash table size
185	*/	185	*/
186	early_init_devtree(__va(dt_ptr));	186	early_init_devtree(__va(dt_ptr));
187		187
188	/* Now we know the logical id of our boot cpu, setup the paca. */	188	/* Now we know the logical id of our boot cpu, setup the paca. */
189	setup_paca(boot_cpuid);	189	setup_paca(boot_cpuid);
190		190
191	/* Fix up paca fields required for the boot cpu */	191	/* Fix up paca fields required for the boot cpu */
192	get_paca()->cpu_start = 1;	192	get_paca()->cpu_start = 1;
193	get_paca()->stab_real = __pa((u64)&initial_stab);	193	get_paca()->stab_real = __pa((u64)&initial_stab);
194	get_paca()->stab_addr = (u64)&initial_stab;	194	get_paca()->stab_addr = (u64)&initial_stab;
195		195
196	/* Probe the machine type */	196	/* Probe the machine type */
197	probe_machine();	197	probe_machine();
198		198
199	setup_kdump_trampoline();	199	setup_kdump_trampoline();
200		200
201	DBG("Found, Initializing memory management...\n");	201	DBG("Found, Initializing memory management...\n");
202		202
203	/*	203	/*
204	* Initialize the MMU Hash table and create the linear mapping	204	* Initialize the MMU Hash table and create the linear mapping
205	* of memory. Has to be done before stab/slb initialization as	205	* of memory. Has to be done before stab/slb initialization as
206	* this is currently where the page size encoding is obtained	206	* this is currently where the page size encoding is obtained
207	*/	207	*/
208	htab_initialize();	208	htab_initialize();
209		209
210	/*	210	/*
211	* Initialize stab / SLB management except on iSeries	211	* Initialize stab / SLB management except on iSeries
212	*/	212	*/
213	if (cpu_has_feature(CPU_FTR_SLB))	213	if (cpu_has_feature(CPU_FTR_SLB))
214	slb_initialize();	214	slb_initialize();
215	else if (!firmware_has_feature(FW_FEATURE_ISERIES))	215	else if (!firmware_has_feature(FW_FEATURE_ISERIES))
216	stab_initialize(get_paca()->stab_real);	216	stab_initialize(get_paca()->stab_real);
217		217
218	DBG(" <- early_setup()\n");	218	DBG(" <- early_setup()\n");
219	}	219	}
220		220
221	#ifdef CONFIG_SMP	221	#ifdef CONFIG_SMP
222	void early_setup_secondary(void)	222	void early_setup_secondary(void)
223	{	223	{
224	struct paca_struct *lpaca = get_paca();	224	struct paca_struct *lpaca = get_paca();
225		225
226	/* Mark enabled in PACA */	226	/* Mark interrupts enabled in PACA */
227	lpaca->proc_enabled = 0;	227	lpaca->soft_enabled = 0;
228		228
229	/* Initialize hash table for that CPU */	229	/* Initialize hash table for that CPU */
230	htab_initialize_secondary();	230	htab_initialize_secondary();
231		231
232	/* Initialize STAB/SLB. We use a virtual address as it works	232	/* Initialize STAB/SLB. We use a virtual address as it works
233	* in real mode on pSeries and we want a virutal address on	233	* in real mode on pSeries and we want a virutal address on
234	* iSeries anyway	234	* iSeries anyway
235	*/	235	*/
236	if (cpu_has_feature(CPU_FTR_SLB))	236	if (cpu_has_feature(CPU_FTR_SLB))
237	slb_initialize();	237	slb_initialize();
238	else	238	else
239	stab_initialize(lpaca->stab_addr);	239	stab_initialize(lpaca->stab_addr);
240	}	240	}
241		241
242	#endif /* CONFIG_SMP */	242	#endif /* CONFIG_SMP */
243		243
244	#if defined(CONFIG_SMP) \|\| defined(CONFIG_KEXEC)	244	#if defined(CONFIG_SMP) \|\| defined(CONFIG_KEXEC)
245	void smp_release_cpus(void)	245	void smp_release_cpus(void)
246	{	246	{
247	extern unsigned long __secondary_hold_spinloop;	247	extern unsigned long __secondary_hold_spinloop;
248	unsigned long *ptr;	248	unsigned long *ptr;
249		249
250	DBG(" -> smp_release_cpus()\n");	250	DBG(" -> smp_release_cpus()\n");
251		251
252	/* All secondary cpus are spinning on a common spinloop, release them	252	/* All secondary cpus are spinning on a common spinloop, release them
253	* all now so they can start to spin on their individual paca	253	* all now so they can start to spin on their individual paca
254	* spinloops. For non SMP kernels, the secondary cpus never get out	254	* spinloops. For non SMP kernels, the secondary cpus never get out
255	* of the common spinloop.	255	* of the common spinloop.
256	* This is useless but harmless on iSeries, secondaries are already	256	* This is useless but harmless on iSeries, secondaries are already
257	* waiting on their paca spinloops. */	257	* waiting on their paca spinloops. */
258		258
259	ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop	259	ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop
260	- PHYSICAL_START);	260	- PHYSICAL_START);
261	*ptr = 1;	261	*ptr = 1;
262	mb();	262	mb();
263		263
264	DBG(" <- smp_release_cpus()\n");	264	DBG(" <- smp_release_cpus()\n");
265	}	265	}
266	#endif /* CONFIG_SMP \|\| CONFIG_KEXEC */	266	#endif /* CONFIG_SMP \|\| CONFIG_KEXEC */
267		267
268	/*	268	/*
269	* Initialize some remaining members of the ppc64_caches and systemcfg	269	* Initialize some remaining members of the ppc64_caches and systemcfg
270	* structures	270	* structures
271	* (at least until we get rid of them completely). This is mostly some	271	* (at least until we get rid of them completely). This is mostly some
272	* cache informations about the CPU that will be used by cache flush	272	* cache informations about the CPU that will be used by cache flush
273	* routines and/or provided to userland	273	* routines and/or provided to userland
274	*/	274	*/
275	static void __init initialize_cache_info(void)	275	static void __init initialize_cache_info(void)
276	{	276	{
277	struct device_node *np;	277	struct device_node *np;
278	unsigned long num_cpus = 0;	278	unsigned long num_cpus = 0;
279		279
280	DBG(" -> initialize_cache_info()\n");	280	DBG(" -> initialize_cache_info()\n");
281		281
282	for (np = NULL; (np = of_find_node_by_type(np, "cpu"));) {	282	for (np = NULL; (np = of_find_node_by_type(np, "cpu"));) {
283	num_cpus += 1;	283	num_cpus += 1;
284		284
285	/* We're assuming all of the CPUs have the same	285	/* We're assuming all of the CPUs have the same
286	* d-cache and i-cache sizes... -Peter	286	* d-cache and i-cache sizes... -Peter
287	*/	287	*/
288		288
289	if ( num_cpus == 1 ) {	289	if ( num_cpus == 1 ) {
290	const u32 sizep, lsizep;	290	const u32 sizep, lsizep;
291	u32 size, lsize;	291	u32 size, lsize;
292	const char dc, ic;	292	const char dc, ic;
293		293
294	/* Then read cache informations */	294	/* Then read cache informations */
295	if (machine_is(powermac)) {	295	if (machine_is(powermac)) {
296	dc = "d-cache-block-size";	296	dc = "d-cache-block-size";
297	ic = "i-cache-block-size";	297	ic = "i-cache-block-size";
298	} else {	298	} else {
299	dc = "d-cache-line-size";	299	dc = "d-cache-line-size";
300	ic = "i-cache-line-size";	300	ic = "i-cache-line-size";
301	}	301	}
302		302
303	size = 0;	303	size = 0;
304	lsize = cur_cpu_spec->dcache_bsize;	304	lsize = cur_cpu_spec->dcache_bsize;
305	sizep = get_property(np, "d-cache-size", NULL);	305	sizep = get_property(np, "d-cache-size", NULL);
306	if (sizep != NULL)	306	if (sizep != NULL)
307	size = *sizep;	307	size = *sizep;
308	lsizep = get_property(np, dc, NULL);	308	lsizep = get_property(np, dc, NULL);
309	if (lsizep != NULL)	309	if (lsizep != NULL)
310	lsize = *lsizep;	310	lsize = *lsizep;
311	if (sizep == 0 \|\| lsizep == 0)	311	if (sizep == 0 \|\| lsizep == 0)
312	DBG("Argh, can't find dcache properties ! "	312	DBG("Argh, can't find dcache properties ! "
313	"sizep: %p, lsizep: %p\n", sizep, lsizep);	313	"sizep: %p, lsizep: %p\n", sizep, lsizep);
314		314
315	ppc64_caches.dsize = size;	315	ppc64_caches.dsize = size;
316	ppc64_caches.dline_size = lsize;	316	ppc64_caches.dline_size = lsize;
317	ppc64_caches.log_dline_size = __ilog2(lsize);	317	ppc64_caches.log_dline_size = __ilog2(lsize);
318	ppc64_caches.dlines_per_page = PAGE_SIZE / lsize;	318	ppc64_caches.dlines_per_page = PAGE_SIZE / lsize;
319		319
320	size = 0;	320	size = 0;
321	lsize = cur_cpu_spec->icache_bsize;	321	lsize = cur_cpu_spec->icache_bsize;
322	sizep = get_property(np, "i-cache-size", NULL);	322	sizep = get_property(np, "i-cache-size", NULL);
323	if (sizep != NULL)	323	if (sizep != NULL)
324	size = *sizep;	324	size = *sizep;
325	lsizep = get_property(np, ic, NULL);	325	lsizep = get_property(np, ic, NULL);
326	if (lsizep != NULL)	326	if (lsizep != NULL)
327	lsize = *lsizep;	327	lsize = *lsizep;
328	if (sizep == 0 \|\| lsizep == 0)	328	if (sizep == 0 \|\| lsizep == 0)
329	DBG("Argh, can't find icache properties ! "	329	DBG("Argh, can't find icache properties ! "
330	"sizep: %p, lsizep: %p\n", sizep, lsizep);	330	"sizep: %p, lsizep: %p\n", sizep, lsizep);
331		331
332	ppc64_caches.isize = size;	332	ppc64_caches.isize = size;
333	ppc64_caches.iline_size = lsize;	333	ppc64_caches.iline_size = lsize;
334	ppc64_caches.log_iline_size = __ilog2(lsize);	334	ppc64_caches.log_iline_size = __ilog2(lsize);
335	ppc64_caches.ilines_per_page = PAGE_SIZE / lsize;	335	ppc64_caches.ilines_per_page = PAGE_SIZE / lsize;
336	}	336	}
337	}	337	}
338		338
339	DBG(" <- initialize_cache_info()\n");	339	DBG(" <- initialize_cache_info()\n");
340	}	340	}
341		341
342		342
343	/*	343	/*
344	* Do some initial setup of the system. The parameters are those which	344	* Do some initial setup of the system. The parameters are those which
345	* were passed in from the bootloader.	345	* were passed in from the bootloader.
346	*/	346	*/
347	void __init setup_system(void)	347	void __init setup_system(void)
348	{	348	{
349	DBG(" -> setup_system()\n");	349	DBG(" -> setup_system()\n");
350		350
351	/*	351	/*
352	* Unflatten the device-tree passed by prom_init or kexec	352	* Unflatten the device-tree passed by prom_init or kexec
353	*/	353	*/
354	unflatten_device_tree();	354	unflatten_device_tree();
355		355
356	/*	356	/*
357	* Fill the ppc64_caches & systemcfg structures with informations	357	* Fill the ppc64_caches & systemcfg structures with informations
358	* retrieved from the device-tree.	358	* retrieved from the device-tree.
359	*/	359	*/
360	initialize_cache_info();	360	initialize_cache_info();
361		361
362	/*	362	/*
363	* Initialize irq remapping subsystem	363	* Initialize irq remapping subsystem
364	*/	364	*/
365	irq_early_init();	365	irq_early_init();
366		366
367	#ifdef CONFIG_PPC_RTAS	367	#ifdef CONFIG_PPC_RTAS
368	/*	368	/*
369	* Initialize RTAS if available	369	* Initialize RTAS if available
370	*/	370	*/
371	rtas_initialize();	371	rtas_initialize();
372	#endif /* CONFIG_PPC_RTAS */	372	#endif /* CONFIG_PPC_RTAS */
373		373
374	/*	374	/*
375	* Check if we have an initrd provided via the device-tree	375	* Check if we have an initrd provided via the device-tree
376	*/	376	*/
377	check_for_initrd();	377	check_for_initrd();
378		378
379	/*	379	/*
380	* Do some platform specific early initializations, that includes	380	* Do some platform specific early initializations, that includes
381	* setting up the hash table pointers. It also sets up some interrupt-mapping	381	* setting up the hash table pointers. It also sets up some interrupt-mapping
382	* related options that will be used by finish_device_tree()	382	* related options that will be used by finish_device_tree()
383	*/	383	*/
384	ppc_md.init_early();	384	ppc_md.init_early();
385		385
386	/*	386	/*
387	* We can discover serial ports now since the above did setup the	387	* We can discover serial ports now since the above did setup the
388	* hash table management for us, thus ioremap works. We do that early	388	* hash table management for us, thus ioremap works. We do that early
389	* so that further code can be debugged	389	* so that further code can be debugged
390	*/	390	*/
391	find_legacy_serial_ports();	391	find_legacy_serial_ports();
392		392
393	/*	393	/*
394	* Register early console	394	* Register early console
395	*/	395	*/
396	register_early_udbg_console();	396	register_early_udbg_console();
397		397
398	/*	398	/*
399	* Initialize xmon	399	* Initialize xmon
400	*/	400	*/
401	xmon_setup();	401	xmon_setup();
402		402
403	check_smt_enabled();	403	check_smt_enabled();
404	smp_setup_cpu_maps();	404	smp_setup_cpu_maps();
405		405
406	#ifdef CONFIG_SMP	406	#ifdef CONFIG_SMP
407	/* Release secondary cpus out of their spinloops at 0x60 now that	407	/* Release secondary cpus out of their spinloops at 0x60 now that
408	* we can map physical -> logical CPU ids	408	* we can map physical -> logical CPU ids
409	*/	409	*/
410	smp_release_cpus();	410	smp_release_cpus();
411	#endif	411	#endif
412		412
413	printk("Starting Linux PPC64 %s\n", init_utsname()->version);	413	printk("Starting Linux PPC64 %s\n", init_utsname()->version);
414		414
415	printk("-----------------------------------------------------\n");	415	printk("-----------------------------------------------------\n");
416	printk("ppc64_pft_size = 0x%lx\n", ppc64_pft_size);	416	printk("ppc64_pft_size = 0x%lx\n", ppc64_pft_size);
417	printk("physicalMemorySize = 0x%lx\n", lmb_phys_mem_size());	417	printk("physicalMemorySize = 0x%lx\n", lmb_phys_mem_size());
418	printk("ppc64_caches.dcache_line_size = 0x%x\n",	418	printk("ppc64_caches.dcache_line_size = 0x%x\n",
419	ppc64_caches.dline_size);	419	ppc64_caches.dline_size);
420	printk("ppc64_caches.icache_line_size = 0x%x\n",	420	printk("ppc64_caches.icache_line_size = 0x%x\n",
421	ppc64_caches.iline_size);	421	ppc64_caches.iline_size);
422	printk("htab_address = 0x%p\n", htab_address);	422	printk("htab_address = 0x%p\n", htab_address);
423	printk("htab_hash_mask = 0x%lx\n", htab_hash_mask);	423	printk("htab_hash_mask = 0x%lx\n", htab_hash_mask);
424	#if PHYSICAL_START > 0	424	#if PHYSICAL_START > 0
425	printk("physical_start = 0x%x\n", PHYSICAL_START);	425	printk("physical_start = 0x%x\n", PHYSICAL_START);
426	#endif	426	#endif
427	printk("-----------------------------------------------------\n");	427	printk("-----------------------------------------------------\n");
428		428
429	DBG(" <- setup_system()\n");	429	DBG(" <- setup_system()\n");
430	}	430	}
431		431
432	#ifdef CONFIG_IRQSTACKS	432	#ifdef CONFIG_IRQSTACKS
433	static void __init irqstack_early_init(void)	433	static void __init irqstack_early_init(void)
434	{	434	{
435	unsigned int i;	435	unsigned int i;
436		436
437	/*	437	/*
438	* interrupt stacks must be under 256MB, we cannot afford to take	438	* interrupt stacks must be under 256MB, we cannot afford to take
439	* SLB misses on them.	439	* SLB misses on them.
440	*/	440	*/
441	for_each_possible_cpu(i) {	441	for_each_possible_cpu(i) {
442	softirq_ctx[i] = (struct thread_info *)	442	softirq_ctx[i] = (struct thread_info *)
443	__va(lmb_alloc_base(THREAD_SIZE,	443	__va(lmb_alloc_base(THREAD_SIZE,
444	THREAD_SIZE, 0x10000000));	444	THREAD_SIZE, 0x10000000));
445	hardirq_ctx[i] = (struct thread_info *)	445	hardirq_ctx[i] = (struct thread_info *)
446	__va(lmb_alloc_base(THREAD_SIZE,	446	__va(lmb_alloc_base(THREAD_SIZE,
447	THREAD_SIZE, 0x10000000));	447	THREAD_SIZE, 0x10000000));
448	}	448	}
449	}	449	}
450	#else	450	#else
451	#define irqstack_early_init()	451	#define irqstack_early_init()
452	#endif	452	#endif
453		453
454	/*	454	/*
455	* Stack space used when we detect a bad kernel stack pointer, and	455	* Stack space used when we detect a bad kernel stack pointer, and
456	* early in SMP boots before relocation is enabled.	456	* early in SMP boots before relocation is enabled.
457	*/	457	*/
458	static void __init emergency_stack_init(void)	458	static void __init emergency_stack_init(void)
459	{	459	{
460	unsigned long limit;	460	unsigned long limit;
461	unsigned int i;	461	unsigned int i;
462		462
463	/*	463	/*
464	* Emergency stacks must be under 256MB, we cannot afford to take	464	* Emergency stacks must be under 256MB, we cannot afford to take
465	* SLB misses on them. The ABI also requires them to be 128-byte	465	* SLB misses on them. The ABI also requires them to be 128-byte
466	* aligned.	466	* aligned.
467	*	467	*
468	* Since we use these as temporary stacks during secondary CPU	468	* Since we use these as temporary stacks during secondary CPU
469	* bringup, we need to get at them in real mode. This means they	469	* bringup, we need to get at them in real mode. This means they
470	* must also be within the RMO region.	470	* must also be within the RMO region.
471	*/	471	*/
472	limit = min(0x10000000UL, lmb.rmo_size);	472	limit = min(0x10000000UL, lmb.rmo_size);
473		473
474	for_each_possible_cpu(i)	474	for_each_possible_cpu(i)
475	paca[i].emergency_sp =	475	paca[i].emergency_sp =
476	__va(lmb_alloc_base(HW_PAGE_SIZE, 128, limit)) + HW_PAGE_SIZE;	476	__va(lmb_alloc_base(HW_PAGE_SIZE, 128, limit)) + HW_PAGE_SIZE;
477	}	477	}
478		478
479	/*	479	/*
480	* Called into from start_kernel, after lock_kernel has been called.	480	* Called into from start_kernel, after lock_kernel has been called.
481	* Initializes bootmem, which is unsed to manage page allocation until	481	* Initializes bootmem, which is unsed to manage page allocation until
482	* mem_init is called.	482	* mem_init is called.
483	*/	483	*/
484	void __init setup_arch(char **cmdline_p)	484	void __init setup_arch(char **cmdline_p)
485	{	485	{
486	ppc64_boot_msg(0x12, "Setup Arch");	486	ppc64_boot_msg(0x12, "Setup Arch");
487		487
488	*cmdline_p = cmd_line;	488	*cmdline_p = cmd_line;
489		489
490	/*	490	/*
491	* Set cache line size based on type of cpu as a default.	491	* Set cache line size based on type of cpu as a default.
492	* Systems with OF can look in the properties on the cpu node(s)	492	* Systems with OF can look in the properties on the cpu node(s)
493	* for a possibly more accurate value.	493	* for a possibly more accurate value.
494	*/	494	*/
495	dcache_bsize = ppc64_caches.dline_size;	495	dcache_bsize = ppc64_caches.dline_size;
496	icache_bsize = ppc64_caches.iline_size;	496	icache_bsize = ppc64_caches.iline_size;
497		497
498	/* reboot on panic */	498	/* reboot on panic */
499	panic_timeout = 180;	499	panic_timeout = 180;
500		500
501	if (ppc_md.panic)	501	if (ppc_md.panic)
502	setup_panic();	502	setup_panic();
503		503
504	init_mm.start_code = PAGE_OFFSET;	504	init_mm.start_code = PAGE_OFFSET;
505	init_mm.end_code = (unsigned long) _etext;	505	init_mm.end_code = (unsigned long) _etext;
506	init_mm.end_data = (unsigned long) _edata;	506	init_mm.end_data = (unsigned long) _edata;
507	init_mm.brk = klimit;	507	init_mm.brk = klimit;
508		508
509	irqstack_early_init();	509	irqstack_early_init();
510	emergency_stack_init();	510	emergency_stack_init();
511		511
512	stabs_alloc();	512	stabs_alloc();
513		513
514	/* set up the bootmem stuff with available memory */	514	/* set up the bootmem stuff with available memory */
515	do_init_bootmem();	515	do_init_bootmem();
516	sparse_init();	516	sparse_init();
517		517
518	#ifdef CONFIG_DUMMY_CONSOLE	518	#ifdef CONFIG_DUMMY_CONSOLE
519	conswitchp = &dummy_con;	519	conswitchp = &dummy_con;
520	#endif	520	#endif
521		521
522	ppc_md.setup_arch();	522	ppc_md.setup_arch();
523		523
524	paging_init();	524	paging_init();
525	ppc64_boot_msg(0x15, "Setup Done");	525	ppc64_boot_msg(0x15, "Setup Done");
526	}	526	}
527		527
528		528
529	/* ToDo: do something useful if ppc_md is not yet setup. */	529	/* ToDo: do something useful if ppc_md is not yet setup. */
530	#define PPC64_LINUX_FUNCTION 0x0f000000	530	#define PPC64_LINUX_FUNCTION 0x0f000000
531	#define PPC64_IPL_MESSAGE 0xc0000000	531	#define PPC64_IPL_MESSAGE 0xc0000000
532	#define PPC64_TERM_MESSAGE 0xb0000000	532	#define PPC64_TERM_MESSAGE 0xb0000000
533		533
534	static void ppc64_do_msg(unsigned int src, const char *msg)	534	static void ppc64_do_msg(unsigned int src, const char *msg)
535	{	535	{
536	if (ppc_md.progress) {	536	if (ppc_md.progress) {
537	char buf[128];	537	char buf[128];
538		538
539	sprintf(buf, "%08X\n", src);	539	sprintf(buf, "%08X\n", src);
540	ppc_md.progress(buf, 0);	540	ppc_md.progress(buf, 0);
541	snprintf(buf, 128, "%s", msg);	541	snprintf(buf, 128, "%s", msg);
542	ppc_md.progress(buf, 0);	542	ppc_md.progress(buf, 0);
543	}	543	}
544	}	544	}
545		545
546	/* Print a boot progress message. */	546	/* Print a boot progress message. */
547	void ppc64_boot_msg(unsigned int src, const char *msg)	547	void ppc64_boot_msg(unsigned int src, const char *msg)
548	{	548	{
549	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_IPL_MESSAGE\|src, msg);	549	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_IPL_MESSAGE\|src, msg);
550	printk("[boot]%04x %s\n", src, msg);	550	printk("[boot]%04x %s\n", src, msg);
551	}	551	}
552		552
553	/* Print a termination message (print only -- does not stop the kernel) */	553	/* Print a termination message (print only -- does not stop the kernel) */
554	void ppc64_terminate_msg(unsigned int src, const char *msg)	554	void ppc64_terminate_msg(unsigned int src, const char *msg)
555	{	555	{
556	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_TERM_MESSAGE\|src, msg);	556	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_TERM_MESSAGE\|src, msg);
557	printk("[terminate]%04x %s\n", src, msg);	557	printk("[terminate]%04x %s\n", src, msg);
558	}	558	}
559		559
560	void cpu_die(void)	560	void cpu_die(void)
561	{	561	{
562	if (ppc_md.cpu_die)	562	if (ppc_md.cpu_die)
563	ppc_md.cpu_die();	563	ppc_md.cpu_die();
564	}	564	}
565		565
566	#ifdef CONFIG_SMP	566	#ifdef CONFIG_SMP
567	void __init setup_per_cpu_areas(void)	567	void __init setup_per_cpu_areas(void)
568	{	568	{
569	int i;	569	int i;
570	unsigned long size;	570	unsigned long size;
571	char *ptr;	571	char *ptr;
572		572
573	/* Copy section for each CPU (we discard the original) */	573	/* Copy section for each CPU (we discard the original) */
574	size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);	574	size = ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES);
575	#ifdef CONFIG_MODULES	575	#ifdef CONFIG_MODULES
576	if (size < PERCPU_ENOUGH_ROOM)	576	if (size < PERCPU_ENOUGH_ROOM)
577	size = PERCPU_ENOUGH_ROOM;	577	size = PERCPU_ENOUGH_ROOM;
578	#endif	578	#endif
579		579
580	for_each_possible_cpu(i) {	580	for_each_possible_cpu(i) {
581	ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);	581	ptr = alloc_bootmem_node(NODE_DATA(cpu_to_node(i)), size);
582	if (!ptr)	582	if (!ptr)
583	panic("Cannot allocate cpu data for CPU %d\n", i);	583	panic("Cannot allocate cpu data for CPU %d\n", i);
584		584
585	paca[i].data_offset = ptr - __per_cpu_start;	585	paca[i].data_offset = ptr - __per_cpu_start;
586	memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);	586	memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
587	}	587	}
588	}	588	}
589	#endif	589	#endif
590		590

arch/powerpc/platforms/iseries/ksyms.c

Diff comments View file @ d04c56f

 /*
  * (C) 2001-2005 PPC 64 Team, IBM Corp
  *
  *      This program is free software; you can redistribute it and/or
  *      modify it under the terms of the GNU General Public License
  *      as published by the Free Software Foundation; either version
  *      2 of the License, or (at your option) any later version.
  */
 #include <linux/module.h>
 #include <asm/hw_irq.h>
 #include <asm/iseries/hv_call_sc.h>
 EXPORT_SYMBOL(HvCall0);
 EXPORT_SYMBOL(HvCall1);
 EXPORT_SYMBOL(HvCall2);
 EXPORT_SYMBOL(HvCall3);
 EXPORT_SYMBOL(HvCall4);
 EXPORT_SYMBOL(HvCall5);
 EXPORT_SYMBOL(HvCall6);
 EXPORT_SYMBOL(HvCall7);
-#ifdef CONFIG_SMP
-EXPORT_SYMBOL(local_get_flags);
-EXPORT_SYMBOL(local_irq_disable);
-EXPORT_SYMBOL(local_irq_restore);
-#endif

arch/powerpc/platforms/iseries/misc.S

Diff comments View file @ d04c56f

1	/*	1	/*
2	* This file contains miscellaneous low-level functions.	2	* This file contains miscellaneous low-level functions.
3	* Copyright (C) 1995-2005 IBM Corp	3	* Copyright (C) 1995-2005 IBM Corp
4	*	4	*
5	* Largely rewritten by Cort Dougan (cort@cs.nmt.edu)	5	* Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6	* and Paul Mackerras.	6	* and Paul Mackerras.
7	* Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)	7	* Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8	* PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)	8	* PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9	*	9	*
10	* This program is free software; you can redistribute it and/or	10	* This program is free software; you can redistribute it and/or
11	* modify it under the terms of the GNU General Public License	11	* modify it under the terms of the GNU General Public License
12	* as published by the Free Software Foundation; either version	12	* as published by the Free Software Foundation; either version
13	* 2 of the License, or (at your option) any later version.	13	* 2 of the License, or (at your option) any later version.
14	*/	14	*/
15		15
16	#include <asm/processor.h>	16	#include <asm/processor.h>
17	#include <asm/asm-offsets.h>	17	#include <asm/asm-offsets.h>
18	#include <asm/ppc_asm.h>	18	#include <asm/ppc_asm.h>
19		19
20	.text	20	.text
21		21
22	/* unsigned long local_save_flags(void) */	22	/* Handle pending interrupts in interrupt context */
23	_GLOBAL(local_get_flags)	23	_GLOBAL(iseries_handle_interrupts)
24	lbz r3,PACAPROCENABLED(r13)
25	blr
26
27	/* unsigned long local_irq_disable(void) */
28	_GLOBAL(local_irq_disable)
29	lbz r3,PACAPROCENABLED(r13)
30	li r4,0
31	stb r4,PACAPROCENABLED(r13)
32	blr /* Done */
33
34	/* void local_irq_restore(unsigned long flags) */
35	_GLOBAL(local_irq_restore)
36	lbz r5,PACAPROCENABLED(r13)
37	/* Check if things are setup the way we want _already_. */
38	cmpw 0,r3,r5
39	beqlr
40	/* are we enabling interrupts? */
41	cmpdi 0,r3,0
42	stb r3,PACAPROCENABLED(r13)
43	beqlr
44	/* Check pending interrupts */
45	/* A decrementer, IPI or PMC interrupt may have occurred
46	* while we were in the hypervisor (which enables) */
47	ld r4,PACALPPACAPTR(r13)
48	ld r4,LPPACAANYINT(r4)
49	cmpdi r4,0
50	beqlr
51
52	/*
53	* Handle pending interrupts in interrupt context
54	*/
55	li r0,0x5555	24	li r0,0x5555
56	sc	25	sc
57	blr	26	blr
58		27

include/asm-powerpc/hw_irq.h

Diff comments View file @ d04c56f

 /*
  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
  */
 #ifndef _ASM_POWERPC_HW_IRQ_H
 #define _ASM_POWERPC_HW_IRQ_H
 #ifdef __KERNEL__
 #include <linux/errno.h>
+#include <linux/compiler.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
 extern void timer_interrupt(struct pt_regs *);
-#ifdef CONFIG_PPC_ISERIES
+#ifdef CONFIG_PPC64
+#include <asm/paca.h>
-extern unsigned long local_get_flags(void);
+static inline unsigned long local_get_flags(void)
-extern unsigned long local_irq_disable(void);
+{
+	return get_paca()->soft_enabled;
+}
+static inline unsigned long local_irq_disable(void)
+{
+	unsigned long flag = get_paca()->soft_enabled;
+	get_paca()->soft_enabled = 0;
+	barrier();
+	return flag;
+}
 extern void local_irq_restore(unsigned long);
+extern void iseries_handle_interrupts(void);
 #define local_irq_enable()	local_irq_restore(1)
 #define local_save_flags(flags)	((flags) = local_get_flags())
 #define local_irq_save(flags)	((flags) = local_irq_disable())
 #define irqs_disabled()		(local_get_flags() == 0)
+#define hard_irq_enable()	__mtmsrd(mfmsr() | MSR_EE, 1)
+#define hard_irq_disable()	__mtmsrd(mfmsr() & ~MSR_EE, 1)
 #else
 #if defined(CONFIG_BOOKE)
 #define SET_MSR_EE(x)	mtmsr(x)
 #define local_irq_restore(flags)	__asm__ __volatile__("wrtee %0" : : "r" (flags) : "memory")
-#elif defined(__powerpc64__)
-#define SET_MSR_EE(x)	__mtmsrd(x, 1)
-#define local_irq_restore(flags) do { \
-	__asm__ __volatile__("": : :"memory"); \
-	__mtmsrd((flags), 1); \
-} while(0)
 #else
 #define SET_MSR_EE(x)	mtmsr(x)
 #define local_irq_restore(flags)	mtmsr(flags)
 #endif
 static inline void local_irq_disable(void)
 {
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 0": : :"memory");
 #else
 	unsigned long msr;
 	__asm__ __volatile__("": : :"memory");
 	msr = mfmsr();
 	SET_MSR_EE(msr & ~MSR_EE);
 #endif
 }
 static inline void local_irq_enable(void)
 {
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 1": : :"memory");
 #else
 	unsigned long msr;
 	__asm__ __volatile__("": : :"memory");
 	msr = mfmsr();
 	SET_MSR_EE(msr | MSR_EE);
 #endif
 }
 static inline void local_irq_save_ptr(unsigned long *flags)
 {
 	unsigned long msr;
 	msr = mfmsr();
 	*flags = msr;
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 0": : :"memory");
 #else
 	SET_MSR_EE(msr & ~MSR_EE);
 #endif
 	__asm__ __volatile__("": : :"memory");
 }
 #define local_save_flags(flags)	((flags) = mfmsr())
 #define local_irq_save(flags)	local_irq_save_ptr(&flags)
 #define irqs_disabled()		((mfmsr() & MSR_EE) == 0)
-#endif /* CONFIG_PPC_ISERIES */
+#endif /* CONFIG_PPC64 */
 #define mask_irq(irq)						\
 	({							\
 	 	irq_desc_t *desc = get_irq_desc(irq);		\
 		if (desc->chip && desc->chip->disable)	\
 			desc->chip->disable(irq);		\
 	})
 #define unmask_irq(irq)						\
 	({							\
 	 	irq_desc_t *desc = get_irq_desc(irq);		\
 		if (desc->chip && desc->chip->enable)	\
 			desc->chip->enable(irq);		\
 	})
 #define ack_irq(irq)						\
 	({							\
 	 	irq_desc_t *desc = get_irq_desc(irq);		\
 		if (desc->chip && desc->chip->ack)	\
 			desc->chip->ack(irq);		\
 	})
 /*
  * interrupt-retrigger: should we handle this via lost interrupts and IPIs
  * or should we not care like we do now ? --BenH.

include/asm-powerpc/paca.h

Diff comments View file @ d04c56f

 /*
  * include/asm-powerpc/paca.h
  *
  * This control block defines the PACA which defines the processor
  * specific data for each logical processor on the system.
  * There are some pointers defined that are utilized by PLIC.
  *
  * C 2001 PPC 64 Team, IBM Corp
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #ifndef _ASM_POWERPC_PACA_H
 #define _ASM_POWERPC_PACA_H
 #ifdef __KERNEL__
 #include	<asm/types.h>
 #include	<asm/lppaca.h>
 #include	<asm/mmu.h>
 register struct paca_struct *local_paca asm("r13");
 #define get_paca()	local_paca
 #define get_lppaca()	(get_paca()->lppaca_ptr)
 #define get_slb_shadow()	(get_paca()->slb_shadow_ptr)
 struct task_struct;
 /*
  * Defines the layout of the paca.
  *
  * This structure is not directly accessed by firmware or the service
  * processor except for the first two pointers that point to the
  * lppaca area and the ItLpRegSave area for this CPU.  The lppaca
  * object is currently contained within the PACA but it doesn't need
  * to be.
  */
 struct paca_struct {
 	/*
 	 * Because hw_cpu_id, unlike other paca fields, is accessed
 	 * routinely from other CPUs (from the IRQ code), we stick to
 	 * read-only (after boot) fields in the first cacheline to
 	 * avoid cacheline bouncing.
 	 */
 	/*
 	 * MAGIC: These first two pointers can't be moved - they're
 	 * accessed by the firmware
 	 */
 	struct lppaca *lppaca_ptr;	/* Pointer to LpPaca for PLIC */
 #ifdef CONFIG_PPC_ISERIES
 	void *reg_save_ptr; /* Pointer to LpRegSave for PLIC */
 #endif /* CONFIG_PPC_ISERIES */
 	/*
 	 * MAGIC: the spinlock functions in arch/powerpc/lib/locks.c
 	 * load lock_token and paca_index with a single lwz
 	 * instruction.  They must travel together and be properly
 	 * aligned.
 	 */
 	u16 lock_token;			/* Constant 0x8000, used in locks */
 	u16 paca_index;			/* Logical processor number */
 	u64 kernel_toc;			/* Kernel TOC address */
 	u64 stab_real;			/* Absolute address of segment table */
 	u64 stab_addr;			/* Virtual address of segment table */
 	void *emergency_sp;		/* pointer to emergency stack */
 	u64 data_offset;		/* per cpu data offset */
 	s16 hw_cpu_id;			/* Physical processor number */
 	u8 cpu_start;			/* At startup, processor spins until */
 					/* this becomes non-zero. */
 	/*
 	 * Now, starting in cacheline 2, the exception save areas
 	 */
 	/* used for most interrupts/exceptions */
 	u64 exgen[10] __attribute__((aligned(0x80)));
 	u64 exmc[10];		/* used for machine checks */
 	u64 exslb[10];		/* used for SLB/segment table misses
  				 * on the linear mapping */
 	mm_context_t context;
 	u16 vmalloc_sllp;
 	u16 slb_cache[SLB_CACHE_ENTRIES];
 	u16 slb_cache_ptr;
 	/*
 	 * then miscellaneous read-write fields
 	 */
 	struct task_struct *__current;	/* Pointer to current */
 	u64 kstack;			/* Saved Kernel stack addr */
 	u64 stab_rr;			/* stab/slb round-robin counter */
 	u64 saved_r1;			/* r1 save for RTAS calls */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
-	u8 proc_enabled;		/* irq soft-enable flag */
+	u8 soft_enabled;		/* irq soft-enable flag */
+	u8 hard_enabled;		/* set if irqs are enabled in MSR */
 	u8 io_sync;			/* writel() needs spin_unlock sync */
 	/* Stuff for accurate time accounting */
 	u64 user_time;			/* accumulated usermode TB ticks */
 	u64 system_time;		/* accumulated system TB ticks */
 	u64 startpurr;			/* PURR/TB value snapshot */
 	struct slb_shadow *slb_shadow_ptr;
 };
 extern struct paca_struct paca[];
 void setup_boot_paca(void);
 #endif /* __KERNEL__ */
 #endif /* _ASM_POWERPC_PACA_H */